agentv 3.9.2 → 3.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/{chunk-OIVGGWJ3.js → chunk-GWHHM6X2.js} +25 -14
  2. package/dist/chunk-GWHHM6X2.js.map +1 -0
  3. package/dist/{chunk-6ZAFWUBT.js → chunk-JLFFYTZA.js} +4 -4
  4. package/dist/{chunk-JGMJL2LV.js → chunk-TXCVDTEE.js} +8 -7
  5. package/dist/{chunk-JGMJL2LV.js.map → chunk-TXCVDTEE.js.map} +1 -1
  6. package/dist/cli.js +3 -3
  7. package/dist/{dist-PUPHGVKL.js → dist-FPC7J7KQ.js} +2 -2
  8. package/dist/index.js +3 -3
  9. package/dist/{interactive-BD56NB23.js → interactive-N463HRIL.js} +3 -3
  10. package/dist/templates/.agents/skills/agentv-chat-to-eval/README.md +84 -0
  11. package/dist/templates/.agents/skills/agentv-chat-to-eval/SKILL.md +144 -0
  12. package/dist/templates/.agents/skills/agentv-chat-to-eval/examples/transcript-json.md +67 -0
  13. package/dist/templates/.agents/skills/agentv-chat-to-eval/examples/transcript-markdown.md +101 -0
  14. package/dist/templates/.agents/skills/agentv-eval-builder/SKILL.md +458 -0
  15. package/dist/templates/.agents/skills/agentv-eval-builder/references/config-schema.json +36 -0
  16. package/dist/templates/.agents/skills/agentv-eval-builder/references/custom-evaluators.md +118 -0
  17. package/dist/templates/.agents/skills/agentv-eval-builder/references/eval-schema.json +12753 -0
  18. package/dist/templates/.agents/skills/agentv-eval-builder/references/rubric-evaluator.md +77 -0
  19. package/dist/templates/.agents/skills/agentv-eval-orchestrator/SKILL.md +50 -0
  20. package/dist/templates/.agents/skills/agentv-prompt-optimizer/SKILL.md +78 -0
  21. package/dist/templates/.agentv/.env.example +25 -0
  22. package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md +177 -0
  23. package/dist/templates/.claude/skills/agentv-eval-builder/references/batch-cli-evaluator.md +316 -0
  24. package/dist/templates/.claude/skills/agentv-eval-builder/references/compare-command.md +137 -0
  25. package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md +215 -0
  26. package/dist/templates/.claude/skills/agentv-eval-builder/references/config-schema.json +27 -0
  27. package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md +115 -0
  28. package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json +278 -0
  29. package/dist/templates/.claude/skills/agentv-eval-builder/references/example-evals.md +333 -0
  30. package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md +79 -0
  31. package/dist/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md +121 -0
  32. package/dist/templates/.claude/skills/agentv-eval-builder/references/tool-trajectory-evaluator.md +298 -0
  33. package/dist/templates/.claude/skills/agentv-prompt-optimizer/SKILL.md +78 -0
  34. package/dist/templates/.github/prompts/agentv-eval-build.prompt.md +5 -0
  35. package/dist/templates/.github/prompts/agentv-optimize.prompt.md +4 -0
  36. package/package.json +3 -3
  37. package/dist/chunk-OIVGGWJ3.js.map +0 -1
  38. /package/dist/{chunk-6ZAFWUBT.js.map → chunk-JLFFYTZA.js.map} +0 -0
  39. /package/dist/{dist-PUPHGVKL.js.map → dist-FPC7J7KQ.js.map} +0 -0
  40. /package/dist/{interactive-BD56NB23.js.map → interactive-N463HRIL.js.map} +0 -0
@@ -301,7 +301,7 @@ var require_dist = __commonJS({
301
301
  }
302
302
  });
303
303
 
304
- // ../../packages/core/dist/chunk-K7JCJIXA.js
304
+ // ../../packages/core/dist/chunk-VCFYWLFV.js
305
305
  import { constants } from "node:fs";
306
306
  import { access, readFile } from "node:fs/promises";
307
307
  import path from "node:path";
@@ -419,7 +419,7 @@ __export(external_exports2, {
419
419
  void: () => voidType
420
420
  });
421
421
 
422
- // ../../packages/core/dist/chunk-K7JCJIXA.js
422
+ // ../../packages/core/dist/chunk-VCFYWLFV.js
423
423
  import { readFile as readFile2 } from "node:fs/promises";
424
424
  import path3 from "node:path";
425
425
  import fg from "fast-glob";
@@ -840,7 +840,12 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
840
840
  `${parsed.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`
841
841
  );
842
842
  }
843
- const provider = parsed.provider.toLowerCase();
843
+ const provider = resolveString(
844
+ parsed.provider,
845
+ env,
846
+ `${parsed.name} provider`,
847
+ true
848
+ ).toLowerCase();
844
849
  const providerBatching = resolveOptionalBoolean(
845
850
  parsed.provider_batching ?? parsed.providerBatching
846
851
  );
@@ -20057,11 +20062,7 @@ var CopilotCliProvider = class {
20057
20062
  }
20058
20063
  }
20059
20064
  if (sessionUpdate === "usage_update") {
20060
- if (tokenUsage) {
20061
- tokenUsage = { input: update.used, output: tokenUsage.output };
20062
- } else {
20063
- tokenUsage = { input: update.used, output: 0 };
20064
- }
20065
+ tokenUsage = { input: update.used, output: 0 };
20065
20066
  if (update.cost && update.cost.currency === "USD") {
20066
20067
  costUsd = (costUsd ?? 0) + update.cost.amount;
20067
20068
  }
@@ -20095,21 +20096,32 @@ var CopilotCliProvider = class {
20095
20096
  sessionId: session.sessionId,
20096
20097
  prompt: promptMessages
20097
20098
  });
20099
+ let promptResponse;
20098
20100
  if (request.signal) {
20099
20101
  const abortHandler = () => {
20100
20102
  killProcess(agentProcess);
20101
20103
  };
20102
20104
  request.signal.addEventListener("abort", abortHandler, { once: true });
20103
20105
  try {
20104
- await this.raceWithTimeout(sendPromise, agentProcess);
20106
+ promptResponse = await this.raceWithTimeout(sendPromise, agentProcess);
20105
20107
  } finally {
20106
20108
  request.signal.removeEventListener("abort", abortHandler);
20107
20109
  }
20108
20110
  } else {
20109
- await this.raceWithTimeout(sendPromise, agentProcess);
20111
+ promptResponse = await this.raceWithTimeout(sendPromise, agentProcess);
20110
20112
  }
20111
20113
  const endTime = (/* @__PURE__ */ new Date()).toISOString();
20112
20114
  const durationMs = Date.now() - startMs;
20115
+ const responseUsage = promptResponse.usage;
20116
+ if (responseUsage && responseUsage.totalTokens > 0) {
20117
+ tokenUsage = {
20118
+ input: responseUsage.inputTokens,
20119
+ output: responseUsage.outputTokens,
20120
+ ...responseUsage.thoughtTokens != null ? { reasoning: responseUsage.thoughtTokens } : {},
20121
+ ...responseUsage.cachedReadTokens != null ? { cached: responseUsage.cachedReadTokens } : {}
20122
+ };
20123
+ request.streamCallbacks?.onLlmCallEnd?.("copilot", tokenUsage);
20124
+ }
20113
20125
  const rejectedCalls = completedToolCalls.filter((tc) => {
20114
20126
  const out = tc.output;
20115
20127
  return out && (out.code === "rejected" || out.code === "denied");
@@ -20167,8 +20179,7 @@ var CopilotCliProvider = class {
20167
20179
  async raceWithTimeout(sendPromise, agentProcess) {
20168
20180
  const timeoutMs = this.config.timeoutMs;
20169
20181
  if (!timeoutMs) {
20170
- await sendPromise;
20171
- return;
20182
+ return sendPromise;
20172
20183
  }
20173
20184
  let timer;
20174
20185
  const timeoutPromise = new Promise((_, reject) => {
@@ -20179,7 +20190,7 @@ var CopilotCliProvider = class {
20179
20190
  timer.unref?.();
20180
20191
  });
20181
20192
  try {
20182
- await Promise.race([sendPromise, timeoutPromise]);
20193
+ return await Promise.race([sendPromise, timeoutPromise]);
20183
20194
  } finally {
20184
20195
  if (timer) clearTimeout(timer);
20185
20196
  }
@@ -31051,4 +31062,4 @@ export {
31051
31062
  OtelStreamingObserver,
31052
31063
  createAgentKernel
31053
31064
  };
31054
- //# sourceMappingURL=chunk-OIVGGWJ3.js.map
31065
+ //# sourceMappingURL=chunk-GWHHM6X2.js.map