agentv 2.8.0-next.1 → 2.9.0-next.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +28 -2
  2. package/dist/{chunk-RCFB5QFS.js → chunk-3INJ7ISP.js} +46 -85
  3. package/dist/chunk-3INJ7ISP.js.map +1 -0
  4. package/dist/{chunk-2SXGPQVR.js → chunk-PC3FAOHT.js} +4 -4
  5. package/dist/chunk-PC3FAOHT.js.map +1 -0
  6. package/dist/{chunk-DJCMBVB3.js → chunk-RJWTL3VS.js} +166 -75
  7. package/dist/chunk-RJWTL3VS.js.map +1 -0
  8. package/dist/cli.js +3 -3
  9. package/dist/{dist-T7REAXNS.js → dist-BGRU67HI.js} +2 -2
  10. package/dist/index.js +3 -3
  11. package/dist/{interactive-TE5SJPJW.js → interactive-7KFUCBIP.js} +3 -3
  12. package/dist/templates/.agents/skills/agentv-eval-builder/SKILL.md +1 -1
  13. package/dist/templates/.agents/skills/agentv-eval-builder/references/custom-evaluators.md +8 -8
  14. package/dist/templates/.agents/skills/agentv-eval-builder/references/eval-schema.json +12683 -237
  15. package/dist/templates/.agentv/config.yaml +1 -1
  16. package/dist/templates/.agentv/targets.yaml +10 -13
  17. package/package.json +2 -2
  18. package/dist/chunk-2SXGPQVR.js.map +0 -1
  19. package/dist/chunk-DJCMBVB3.js.map +0 -1
  20. package/dist/chunk-RCFB5QFS.js.map +0 -1
  21. package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md +0 -202
  22. package/dist/templates/.claude/skills/agentv-eval-builder/references/batch-cli-evaluator.md +0 -316
  23. package/dist/templates/.claude/skills/agentv-eval-builder/references/compare-command.md +0 -137
  24. package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md +0 -215
  25. package/dist/templates/.claude/skills/agentv-eval-builder/references/config-schema.json +0 -27
  26. package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md +0 -118
  27. package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json +0 -278
  28. package/dist/templates/.claude/skills/agentv-eval-builder/references/example-evals.md +0 -333
  29. package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md +0 -77
  30. package/dist/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md +0 -121
  31. package/dist/templates/.claude/skills/agentv-eval-builder/references/tool-trajectory-evaluator.md +0 -298
  32. package/dist/templates/.claude/skills/agentv-prompt-optimizer/SKILL.md +0 -78
  33. package/dist/templates/.github/prompts/agentv-eval-build.prompt.md +0 -5
  34. package/dist/templates/.github/prompts/agentv-optimize.prompt.md +0 -4
  35. /package/dist/{dist-T7REAXNS.js.map → dist-BGRU67HI.js.map} +0 -0
  36. /package/dist/{interactive-TE5SJPJW.js.map → interactive-7KFUCBIP.js.map} +0 -0
@@ -148,7 +148,7 @@ var require_dist = __commonJS({
148
148
  }
149
149
  });
150
150
 
151
- // ../../packages/core/dist/chunk-P2465XAH.js
151
+ // ../../packages/core/dist/chunk-7Q4PH265.js
152
152
  import { constants } from "node:fs";
153
153
  import { access, readFile } from "node:fs/promises";
154
154
  import path from "node:path";
@@ -4195,7 +4195,7 @@ var coerce = {
4195
4195
  };
4196
4196
  var NEVER = INVALID;
4197
4197
 
4198
- // ../../packages/core/dist/chunk-P2465XAH.js
4198
+ // ../../packages/core/dist/chunk-7Q4PH265.js
4199
4199
  var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
4200
4200
  var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
4201
4201
  var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
@@ -34030,14 +34030,16 @@ function computeTraceSummary(messages) {
34030
34030
  }
34031
34031
  const toolNames = Object.keys(toolCallCounts).sort();
34032
34032
  return {
34033
- eventCount: totalToolCalls,
34034
- toolNames,
34035
- toolCallsByName: toolCallCounts,
34036
- errorCount: 0,
34033
+ trace: {
34034
+ eventCount: totalToolCalls,
34035
+ toolNames,
34036
+ toolCallsByName: toolCallCounts,
34037
+ errorCount: 0,
34038
+ llmCallCount,
34039
+ ...hasAnyDuration ? { toolDurations } : {}
34040
+ },
34037
34041
  startTime: earliestStart?.toISOString(),
34038
- endTime: latestEnd?.toISOString(),
34039
- llmCallCount,
34040
- ...hasAnyDuration ? { toolDurations } : {}
34042
+ endTime: latestEnd?.toISOString()
34041
34043
  };
34042
34044
  }
34043
34045
  var DEFAULT_EXPLORATION_TOOLS = [
@@ -34060,9 +34062,9 @@ function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS)
34060
34062
  );
34061
34063
  return explorationCalls / summary.eventCount;
34062
34064
  }
34063
- function tokensPerTool(summary) {
34064
- if (!summary.tokenUsage || summary.eventCount === 0) return void 0;
34065
- const totalTokens = summary.tokenUsage.input + summary.tokenUsage.output;
34065
+ function tokensPerTool(summary, tokenUsage) {
34066
+ if (!tokenUsage || summary.eventCount === 0) return void 0;
34067
+ const totalTokens = tokenUsage.input + tokenUsage.output;
34066
34068
  return totalTokens / summary.eventCount;
34067
34069
  }
34068
34070
  function avgToolDurationMs(summary) {
@@ -34078,16 +34080,15 @@ function avgToolDurationMs(summary) {
34078
34080
  if (totalCalls === 0) return void 0;
34079
34081
  return totalDuration / totalCalls;
34080
34082
  }
34081
- function mergeExecutionMetrics(summary, metrics) {
34082
- if (!metrics) return summary;
34083
+ function mergeExecutionMetrics(computed, metrics) {
34084
+ if (!metrics) return computed;
34083
34085
  return {
34084
- ...summary,
34086
+ trace: computed.trace,
34085
34087
  tokenUsage: metrics.tokenUsage,
34086
34088
  costUsd: metrics.costUsd,
34087
34089
  durationMs: metrics.durationMs,
34088
- // Provider-level timing takes precedence over span-derived timing
34089
- startTime: metrics.startTime ?? summary.startTime,
34090
- endTime: metrics.endTime ?? summary.endTime
34090
+ startTime: metrics.startTime ?? computed.startTime,
34091
+ endTime: metrics.endTime ?? computed.endTime
34091
34092
  };
34092
34093
  }
34093
34094
  var ANSI_YELLOW = "\x1B[33m";
@@ -36973,10 +36974,13 @@ async function invokeModel(options) {
36973
36974
  }
36974
36975
  function mapResponse(result) {
36975
36976
  const content = result.text ?? "";
36977
+ const rawUsage = result.totalUsage ?? result.usage;
36978
+ const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
36976
36979
  return {
36977
36980
  raw: result,
36978
- usage: toJsonObject(result.totalUsage ?? result.usage),
36979
- output: [{ role: "assistant", content }]
36981
+ usage: toJsonObject(rawUsage),
36982
+ output: [{ role: "assistant", content }],
36983
+ tokenUsage
36980
36984
  };
36981
36985
  }
36982
36986
  function toJsonObject(value) {
@@ -42100,6 +42104,8 @@ async function createTargetProxy(options) {
42100
42104
  const token = randomBytes(32).toString("hex");
42101
42105
  let callCount = 0;
42102
42106
  let isShutdown = false;
42107
+ let totalInputTokens = 0;
42108
+ let totalOutputTokens = 0;
42103
42109
  const targetsList = availableTargets ?? [defaultProvider.targetName];
42104
42110
  function resolveProvider(targetName) {
42105
42111
  if (targetName === void 0 || targetName === defaultProvider.targetName) {
@@ -42178,11 +42184,16 @@ async function createTargetProxy(options) {
42178
42184
  evalCaseId: request.evalCaseId ?? "proxy",
42179
42185
  attempt: request.attempt ?? 1
42180
42186
  });
42187
+ if (response.tokenUsage) {
42188
+ totalInputTokens += response.tokenUsage.input;
42189
+ totalOutputTokens += response.tokenUsage.output;
42190
+ }
42181
42191
  const output = response.output ?? [];
42182
42192
  const rawText = extractLastAssistantContent2(output);
42183
42193
  const result = {
42184
42194
  output,
42185
- rawText
42195
+ rawText,
42196
+ tokenUsage: response.tokenUsage
42186
42197
  };
42187
42198
  sendJson(res, 200, result);
42188
42199
  } catch (error40) {
@@ -42229,10 +42240,15 @@ async function createTargetProxy(options) {
42229
42240
  evalCaseId: request.evalCaseId ?? "proxy",
42230
42241
  attempt: request.attempt ?? 1
42231
42242
  });
42243
+ if (response.tokenUsage) {
42244
+ totalInputTokens += response.tokenUsage.input;
42245
+ totalOutputTokens += response.tokenUsage.output;
42246
+ }
42232
42247
  const output = response.output ?? [];
42233
42248
  responses.push({
42234
42249
  output,
42235
- rawText: extractLastAssistantContent2(output)
42250
+ rawText: extractLastAssistantContent2(output),
42251
+ tokenUsage: response.tokenUsage
42236
42252
  });
42237
42253
  } catch (error40) {
42238
42254
  const message = error40 instanceof Error ? error40.message : String(error40);
@@ -42271,7 +42287,8 @@ async function createTargetProxy(options) {
42271
42287
  },
42272
42288
  getUsageMetadata: () => ({
42273
42289
  callCount,
42274
- maxCalls
42290
+ maxCalls,
42291
+ tokenUsage: totalInputTokens > 0 || totalOutputTokens > 0 ? { input: totalInputTokens, output: totalOutputTokens } : void 0
42275
42292
  })
42276
42293
  };
42277
42294
  }
@@ -42392,6 +42409,11 @@ var CodeEvaluator = class {
42392
42409
  ),
42393
42410
  input: context.evalCase.input,
42394
42411
  trace: context.trace ?? null,
42412
+ tokenUsage: context.tokenUsage ?? null,
42413
+ costUsd: context.costUsd ?? null,
42414
+ durationMs: context.durationMs ?? null,
42415
+ startTime: context.startTime ?? null,
42416
+ endTime: context.endTime ?? null,
42395
42417
  fileChanges: context.fileChanges ?? null,
42396
42418
  workspacePath: context.workspacePath ?? null,
42397
42419
  config: this.config ?? null
@@ -42450,7 +42472,8 @@ var CodeEvaluator = class {
42450
42472
  expectedAspectCount: hits.length + misses.length || 1,
42451
42473
  reasoning,
42452
42474
  evaluatorRawRequest,
42453
- ...details ? { details } : {}
42475
+ ...details ? { details } : {},
42476
+ tokenUsage: proxyUsage?.tokenUsage
42454
42477
  };
42455
42478
  } catch (error40) {
42456
42479
  const message = error40 instanceof Error ? error40.message : String(error40);
@@ -42472,7 +42495,8 @@ var CodeEvaluator = class {
42472
42495
  }
42473
42496
  } : {},
42474
42497
  error: message
42475
- }
42498
+ },
42499
+ tokenUsage: proxyUsage?.tokenUsage
42476
42500
  };
42477
42501
  } finally {
42478
42502
  if (proxyShutdown) {
@@ -42600,7 +42624,7 @@ ${context.fileChanges}`;
42600
42624
  target: judgeProvider.targetName
42601
42625
  };
42602
42626
  try {
42603
- const { data } = await this.runWithRetry({
42627
+ const { data, tokenUsage } = await this.runWithRetry({
42604
42628
  context,
42605
42629
  judgeProvider,
42606
42630
  systemPrompt,
@@ -42619,7 +42643,8 @@ ${context.fileChanges}`;
42619
42643
  misses,
42620
42644
  expectedAspectCount,
42621
42645
  reasoning,
42622
- evaluatorRawRequest
42646
+ evaluatorRawRequest,
42647
+ tokenUsage
42623
42648
  };
42624
42649
  } catch {
42625
42650
  return {
@@ -42649,7 +42674,7 @@ ${context.fileChanges}`;
42649
42674
  systemPrompt,
42650
42675
  target: judgeProvider.targetName
42651
42676
  };
42652
- const { data } = await this.runWithRetry({
42677
+ const { data, tokenUsage } = await this.runWithRetry({
42653
42678
  context,
42654
42679
  judgeProvider,
42655
42680
  systemPrompt,
@@ -42664,7 +42689,8 @@ ${context.fileChanges}`;
42664
42689
  misses,
42665
42690
  expectedAspectCount: rubrics.length,
42666
42691
  reasoning: data.overall_reasoning,
42667
- evaluatorRawRequest
42692
+ evaluatorRawRequest,
42693
+ tokenUsage
42668
42694
  };
42669
42695
  }
42670
42696
  /**
@@ -42679,7 +42705,7 @@ ${context.fileChanges}`;
42679
42705
  systemPrompt,
42680
42706
  target: judgeProvider.targetName
42681
42707
  };
42682
- const { data } = await this.runWithRetry({
42708
+ const { data, tokenUsage } = await this.runWithRetry({
42683
42709
  context,
42684
42710
  judgeProvider,
42685
42711
  systemPrompt,
@@ -42695,7 +42721,8 @@ ${context.fileChanges}`;
42695
42721
  expectedAspectCount: rubrics.length,
42696
42722
  reasoning: data.overall_reasoning,
42697
42723
  evaluatorRawRequest,
42698
- details
42724
+ details,
42725
+ tokenUsage
42699
42726
  };
42700
42727
  }
42701
42728
  /**
@@ -42779,15 +42806,17 @@ ${context.fileChanges}`;
42779
42806
  try {
42780
42807
  const model = judgeProvider.asLanguageModel?.();
42781
42808
  if (model) {
42782
- const { text: text2 } = await generateText({
42809
+ const result = await generateText({
42783
42810
  model,
42784
42811
  system: systemPrompt,
42785
42812
  prompt: userPrompt,
42786
42813
  ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
42787
42814
  ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
42788
42815
  });
42789
- const data2 = schema.parse(parseJsonFromText(text2));
42790
- return { data: data2 };
42816
+ const data2 = schema.parse(parseJsonFromText(result.text));
42817
+ const rawUsage = result.usage;
42818
+ const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
42819
+ return { data: data2, tokenUsage };
42791
42820
  }
42792
42821
  const response = await judgeProvider.invoke({
42793
42822
  question: userPrompt,
@@ -42798,7 +42827,7 @@ ${context.fileChanges}`;
42798
42827
  temperature: this.temperature
42799
42828
  });
42800
42829
  const data = schema.parse(parseJsonFromText(extractLastAssistantContent(response.output)));
42801
- return { data, providerResponse: response };
42830
+ return { data, providerResponse: response, tokenUsage: response.tokenUsage };
42802
42831
  } catch (e) {
42803
42832
  lastError = e instanceof Error ? e : new Error(String(e));
42804
42833
  }
@@ -43002,7 +43031,8 @@ var CompositeEvaluator = class {
43002
43031
  reasoning: member.result.reasoning,
43003
43032
  evaluatorRawRequest: member.result.evaluatorRawRequest,
43004
43033
  scores: member.result.scores,
43005
- details: member.result.details
43034
+ details: member.result.details,
43035
+ tokenUsage: member.result.tokenUsage
43006
43036
  });
43007
43037
  }
43008
43038
  const finalScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
@@ -43050,7 +43080,8 @@ var CompositeEvaluator = class {
43050
43080
  reasoning: member.result.reasoning,
43051
43081
  evaluatorRawRequest: member.result.evaluatorRawRequest,
43052
43082
  scores: member.result.scores,
43053
- details: member.result.details
43083
+ details: member.result.details,
43084
+ tokenUsage: member.result.tokenUsage
43054
43085
  });
43055
43086
  }
43056
43087
  const totalCount = results.length;
@@ -43227,7 +43258,7 @@ var CostEvaluator = class {
43227
43258
  }
43228
43259
  evaluate(context) {
43229
43260
  const { budget } = this.config;
43230
- const costUsd = context.trace?.costUsd;
43261
+ const costUsd = context.costUsd;
43231
43262
  if (costUsd === void 0) {
43232
43263
  return {
43233
43264
  score: 0,
@@ -43268,7 +43299,7 @@ var ExecutionMetricsEvaluator = class {
43268
43299
  this.config = options.config;
43269
43300
  }
43270
43301
  evaluate(context) {
43271
- const { trace: trace2 } = context;
43302
+ const { trace: trace2, tokenUsage, costUsd, durationMs } = context;
43272
43303
  const {
43273
43304
  max_tool_calls,
43274
43305
  max_llm_calls,
@@ -43278,7 +43309,8 @@ var ExecutionMetricsEvaluator = class {
43278
43309
  target_exploration_ratio,
43279
43310
  exploration_tolerance = 0.2
43280
43311
  } = this.config;
43281
- if (!trace2) {
43312
+ const needsTrace = max_tool_calls !== void 0 || max_llm_calls !== void 0 || target_exploration_ratio !== void 0;
43313
+ if (needsTrace && !trace2) {
43282
43314
  return {
43283
43315
  score: 0,
43284
43316
  verdict: "fail",
@@ -43293,11 +43325,12 @@ var ExecutionMetricsEvaluator = class {
43293
43325
  }
43294
43326
  };
43295
43327
  }
43328
+ const narrowedTrace = trace2;
43296
43329
  const hits = [];
43297
43330
  const misses = [];
43298
43331
  const actualMetrics = {};
43299
- if (max_tool_calls !== void 0) {
43300
- const toolCalls = trace2.eventCount;
43332
+ if (max_tool_calls !== void 0 && narrowedTrace) {
43333
+ const toolCalls = narrowedTrace.eventCount;
43301
43334
  actualMetrics.tool_calls = toolCalls;
43302
43335
  if (toolCalls <= max_tool_calls) {
43303
43336
  hits.push(`Tool calls ${toolCalls} <= ${max_tool_calls} max`);
@@ -43305,8 +43338,8 @@ var ExecutionMetricsEvaluator = class {
43305
43338
  misses.push(`Tool calls ${toolCalls} > ${max_tool_calls} max`);
43306
43339
  }
43307
43340
  }
43308
- if (max_llm_calls !== void 0) {
43309
- const llmCalls = trace2.llmCallCount;
43341
+ if (max_llm_calls !== void 0 && narrowedTrace) {
43342
+ const llmCalls = narrowedTrace.llmCallCount;
43310
43343
  if (llmCalls === void 0) {
43311
43344
  misses.push("LLM call count data not available");
43312
43345
  } else {
@@ -43319,7 +43352,6 @@ var ExecutionMetricsEvaluator = class {
43319
43352
  }
43320
43353
  }
43321
43354
  if (max_tokens !== void 0) {
43322
- const tokenUsage = trace2.tokenUsage;
43323
43355
  if (!tokenUsage) {
43324
43356
  misses.push("Token usage data not available");
43325
43357
  } else {
@@ -43333,7 +43365,6 @@ var ExecutionMetricsEvaluator = class {
43333
43365
  }
43334
43366
  }
43335
43367
  if (max_cost_usd !== void 0) {
43336
- const costUsd = trace2.costUsd;
43337
43368
  if (costUsd === void 0) {
43338
43369
  misses.push("Cost data not available");
43339
43370
  } else {
@@ -43347,7 +43378,6 @@ var ExecutionMetricsEvaluator = class {
43347
43378
  }
43348
43379
  }
43349
43380
  if (max_duration_ms !== void 0) {
43350
- const durationMs = trace2.durationMs;
43351
43381
  if (durationMs === void 0) {
43352
43382
  misses.push("Duration data not available");
43353
43383
  } else {
@@ -43359,8 +43389,8 @@ var ExecutionMetricsEvaluator = class {
43359
43389
  }
43360
43390
  }
43361
43391
  }
43362
- if (target_exploration_ratio !== void 0) {
43363
- const ratio = explorationRatio(trace2);
43392
+ if (target_exploration_ratio !== void 0 && narrowedTrace) {
43393
+ const ratio = explorationRatio(narrowedTrace);
43364
43394
  if (ratio === void 0) {
43365
43395
  misses.push("Exploration ratio not available (no tool calls)");
43366
43396
  } else {
@@ -43870,7 +43900,7 @@ var LatencyEvaluator = class {
43870
43900
  }
43871
43901
  evaluate(context) {
43872
43902
  const { threshold } = this.config;
43873
- const durationMs = context.trace?.durationMs;
43903
+ const durationMs = context.durationMs;
43874
43904
  if (durationMs === void 0) {
43875
43905
  return {
43876
43906
  score: 0,
@@ -44505,7 +44535,7 @@ var TokenUsageEvaluator = class {
44505
44535
  this.config = options.config;
44506
44536
  }
44507
44537
  evaluate(context) {
44508
- const usage = context.trace?.tokenUsage;
44538
+ const usage = context.tokenUsage;
44509
44539
  const maxTotal = this.config.max_total;
44510
44540
  const maxInput = this.config.max_input;
44511
44541
  const maxOutput = this.config.max_output;
@@ -46149,7 +46179,7 @@ async function runEvaluation(options) {
46149
46179
  caseCost = trialCostSum;
46150
46180
  }
46151
46181
  } else {
46152
- caseCost = result.trace?.costUsd;
46182
+ caseCost = result.costUsd;
46153
46183
  }
46154
46184
  if (caseCost !== void 0) {
46155
46185
  cumulativeBudgetCost += caseCost;
@@ -46310,17 +46340,18 @@ async function runBatchEvaluation(options) {
46310
46340
  const providerResponse = batchResponse[i];
46311
46341
  const output = providerResponse.output;
46312
46342
  const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
46313
- const baseSummary = output ? computeTraceSummary(output) : hasExecutionMetrics ? {
46314
- eventCount: 0,
46315
- toolNames: [],
46316
- toolCallsByName: {},
46317
- errorCount: 0
46318
- } : void 0;
46319
- const trace2 = baseSummary ? mergeExecutionMetrics(baseSummary, {
46343
+ const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
46344
+ const merged = computed ? mergeExecutionMetrics(computed, {
46320
46345
  tokenUsage: providerResponse.tokenUsage,
46321
46346
  costUsd: providerResponse.costUsd,
46322
46347
  durationMs: providerResponse.durationMs
46323
46348
  }) : void 0;
46349
+ const trace2 = merged?.trace;
46350
+ const costUsd = merged?.costUsd;
46351
+ const durationMs = merged?.durationMs;
46352
+ const tokenUsage = merged?.tokenUsage;
46353
+ const startTime = merged?.startTime;
46354
+ const endTime = merged?.endTime;
46324
46355
  const candidate = extractLastAssistantContent(output);
46325
46356
  const providerError = extractProviderError(providerResponse);
46326
46357
  let result;
@@ -46339,6 +46370,11 @@ async function runBatchEvaluation(options) {
46339
46370
  agentTimeoutMs,
46340
46371
  output,
46341
46372
  trace: trace2,
46373
+ costUsd,
46374
+ durationMs,
46375
+ tokenUsage,
46376
+ startTime,
46377
+ endTime,
46342
46378
  targetResolver,
46343
46379
  availableTargets
46344
46380
  });
@@ -46575,17 +46611,18 @@ async function runEvalCase(options) {
46575
46611
  }
46576
46612
  const output = providerResponse.output;
46577
46613
  const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
46578
- const baseSummary = output ? computeTraceSummary(output) : hasExecutionMetrics ? {
46579
- eventCount: 0,
46580
- toolNames: [],
46581
- toolCallsByName: {},
46582
- errorCount: 0
46583
- } : void 0;
46584
- const trace2 = baseSummary ? mergeExecutionMetrics(baseSummary, {
46614
+ const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
46615
+ const merged = computed ? mergeExecutionMetrics(computed, {
46585
46616
  tokenUsage: providerResponse.tokenUsage,
46586
46617
  costUsd: providerResponse.costUsd,
46587
46618
  durationMs: providerResponse.durationMs
46588
46619
  }) : void 0;
46620
+ const trace2 = merged?.trace;
46621
+ const costUsd = merged?.costUsd;
46622
+ const durationMs = merged?.durationMs;
46623
+ const tokenUsage = merged?.tokenUsage;
46624
+ const startTime = merged?.startTime;
46625
+ const endTime = merged?.endTime;
46589
46626
  const candidate = extractLastAssistantContent(output);
46590
46627
  let fileChanges;
46591
46628
  if (baselineCommit && workspacePath) {
@@ -46630,6 +46667,11 @@ async function runEvalCase(options) {
46630
46667
  agentTimeoutMs,
46631
46668
  output,
46632
46669
  trace: trace2,
46670
+ costUsd,
46671
+ durationMs,
46672
+ tokenUsage,
46673
+ startTime,
46674
+ endTime,
46633
46675
  targetResolver,
46634
46676
  availableTargets,
46635
46677
  fileChanges,
@@ -46686,7 +46728,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
46686
46728
  };
46687
46729
  const result = await runEvalCase(trialOptions);
46688
46730
  allResults.push(result);
46689
- const trialCost = result.trace?.costUsd;
46731
+ const trialCost = result.costUsd;
46690
46732
  const trialVerdict = scoreToVerdict(result.score);
46691
46733
  const trial = {
46692
46734
  attempt,
@@ -46742,6 +46784,11 @@ async function evaluateCandidate(options) {
46742
46784
  agentTimeoutMs,
46743
46785
  output,
46744
46786
  trace: trace2,
46787
+ costUsd,
46788
+ durationMs,
46789
+ tokenUsage,
46790
+ startTime,
46791
+ endTime,
46745
46792
  targetResolver,
46746
46793
  availableTargets,
46747
46794
  fileChanges,
@@ -46762,6 +46809,11 @@ async function evaluateCandidate(options) {
46762
46809
  agentTimeoutMs,
46763
46810
  output,
46764
46811
  trace: trace2,
46812
+ costUsd,
46813
+ durationMs,
46814
+ tokenUsage,
46815
+ startTime,
46816
+ endTime,
46765
46817
  targetResolver,
46766
46818
  availableTargets,
46767
46819
  fileChanges,
@@ -46805,6 +46857,11 @@ async function evaluateCandidate(options) {
46805
46857
  answer: candidate,
46806
46858
  target: target.name,
46807
46859
  reasoning: score.reasoning,
46860
+ tokenUsage,
46861
+ costUsd,
46862
+ durationMs,
46863
+ startTime,
46864
+ endTime,
46808
46865
  requests,
46809
46866
  input,
46810
46867
  scores,
@@ -46828,6 +46885,11 @@ async function runEvaluatorsForCase(options) {
46828
46885
  agentTimeoutMs,
46829
46886
  output,
46830
46887
  trace: trace2,
46888
+ costUsd,
46889
+ durationMs,
46890
+ tokenUsage,
46891
+ startTime,
46892
+ endTime,
46831
46893
  targetResolver,
46832
46894
  availableTargets,
46833
46895
  fileChanges,
@@ -46849,6 +46911,11 @@ async function runEvaluatorsForCase(options) {
46849
46911
  agentTimeoutMs,
46850
46912
  output,
46851
46913
  trace: trace2,
46914
+ costUsd,
46915
+ durationMs,
46916
+ tokenUsage,
46917
+ startTime,
46918
+ endTime,
46852
46919
  targetResolver,
46853
46920
  availableTargets,
46854
46921
  fileChanges,
@@ -46871,6 +46938,11 @@ async function runEvaluatorsForCase(options) {
46871
46938
  judgeProvider,
46872
46939
  output,
46873
46940
  trace: trace2,
46941
+ tokenUsage,
46942
+ costUsd,
46943
+ durationMs,
46944
+ startTime,
46945
+ endTime,
46874
46946
  targetResolver,
46875
46947
  availableTargets,
46876
46948
  fileChanges,
@@ -46894,6 +46966,11 @@ async function runEvaluatorList(options) {
46894
46966
  agentTimeoutMs,
46895
46967
  output,
46896
46968
  trace: trace2,
46969
+ costUsd,
46970
+ durationMs,
46971
+ tokenUsage,
46972
+ startTime,
46973
+ endTime,
46897
46974
  targetResolver,
46898
46975
  availableTargets,
46899
46976
  fileChanges,
@@ -46912,6 +46989,11 @@ async function runEvaluatorList(options) {
46912
46989
  judgeProvider,
46913
46990
  output,
46914
46991
  trace: trace2,
46992
+ tokenUsage,
46993
+ costUsd,
46994
+ durationMs,
46995
+ startTime,
46996
+ endTime,
46915
46997
  targetResolver,
46916
46998
  availableTargets,
46917
46999
  fileChanges,
@@ -46951,7 +47033,8 @@ async function runEvaluatorList(options) {
46951
47033
  reasoning: score2.reasoning,
46952
47034
  evaluatorProviderRequest: score2.evaluatorRawRequest,
46953
47035
  details: score2.details,
46954
- scores: mapChildResults(score2.scores)
47036
+ scores: mapChildResults(score2.scores),
47037
+ tokenUsage: score2.tokenUsage
46955
47038
  });
46956
47039
  } catch (error40) {
46957
47040
  const message = error40 instanceof Error ? error40.message : String(error40);
@@ -47199,7 +47282,8 @@ function mapChildResults(children) {
47199
47282
  reasoning: child.reasoning,
47200
47283
  evaluatorProviderRequest: child.evaluatorRawRequest,
47201
47284
  scores: mapChildResults(child.scores),
47202
- details: child.details
47285
+ details: child.details,
47286
+ tokenUsage: child.tokenUsage
47203
47287
  }));
47204
47288
  }
47205
47289
  function computeWeightedMean(entries) {
@@ -47562,7 +47646,13 @@ var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
47562
47646
  "beforeEachOutput",
47563
47647
  "afterAllOutput",
47564
47648
  "afterEachOutput",
47565
- "fileChanges"
47649
+ "fileChanges",
47650
+ // Promoted execution metrics (debug, not needed for regression comparison)
47651
+ "tokenUsage",
47652
+ "costUsd",
47653
+ "durationMs",
47654
+ "startTime",
47655
+ "endTime"
47566
47656
  ]);
47567
47657
  var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "evaluatorProviderRequest"]);
47568
47658
  function trimEvaluatorResult(result) {
@@ -47683,8 +47773,8 @@ var OtelTraceExporter = class {
47683
47773
  const api = this.api;
47684
47774
  const tracer = this.tracer;
47685
47775
  const captureContent = this.options.captureContent ?? false;
47686
- const startHr = toHrTime(result.trace?.startTime ?? result.timestamp);
47687
- const endHr = toHrTime(result.trace?.endTime ?? result.timestamp);
47776
+ const startHr = toHrTime(result.startTime ?? result.timestamp);
47777
+ const endHr = toHrTime(result.endTime ?? result.timestamp);
47688
47778
  let parentCtx = api.ROOT_CONTEXT;
47689
47779
  const traceparent = process.env.TRACEPARENT;
47690
47780
  if (traceparent && this.W3CPropagator) {
@@ -47713,12 +47803,13 @@ var OtelTraceExporter = class {
47713
47803
  if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
47714
47804
  rootSpan.setAttribute("agentv.score", result.score);
47715
47805
  if (captureContent) rootSpan.setAttribute("agentv.answer", result.answer);
47806
+ if (result.durationMs != null)
47807
+ rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
47808
+ if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
47716
47809
  if (result.trace) {
47717
47810
  const t = result.trace;
47718
47811
  rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
47719
47812
  rootSpan.setAttribute("agentv.trace.tool_names", t.toolNames.join(","));
47720
- if (t.durationMs != null) rootSpan.setAttribute("agentv.trace.duration_ms", t.durationMs);
47721
- if (t.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", t.costUsd);
47722
47813
  if (t.llmCallCount != null)
47723
47814
  rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
47724
47815
  }
@@ -48105,4 +48196,4 @@ export {
48105
48196
  OtelStreamingObserver,
48106
48197
  createAgentKernel
48107
48198
  };
48108
- //# sourceMappingURL=chunk-DJCMBVB3.js.map
48199
+ //# sourceMappingURL=chunk-RJWTL3VS.js.map