@agentv/core 2.8.0-next.1 → 2.9.0-next.1

package/dist/evaluation/validation/index.js

@@ -6,7 +6,7 @@ import {
   findGitRoot,
   isEvaluatorKind,
   resolveFileReference
-} from "../../chunk-P2465XAH.js";
+} from "../../chunk-7Q4PH265.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";

package/dist/index.cjs

@@ -1686,14 +1686,16 @@ function computeTraceSummary(messages) {
   }
   const toolNames = Object.keys(toolCallCounts).sort();
   return {
-    eventCount: totalToolCalls,
-    toolNames,
-    toolCallsByName: toolCallCounts,
-    errorCount: 0,
+    trace: {
+      eventCount: totalToolCalls,
+      toolNames,
+      toolCallsByName: toolCallCounts,
+      errorCount: 0,
+      llmCallCount,
+      ...hasAnyDuration ? { toolDurations } : {}
+    },
     startTime: earliestStart?.toISOString(),
-    endTime: latestEnd?.toISOString(),
-    llmCallCount,
-    ...hasAnyDuration ? { toolDurations } : {}
+    endTime: latestEnd?.toISOString()
   };
 }
 var DEFAULT_EXPLORATION_TOOLS = [
@@ -1716,9 +1718,9 @@ function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS)
   );
   return explorationCalls / summary.eventCount;
 }
-function tokensPerTool(summary) {
-  if (!summary.tokenUsage || summary.eventCount === 0) return void 0;
-  const totalTokens = summary.tokenUsage.input + summary.tokenUsage.output;
+function tokensPerTool(summary, tokenUsage) {
+  if (!tokenUsage || summary.eventCount === 0) return void 0;
+  const totalTokens = tokenUsage.input + tokenUsage.output;
   return totalTokens / summary.eventCount;
 }
 function avgToolDurationMs(summary) {
@@ -1734,16 +1736,15 @@ function avgToolDurationMs(summary) {
   if (totalCalls === 0) return void 0;
   return totalDuration / totalCalls;
 }
-function mergeExecutionMetrics(summary, metrics) {
-  if (!metrics) return summary;
+function mergeExecutionMetrics(computed, metrics) {
+  if (!metrics) return computed;
   return {
-    ...summary,
+    trace: computed.trace,
     tokenUsage: metrics.tokenUsage,
     costUsd: metrics.costUsd,
     durationMs: metrics.durationMs,
-    // Provider-level timing takes precedence over span-derived timing
-    startTime: metrics.startTime ?? summary.startTime,
-    endTime: metrics.endTime ?? summary.endTime
+    startTime: metrics.startTime ?? computed.startTime,
+    endTime: metrics.endTime ?? computed.endTime
   };
 }
 
@@ -4815,10 +4816,13 @@ async function invokeModel(options) {
 }
 function mapResponse(result) {
   const content = result.text ?? "";
+  const rawUsage = result.totalUsage ?? result.usage;
+  const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
   return {
     raw: result,
-    usage: toJsonObject(result.totalUsage ?? result.usage),
-    output: [{ role: "assistant", content }]
+    usage: toJsonObject(rawUsage),
+    output: [{ role: "assistant", content }],
+    tokenUsage
   };
 }
 function toJsonObject(value) {
@@ -11208,6 +11212,8 @@ async function createTargetProxy(options) {
   const token = (0, import_node_crypto7.randomBytes)(32).toString("hex");
   let callCount = 0;
   let isShutdown = false;
+  let totalInputTokens = 0;
+  let totalOutputTokens = 0;
   const targetsList = availableTargets ?? [defaultProvider.targetName];
   function resolveProvider(targetName) {
     if (targetName === void 0 || targetName === defaultProvider.targetName) {
@@ -11286,11 +11292,16 @@ async function createTargetProxy(options) {
         evalCaseId: request.evalCaseId ?? "proxy",
         attempt: request.attempt ?? 1
       });
+      if (response.tokenUsage) {
+        totalInputTokens += response.tokenUsage.input;
+        totalOutputTokens += response.tokenUsage.output;
+      }
       const output = response.output ?? [];
       const rawText = extractLastAssistantContent(output);
       const result = {
         output,
-        rawText
+        rawText,
+        tokenUsage: response.tokenUsage
       };
       sendJson(res, 200, result);
     } catch (error) {
@@ -11337,10 +11348,15 @@ async function createTargetProxy(options) {
             evalCaseId: request.evalCaseId ?? "proxy",
             attempt: request.attempt ?? 1
           });
+          if (response.tokenUsage) {
+            totalInputTokens += response.tokenUsage.input;
+            totalOutputTokens += response.tokenUsage.output;
+          }
           const output = response.output ?? [];
           responses.push({
             output,
-            rawText: extractLastAssistantContent(output)
+            rawText: extractLastAssistantContent(output),
+            tokenUsage: response.tokenUsage
           });
         } catch (error) {
           const message = error instanceof Error ? error.message : String(error);
@@ -11379,7 +11395,8 @@ async function createTargetProxy(options) {
     },
     getUsageMetadata: () => ({
       callCount,
-      maxCalls
+      maxCalls,
+      tokenUsage: totalInputTokens > 0 || totalOutputTokens > 0 ? { input: totalInputTokens, output: totalOutputTokens } : void 0
     })
   };
 }
@@ -11504,6 +11521,11 @@ var CodeEvaluator = class {
       ),
       input: context2.evalCase.input,
       trace: context2.trace ?? null,
+      tokenUsage: context2.tokenUsage ?? null,
+      costUsd: context2.costUsd ?? null,
+      durationMs: context2.durationMs ?? null,
+      startTime: context2.startTime ?? null,
+      endTime: context2.endTime ?? null,
       fileChanges: context2.fileChanges ?? null,
       workspacePath: context2.workspacePath ?? null,
       config: this.config ?? null
@@ -11562,7 +11584,8 @@ var CodeEvaluator = class {
         expectedAspectCount: hits.length + misses.length || 1,
         reasoning,
         evaluatorRawRequest,
-        ...details ? { details } : {}
+        ...details ? { details } : {},
+        tokenUsage: proxyUsage?.tokenUsage
       };
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -11584,7 +11607,8 @@ var CodeEvaluator = class {
             }
           } : {},
           error: message
-        }
+        },
+        tokenUsage: proxyUsage?.tokenUsage
       };
     } finally {
       if (proxyShutdown) {
@@ -11748,7 +11772,7 @@ ${context2.fileChanges}`;
       target: judgeProvider.targetName
     };
     try {
-      const { data } = await this.runWithRetry({
+      const { data, tokenUsage } = await this.runWithRetry({
         context: context2,
         judgeProvider,
         systemPrompt,
@@ -11767,7 +11791,8 @@ ${context2.fileChanges}`;
         misses,
         expectedAspectCount,
         reasoning,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        tokenUsage
       };
     } catch {
       return {
@@ -11797,7 +11822,7 @@ ${context2.fileChanges}`;
       systemPrompt,
       target: judgeProvider.targetName
     };
-    const { data } = await this.runWithRetry({
+    const { data, tokenUsage } = await this.runWithRetry({
       context: context2,
       judgeProvider,
       systemPrompt,
@@ -11812,7 +11837,8 @@ ${context2.fileChanges}`;
       misses,
       expectedAspectCount: rubrics.length,
       reasoning: data.overall_reasoning,
-      evaluatorRawRequest
+      evaluatorRawRequest,
+      tokenUsage
     };
   }
   /**
@@ -11827,7 +11853,7 @@ ${context2.fileChanges}`;
       systemPrompt,
       target: judgeProvider.targetName
     };
-    const { data } = await this.runWithRetry({
+    const { data, tokenUsage } = await this.runWithRetry({
       context: context2,
       judgeProvider,
       systemPrompt,
@@ -11843,7 +11869,8 @@ ${context2.fileChanges}`;
       expectedAspectCount: rubrics.length,
       reasoning: data.overall_reasoning,
       evaluatorRawRequest,
-      details
+      details,
+      tokenUsage
     };
   }
   /**
@@ -11927,15 +11954,17 @@ ${context2.fileChanges}`;
       try {
         const model = judgeProvider.asLanguageModel?.();
         if (model) {
-          const { text } = await (0, import_ai2.generateText)({
+          const result = await (0, import_ai2.generateText)({
             model,
             system: systemPrompt,
             prompt: userPrompt,
             ...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
             ...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
           });
-          const data2 = schema.parse(parseJsonFromText(text));
-          return { data: data2 };
+          const data2 = schema.parse(parseJsonFromText(result.text));
+          const rawUsage = result.usage;
+          const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
+          return { data: data2, tokenUsage };
         }
         const response = await judgeProvider.invoke({
           question: userPrompt,
@@ -11946,7 +11975,7 @@ ${context2.fileChanges}`;
           temperature: this.temperature
         });
         const data = schema.parse(parseJsonFromText(extractLastAssistantContent2(response.output)));
-        return { data, providerResponse: response };
+        return { data, providerResponse: response, tokenUsage: response.tokenUsage };
       } catch (e) {
         lastError = e instanceof Error ? e : new Error(String(e));
       }
@@ -12152,7 +12181,8 @@ var CompositeEvaluator = class {
         reasoning: member.result.reasoning,
         evaluatorRawRequest: member.result.evaluatorRawRequest,
         scores: member.result.scores,
-        details: member.result.details
+        details: member.result.details,
+        tokenUsage: member.result.tokenUsage
       });
     }
     const finalScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
@@ -12200,7 +12230,8 @@ var CompositeEvaluator = class {
         reasoning: member.result.reasoning,
         evaluatorRawRequest: member.result.evaluatorRawRequest,
         scores: member.result.scores,
-        details: member.result.details
+        details: member.result.details,
+        tokenUsage: member.result.tokenUsage
       });
     }
     const totalCount = results.length;
@@ -12379,7 +12410,7 @@ var CostEvaluator = class {
   }
   evaluate(context2) {
     const { budget } = this.config;
-    const costUsd = context2.trace?.costUsd;
+    const costUsd = context2.costUsd;
     if (costUsd === void 0) {
       return {
         score: 0,
@@ -12422,7 +12453,7 @@ var ExecutionMetricsEvaluator = class {
     this.config = options.config;
   }
   evaluate(context2) {
-    const { trace: trace2 } = context2;
+    const { trace: trace2, tokenUsage, costUsd, durationMs } = context2;
     const {
       max_tool_calls,
       max_llm_calls,
@@ -12432,7 +12463,8 @@ var ExecutionMetricsEvaluator = class {
       target_exploration_ratio,
       exploration_tolerance = 0.2
     } = this.config;
-    if (!trace2) {
+    const needsTrace = max_tool_calls !== void 0 || max_llm_calls !== void 0 || target_exploration_ratio !== void 0;
+    if (needsTrace && !trace2) {
       return {
         score: 0,
         verdict: "fail",
@@ -12447,11 +12479,12 @@ var ExecutionMetricsEvaluator = class {
         }
       };
     }
+    const narrowedTrace = trace2;
     const hits = [];
     const misses = [];
     const actualMetrics = {};
-    if (max_tool_calls !== void 0) {
-      const toolCalls = trace2.eventCount;
+    if (max_tool_calls !== void 0 && narrowedTrace) {
+      const toolCalls = narrowedTrace.eventCount;
       actualMetrics.tool_calls = toolCalls;
       if (toolCalls <= max_tool_calls) {
         hits.push(`Tool calls ${toolCalls} <= ${max_tool_calls} max`);
@@ -12459,8 +12492,8 @@ var ExecutionMetricsEvaluator = class {
         misses.push(`Tool calls ${toolCalls} > ${max_tool_calls} max`);
       }
     }
-    if (max_llm_calls !== void 0) {
-      const llmCalls = trace2.llmCallCount;
+    if (max_llm_calls !== void 0 && narrowedTrace) {
+      const llmCalls = narrowedTrace.llmCallCount;
       if (llmCalls === void 0) {
         misses.push("LLM call count data not available");
       } else {
@@ -12473,7 +12506,6 @@ var ExecutionMetricsEvaluator = class {
       }
     }
     if (max_tokens !== void 0) {
-      const tokenUsage = trace2.tokenUsage;
       if (!tokenUsage) {
         misses.push("Token usage data not available");
       } else {
@@ -12487,7 +12519,6 @@ var ExecutionMetricsEvaluator = class {
       }
     }
     if (max_cost_usd !== void 0) {
-      const costUsd = trace2.costUsd;
       if (costUsd === void 0) {
         misses.push("Cost data not available");
       } else {
@@ -12501,7 +12532,6 @@ var ExecutionMetricsEvaluator = class {
       }
     }
     if (max_duration_ms !== void 0) {
-      const durationMs = trace2.durationMs;
       if (durationMs === void 0) {
         misses.push("Duration data not available");
       } else {
@@ -12513,8 +12543,8 @@ var ExecutionMetricsEvaluator = class {
         }
       }
     }
-    if (target_exploration_ratio !== void 0) {
-      const ratio = explorationRatio(trace2);
+    if (target_exploration_ratio !== void 0 && narrowedTrace) {
+      const ratio = explorationRatio(narrowedTrace);
       if (ratio === void 0) {
         misses.push("Exploration ratio not available (no tool calls)");
       } else {
@@ -13028,7 +13058,7 @@ var LatencyEvaluator = class {
   }
   evaluate(context2) {
     const { threshold } = this.config;
-    const durationMs = context2.trace?.durationMs;
+    const durationMs = context2.durationMs;
     if (durationMs === void 0) {
       return {
         score: 0,
@@ -13673,7 +13703,7 @@ var TokenUsageEvaluator = class {
     this.config = options.config;
   }
   evaluate(context2) {
-    const usage = context2.trace?.tokenUsage;
+    const usage = context2.tokenUsage;
     const maxTotal = this.config.max_total;
     const maxInput = this.config.max_input;
     const maxOutput = this.config.max_output;
@@ -15366,7 +15396,7 @@ async function runEvaluation(options) {
               caseCost = trialCostSum;
             }
           } else {
-            caseCost = result.trace?.costUsd;
+            caseCost = result.costUsd;
           }
           if (caseCost !== void 0) {
             cumulativeBudgetCost += caseCost;
@@ -15527,17 +15557,18 @@ async function runBatchEvaluation(options) {
     const providerResponse = batchResponse[i];
     const output = providerResponse.output;
     const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-    const baseSummary = output ? computeTraceSummary(output) : hasExecutionMetrics ? {
-      eventCount: 0,
-      toolNames: [],
-      toolCallsByName: {},
-      errorCount: 0
-    } : void 0;
-    const trace2 = baseSummary ? mergeExecutionMetrics(baseSummary, {
+    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+    const merged = computed ? mergeExecutionMetrics(computed, {
       tokenUsage: providerResponse.tokenUsage,
       costUsd: providerResponse.costUsd,
       durationMs: providerResponse.durationMs
     }) : void 0;
+    const trace2 = merged?.trace;
+    const costUsd = merged?.costUsd;
+    const durationMs = merged?.durationMs;
+    const tokenUsage = merged?.tokenUsage;
+    const startTime = merged?.startTime;
+    const endTime = merged?.endTime;
     const candidate = extractLastAssistantContent2(output);
     const providerError = extractProviderError(providerResponse);
     let result;
@@ -15556,6 +15587,11 @@ async function runBatchEvaluation(options) {
         agentTimeoutMs,
         output,
         trace: trace2,
+        costUsd,
+        durationMs,
+        tokenUsage,
+        startTime,
+        endTime,
         targetResolver,
         availableTargets
       });
@@ -15792,17 +15828,18 @@ async function runEvalCase(options) {
   }
   const output = providerResponse.output;
   const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-  const baseSummary = output ? computeTraceSummary(output) : hasExecutionMetrics ? {
-    eventCount: 0,
-    toolNames: [],
-    toolCallsByName: {},
-    errorCount: 0
-  } : void 0;
-  const trace2 = baseSummary ? mergeExecutionMetrics(baseSummary, {
+  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+  const merged = computed ? mergeExecutionMetrics(computed, {
     tokenUsage: providerResponse.tokenUsage,
     costUsd: providerResponse.costUsd,
     durationMs: providerResponse.durationMs
   }) : void 0;
+  const trace2 = merged?.trace;
+  const costUsd = merged?.costUsd;
+  const durationMs = merged?.durationMs;
+  const tokenUsage = merged?.tokenUsage;
+  const startTime = merged?.startTime;
+  const endTime = merged?.endTime;
   const candidate = extractLastAssistantContent2(output);
   let fileChanges;
   if (baselineCommit && workspacePath) {
@@ -15847,6 +15884,11 @@ async function runEvalCase(options) {
       agentTimeoutMs,
       output,
       trace: trace2,
+      costUsd,
+      durationMs,
+      tokenUsage,
+      startTime,
+      endTime,
       targetResolver,
       availableTargets,
       fileChanges,
@@ -15903,7 +15945,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
     };
     const result = await runEvalCase(trialOptions);
     allResults.push(result);
-    const trialCost = result.trace?.costUsd;
+    const trialCost = result.costUsd;
     const trialVerdict = scoreToVerdict(result.score);
     const trial = {
       attempt,
@@ -15959,6 +16001,11 @@ async function evaluateCandidate(options) {
     agentTimeoutMs,
     output,
     trace: trace2,
+    costUsd,
+    durationMs,
+    tokenUsage,
+    startTime,
+    endTime,
     targetResolver,
     availableTargets,
     fileChanges,
@@ -15979,6 +16026,11 @@ async function evaluateCandidate(options) {
     agentTimeoutMs,
     output,
     trace: trace2,
+    costUsd,
+    durationMs,
+    tokenUsage,
+    startTime,
+    endTime,
     targetResolver,
     availableTargets,
     fileChanges,
@@ -16022,6 +16074,11 @@ async function evaluateCandidate(options) {
     answer: candidate,
     target: target.name,
     reasoning: score.reasoning,
+    tokenUsage,
+    costUsd,
+    durationMs,
+    startTime,
+    endTime,
     requests,
     input,
     scores,
@@ -16045,6 +16102,11 @@ async function runEvaluatorsForCase(options) {
     agentTimeoutMs,
     output,
     trace: trace2,
+    costUsd,
+    durationMs,
+    tokenUsage,
+    startTime,
+    endTime,
     targetResolver,
     availableTargets,
     fileChanges,
@@ -16066,6 +16128,11 @@ async function runEvaluatorsForCase(options) {
       agentTimeoutMs,
       output,
       trace: trace2,
+      costUsd,
+      durationMs,
+      tokenUsage,
+      startTime,
+      endTime,
       targetResolver,
       availableTargets,
       fileChanges,
@@ -16088,6 +16155,11 @@ async function runEvaluatorsForCase(options) {
     judgeProvider,
     output,
     trace: trace2,
+    tokenUsage,
+    costUsd,
+    durationMs,
+    startTime,
+    endTime,
     targetResolver,
     availableTargets,
     fileChanges,
@@ -16111,6 +16183,11 @@ async function runEvaluatorList(options) {
     agentTimeoutMs,
     output,
     trace: trace2,
+    costUsd,
+    durationMs,
+    tokenUsage,
+    startTime,
+    endTime,
     targetResolver,
     availableTargets,
     fileChanges,
@@ -16129,6 +16206,11 @@ async function runEvaluatorList(options) {
     judgeProvider,
     output,
     trace: trace2,
+    tokenUsage,
+    costUsd,
+    durationMs,
+    startTime,
+    endTime,
     targetResolver,
     availableTargets,
     fileChanges,
@@ -16168,7 +16250,8 @@ async function runEvaluatorList(options) {
         reasoning: score2.reasoning,
         evaluatorProviderRequest: score2.evaluatorRawRequest,
         details: score2.details,
-        scores: mapChildResults(score2.scores)
+        scores: mapChildResults(score2.scores),
+        tokenUsage: score2.tokenUsage
       });
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -16416,7 +16499,8 @@ function mapChildResults(children) {
     reasoning: child.reasoning,
     evaluatorProviderRequest: child.evaluatorRawRequest,
     scores: mapChildResults(child.scores),
-    details: child.details
+    details: child.details,
+    tokenUsage: child.tokenUsage
   }));
 }
 function computeWeightedMean(entries) {
@@ -16796,7 +16880,13 @@ var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
   "beforeEachOutput",
   "afterAllOutput",
   "afterEachOutput",
-  "fileChanges"
+  "fileChanges",
+  // Promoted execution metrics (debug, not needed for regression comparison)
+  "tokenUsage",
+  "costUsd",
+  "durationMs",
+  "startTime",
+  "endTime"
 ]);
 var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "evaluatorProviderRequest"]);
 function trimEvaluatorResult(result) {
@@ -16919,8 +17009,8 @@ var OtelTraceExporter = class {
     const api = this.api;
     const tracer = this.tracer;
     const captureContent = this.options.captureContent ?? false;
-    const startHr = toHrTime(result.trace?.startTime ?? result.timestamp);
-    const endHr = toHrTime(result.trace?.endTime ?? result.timestamp);
+    const startHr = toHrTime(result.startTime ?? result.timestamp);
+    const endHr = toHrTime(result.endTime ?? result.timestamp);
     let parentCtx = api.ROOT_CONTEXT;
     const traceparent = process.env.TRACEPARENT;
     if (traceparent && this.W3CPropagator) {
@@ -16949,12 +17039,13 @@ var OtelTraceExporter = class {
         if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
         rootSpan.setAttribute("agentv.score", result.score);
         if (captureContent) rootSpan.setAttribute("agentv.answer", result.answer);
+        if (result.durationMs != null)
+          rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
+        if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
         if (result.trace) {
           const t = result.trace;
           rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
           rootSpan.setAttribute("agentv.trace.tool_names", t.toolNames.join(","));
-          if (t.durationMs != null) rootSpan.setAttribute("agentv.trace.duration_ms", t.durationMs);
-          if (t.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", t.costUsd);
           if (t.llmCallCount != null)
             rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
         }