@agentv/core 3.6.0 → 3.8.0

package/dist/index.cjs

@@ -1786,12 +1786,10 @@ function computeTraceSummary(messages) {
       }
     }
   }
-  const toolNames = Object.keys(toolCallCounts).sort();
   return {
     trace: {
       eventCount: totalToolCalls,
-      toolNames,
-      toolCallsByName: toolCallCounts,
+      toolCalls: toolCallCounts,
       errorCount: 0,
       llmCallCount,
       ...hasAnyDuration ? { toolDurations } : {}
@@ -1815,7 +1813,7 @@ var DEFAULT_EXPLORATION_TOOLS = [
 function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS) {
   if (summary.eventCount === 0) return void 0;
   const explorationCalls = explorationTools.reduce(
-    (sum, tool2) => sum + (summary.toolCallsByName[tool2] ?? 0),
+    (sum, tool2) => sum + (summary.toolCalls[tool2] ?? 0),
     0
   );
   return explorationCalls / summary.eventCount;
@@ -15832,11 +15830,9 @@ var ToolTrajectoryEvaluator = class {
     for (const call of toolCalls) {
       toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
     }
-    const toolNames = Object.keys(toolCallsByName).sort();
     return {
       eventCount: toolCalls.length,
-      toolNames,
-      toolCallsByName,
+      toolCalls: toolCallsByName,
       errorCount: 0
     };
   }
@@ -15854,7 +15850,7 @@ var ToolTrajectoryEvaluator = class {
     const assertions = [];
     for (const toolName of toolNames) {
       const required = minimums[toolName];
-      const actual = summary.toolCallsByName[toolName] ?? 0;
+      const actual = summary.toolCalls[toolName] ?? 0;
       if (actual >= required) {
         assertions.push({
           text: `${toolName}: called ${actual} times (required >=${required})`,
@@ -18234,7 +18230,7 @@ async function runEvaluation(options) {
             dataset: evalCase.dataset,
             score: 0,
             assertions: [],
-            outputText: "",
+            output: [],
             target: target.name,
             error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
             budgetExceeded: true,
@@ -18270,7 +18266,7 @@ async function runEvaluation(options) {
             dataset: evalCase.dataset,
             score: 0,
             assertions: [],
-            outputText: "",
+            output: [],
             target: target.name,
             error: errorMsg,
             executionStatus: "execution_error",
@@ -18535,7 +18531,7 @@ async function runBatchEvaluation(options) {
     const providerResponse = batchResponse[i];
     const output = providerResponse.output;
     const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
     const merged = computed ? mergeExecutionMetrics(computed, {
       tokenUsage: providerResponse.tokenUsage,
       costUsd: providerResponse.costUsd,
@@ -18932,7 +18928,7 @@ async function runEvalCase(options) {
   }
   const output = providerResponse.output;
   const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
   const merged = computed ? mergeExecutionMetrics(computed, {
     tokenUsage: providerResponse.tokenUsage,
     costUsd: providerResponse.costUsd,
@@ -19237,7 +19233,6 @@ async function evaluateCandidate(options) {
     conversationId: evalCase.conversation_id,
     score: score.score,
     assertions: score.assertions,
-    outputText: candidate,
     target: target.name,
     tokenUsage,
     costUsd,
@@ -19248,7 +19243,7 @@ async function evaluateCandidate(options) {
     input,
     scores,
     trace: trace2,
-    output,
+    output: output ?? [{ role: "assistant", content: candidate }],
     fileChanges,
     executionStatus: classifyQualityStatus(score.score)
   };
@@ -19413,7 +19408,7 @@ async function runEvaluatorList(options) {
         weight,
         verdict: score2.verdict,
         assertions: score2.assertions,
-        evaluatorProviderRequest: score2.evaluatorRawRequest,
+        input: score2.evaluatorRawRequest,
         details: score2.details,
         scores: mapChildResults(score2.scores),
         tokenUsage: score2.tokenUsage,
@@ -19593,7 +19588,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
-    outputText: `Error occurred: ${message}`,
+    output: [{ role: "assistant", content: `Error occurred: ${message}` }],
     target: targetName,
     requests,
     input,
@@ -19637,7 +19632,7 @@ function buildResultInput(promptInputs) {
       content: message.content
     }));
   }
-  return promptInputs.question;
+  return [{ role: "user", content: promptInputs.question }];
 }
 function aggregateEvaluatorTokenUsage(scores) {
   if (!scores || scores.length === 0) return void 0;
@@ -19703,7 +19698,7 @@ function mapChildResults(children) {
     weight: child.weight,
     verdict: child.verdict,
     assertions: child.assertions,
-    evaluatorProviderRequest: child.evaluatorRawRequest,
+    input: child.evaluatorRawRequest,
     scores: mapChildResults(child.scores),
     details: child.details,
     tokenUsage: child.tokenUsage
@@ -20131,7 +20126,6 @@ function shouldSkipCacheForTemperature(targetConfig) {
 
 // src/evaluation/baseline.ts
 var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
-  "outputText",
   "requests",
   "trace",
   "workspacePath",
@@ -20148,7 +20142,7 @@ var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
   "startTime",
   "endTime"
 ]);
-var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "evaluatorProviderRequest"]);
+var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "input"]);
 function trimEvaluatorResult(result) {
   const trimmed = {};
   for (const [key, value] of Object.entries(result)) {
@@ -20305,14 +20299,21 @@ var OtelTraceExporter = class {
         rootSpan.setAttribute("agentv.target", result.target);
         if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
         rootSpan.setAttribute("agentv.score", result.score);
-        if (captureContent) rootSpan.setAttribute("agentv.output_text", result.outputText);
+        if (captureContent && result.output.length > 0) {
+          const lastMsg = result.output[result.output.length - 1];
+          const text = typeof lastMsg.content === "string" ? lastMsg.content : JSON.stringify(lastMsg.content);
+          rootSpan.setAttribute("agentv.output_text", text);
+        }
         if (result.durationMs != null)
           rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
         if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
         if (result.trace) {
           const t = result.trace;
           rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
-          rootSpan.setAttribute("agentv.trace.tool_names", t.toolNames.join(","));
+          rootSpan.setAttribute(
+            "agentv.trace.tool_names",
+            Object.keys(t.toolCalls).sort().join(",")
+          );
           if (t.llmCallCount != null)
             rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
         }