@agentv/core 3.5.0 → 3.7.0

package/dist/evaluation/validation/index.js

@@ -6,7 +6,7 @@ import {
   findGitRoot,
   isEvaluatorKind,
   resolveFileReference
-} from "../../chunk-EFR4JHPL.js";
+} from "../../chunk-2IZOTQ25.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";

package/dist/index.cjs

@@ -1786,12 +1786,10 @@ function computeTraceSummary(messages) {
       }
     }
   }
-  const toolNames = Object.keys(toolCallCounts).sort();
   return {
     trace: {
       eventCount: totalToolCalls,
-      toolNames,
-      toolCallsByName: toolCallCounts,
+      toolCalls: toolCallCounts,
       errorCount: 0,
       llmCallCount,
       ...hasAnyDuration ? { toolDurations } : {}
@@ -1815,7 +1813,7 @@ var DEFAULT_EXPLORATION_TOOLS = [
 function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS) {
   if (summary.eventCount === 0) return void 0;
   const explorationCalls = explorationTools.reduce(
-    (sum, tool2) => sum + (summary.toolCallsByName[tool2] ?? 0),
+    (sum, tool2) => sum + (summary.toolCalls[tool2] ?? 0),
     0
   );
   return explorationCalls / summary.eventCount;
@@ -2445,14 +2443,8 @@ var import_promises5 = require("fs/promises");
 
 // src/evaluation/template-variables.ts
 var TEMPLATE_VARIABLES = {
-  /** @deprecated Use OUTPUT_TEXT instead */
-  ANSWER: "answer",
   EXPECTED_OUTPUT: "expected_output",
-  /** @deprecated Use INPUT_TEXT instead */
-  QUESTION: "question",
   CRITERIA: "criteria",
-  /** @deprecated Use EXPECTED_OUTPUT_TEXT instead */
-  REFERENCE_ANSWER: "reference_answer",
   INPUT: "input",
   OUTPUT: "output",
   FILE_CHANGES: "file_changes",
@@ -2462,9 +2454,8 @@ var TEMPLATE_VARIABLES = {
 };
 var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
-  TEMPLATE_VARIABLES.ANSWER,
-  TEMPLATE_VARIABLES.EXPECTED_OUTPUT,
-  TEMPLATE_VARIABLES.OUTPUT_TEXT
+  TEMPLATE_VARIABLES.OUTPUT_TEXT,
+  TEMPLATE_VARIABLES.EXPECTED_OUTPUT
 ]);
 
 // src/evaluation/validation/prompt-validator.ts
@@ -2487,13 +2478,13 @@ function validateTemplateVariables(content, source) {
     }
     match = variablePattern.exec(content);
   }
-  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.ANSWER) || foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
+  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
   const hasExpectedOutput = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT);
   const hasRequiredFields = hasCandidateAnswer || hasExpectedOutput;
   if (!hasRequiredFields) {
     throw new Error(
       `Missing required fields. Must include at least one of:
-  - {{ ${TEMPLATE_VARIABLES.ANSWER} }} or {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
+  - {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
   - {{ ${TEMPLATE_VARIABLES.EXPECTED_OUTPUT} }}`
     );
   }
@@ -5795,6 +5786,8 @@ async function invokeModel(options) {
   const { model, request, defaults, retryConfig, providerOptions } = options;
   const chatPrompt = buildChatPrompt(request);
   const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
+  const startTime = (/* @__PURE__ */ new Date()).toISOString();
+  const startMs = Date.now();
   const result = await withRetry(
     () => (0, import_ai.generateText)({
       model,
@@ -5808,9 +5801,11 @@ async function invokeModel(options) {
     retryConfig,
     request.signal
   );
-  return mapResponse(result);
+  const endTime = (/* @__PURE__ */ new Date()).toISOString();
+  const durationMs = Date.now() - startMs;
+  return mapResponse(result, { durationMs, startTime, endTime });
 }
-function mapResponse(result) {
+function mapResponse(result, timing) {
   const content = result.text ?? "";
   const rawUsage = result.totalUsage ?? result.usage;
   const reasoning = rawUsage?.outputTokenDetails?.reasoningTokens ?? void 0;
@@ -5825,7 +5820,10 @@ function mapResponse(result) {
     raw: result,
     usage: toJsonObject(rawUsage),
     output: [{ role: "assistant", content }],
-    tokenUsage
+    tokenUsage,
+    durationMs: timing?.durationMs,
+    startTime: timing?.startTime,
+    endTime: timing?.endTime
   };
 }
 function toJsonObject(value) {
@@ -6703,10 +6701,12 @@ var ClaudeSdkProvider = class {
             if (usage) {
               const inputTokens = (usage.input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0);
               const outputTokens = usage.output_tokens ?? 0;
+              const reasoningTokens = usage.reasoning_tokens ?? void 0;
               tokenUsage = {
                 input: inputTokens,
                 output: outputTokens,
-                cached: usage.cache_read_input_tokens ?? void 0
+                cached: usage.cache_read_input_tokens ?? void 0,
+                reasoning: reasoningTokens
               };
               request.streamCallbacks?.onLlmCallEnd?.(this.config.model ?? "claude", tokenUsage);
             }
@@ -7720,7 +7720,8 @@ ${basePrompt}` : basePrompt;
           onUsage({
             input: usage.input_tokens ?? 0,
             output: usage.output_tokens ?? 0,
-            cached: usage.cached_input_tokens ?? void 0
+            cached: usage.cached_input_tokens ?? void 0,
+            reasoning: usage.reasoning_tokens ?? void 0
           });
         }
       }
@@ -9735,10 +9736,12 @@ function extractTokenUsage(events) {
           output: output ?? 0
         };
         const cached = toFiniteNumber(u.cache_read_input_tokens ?? u.cached ?? u.cachedTokens);
-        if (cached !== void 0) {
-          return { ...result, cached };
-        }
-        return result;
+        const reasoning = toFiniteNumber(u.reasoning_tokens ?? u.reasoningTokens ?? u.reasoning);
+        return {
+          ...result,
+          ...cached !== void 0 ? { cached } : {},
+          ...reasoning !== void 0 ? { reasoning } : {}
+        };
       }
     }
     const messages = record.messages;
@@ -13265,11 +13268,9 @@ var CodeEvaluator = class {
       }
     }
     const payload = {
-      question: context2.evalCase.question,
       criteria: context2.evalCase.criteria,
       expectedOutput: context2.evalCase.expected_output,
-      referenceAnswer: context2.evalCase.reference_answer,
-      answer: context2.candidate,
+      outputText: context2.candidate,
       output: outputForPayload,
       outputPath,
       guidelineFiles: context2.evalCase.guideline_paths,
@@ -13286,9 +13287,7 @@ var CodeEvaluator = class {
       fileChanges: context2.fileChanges ?? null,
       workspacePath: context2.workspacePath ?? null,
       config: this.config ?? null,
-      // Text convenience accessors (new names, always strings)
       inputText: context2.evalCase.question,
-      outputText: context2.candidate,
       expectedOutputText: context2.evalCase.reference_answer ?? ""
     };
     const inputPayload = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
@@ -13488,13 +13487,13 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 {{${TEMPLATE_VARIABLES.CRITERIA}}}
 
 [[ ## question ## ]]
-{{${TEMPLATE_VARIABLES.QUESTION}}}
+{{${TEMPLATE_VARIABLES.INPUT_TEXT}}}
 
 [[ ## reference_answer ## ]]
-{{${TEMPLATE_VARIABLES.REFERENCE_ANSWER}}}
+{{${TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT}}}
 
 [[ ## answer ## ]]
-{{${TEMPLATE_VARIABLES.ANSWER}}}`;
+{{${TEMPLATE_VARIABLES.OUTPUT_TEXT}}}`;
 var freeformEvaluationSchema = import_zod4.z.object({
   score: import_zod4.z.number().min(0).max(1).describe("Score between 0.0 and 1.0"),
   assertions: import_zod4.z.array(
@@ -13572,12 +13571,8 @@ var LlmGraderEvaluator = class {
         2
       ),
       [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify(context2.output ?? [], null, 2),
-      [TEMPLATE_VARIABLES.ANSWER]: context2.candidate.trim(),
-      [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
-      [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
-      // Text convenience accessors (new names, always strings)
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim()
@@ -13882,10 +13877,10 @@ ${context2.fileChanges}`;
   buildAgentUserPrompt(context2) {
     const formattedQuestion = context2.promptInputs.question && context2.promptInputs.question.trim().length > 0 ? context2.promptInputs.question : context2.evalCase.question;
     const variables = {
-      [TEMPLATE_VARIABLES.ANSWER]: context2.candidate.trim(),
-      [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
-      [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? ""
     };
     if (this.evaluatorTemplate) {
@@ -13938,10 +13933,10 @@ ${context2.fileChanges}`;
     const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
     if (this.evaluatorTemplate) {
       const variables = {
-        [TEMPLATE_VARIABLES.ANSWER]: context2.candidate.trim(),
-        [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context2.evalCase.reference_answer ?? "").trim(),
         [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
-        [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
+        [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
+        [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
+        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim(),
         [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? ""
       };
       const customPrompt = substituteVariables(this.evaluatorTemplate, variables);
@@ -15369,7 +15364,10 @@ var COPILOT_MATCHER = {
   skillTools: ["Skill", "skill"],
   skillInputField: "skill",
   readTools: ["Read File", "readFile", "Read", "readTextFile"],
-  readInputField: "file_path"
+  readInputField: "file_path",
+  skillToolPrefixes: ["Using skill: "],
+  readToolPrefixes: ["Viewing "],
+  readInputFields: ["file_path", "path"]
 };
 var PROVIDER_TOOL_SEMANTICS = {
   claude: CLAUDE_MATCHER,
@@ -15411,12 +15409,22 @@ var SkillTriggerEvaluator = class {
           triggered = true;
           evidence = `Skill tool invoked with ${matcher.skillInputField}="${skillArg}"`;
         }
+      } else if (matcher.skillToolPrefixes?.some(
+        (prefix) => firstTool.tool.startsWith(prefix) && firstTool.tool.includes(skillName)
+      )) {
+        triggered = true;
+        evidence = `Skill tool invoked via tool name "${firstTool.tool}"`;
       } else if (matcher.readTools.includes(firstTool.tool)) {
-        const filePath = String(input[matcher.readInputField] ?? "");
+        const filePath = this.readPathFromInput(input, matcher);
         if (filePath.includes(skillName)) {
           triggered = true;
           evidence = `Read tool loaded skill file: ${filePath}`;
         }
+      } else if (matcher.readToolPrefixes?.some(
+        (prefix) => firstTool.tool.startsWith(prefix) && firstTool.tool.includes(skillName)
+      )) {
+        triggered = true;
+        evidence = `Read tool loaded skill file via tool name "${firstTool.tool}"`;
       }
     }
     const pass = triggered === shouldTrigger;
@@ -15445,6 +15453,16 @@ var SkillTriggerEvaluator = class {
       expectedAspectCount: 1
     };
   }
+  readPathFromInput(input, matcher) {
+    const fields = matcher.readInputFields ?? [matcher.readInputField];
+    for (const field of fields) {
+      const value = input[field];
+      if (value !== void 0 && value !== null) {
+        return String(value);
+      }
+    }
+    return "";
+  }
 };
 
 // src/evaluation/evaluators/llm-grader-prompt.ts
@@ -15479,12 +15497,8 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evalua
     [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(evalCase.input_segments, null, 2),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(evalCase.expected_output, null, 2),
     [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify([], null, 2),
-    [TEMPLATE_VARIABLES.ANSWER]: candidate.trim(),
-    [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
-    [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
-    // Text convenience accessors (new names, always strings)
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (evalCase.reference_answer ?? "").trim()
@@ -15816,11 +15830,9 @@ var ToolTrajectoryEvaluator = class {
     for (const call of toolCalls) {
       toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
     }
-    const toolNames = Object.keys(toolCallsByName).sort();
     return {
       eventCount: toolCalls.length,
-      toolNames,
-      toolCallsByName,
+      toolCalls: toolCallsByName,
       errorCount: 0
     };
   }
@@ -15838,7 +15850,7 @@ var ToolTrajectoryEvaluator = class {
     const assertions = [];
     for (const toolName of toolNames) {
       const required = minimums[toolName];
-      const actual = summary.toolCallsByName[toolName] ?? 0;
+      const actual = summary.toolCalls[toolName] ?? 0;
       if (actual >= required) {
         assertions.push({
           text: `${toolName}: called ${actual} times (required >=${required})`,
@@ -16541,11 +16553,9 @@ async function resolveCustomPrompt(promptConfig, context2, timeoutMs) {
 }
 async function executePromptTemplate(script, context2, config, timeoutMs) {
   const payload = {
-    question: context2.evalCase.question,
     criteria: context2.evalCase.criteria,
     expectedOutput: context2.evalCase.expected_output,
-    referenceAnswer: context2.evalCase.reference_answer,
-    answer: context2.candidate,
+    outputText: context2.candidate,
     output: context2.output ?? null,
     guidelineFiles: context2.evalCase.guideline_paths,
     inputFiles: context2.evalCase.file_paths.filter(
@@ -16556,9 +16566,7 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
     fileChanges: context2.fileChanges ?? null,
     workspacePath: context2.workspacePath ?? null,
     config: config ?? context2.config ?? null,
-    // Text convenience accessors (new names, always strings)
     inputText: context2.evalCase.question,
-    outputText: context2.candidate,
     expectedOutputText: context2.evalCase.reference_answer ?? ""
   };
   const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
@@ -18222,7 +18230,7 @@ async function runEvaluation(options) {
             dataset: evalCase.dataset,
             score: 0,
             assertions: [],
-            answer: "",
+            outputText: "",
             target: target.name,
             error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
             budgetExceeded: true,
@@ -18258,7 +18266,7 @@ async function runEvaluation(options) {
             dataset: evalCase.dataset,
             score: 0,
             assertions: [],
-            answer: "",
+            outputText: "",
             target: target.name,
             error: errorMsg,
             executionStatus: "execution_error",
@@ -18523,7 +18531,7 @@ async function runBatchEvaluation(options) {
     const providerResponse = batchResponse[i];
     const output = providerResponse.output;
     const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
     const merged = computed ? mergeExecutionMetrics(computed, {
       tokenUsage: providerResponse.tokenUsage,
       costUsd: providerResponse.costUsd,
@@ -18920,7 +18928,7 @@ async function runEvalCase(options) {
   }
   const output = providerResponse.output;
   const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
   const merged = computed ? mergeExecutionMetrics(computed, {
     tokenUsage: providerResponse.tokenUsage,
     costUsd: providerResponse.costUsd,
@@ -19225,7 +19233,7 @@ async function evaluateCandidate(options) {
     conversationId: evalCase.conversation_id,
     score: score.score,
     assertions: score.assertions,
-    answer: candidate,
+    outputText: candidate,
     target: target.name,
     tokenUsage,
     costUsd,
@@ -19581,7 +19589,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
-    answer: `Error occurred: ${message}`,
+    outputText: `Error occurred: ${message}`,
     target: targetName,
     requests,
     input,
@@ -20119,7 +20127,7 @@ function shouldSkipCacheForTemperature(targetConfig) {
 
 // src/evaluation/baseline.ts
 var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
-  "answer",
+  "outputText",
   "requests",
   "trace",
   "workspacePath",
@@ -20293,14 +20301,17 @@ var OtelTraceExporter = class {
         rootSpan.setAttribute("agentv.target", result.target);
         if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
         rootSpan.setAttribute("agentv.score", result.score);
-        if (captureContent) rootSpan.setAttribute("agentv.answer", result.answer);
+        if (captureContent) rootSpan.setAttribute("agentv.output_text", result.outputText);
         if (result.durationMs != null)
           rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
         if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
         if (result.trace) {
           const t = result.trace;
           rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
-          rootSpan.setAttribute("agentv.trace.tool_names", t.toolNames.join(","));
+          rootSpan.setAttribute(
+            "agentv.trace.tool_names",
+            Object.keys(t.toolCalls).sort().join(",")
+          );
           if (t.llmCallCount != null)
             rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
         }