@agentv/core 3.5.0 → 3.7.0

package/dist/index.d.cts

@@ -251,10 +251,8 @@ interface TokenUsage {
 interface TraceSummary {
     /** Total number of events in trace */
     readonly eventCount: number;
-    /** Unique tool names, sorted alphabetically */
-    readonly toolNames: readonly string[];
     /** Map of tool name to call count */
-    readonly toolCallsByName: Readonly<Record<string, number>>;
+    readonly toolCalls: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
     /** Per-tool duration arrays in milliseconds (optional) */
@@ -1134,7 +1132,7 @@ interface EvaluationResult {
     readonly conversationId?: string;
     readonly score: number;
     readonly assertions: readonly AssertionEntry[];
-    readonly answer: string;
+    readonly outputText: string;
     readonly target: string;
     /** Token usage metrics from provider (optional) */
     readonly tokenUsage?: TokenUsage;
@@ -2412,6 +2410,7 @@ declare class SkillTriggerEvaluator implements Evaluator {
     constructor(config: SkillTriggerEvaluatorConfig);
     private resolveMatcher;
     evaluate(context: EvaluationContext): EvaluationScore;
+    private readPathFromInput;
 }
 
 interface LlmGraderPromptAssembly {

package/dist/index.d.ts

@@ -251,10 +251,8 @@ interface TokenUsage {
 interface TraceSummary {
     /** Total number of events in trace */
     readonly eventCount: number;
-    /** Unique tool names, sorted alphabetically */
-    readonly toolNames: readonly string[];
     /** Map of tool name to call count */
-    readonly toolCallsByName: Readonly<Record<string, number>>;
+    readonly toolCalls: Readonly<Record<string, number>>;
     /** Number of error events */
     readonly errorCount: number;
     /** Per-tool duration arrays in milliseconds (optional) */
@@ -1134,7 +1132,7 @@ interface EvaluationResult {
     readonly conversationId?: string;
     readonly score: number;
     readonly assertions: readonly AssertionEntry[];
-    readonly answer: string;
+    readonly outputText: string;
     readonly target: string;
     /** Token usage metrics from provider (optional) */
     readonly tokenUsage?: TokenUsage;
@@ -2412,6 +2410,7 @@ declare class SkillTriggerEvaluator implements Evaluator {
     constructor(config: SkillTriggerEvaluatorConfig);
     private resolveMatcher;
     evaluate(context: EvaluationContext): EvaluationScore;
+    private readPathFromInput;
 }
 
 interface LlmGraderPromptAssembly {

package/dist/index.js

@@ -16,7 +16,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-EFR4JHPL.js";
+} from "./chunk-2IZOTQ25.js";
 import {
   AgentvProvider
 } from "./chunk-W5YDZWT4.js";
@@ -83,12 +83,10 @@ function computeTraceSummary(messages) {
       }
     }
   }
-  const toolNames = Object.keys(toolCallCounts).sort();
   return {
     trace: {
       eventCount: totalToolCalls,
-      toolNames,
-      toolCallsByName: toolCallCounts,
+      toolCalls: toolCallCounts,
       errorCount: 0,
       llmCallCount,
       ...hasAnyDuration ? { toolDurations } : {}
@@ -112,7 +110,7 @@ var DEFAULT_EXPLORATION_TOOLS = [
 function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS) {
   if (summary.eventCount === 0) return void 0;
   const explorationCalls = explorationTools.reduce(
-    (sum, tool2) => sum + (summary.toolCallsByName[tool2] ?? 0),
+    (sum, tool2) => sum + (summary.toolCalls[tool2] ?? 0),
     0
   );
   return explorationCalls / summary.eventCount;
@@ -742,14 +740,8 @@ import { readFile as readFile4 } from "node:fs/promises";
 
 // src/evaluation/template-variables.ts
 var TEMPLATE_VARIABLES = {
-  /** @deprecated Use OUTPUT_TEXT instead */
-  ANSWER: "answer",
   EXPECTED_OUTPUT: "expected_output",
-  /** @deprecated Use INPUT_TEXT instead */
-  QUESTION: "question",
   CRITERIA: "criteria",
-  /** @deprecated Use EXPECTED_OUTPUT_TEXT instead */
-  REFERENCE_ANSWER: "reference_answer",
   INPUT: "input",
   OUTPUT: "output",
   FILE_CHANGES: "file_changes",
@@ -759,9 +751,8 @@ var TEMPLATE_VARIABLES = {
 };
 var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
-  TEMPLATE_VARIABLES.ANSWER,
-  TEMPLATE_VARIABLES.EXPECTED_OUTPUT,
-  TEMPLATE_VARIABLES.OUTPUT_TEXT
+  TEMPLATE_VARIABLES.OUTPUT_TEXT,
+  TEMPLATE_VARIABLES.EXPECTED_OUTPUT
 ]);
 
 // src/evaluation/validation/prompt-validator.ts
@@ -784,13 +775,13 @@ function validateTemplateVariables(content, source) {
     }
     match = variablePattern.exec(content);
   }
-  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.ANSWER) || foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
+  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
   const hasExpectedOutput = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT);
   const hasRequiredFields = hasCandidateAnswer || hasExpectedOutput;
   if (!hasRequiredFields) {
     throw new Error(
       `Missing required fields. Must include at least one of:
-  - {{ ${TEMPLATE_VARIABLES.ANSWER} }} or {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
+  - {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
   - {{ ${TEMPLATE_VARIABLES.EXPECTED_OUTPUT} }}`
     );
   }
@@ -3974,6 +3965,8 @@ async function invokeModel(options) {
   const { model, request, defaults, retryConfig, providerOptions } = options;
   const chatPrompt = buildChatPrompt(request);
   const { temperature, maxOutputTokens } = resolveModelSettings(request, defaults);
+  const startTime = (/* @__PURE__ */ new Date()).toISOString();
+  const startMs = Date.now();
   const result = await withRetry(
     () => generateText({
       model,
@@ -3987,9 +3980,11 @@ async function invokeModel(options) {
     retryConfig,
     request.signal
   );
-  return mapResponse(result);
+  const endTime = (/* @__PURE__ */ new Date()).toISOString();
+  const durationMs = Date.now() - startMs;
+  return mapResponse(result, { durationMs, startTime, endTime });
 }
-function mapResponse(result) {
+function mapResponse(result, timing) {
   const content = result.text ?? "";
   const rawUsage = result.totalUsage ?? result.usage;
   const reasoning = rawUsage?.outputTokenDetails?.reasoningTokens ?? void 0;
@@ -4004,7 +3999,10 @@ function mapResponse(result) {
     raw: result,
     usage: toJsonObject(rawUsage),
     output: [{ role: "assistant", content }],
-    tokenUsage
+    tokenUsage,
+    durationMs: timing?.durationMs,
+    startTime: timing?.startTime,
+    endTime: timing?.endTime
   };
 }
 function toJsonObject(value) {
@@ -4882,10 +4880,12 @@ var ClaudeSdkProvider = class {
             if (usage) {
               const inputTokens = (usage.input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0);
               const outputTokens = usage.output_tokens ?? 0;
+              const reasoningTokens = usage.reasoning_tokens ?? void 0;
               tokenUsage = {
                 input: inputTokens,
                 output: outputTokens,
-                cached: usage.cache_read_input_tokens ?? void 0
+                cached: usage.cache_read_input_tokens ?? void 0,
+                reasoning: reasoningTokens
               };
               request.streamCallbacks?.onLlmCallEnd?.(this.config.model ?? "claude", tokenUsage);
             }
@@ -5899,7 +5899,8 @@ ${basePrompt}` : basePrompt;
           onUsage({
             input: usage.input_tokens ?? 0,
             output: usage.output_tokens ?? 0,
-            cached: usage.cached_input_tokens ?? void 0
+            cached: usage.cached_input_tokens ?? void 0,
+            reasoning: usage.reasoning_tokens ?? void 0
           });
         }
       }
@@ -7913,10 +7914,12 @@ function extractTokenUsage(events) {
           output: output ?? 0
         };
         const cached = toFiniteNumber(u.cache_read_input_tokens ?? u.cached ?? u.cachedTokens);
-        if (cached !== void 0) {
-          return { ...result, cached };
-        }
-        return result;
+        const reasoning = toFiniteNumber(u.reasoning_tokens ?? u.reasoningTokens ?? u.reasoning);
+        return {
+          ...result,
+          ...cached !== void 0 ? { cached } : {},
+          ...reasoning !== void 0 ? { reasoning } : {}
+        };
       }
     }
     const messages = record.messages;
@@ -10245,11 +10248,9 @@ var CodeEvaluator = class {
       }
     }
     const payload = {
-      question: context.evalCase.question,
       criteria: context.evalCase.criteria,
       expectedOutput: context.evalCase.expected_output,
-      referenceAnswer: context.evalCase.reference_answer,
-      answer: context.candidate,
+      outputText: context.candidate,
       output: outputForPayload,
       outputPath,
       guidelineFiles: context.evalCase.guideline_paths,
@@ -10266,9 +10267,7 @@ var CodeEvaluator = class {
       fileChanges: context.fileChanges ?? null,
       workspacePath: context.workspacePath ?? null,
       config: this.config ?? null,
-      // Text convenience accessors (new names, always strings)
       inputText: context.evalCase.question,
-      outputText: context.candidate,
       expectedOutputText: context.evalCase.reference_answer ?? ""
     };
     const inputPayload = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
@@ -10436,13 +10435,13 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 {{${TEMPLATE_VARIABLES.CRITERIA}}}
 
 [[ ## question ## ]]
-{{${TEMPLATE_VARIABLES.QUESTION}}}
+{{${TEMPLATE_VARIABLES.INPUT_TEXT}}}
 
 [[ ## reference_answer ## ]]
-{{${TEMPLATE_VARIABLES.REFERENCE_ANSWER}}}
+{{${TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT}}}
 
 [[ ## answer ## ]]
-{{${TEMPLATE_VARIABLES.ANSWER}}}`;
+{{${TEMPLATE_VARIABLES.OUTPUT_TEXT}}}`;
 var freeformEvaluationSchema = z3.object({
   score: z3.number().min(0).max(1).describe("Score between 0.0 and 1.0"),
   assertions: z3.array(
@@ -10520,12 +10519,8 @@ var LlmGraderEvaluator = class {
         2
       ),
       [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify(context.output ?? [], null, 2),
-      [TEMPLATE_VARIABLES.ANSWER]: context.candidate.trim(),
-      [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
-      [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
-      // Text convenience accessors (new names, always strings)
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim()
@@ -10830,10 +10825,10 @@ ${context.fileChanges}`;
   buildAgentUserPrompt(context) {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const variables = {
-      [TEMPLATE_VARIABLES.ANSWER]: context.candidate.trim(),
-      [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
-      [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? ""
     };
     if (this.evaluatorTemplate) {
@@ -10886,10 +10881,10 @@ ${context.fileChanges}`;
     const rubrics = config?.type === "llm-grader" || config?.type === "llm-judge" ? config.rubrics : void 0;
     if (this.evaluatorTemplate) {
       const variables = {
-        [TEMPLATE_VARIABLES.ANSWER]: context.candidate.trim(),
-        [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context.evalCase.reference_answer ?? "").trim(),
         [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
-        [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
+        [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
+        [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
+        [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context.evalCase.reference_answer ?? "").trim(),
         [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? ""
       };
       const customPrompt = substituteVariables(this.evaluatorTemplate, variables);
@@ -12317,7 +12312,10 @@ var COPILOT_MATCHER = {
   skillTools: ["Skill", "skill"],
   skillInputField: "skill",
   readTools: ["Read File", "readFile", "Read", "readTextFile"],
-  readInputField: "file_path"
+  readInputField: "file_path",
+  skillToolPrefixes: ["Using skill: "],
+  readToolPrefixes: ["Viewing "],
+  readInputFields: ["file_path", "path"]
 };
 var PROVIDER_TOOL_SEMANTICS = {
   claude: CLAUDE_MATCHER,
@@ -12359,12 +12357,22 @@ var SkillTriggerEvaluator = class {
           triggered = true;
           evidence = `Skill tool invoked with ${matcher.skillInputField}="${skillArg}"`;
         }
+      } else if (matcher.skillToolPrefixes?.some(
+        (prefix) => firstTool.tool.startsWith(prefix) && firstTool.tool.includes(skillName)
+      )) {
+        triggered = true;
+        evidence = `Skill tool invoked via tool name "${firstTool.tool}"`;
       } else if (matcher.readTools.includes(firstTool.tool)) {
-        const filePath = String(input[matcher.readInputField] ?? "");
+        const filePath = this.readPathFromInput(input, matcher);
         if (filePath.includes(skillName)) {
           triggered = true;
           evidence = `Read tool loaded skill file: ${filePath}`;
         }
+      } else if (matcher.readToolPrefixes?.some(
+        (prefix) => firstTool.tool.startsWith(prefix) && firstTool.tool.includes(skillName)
+      )) {
+        triggered = true;
+        evidence = `Read tool loaded skill file via tool name "${firstTool.tool}"`;
       }
     }
     const pass = triggered === shouldTrigger;
@@ -12393,6 +12401,16 @@ var SkillTriggerEvaluator = class {
       expectedAspectCount: 1
     };
   }
+  readPathFromInput(input, matcher) {
+    const fields = matcher.readInputFields ?? [matcher.readInputField];
+    for (const field of fields) {
+      const value = input[field];
+      if (value !== void 0 && value !== null) {
+        return String(value);
+      }
+    }
+    return "";
+  }
 };
 
 // src/evaluation/evaluators/llm-grader-prompt.ts
@@ -12427,12 +12445,8 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evalua
     [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(evalCase.input_segments, null, 2),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(evalCase.expected_output, null, 2),
     [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify([], null, 2),
-    [TEMPLATE_VARIABLES.ANSWER]: candidate.trim(),
-    [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
-    [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
-    // Text convenience accessors (new names, always strings)
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (evalCase.reference_answer ?? "").trim()
@@ -12764,11 +12778,9 @@ var ToolTrajectoryEvaluator = class {
     for (const call of toolCalls) {
       toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
     }
-    const toolNames = Object.keys(toolCallsByName).sort();
     return {
       eventCount: toolCalls.length,
-      toolNames,
-      toolCallsByName,
+      toolCalls: toolCallsByName,
       errorCount: 0
     };
   }
@@ -12786,7 +12798,7 @@ var ToolTrajectoryEvaluator = class {
     const assertions = [];
     for (const toolName of toolNames) {
       const required = minimums[toolName];
-      const actual = summary.toolCallsByName[toolName] ?? 0;
+      const actual = summary.toolCalls[toolName] ?? 0;
       if (actual >= required) {
         assertions.push({
           text: `${toolName}: called ${actual} times (required >=${required})`,
@@ -13489,11 +13501,9 @@ async function resolveCustomPrompt(promptConfig, context, timeoutMs) {
 }
 async function executePromptTemplate(script, context, config, timeoutMs) {
   const payload = {
-    question: context.evalCase.question,
     criteria: context.evalCase.criteria,
     expectedOutput: context.evalCase.expected_output,
-    referenceAnswer: context.evalCase.reference_answer,
-    answer: context.candidate,
+    outputText: context.candidate,
     output: context.output ?? null,
     guidelineFiles: context.evalCase.guideline_paths,
     inputFiles: context.evalCase.file_paths.filter(
@@ -13504,9 +13514,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
     fileChanges: context.fileChanges ?? null,
     workspacePath: context.workspacePath ?? null,
     config: config ?? context.config ?? null,
-    // Text convenience accessors (new names, always strings)
     inputText: context.evalCase.question,
-    outputText: context.candidate,
     expectedOutputText: context.evalCase.reference_answer ?? ""
   };
   const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
@@ -15170,7 +15178,7 @@ async function runEvaluation(options) {
             dataset: evalCase.dataset,
             score: 0,
             assertions: [],
-            answer: "",
+            outputText: "",
             target: target.name,
             error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
             budgetExceeded: true,
@@ -15206,7 +15214,7 @@ async function runEvaluation(options) {
             dataset: evalCase.dataset,
             score: 0,
             assertions: [],
-            answer: "",
+            outputText: "",
             target: target.name,
             error: errorMsg,
             executionStatus: "execution_error",
@@ -15471,7 +15479,7 @@ async function runBatchEvaluation(options) {
     const providerResponse = batchResponse[i];
     const output = providerResponse.output;
     const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+    const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
     const merged = computed ? mergeExecutionMetrics(computed, {
       tokenUsage: providerResponse.tokenUsage,
       costUsd: providerResponse.costUsd,
@@ -15868,7 +15876,7 @@ async function runEvalCase(options) {
   }
   const output = providerResponse.output;
   const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
-  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
+  const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
   const merged = computed ? mergeExecutionMetrics(computed, {
     tokenUsage: providerResponse.tokenUsage,
     costUsd: providerResponse.costUsd,
@@ -16173,7 +16181,7 @@ async function evaluateCandidate(options) {
     conversationId: evalCase.conversation_id,
     score: score.score,
     assertions: score.assertions,
-    answer: candidate,
+    outputText: candidate,
     target: target.name,
     tokenUsage,
     costUsd,
@@ -16529,7 +16537,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
-    answer: `Error occurred: ${message}`,
+    outputText: `Error occurred: ${message}`,
     target: targetName,
     requests,
     input,
@@ -17067,7 +17075,7 @@ function shouldSkipCacheForTemperature(targetConfig) {
 
 // src/evaluation/baseline.ts
 var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
-  "answer",
+  "outputText",
   "requests",
   "trace",
   "workspacePath",
@@ -17241,14 +17249,17 @@ var OtelTraceExporter = class {
         rootSpan.setAttribute("agentv.target", result.target);
         if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
         rootSpan.setAttribute("agentv.score", result.score);
-        if (captureContent) rootSpan.setAttribute("agentv.answer", result.answer);
+        if (captureContent) rootSpan.setAttribute("agentv.output_text", result.outputText);
         if (result.durationMs != null)
           rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
         if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
         if (result.trace) {
           const t = result.trace;
           rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
-          rootSpan.setAttribute("agentv.trace.tool_names", t.toolNames.join(","));
+          rootSpan.setAttribute(
+            "agentv.trace.tool_names",
+            Object.keys(t.toolCalls).sort().join(",")
+          );
           if (t.llmCallCount != null)
             rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
         }