agentv 3.2.4 → 3.3.0

package/dist/{chunk-VBGYESW7.js → chunk-5M3K2DMV.js}

@@ -301,7 +301,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-5SQK5FXC.js
+// ../../packages/core/dist/chunk-C4MKEQR5.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -419,7 +419,7 @@ __export(external_exports2, {
   void: () => voidType
 });
 
-// ../../packages/core/dist/chunk-5SQK5FXC.js
+// ../../packages/core/dist/chunk-C4MKEQR5.js
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
@@ -14657,19 +14657,26 @@ function logWarning(message) {
   console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET3}`);
 }
 var TEMPLATE_VARIABLES = {
+  /** @deprecated Use OUTPUT_TEXT instead */
   ANSWER: "answer",
   EXPECTED_OUTPUT: "expected_output",
+  /** @deprecated Use INPUT_TEXT instead */
   QUESTION: "question",
   CRITERIA: "criteria",
+  /** @deprecated Use EXPECTED_OUTPUT_TEXT instead */
   REFERENCE_ANSWER: "reference_answer",
   INPUT: "input",
   OUTPUT: "output",
-  FILE_CHANGES: "file_changes"
+  FILE_CHANGES: "file_changes",
+  INPUT_TEXT: "input_text",
+  OUTPUT_TEXT: "output_text",
+  EXPECTED_OUTPUT_TEXT: "expected_output_text"
 };
 var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
   TEMPLATE_VARIABLES.ANSWER,
-  TEMPLATE_VARIABLES.EXPECTED_OUTPUT
+  TEMPLATE_VARIABLES.EXPECTED_OUTPUT,
+  TEMPLATE_VARIABLES.OUTPUT_TEXT
 ]);
 var ANSI_YELLOW3 = "\x1B[33m";
 var ANSI_RESET4 = "\x1B[0m";
@@ -14690,13 +14697,13 @@ function validateTemplateVariables(content, source) {
     }
     match = variablePattern.exec(content);
   }
-  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.ANSWER);
+  const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.ANSWER) || foundVariables.has(TEMPLATE_VARIABLES.OUTPUT_TEXT);
   const hasExpectedOutput = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_OUTPUT);
   const hasRequiredFields = hasCandidateAnswer || hasExpectedOutput;
   if (!hasRequiredFields) {
     throw new Error(
       `Missing required fields. Must include at least one of:
-  - {{ ${TEMPLATE_VARIABLES.ANSWER} }}
+  - {{ ${TEMPLATE_VARIABLES.ANSWER} }} or {{ ${TEMPLATE_VARIABLES.OUTPUT_TEXT} }}
   - {{ ${TEMPLATE_VARIABLES.EXPECTED_OUTPUT} }}`
     );
   }
@@ -17856,7 +17863,14 @@ async function invokeModel(options) {
 function mapResponse(result) {
   const content = result.text ?? "";
   const rawUsage = result.totalUsage ?? result.usage;
-  const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
+  const reasoning = rawUsage?.outputTokenDetails?.reasoningTokens ?? void 0;
+  const cached = rawUsage?.inputTokenDetails?.cacheReadTokens ?? void 0;
+  const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? {
+    input: rawUsage.inputTokens,
+    output: rawUsage.outputTokens,
+    ...reasoning != null ? { reasoning } : {},
+    ...cached != null ? { cached } : {}
+  } : void 0;
   return {
     raw: result,
     usage: toJsonObject(rawUsage),
@@ -18191,10 +18205,12 @@ var ClaudeCliProvider = class {
             if (usage) {
               const inputTokens = (usage.input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0);
               const outputTokens = usage.output_tokens ?? 0;
+              const reasoningTokens = usage.reasoning_tokens ?? void 0;
               tokenUsage = {
                 input: inputTokens,
                 output: outputTokens,
-                cached: usage.cache_read_input_tokens ?? void 0
+                cached: usage.cache_read_input_tokens ?? void 0,
+                reasoning: reasoningTokens
               };
               request.streamCallbacks?.onLlmCallEnd?.(this.config.model ?? "claude", tokenUsage);
             }
@@ -23859,7 +23875,11 @@ var CodeEvaluator = class {
       endTime: context2.endTime ?? null,
       fileChanges: context2.fileChanges ?? null,
       workspacePath: context2.workspacePath ?? null,
-      config: this.config ?? null
+      config: this.config ?? null,
+      // Text convenience accessors (new names, always strings)
+      inputText: context2.evalCase.question,
+      outputText: context2.candidate,
+      expectedOutputText: context2.evalCase.reference_answer ?? ""
     };
     const inputPayload = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
     let proxyEnv;
@@ -24101,7 +24121,11 @@ var LlmGraderEvaluator = class {
       [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
       [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
-      [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? ""
+      [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+      // Text convenience accessors (new names, always strings)
+      [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
+      [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
+      [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (context2.evalCase.reference_answer ?? "").trim()
     };
     const systemPrompt = buildOutputSchema();
     const evaluatorTemplate = context2.evaluatorTemplateOverride ?? this.evaluatorTemplate ?? DEFAULT_EVALUATOR_TEMPLATE;
@@ -25941,28 +25965,60 @@ var LatencyEvaluator = class {
     };
   }
 };
+var CLAUDE_MATCHER = {
+  skillTools: ["Skill"],
+  skillInputField: "skill",
+  readTools: ["Read"],
+  readInputField: "file_path"
+};
+var COPILOT_MATCHER = {
+  skillTools: ["Skill", "skill"],
+  skillInputField: "skill",
+  readTools: ["Read File", "readFile", "Read", "readTextFile"],
+  readInputField: "file_path"
+};
+var PROVIDER_TOOL_SEMANTICS = {
+  claude: CLAUDE_MATCHER,
+  "claude-cli": CLAUDE_MATCHER,
+  "claude-sdk": CLAUDE_MATCHER,
+  "pi-coding-agent": CLAUDE_MATCHER,
+  "pi-agent-sdk": CLAUDE_MATCHER,
+  "copilot-cli": COPILOT_MATCHER,
+  "copilot-sdk": COPILOT_MATCHER,
+  vscode: COPILOT_MATCHER,
+  "vscode-insiders": COPILOT_MATCHER
+};
 var SkillTriggerEvaluator = class {
   kind = "skill-trigger";
   config;
   constructor(config) {
     this.config = config;
   }
+  resolveMatcher(providerKind) {
+    if (providerKind) {
+      const match = PROVIDER_TOOL_SEMANTICS[providerKind];
+      if (match) return match;
+    }
+    return CLAUDE_MATCHER;
+  }
   evaluate(context2) {
     const skillName = this.config.skill;
     const shouldTrigger = this.config.should_trigger !== false;
+    const providerKind = context2.provider?.kind;
+    const matcher = this.resolveMatcher(providerKind);
     const firstTool = (context2.output ?? []).flatMap((msg) => msg.toolCalls ?? [])[0];
     let triggered = false;
     let evidence = "";
     if (firstTool) {
       const input = firstTool.input ?? {};
-      if (firstTool.tool === "Skill") {
-        const skillArg = String(input.skill ?? "");
+      if (matcher.skillTools.includes(firstTool.tool)) {
+        const skillArg = String(input[matcher.skillInputField] ?? "");
         if (skillArg.includes(skillName)) {
           triggered = true;
-          evidence = `Skill tool invoked with skill="${skillArg}"`;
+          evidence = `Skill tool invoked with ${matcher.skillInputField}="${skillArg}"`;
         }
-      } else if (firstTool.tool === "Read") {
-        const filePath = String(input.file_path ?? "");
+      } else if (matcher.readTools.includes(firstTool.tool)) {
+        const filePath = String(input[matcher.readInputField] ?? "");
         if (filePath.includes(skillName)) {
           triggered = true;
           evidence = `Read tool loaded skill file: ${filePath}`;
@@ -25987,7 +26043,7 @@ var SkillTriggerEvaluator = class {
       verdict: "fail",
       hits: [],
       misses: [
-        shouldTrigger ? firstTool ? `First tool was "${firstTool.tool}" \u2014 not Skill/Read for "${skillName}"` : "No tool calls recorded" : evidence || `Skill "${skillName}" triggered unexpectedly`
+        shouldTrigger ? firstTool ? `First tool was "${firstTool.tool}" \u2014 not a skill/read tool for "${skillName}"` : "No tool calls recorded" : evidence || `Skill "${skillName}" triggered unexpectedly`
       ],
       expectedAspectCount: 1,
       reasoning: shouldTrigger ? `Skill "${skillName}" was not triggered` : "False trigger: skill fired when it should not have"
@@ -26029,7 +26085,11 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evalua
     [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
     [TEMPLATE_VARIABLES.QUESTION]: formattedQuestion.trim(),
-    [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? ""
+    [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
+    // Text convenience accessors (new names, always strings)
+    [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
+    [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
+    [TEMPLATE_VARIABLES.EXPECTED_OUTPUT_TEXT]: (evalCase.reference_answer ?? "").trim()
   };
   const systemPrompt = buildOutputSchema();
   const template = evaluatorTemplateOverride ?? DEFAULT_EVALUATOR_TEMPLATE;
@@ -27020,7 +27080,11 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
     trace: context2.trace ?? null,
     fileChanges: context2.fileChanges ?? null,
     workspacePath: context2.workspacePath ?? null,
-    config: config ?? context2.config ?? null
+    config: config ?? context2.config ?? null,
+    // Text convenience accessors (new names, always strings)
+    inputText: context2.evalCase.question,
+    outputText: context2.candidate,
+    expectedOutputText: context2.evalCase.reference_answer ?? ""
   };
   const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
   const scriptPath = script[script.length - 1];
@@ -29279,6 +29343,7 @@ async function runEvalCase(options) {
     } catch {
     }
   }
+  const caseStartMs = Date.now();
   const attemptBudget = (maxRetries ?? 0) + 1;
   let attempt = 0;
   let providerResponse = cachedResponse;
@@ -29427,9 +29492,22 @@ async function runEvalCase(options) {
       fileChanges,
       workspacePath
     });
+    const totalDurationMs = Date.now() - caseStartMs;
+    const graderTokens = aggregateEvaluatorTokenUsage(result.scores);
+    const evalRunTokenUsage = tokenUsage || graderTokens ? {
+      input: (tokenUsage?.input ?? 0) + (graderTokens?.input ?? 0),
+      output: (tokenUsage?.output ?? 0) + (graderTokens?.output ?? 0),
+      ...tokenUsage?.reasoning != null || graderTokens?.reasoning != null ? { reasoning: (tokenUsage?.reasoning ?? 0) + (graderTokens?.reasoning ?? 0) } : {},
+      ...tokenUsage?.cached != null || graderTokens?.cached != null ? { cached: (tokenUsage?.cached ?? 0) + (graderTokens?.cached ?? 0) } : {}
+    } : void 0;
+    const evalRun = {
+      durationMs: totalDurationMs,
+      ...evalRunTokenUsage ? { tokenUsage: evalRunTokenUsage } : {}
+    };
     const executionStatus = providerError ? "execution_error" : classifyQualityStatus(result.score);
     const finalResult = providerError ? {
       ...result,
+      evalRun,
       error: providerError,
       executionStatus,
       failureStage: "agent",
@@ -29438,7 +29516,7 @@ async function runEvalCase(options) {
       beforeAllOutput,
       beforeEachOutput,
       afterEachOutput
-    } : { ...result, executionStatus, beforeAllOutput, beforeEachOutput, afterEachOutput };
+    } : { ...result, evalRun, executionStatus, beforeAllOutput, beforeEachOutput, afterEachOutput };
     const isFailure = !!finalResult.error || finalResult.score < 0.5;
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup) {
@@ -29458,6 +29536,7 @@ async function runEvalCase(options) {
     }
     return finalResult;
   } catch (error) {
+    const evalRun = { durationMs: Date.now() - caseStartMs };
     const errorResult = buildErrorResult(
       evalCase,
       target.name,
@@ -29473,10 +29552,10 @@ async function runEvalCase(options) {
         await cleanupWorkspace(workspacePath).catch(() => {
         });
       } else {
-        return { ...errorResult, workspacePath, beforeEachOutput, afterEachOutput };
+        return { ...errorResult, evalRun, workspacePath, beforeEachOutput, afterEachOutput };
       }
     }
-    return { ...errorResult, beforeEachOutput, afterEachOutput };
+    return { ...errorResult, evalRun, beforeEachOutput, afterEachOutput };
   }
 }
 async function runEvalCaseWithTrials(options, trialsConfig) {
@@ -30051,6 +30130,44 @@ function buildResultInput(promptInputs) {
   }
   return promptInputs.question;
 }
+function aggregateEvaluatorTokenUsage(scores) {
+  if (!scores || scores.length === 0) return void 0;
+  let hasAny = false;
+  let input = 0;
+  let output = 0;
+  let reasoning = 0;
+  let cached = 0;
+  let hasReasoning = false;
+  let hasCached = false;
+  const visit = (items) => {
+    for (const item of items) {
+      if (item.tokenUsage) {
+        hasAny = true;
+        input += item.tokenUsage.input;
+        output += item.tokenUsage.output;
+        if (item.tokenUsage.reasoning != null) {
+          hasReasoning = true;
+          reasoning += item.tokenUsage.reasoning;
+        }
+        if (item.tokenUsage.cached != null) {
+          hasCached = true;
+          cached += item.tokenUsage.cached;
+        }
+      }
+      if (item.scores) {
+        visit(item.scores);
+      }
+    }
+  };
+  visit(scores);
+  if (!hasAny) return void 0;
+  return {
+    input,
+    output,
+    ...hasReasoning ? { reasoning } : {},
+    ...hasCached ? { cached } : {}
+  };
+}
 function isTimeoutLike(error) {
   if (!error) {
     return false;
@@ -31086,4 +31203,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-VBGYESW7.js.map
+//# sourceMappingURL=chunk-5M3K2DMV.js.map