agentv 4.19.0 → 4.20.0

package/dist/{chunk-PTYQS37Y.js → chunk-LP4Y5D2Z.js}

@@ -8225,7 +8225,7 @@ var _a20;
 _a20 = symbol20;
 var defaultDownload2 = createDownload();
 
-// ../../packages/core/dist/chunk-QXX3IBYV.js
+// ../../packages/core/dist/chunk-ELF6SQAK.js
 import path46 from "node:path";
 import { pathToFileURL as pathToFileURL2 } from "node:url";
 import { existsSync as existsSync6 } from "node:fs";
@@ -12915,7 +12915,7 @@ var openrouter = createOpenRouter({
   // strict for OpenRouter API
 });
 
-// ../../packages/core/dist/chunk-QXX3IBYV.js
+// ../../packages/core/dist/chunk-ELF6SQAK.js
 import { spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { createWriteStream } from "node:fs";
@@ -14419,7 +14419,7 @@ var RequestError = class _RequestError extends Error {
   }
 };
 
-// ../../packages/core/dist/chunk-QXX3IBYV.js
+// ../../packages/core/dist/chunk-ELF6SQAK.js
 import { exec as execCallback } from "node:child_process";
 import { readdirSync, statSync } from "node:fs";
 import { readFile as readFile22, readdir, stat } from "node:fs/promises";
@@ -15461,6 +15461,7 @@ var TEMPLATE_VARIABLES = {
   INPUT: "input",
   OUTPUT: "output",
   FILE_CHANGES: "file_changes",
+  TOOL_CALLS: "tool_calls",
   /** @deprecated Use INPUT instead — resolves to the same text value. */
   INPUT_TEXT: "input_text",
   /** @deprecated Use OUTPUT instead — resolves to the same text value. */
@@ -15637,6 +15638,7 @@ var LlmGrader = class {
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+      [TEMPLATE_VARIABLES.TOOL_CALLS]: context2.toolCalls ?? "",
       // Deprecated aliases — same values as the primary variables above
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
@@ -15651,6 +15653,12 @@ var LlmGrader = class {
 
 [[ ## file_changes ## ]]
 ${context2.fileChanges}`;
+    }
+    if (context2.toolCalls && !context2.graderTemplateOverride && !this.graderTemplate) {
+      userPrompt += `
+
+[[ ## tool_calls ## ]]
+${context2.toolCalls}`;
     }
     const graderRawRequest = {
       userPrompt,
@@ -15972,6 +15980,7 @@ ${context2.fileChanges}`;
       [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+      [TEMPLATE_VARIABLES.TOOL_CALLS]: context2.toolCalls ?? "",
       // Deprecated aliases
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
@@ -16000,6 +16009,9 @@ ${context2.fileChanges}`;
     if (context2.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
     }
+    if (context2.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+    }
     if (rubrics && rubrics.length > 0) {
       parts.push("[[ ## rubrics ## ]]");
       for (const rubric of rubrics) {
@@ -16033,6 +16045,7 @@ ${context2.fileChanges}`;
         [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
         [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
         [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+        [TEMPLATE_VARIABLES.TOOL_CALLS]: context2.toolCalls ?? "",
         // Deprecated aliases
         [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
         [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
@@ -16062,6 +16075,9 @@ ${outputSchema2}`;
     if (context2.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
     }
+    if (context2.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+    }
     if (rubrics && rubrics.length > 0) {
       parts.push("[[ ## rubrics ## ]]");
       for (const rubric of rubrics) {
@@ -16154,6 +16170,9 @@ ${outputSchema2}`;
     if (context2.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
     }
+    if (context2.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+    }
     parts.push("[[ ## scoring_criteria ## ]]");
     for (const rubric of rubrics) {
       const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
@@ -16196,6 +16215,9 @@ ${outputSchema2}`;
     if (context2.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
     }
+    if (context2.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+    }
     parts.push("[[ ## rubrics ## ]]");
     for (const rubric of rubrics) {
       const requiredLabel = rubric.required ? " (REQUIRED)" : "";
@@ -17694,6 +17716,52 @@ var LatencyGrader = class {
     };
   }
 };
+var KEY_INPUT_FIELDS = /* @__PURE__ */ new Map([
+  ["Skill", ["skill"]],
+  ["Read", ["file_path"]],
+  ["Write", ["file_path"]],
+  ["Edit", ["file_path"]],
+  ["Bash", ["command"]],
+  ["Grep", ["pattern"]],
+  ["Glob", ["pattern"]]
+]);
+var MAX_FALLBACK_LENGTH = 120;
+function formatToolCalls(output) {
+  if (!output) return "";
+  const lines = [];
+  for (const message of output) {
+    if (!message.toolCalls) continue;
+    for (const call of message.toolCalls) {
+      const toolName = call.tool ?? "unknown";
+      const detail = extractKeyDetail(toolName, call.input);
+      lines.push(detail ? `- ${toolName}: ${detail}` : `- ${toolName}`);
+    }
+  }
+  return lines.length > 0 ? lines.join("\n") : "";
+}
+function extractKeyDetail(toolName, input) {
+  if (!input || typeof input !== "object") return "";
+  const record = input;
+  const knownFields = KEY_INPUT_FIELDS.get(toolName);
+  if (knownFields) {
+    for (const field of knownFields) {
+      const value = record[field];
+      if (typeof value === "string" && value.length > 0) {
+        return truncate(value);
+      }
+    }
+  }
+  for (const value of Object.values(record)) {
+    if (typeof value === "string" && value.length > 0 && value.length <= MAX_FALLBACK_LENGTH) {
+      return truncate(value);
+    }
+  }
+  return "";
+}
+function truncate(value, maxLen = 120) {
+  if (value.length <= maxLen) return value;
+  return `${value.slice(0, maxLen)}\u2026`;
+}
 var SkillTriggerGrader = class {
   kind = "skill-trigger";
   config;
@@ -17767,19 +17835,27 @@ function assembleLlmGraderPrompt(input) {
     promptInputs,
     evaluatorConfig,
     fileChanges,
+    toolCalls,
     graderTemplateOverride
   } = input;
   const rubrics = evaluatorConfig?.rubrics;
   if (rubrics && rubrics.length > 0) {
     const hasScoreRanges = rubrics.some((r) => r.score_ranges && r.score_ranges.length > 0);
     if (hasScoreRanges) {
-      return assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges);
+      return assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls);
     }
-    return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges);
+    return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls);
   }
-  return assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride);
+  return assembleFreeform(
+    evalCase,
+    candidate,
+    promptInputs,
+    fileChanges,
+    toolCalls,
+    graderTemplateOverride
+  );
 }
-function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride) {
+function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, toolCalls, graderTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
     [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
@@ -17787,6 +17863,7 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, grader
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
+    [TEMPLATE_VARIABLES.TOOL_CALLS]: toolCalls ?? "",
     // Deprecated aliases
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
@@ -17800,6 +17877,12 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, grader
 
 [[ ## file_changes ## ]]
 ${fileChanges}`;
+  }
+  if (toolCalls && !graderTemplateOverride) {
+    userPrompt += `
+
+[[ ## tool_calls ## ]]
+${toolCalls}`;
   }
   return {
     systemPrompt,
@@ -17808,7 +17891,7 @@ ${fileChanges}`;
     mode: "freeform"
   };
 }
-function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges) {
+function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
     "You are an expert grader. Evaluate the candidate answer against each rubric item below.",
@@ -17827,6 +17910,9 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
   if (fileChanges) {
     parts.push("[[ ## file_changes ## ]]", fileChanges, "");
   }
+  if (toolCalls) {
+    parts.push("[[ ## tool_calls ## ]]", toolCalls, "");
+  }
   parts.push("[[ ## rubrics ## ]]");
   for (const rubric of rubrics) {
     const requiredLabel = rubric.required ? " (REQUIRED)" : "";
@@ -17843,7 +17929,7 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
     mode: "checklist"
   };
 }
-function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges) {
+function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
     "You are an expert grader. Score the candidate answer on each criterion below using the provided score ranges.",
@@ -17863,6 +17949,9 @@ function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChan
   if (fileChanges) {
     parts.push("[[ ## file_changes ## ]]", fileChanges, "");
   }
+  if (toolCalls) {
+    parts.push("[[ ## tool_calls ## ]]", toolCalls, "");
+  }
   parts.push("[[ ## scoring_criteria ## ]]");
   for (const rubric of rubrics) {
     const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
@@ -30105,7 +30194,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     return { tests: await loadTestsFromAgentSkills(evalFilePath) };
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-XFQ6S4DT-S7P2UUBX.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-32COE32J-TCT4RIRT.js");
     return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
   }
   const { tests, parsed, suiteWorkspacePath } = await loadTestsFromYaml(
@@ -30140,7 +30229,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
     return loadTestsFromAgentSkills(evalFilePath);
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-XFQ6S4DT-S7P2UUBX.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-32COE32J-TCT4RIRT.js");
     const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
     return suite.tests;
   }
@@ -30743,6 +30832,7 @@ async function runEvaluation(options) {
     trials,
     streamCallbacks,
     budgetUsd,
+    runBudgetTracker,
     failOnError,
     poolWorkspaces,
     poolMaxSlots: configPoolMaxSlots,
@@ -31077,8 +31167,14 @@ async function runEvaluation(options) {
         }
       }
       return { ok: allPassed, depResults };
+    }, extractEvaluationCostUsd2 = function(result) {
+      if (result.trials && result.trials.length > 0) {
+        const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
+        return trialCostSum > 0 ? trialCostSum : void 0;
+      }
+      return result.costUsd;
     };
-    var toDependencyResult = toDependencyResult2, checkDependencies = checkDependencies2;
+    var toDependencyResult = toDependencyResult2, checkDependencies = checkDependencies2, extractEvaluationCostUsd = extractEvaluationCostUsd2;
     if (suiteWorkspaceFile && sharedWorkspacePath) {
       const copiedWorkspaceFile = path44.join(sharedWorkspacePath, path44.basename(suiteWorkspaceFile));
       try {
@@ -31271,6 +31367,42 @@ async function runEvaluation(options) {
     async function dispatchTest(evalCase, depResults) {
       const workerId = nextWorkerId++;
       workerIdByEvalId.set(evalCase.id, workerId);
+      if (runBudgetTracker?.isExceeded()) {
+        const budgetResult = {
+          timestamp: (now2 ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          suite: evalCase.suite,
+          category: evalCase.category,
+          score: 0,
+          assertions: [],
+          output: [],
+          target: target.name,
+          error: `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`,
+          budgetExceeded: true,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "budget_exceeded",
+          executionError: {
+            message: `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`,
+            stage: "setup"
+          }
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: budgetResult.error,
+            score: budgetResult.score,
+            executionStatus: budgetResult.executionStatus
+          });
+        }
+        if (onResult) {
+          await onResult(budgetResult);
+        }
+        return budgetResult;
+      }
       if (budgetUsd !== void 0 && budgetExhausted) {
         const budgetResult = {
           timestamp: (now2 ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
@@ -31384,22 +31516,17 @@ async function runEvaluation(options) {
           ...depResults && Object.keys(depResults).length > 0 ? { dependencyResults: depResults } : {}
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
-        if (budgetUsd !== void 0) {
-          let caseCost;
-          if (result.trials && result.trials.length > 0) {
-            const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
-            if (trialCostSum > 0) {
-              caseCost = trialCostSum;
-            }
-          } else {
-            caseCost = result.costUsd;
-          }
-          if (caseCost !== void 0) {
+        const caseCost = extractEvaluationCostUsd2(result);
+        if (caseCost !== void 0) {
+          if (budgetUsd !== void 0) {
             cumulativeBudgetCost += caseCost;
             if (cumulativeBudgetCost >= budgetUsd) {
               budgetExhausted = true;
             }
           }
+          if (runBudgetTracker) {
+            runBudgetTracker.add(caseCost);
+          }
         }
         if (failOnError === true && result.executionStatus === "execution_error") {
           failOnErrorTriggered = true;
@@ -32207,6 +32334,7 @@ async function runEvalCase(options) {
     fileChanges = fileChanges ? `${fileChanges}
 ${providerFileChanges}` : providerFileChanges;
   }
+  const toolCalls = formatToolCalls(output);
   const providerError = extractProviderError(providerResponse);
   const targetAfterEachHook = options.targetHooks?.after_each;
   if (workspacePath && hasHookCommand(targetAfterEachHook)) {
@@ -32290,6 +32418,7 @@ ${providerFileChanges}` : providerFileChanges;
       targetResolver,
       availableTargets,
       fileChanges,
+      toolCalls,
       workspacePath,
       dockerConfig: evalCase.workspace?.docker,
       verbose,
@@ -32487,6 +32616,7 @@ async function evaluateCandidate(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold: evalThreshold,
@@ -32515,6 +32645,7 @@ async function evaluateCandidate(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold: evalThreshold,
@@ -32592,6 +32723,7 @@ async function runEvaluatorsForCase(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold,
@@ -32621,6 +32753,7 @@ async function runEvaluatorsForCase(options) {
       targetResolver,
       availableTargets,
       fileChanges,
+      toolCalls,
       workspacePath,
       dockerConfig,
       threshold,
@@ -32652,6 +32785,7 @@ async function runEvaluatorsForCase(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults,
@@ -32693,6 +32827,7 @@ async function runEvaluatorList(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults
@@ -32718,6 +32853,7 @@ async function runEvaluatorList(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults
@@ -33778,6 +33914,7 @@ export {
   ExecutionMetricsGrader,
   FieldAccuracyGrader,
   LatencyGrader,
+  formatToolCalls,
   SkillTriggerGrader,
   assembleLlmGraderPrompt,
   TokenUsageGrader,
@@ -33864,4 +34001,4 @@ export {
   loadTsEvalFile,
   loadTsEvalSuite
 };
-//# sourceMappingURL=chunk-PTYQS37Y.js.map
+//# sourceMappingURL=chunk-LP4Y5D2Z.js.map