@agentv/core 4.19.0 → 4.20.0-next.1

package/dist/index.cjs

@@ -1448,6 +1448,7 @@ var init_template_variables = __esm({
       INPUT: "input",
       OUTPUT: "output",
       FILE_CHANGES: "file_changes",
+      TOOL_CALLS: "tool_calls",
       /** @deprecated Use INPUT instead — resolves to the same text value. */
       INPUT_TEXT: "input_text",
       /** @deprecated Use OUTPUT instead — resolves to the same text value. */
@@ -5832,6 +5833,7 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
           [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
           [TEMPLATE_VARIABLES.CRITERIA]: context2.evalCase.criteria.trim(),
           [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+          [TEMPLATE_VARIABLES.TOOL_CALLS]: context2.toolCalls ?? "",
           // Deprecated aliases — same values as the primary variables above
           [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
           [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
@@ -5846,6 +5848,12 @@ Be concise and focused in your evaluation. Provide succinct, specific feedback r
 
 [[ ## file_changes ## ]]
 ${context2.fileChanges}`;
+        }
+        if (context2.toolCalls && !context2.graderTemplateOverride && !this.graderTemplate) {
+          userPrompt += `
+
+[[ ## tool_calls ## ]]
+${context2.toolCalls}`;
         }
         const graderRawRequest = {
           userPrompt,
@@ -6167,6 +6175,7 @@ ${context2.fileChanges}`;
           [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
           [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
           [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+          [TEMPLATE_VARIABLES.TOOL_CALLS]: context2.toolCalls ?? "",
           // Deprecated aliases
           [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
           [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
@@ -6195,6 +6204,9 @@ ${context2.fileChanges}`;
         if (context2.fileChanges) {
           parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
         }
+        if (context2.toolCalls) {
+          parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+        }
         if (rubrics && rubrics.length > 0) {
           parts.push("[[ ## rubrics ## ]]");
           for (const rubric of rubrics) {
@@ -6228,6 +6240,7 @@ ${context2.fileChanges}`;
             [TEMPLATE_VARIABLES.OUTPUT]: context2.candidate.trim(),
             [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context2.evalCase.reference_answer ?? "").trim(),
             [TEMPLATE_VARIABLES.FILE_CHANGES]: context2.fileChanges ?? "",
+            [TEMPLATE_VARIABLES.TOOL_CALLS]: context2.toolCalls ?? "",
             // Deprecated aliases
             [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
             [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context2.candidate.trim(),
@@ -6257,6 +6270,9 @@ ${outputSchema}`;
         if (context2.fileChanges) {
           parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
         }
+        if (context2.toolCalls) {
+          parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+        }
         if (rubrics && rubrics.length > 0) {
           parts.push("[[ ## rubrics ## ]]");
           for (const rubric of rubrics) {
@@ -6349,6 +6365,9 @@ ${outputSchema}`;
         if (context2.fileChanges) {
           parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
         }
+        if (context2.toolCalls) {
+          parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+        }
         parts.push("[[ ## scoring_criteria ## ]]");
         for (const rubric of rubrics) {
           const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
@@ -6391,6 +6410,9 @@ ${outputSchema}`;
         if (context2.fileChanges) {
           parts.push("[[ ## file_changes ## ]]", context2.fileChanges, "");
         }
+        if (context2.toolCalls) {
+          parts.push("[[ ## tool_calls ## ]]", context2.toolCalls, "");
+        }
         parts.push("[[ ## rubrics ## ]]");
         for (const rubric of rubrics) {
           const requiredLabel = rubric.required ? " (REQUIRED)" : "";
@@ -7518,6 +7540,61 @@ var init_latency = __esm({
   }
 });
 
+// src/evaluation/graders/format-tool-calls.ts
+function formatToolCalls(output) {
+  if (!output) return "";
+  const lines = [];
+  for (const message of output) {
+    if (!message.toolCalls) continue;
+    for (const call of message.toolCalls) {
+      const toolName = call.tool ?? "unknown";
+      const detail = extractKeyDetail(toolName, call.input);
+      lines.push(detail ? `- ${toolName}: ${detail}` : `- ${toolName}`);
+    }
+  }
+  return lines.length > 0 ? lines.join("\n") : "";
+}
+function extractKeyDetail(toolName, input) {
+  if (!input || typeof input !== "object") return "";
+  const record = input;
+  const knownFields = KEY_INPUT_FIELDS.get(toolName);
+  if (knownFields) {
+    for (const field of knownFields) {
+      const value = record[field];
+      if (typeof value === "string" && value.length > 0) {
+        return truncate(value);
+      }
+    }
+  }
+  for (const value of Object.values(record)) {
+    if (typeof value === "string" && value.length > 0 && value.length <= MAX_FALLBACK_LENGTH) {
+      return truncate(value);
+    }
+  }
+  return "";
+}
+function truncate(value, maxLen = 120) {
+  if (value.length <= maxLen) return value;
+  return `${value.slice(0, maxLen)}\u2026`;
+}
+var KEY_INPUT_FIELDS, MAX_FALLBACK_LENGTH;
+var init_format_tool_calls = __esm({
+  "src/evaluation/graders/format-tool-calls.ts"() {
+    "use strict";
+    init_cjs_shims();
+    KEY_INPUT_FIELDS = /* @__PURE__ */ new Map([
+      ["Skill", ["skill"]],
+      ["Read", ["file_path"]],
+      ["Write", ["file_path"]],
+      ["Edit", ["file_path"]],
+      ["Bash", ["command"]],
+      ["Grep", ["pattern"]],
+      ["Glob", ["pattern"]]
+    ]);
+    MAX_FALLBACK_LENGTH = 120;
+  }
+});
+
 // src/evaluation/graders/skill-trigger.ts
 var SkillTriggerGrader;
 var init_skill_trigger = __esm({
@@ -7601,19 +7678,27 @@ function assembleLlmGraderPrompt(input) {
     promptInputs,
     evaluatorConfig,
     fileChanges,
+    toolCalls,
     graderTemplateOverride
   } = input;
   const rubrics = evaluatorConfig?.rubrics;
   if (rubrics && rubrics.length > 0) {
     const hasScoreRanges = rubrics.some((r) => r.score_ranges && r.score_ranges.length > 0);
     if (hasScoreRanges) {
-      return assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges);
+      return assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls);
     }
-    return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges);
+    return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls);
   }
-  return assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride);
+  return assembleFreeform(
+    evalCase,
+    candidate,
+    promptInputs,
+    fileChanges,
+    toolCalls,
+    graderTemplateOverride
+  );
 }
-function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride) {
+function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, toolCalls, graderTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
     [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
@@ -7621,6 +7706,7 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, grader
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
+    [TEMPLATE_VARIABLES.TOOL_CALLS]: toolCalls ?? "",
     // Deprecated aliases
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
@@ -7634,6 +7720,12 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, grader
 
 [[ ## file_changes ## ]]
 ${fileChanges}`;
+  }
+  if (toolCalls && !graderTemplateOverride) {
+    userPrompt += `
+
+[[ ## tool_calls ## ]]
+${toolCalls}`;
   }
   return {
     systemPrompt,
@@ -7642,7 +7734,7 @@ ${fileChanges}`;
     mode: "freeform"
   };
 }
-function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges) {
+function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
     "You are an expert grader. Evaluate the candidate answer against each rubric item below.",
@@ -7661,6 +7753,9 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
   if (fileChanges) {
     parts.push("[[ ## file_changes ## ]]", fileChanges, "");
   }
+  if (toolCalls) {
+    parts.push("[[ ## tool_calls ## ]]", toolCalls, "");
+  }
   parts.push("[[ ## rubrics ## ]]");
   for (const rubric of rubrics) {
     const requiredLabel = rubric.required ? " (REQUIRED)" : "";
@@ -7677,7 +7772,7 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
     mode: "checklist"
   };
 }
-function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges) {
+function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
     "You are an expert grader. Score the candidate answer on each criterion below using the provided score ranges.",
@@ -7697,6 +7792,9 @@ function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChan
   if (fileChanges) {
     parts.push("[[ ## file_changes ## ]]", fileChanges, "");
   }
+  if (toolCalls) {
+    parts.push("[[ ## tool_calls ## ]]", toolCalls, "");
+  }
   parts.push("[[ ## scoring_criteria ## ]]");
   for (const rubric of rubrics) {
     const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
@@ -8470,6 +8568,7 @@ var init_graders = __esm({
     init_field_accuracy();
     init_latency();
     init_llm_grader();
+    init_format_tool_calls();
     init_skill_trigger();
     init_llm_grader_prompt();
     init_token_usage();
@@ -19200,6 +19299,7 @@ async function runEvaluation(options) {
     trials,
     streamCallbacks,
     budgetUsd,
+    runBudgetTracker,
     failOnError,
     poolWorkspaces,
     poolMaxSlots: configPoolMaxSlots,
@@ -19534,8 +19634,14 @@ async function runEvaluation(options) {
         }
       }
       return { ok: allPassed, depResults };
+    }, extractEvaluationCostUsd2 = function(result) {
+      if (result.trials && result.trials.length > 0) {
+        const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
+        return trialCostSum > 0 ? trialCostSum : void 0;
+      }
+      return result.costUsd;
     };
-    var toDependencyResult = toDependencyResult2, checkDependencies = checkDependencies2;
+    var toDependencyResult = toDependencyResult2, checkDependencies = checkDependencies2, extractEvaluationCostUsd = extractEvaluationCostUsd2;
     if (suiteWorkspaceFile && sharedWorkspacePath) {
       const copiedWorkspaceFile = import_node_path47.default.join(sharedWorkspacePath, import_node_path47.default.basename(suiteWorkspaceFile));
       try {
@@ -19728,6 +19834,42 @@ async function runEvaluation(options) {
     async function dispatchTest(evalCase, depResults) {
       const workerId = nextWorkerId++;
       workerIdByEvalId.set(evalCase.id, workerId);
+      if (runBudgetTracker?.isExceeded()) {
+        const budgetResult = {
+          timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          suite: evalCase.suite,
+          category: evalCase.category,
+          score: 0,
+          assertions: [],
+          output: [],
+          target: target.name,
+          error: `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`,
+          budgetExceeded: true,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "budget_exceeded",
+          executionError: {
+            message: `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`,
+            stage: "setup"
+          }
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: budgetResult.error,
+            score: budgetResult.score,
+            executionStatus: budgetResult.executionStatus
+          });
+        }
+        if (onResult) {
+          await onResult(budgetResult);
+        }
+        return budgetResult;
+      }
       if (budgetUsd !== void 0 && budgetExhausted) {
         const budgetResult = {
           timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
@@ -19841,22 +19983,17 @@ async function runEvaluation(options) {
           ...depResults && Object.keys(depResults).length > 0 ? { dependencyResults: depResults } : {}
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
-        if (budgetUsd !== void 0) {
-          let caseCost;
-          if (result.trials && result.trials.length > 0) {
-            const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
-            if (trialCostSum > 0) {
-              caseCost = trialCostSum;
-            }
-          } else {
-            caseCost = result.costUsd;
-          }
-          if (caseCost !== void 0) {
+        const caseCost = extractEvaluationCostUsd2(result);
+        if (caseCost !== void 0) {
+          if (budgetUsd !== void 0) {
             cumulativeBudgetCost += caseCost;
             if (cumulativeBudgetCost >= budgetUsd) {
               budgetExhausted = true;
             }
           }
+          if (runBudgetTracker) {
+            runBudgetTracker.add(caseCost);
+          }
         }
         if (failOnError === true && result.executionStatus === "execution_error") {
           failOnErrorTriggered = true;
@@ -20664,6 +20801,7 @@ async function runEvalCase(options) {
     fileChanges = fileChanges ? `${fileChanges}
 ${providerFileChanges}` : providerFileChanges;
   }
+  const toolCalls = formatToolCalls(output);
   const providerError = extractProviderError(providerResponse);
   const targetAfterEachHook = options.targetHooks?.after_each;
   if (workspacePath && hasHookCommand(targetAfterEachHook)) {
@@ -20747,6 +20885,7 @@ ${providerFileChanges}` : providerFileChanges;
       targetResolver,
       availableTargets,
       fileChanges,
+      toolCalls,
       workspacePath,
       dockerConfig: evalCase.workspace?.docker,
       verbose,
@@ -20944,6 +21083,7 @@ async function evaluateCandidate(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold: evalThreshold,
@@ -20972,6 +21112,7 @@ async function evaluateCandidate(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold: evalThreshold,
@@ -21049,6 +21190,7 @@ async function runEvaluatorsForCase(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold,
@@ -21078,6 +21220,7 @@ async function runEvaluatorsForCase(options) {
       targetResolver,
       availableTargets,
       fileChanges,
+      toolCalls,
       workspacePath,
       dockerConfig,
       threshold,
@@ -21109,6 +21252,7 @@ async function runEvaluatorsForCase(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults,
@@ -21150,6 +21294,7 @@ async function runEvaluatorList(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults
@@ -21175,6 +21320,7 @@ async function runEvaluatorList(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults
@@ -24109,6 +24255,7 @@ __export(index_exports, {
   ProviderRegistry: () => ProviderRegistry,
   RepoManager: () => RepoManager,
   ResponseCache: () => ResponseCache,
+  RunBudgetTracker: () => RunBudgetTracker,
   SkillTriggerGrader: () => SkillTriggerGrader,
   TEST_MESSAGE_ROLES: () => TEST_MESSAGE_ROLES,
   TemplateNotDirectoryError: () => TemplateNotDirectoryError,
@@ -24180,6 +24327,7 @@ __export(index_exports, {
   extractWorkersFromSuite: () => extractWorkersFromSuite,
   fileExists: () => fileExists2,
   findGitRoot: () => findGitRoot,
+  formatToolCalls: () => formatToolCalls,
   freeformEvaluationSchema: () => freeformEvaluationSchema,
   generateRubrics: () => generateRubrics,
   getAgentvConfigDir: () => getAgentvConfigDir,
@@ -25857,6 +26005,31 @@ init_assertion_discovery();
 init_assertions();
 init_grader_discovery();
 
+// src/evaluation/run-budget-tracker.ts
+init_cjs_shims();
+var RunBudgetTracker = class {
+  constructor(capUsd) {
+    this.capUsd = capUsd;
+  }
+  cumulative = 0;
+  /** Accumulate cost from a completed test or file. */
+  add(costUsd) {
+    this.cumulative += costUsd;
+  }
+  /** True when cumulative cost meets or exceeds the cap. */
+  isExceeded() {
+    return this.cumulative >= this.capUsd;
+  }
+  /** Current accumulated cost. */
+  get currentCostUsd() {
+    return this.cumulative;
+  }
+  /** The configured cap. */
+  get budgetCapUsd() {
+    return this.capUsd;
+  }
+};
+
 // src/import/index.ts
 init_cjs_shims();
 
@@ -26525,6 +26698,7 @@ function createAgentKernel() {
   ProviderRegistry,
   RepoManager,
   ResponseCache,
+  RunBudgetTracker,
   SkillTriggerGrader,
   TEST_MESSAGE_ROLES,
   TemplateNotDirectoryError,
@@ -26596,6 +26770,7 @@ function createAgentKernel() {
   extractWorkersFromSuite,
   fileExists,
   findGitRoot,
+  formatToolCalls,
   freeformEvaluationSchema,
   generateRubrics,
   getAgentvConfigDir,