@agentv/core 4.19.0-next.1 → 4.20.0-next.1

package/dist/{chunk-QXX3IBYV.js → chunk-ELF6SQAK.js}

@@ -1017,6 +1017,7 @@ var TEMPLATE_VARIABLES = {
   INPUT: "input",
   OUTPUT: "output",
   FILE_CHANGES: "file_changes",
+  TOOL_CALLS: "tool_calls",
   /** @deprecated Use INPUT instead — resolves to the same text value. */
   INPUT_TEXT: "input_text",
   /** @deprecated Use OUTPUT instead — resolves to the same text value. */
@@ -1195,6 +1196,7 @@ var LlmGrader = class {
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.CRITERIA]: context.evalCase.criteria.trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
+      [TEMPLATE_VARIABLES.TOOL_CALLS]: context.toolCalls ?? "",
       // Deprecated aliases — same values as the primary variables above
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
@@ -1209,6 +1211,12 @@ var LlmGrader = class {
 
 [[ ## file_changes ## ]]
 ${context.fileChanges}`;
+    }
+    if (context.toolCalls && !context.graderTemplateOverride && !this.graderTemplate) {
+      userPrompt += `
+
+[[ ## tool_calls ## ]]
+${context.toolCalls}`;
     }
     const graderRawRequest = {
       userPrompt,
@@ -1530,6 +1538,7 @@ ${context.fileChanges}`;
       [TEMPLATE_VARIABLES.OUTPUT]: context.candidate.trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
+      [TEMPLATE_VARIABLES.TOOL_CALLS]: context.toolCalls ?? "",
       // Deprecated aliases
       [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
       [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
@@ -1558,6 +1567,9 @@ ${context.fileChanges}`;
     if (context.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context.fileChanges, "");
     }
+    if (context.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context.toolCalls, "");
+    }
     if (rubrics && rubrics.length > 0) {
       parts.push("[[ ## rubrics ## ]]");
       for (const rubric of rubrics) {
@@ -1591,6 +1603,7 @@ ${context.fileChanges}`;
         [TEMPLATE_VARIABLES.OUTPUT]: context.candidate.trim(),
         [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (context.evalCase.reference_answer ?? "").trim(),
         [TEMPLATE_VARIABLES.FILE_CHANGES]: context.fileChanges ?? "",
+        [TEMPLATE_VARIABLES.TOOL_CALLS]: context.toolCalls ?? "",
         // Deprecated aliases
         [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
         [TEMPLATE_VARIABLES.OUTPUT_TEXT]: context.candidate.trim(),
@@ -1620,6 +1633,9 @@ ${outputSchema}`;
     if (context.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context.fileChanges, "");
     }
+    if (context.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context.toolCalls, "");
+    }
     if (rubrics && rubrics.length > 0) {
       parts.push("[[ ## rubrics ## ]]");
       for (const rubric of rubrics) {
@@ -1712,6 +1728,9 @@ ${outputSchema}`;
     if (context.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context.fileChanges, "");
     }
+    if (context.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context.toolCalls, "");
+    }
     parts.push("[[ ## scoring_criteria ## ]]");
     for (const rubric of rubrics) {
       const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
@@ -1754,6 +1773,9 @@ ${outputSchema}`;
     if (context.fileChanges) {
       parts.push("[[ ## file_changes ## ]]", context.fileChanges, "");
     }
+    if (context.toolCalls) {
+      parts.push("[[ ## tool_calls ## ]]", context.toolCalls, "");
+    }
     parts.push("[[ ## rubrics ## ]]");
     for (const rubric of rubrics) {
       const requiredLabel = rubric.required ? " (REQUIRED)" : "";
@@ -3265,6 +3287,54 @@ var LatencyGrader = class {
   }
 };
 
+// src/evaluation/graders/format-tool-calls.ts
+var KEY_INPUT_FIELDS = /* @__PURE__ */ new Map([
+  ["Skill", ["skill"]],
+  ["Read", ["file_path"]],
+  ["Write", ["file_path"]],
+  ["Edit", ["file_path"]],
+  ["Bash", ["command"]],
+  ["Grep", ["pattern"]],
+  ["Glob", ["pattern"]]
+]);
+var MAX_FALLBACK_LENGTH = 120;
+function formatToolCalls(output) {
+  if (!output) return "";
+  const lines = [];
+  for (const message of output) {
+    if (!message.toolCalls) continue;
+    for (const call of message.toolCalls) {
+      const toolName = call.tool ?? "unknown";
+      const detail = extractKeyDetail(toolName, call.input);
+      lines.push(detail ? `- ${toolName}: ${detail}` : `- ${toolName}`);
+    }
+  }
+  return lines.length > 0 ? lines.join("\n") : "";
+}
+function extractKeyDetail(toolName, input) {
+  if (!input || typeof input !== "object") return "";
+  const record = input;
+  const knownFields = KEY_INPUT_FIELDS.get(toolName);
+  if (knownFields) {
+    for (const field of knownFields) {
+      const value = record[field];
+      if (typeof value === "string" && value.length > 0) {
+        return truncate(value);
+      }
+    }
+  }
+  for (const value of Object.values(record)) {
+    if (typeof value === "string" && value.length > 0 && value.length <= MAX_FALLBACK_LENGTH) {
+      return truncate(value);
+    }
+  }
+  return "";
+}
+function truncate(value, maxLen = 120) {
+  if (value.length <= maxLen) return value;
+  return `${value.slice(0, maxLen)}\u2026`;
+}
+
 // src/evaluation/graders/skill-trigger.ts
 var SkillTriggerGrader = class {
   kind = "skill-trigger";
@@ -3341,19 +3411,27 @@ function assembleLlmGraderPrompt(input) {
     promptInputs,
     evaluatorConfig,
     fileChanges,
+    toolCalls,
     graderTemplateOverride
   } = input;
   const rubrics = evaluatorConfig?.rubrics;
   if (rubrics && rubrics.length > 0) {
     const hasScoreRanges = rubrics.some((r) => r.score_ranges && r.score_ranges.length > 0);
     if (hasScoreRanges) {
-      return assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges);
+      return assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls);
     }
-    return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges);
+    return assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls);
   }
-  return assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride);
+  return assembleFreeform(
+    evalCase,
+    candidate,
+    promptInputs,
+    fileChanges,
+    toolCalls,
+    graderTemplateOverride
+  );
 }
-function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, graderTemplateOverride) {
+function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, toolCalls, graderTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
     [TEMPLATE_VARIABLES.INPUT]: formattedQuestion.trim(),
@@ -3361,6 +3439,7 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, grader
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: (evalCase.reference_answer ?? "").trim(),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
     [TEMPLATE_VARIABLES.FILE_CHANGES]: fileChanges ?? "",
+    [TEMPLATE_VARIABLES.TOOL_CALLS]: toolCalls ?? "",
     // Deprecated aliases
     [TEMPLATE_VARIABLES.INPUT_TEXT]: formattedQuestion.trim(),
     [TEMPLATE_VARIABLES.OUTPUT_TEXT]: candidate.trim(),
@@ -3374,6 +3453,12 @@ function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, grader
 
 [[ ## file_changes ## ]]
 ${fileChanges}`;
+  }
+  if (toolCalls && !graderTemplateOverride) {
+    userPrompt += `
+
+[[ ## tool_calls ## ]]
+${toolCalls}`;
   }
   return {
     systemPrompt,
@@ -3382,7 +3467,7 @@ ${fileChanges}`;
     mode: "freeform"
   };
 }
-function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges) {
+function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
     "You are an expert grader. Evaluate the candidate answer against each rubric item below.",
@@ -3401,6 +3486,9 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
   if (fileChanges) {
     parts.push("[[ ## file_changes ## ]]", fileChanges, "");
   }
+  if (toolCalls) {
+    parts.push("[[ ## tool_calls ## ]]", toolCalls, "");
+  }
   parts.push("[[ ## rubrics ## ]]");
   for (const rubric of rubrics) {
     const requiredLabel = rubric.required ? " (REQUIRED)" : "";
@@ -3417,7 +3505,7 @@ function assembleChecklist(evalCase, candidate, promptInputs, rubrics, fileChang
     mode: "checklist"
   };
 }
-function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges) {
+function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChanges, toolCalls) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const parts = [
     "You are an expert grader. Score the candidate answer on each criterion below using the provided score ranges.",
@@ -3437,6 +3525,9 @@ function assembleScoreRange(evalCase, candidate, promptInputs, rubrics, fileChan
   if (fileChanges) {
     parts.push("[[ ## file_changes ## ]]", fileChanges, "");
   }
+  if (toolCalls) {
+    parts.push("[[ ## tool_calls ## ]]", toolCalls, "");
+  }
   parts.push("[[ ## scoring_criteria ## ]]");
   for (const rubric of rubrics) {
     const weightLabel = rubric.weight !== 1 ? ` (weight: ${rubric.weight})` : "";
@@ -15999,7 +16090,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     return { tests: await loadTestsFromAgentSkills(evalFilePath) };
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-XFQ6S4DT.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-32COE32J.js");
     return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
   }
   const { tests, parsed, suiteWorkspacePath } = await loadTestsFromYaml(
@@ -16034,7 +16125,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
     return loadTestsFromAgentSkills(evalFilePath);
   }
   if (format === "typescript") {
-    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-XFQ6S4DT.js");
+    const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-32COE32J.js");
     const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
     return suite.tests;
   }
@@ -16639,6 +16730,7 @@ async function runEvaluation(options) {
     trials,
     streamCallbacks,
     budgetUsd,
+    runBudgetTracker,
     failOnError,
     poolWorkspaces,
     poolMaxSlots: configPoolMaxSlots,
@@ -16973,8 +17065,14 @@ async function runEvaluation(options) {
         }
       }
       return { ok: allPassed, depResults };
+    }, extractEvaluationCostUsd2 = function(result) {
+      if (result.trials && result.trials.length > 0) {
+        const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
+        return trialCostSum > 0 ? trialCostSum : void 0;
+      }
+      return result.costUsd;
     };
-    var toDependencyResult = toDependencyResult2, checkDependencies = checkDependencies2;
+    var toDependencyResult = toDependencyResult2, checkDependencies = checkDependencies2, extractEvaluationCostUsd = extractEvaluationCostUsd2;
     if (suiteWorkspaceFile && sharedWorkspacePath) {
       const copiedWorkspaceFile = path44.join(sharedWorkspacePath, path44.basename(suiteWorkspaceFile));
       try {
@@ -17167,6 +17265,42 @@ async function runEvaluation(options) {
     async function dispatchTest(evalCase, depResults) {
       const workerId = nextWorkerId++;
       workerIdByEvalId.set(evalCase.id, workerId);
+      if (runBudgetTracker?.isExceeded()) {
+        const budgetResult = {
+          timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          suite: evalCase.suite,
+          category: evalCase.category,
+          score: 0,
+          assertions: [],
+          output: [],
+          target: target.name,
+          error: `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`,
+          budgetExceeded: true,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "budget_exceeded",
+          executionError: {
+            message: `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`,
+            stage: "setup"
+          }
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: budgetResult.error,
+            score: budgetResult.score,
+            executionStatus: budgetResult.executionStatus
+          });
+        }
+        if (onResult) {
+          await onResult(budgetResult);
+        }
+        return budgetResult;
+      }
       if (budgetUsd !== void 0 && budgetExhausted) {
         const budgetResult = {
           timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
@@ -17280,22 +17414,17 @@ async function runEvaluation(options) {
           ...depResults && Object.keys(depResults).length > 0 ? { dependencyResults: depResults } : {}
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
-        if (budgetUsd !== void 0) {
-          let caseCost;
-          if (result.trials && result.trials.length > 0) {
-            const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
-            if (trialCostSum > 0) {
-              caseCost = trialCostSum;
-            }
-          } else {
-            caseCost = result.costUsd;
-          }
-          if (caseCost !== void 0) {
+        const caseCost = extractEvaluationCostUsd2(result);
+        if (caseCost !== void 0) {
+          if (budgetUsd !== void 0) {
             cumulativeBudgetCost += caseCost;
             if (cumulativeBudgetCost >= budgetUsd) {
               budgetExhausted = true;
             }
           }
+          if (runBudgetTracker) {
+            runBudgetTracker.add(caseCost);
+          }
         }
         if (failOnError === true && result.executionStatus === "execution_error") {
           failOnErrorTriggered = true;
@@ -18103,6 +18232,7 @@ async function runEvalCase(options) {
     fileChanges = fileChanges ? `${fileChanges}
 ${providerFileChanges}` : providerFileChanges;
   }
+  const toolCalls = formatToolCalls(output);
   const providerError = extractProviderError(providerResponse);
   const targetAfterEachHook = options.targetHooks?.after_each;
   if (workspacePath && hasHookCommand(targetAfterEachHook)) {
@@ -18186,6 +18316,7 @@ ${providerFileChanges}` : providerFileChanges;
       targetResolver,
       availableTargets,
       fileChanges,
+      toolCalls,
       workspacePath,
       dockerConfig: evalCase.workspace?.docker,
       verbose,
@@ -18383,6 +18514,7 @@ async function evaluateCandidate(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold: evalThreshold,
@@ -18411,6 +18543,7 @@ async function evaluateCandidate(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold: evalThreshold,
@@ -18488,6 +18621,7 @@ async function runEvaluatorsForCase(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     threshold,
@@ -18517,6 +18651,7 @@ async function runEvaluatorsForCase(options) {
       targetResolver,
       availableTargets,
       fileChanges,
+      toolCalls,
       workspacePath,
       dockerConfig,
       threshold,
@@ -18548,6 +18683,7 @@ async function runEvaluatorsForCase(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults,
@@ -18589,6 +18725,7 @@ async function runEvaluatorList(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults
@@ -18614,6 +18751,7 @@ async function runEvaluatorList(options) {
     targetResolver,
     availableTargets,
     fileChanges,
+    toolCalls,
     workspacePath,
     dockerConfig,
     dependencyResults
@@ -19669,6 +19807,7 @@ export {
   ExecutionMetricsGrader,
   FieldAccuracyGrader,
   LatencyGrader,
+  formatToolCalls,
   SkillTriggerGrader,
   assembleLlmGraderPrompt,
   TokenUsageGrader,
@@ -19737,4 +19876,4 @@ export {
   loadTestById,
   loadEvalCaseById
 };
-//# sourceMappingURL=chunk-QXX3IBYV.js.map
+//# sourceMappingURL=chunk-ELF6SQAK.js.map