@agentv/core 2.7.1-next.5 → 2.8.0-next.1

package/dist/index.js

@@ -17,7 +17,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-6W5E3VR6.js";
+} from "./chunk-P2465XAH.js";
 import {
   OtlpJsonFileExporter
 } from "./chunk-HFSYZHGF.js";
@@ -538,6 +538,24 @@ function extractCacheConfig(suite) {
   const resolvedCachePath = typeof cachePath === "string" && cachePath.trim().length > 0 ? cachePath.trim() : void 0;
   return { enabled: cache, cachePath: resolvedCachePath };
 }
+function extractTotalBudgetUsd(suite) {
+  const execution = suite.execution;
+  if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
+    return void 0;
+  }
+  const executionObj = execution;
+  const rawBudget = executionObj.total_budget_usd ?? executionObj.totalBudgetUsd;
+  if (rawBudget === void 0 || rawBudget === null) {
+    return void 0;
+  }
+  if (typeof rawBudget === "number" && rawBudget > 0) {
+    return rawBudget;
+  }
+  logWarning(
+    `Invalid execution.total_budget_usd: ${rawBudget}. Must be a positive number. Ignoring.`
+  );
+  return void 0;
+}
 function logWarning(message) {
   console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET2}`);
 }
@@ -670,24 +688,24 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       continue;
     }
     if (typeValue === "code_judge") {
-      let script;
-      const rawScript = rawEvaluator.script;
-      if (typeof rawScript === "string") {
-        const trimmed = rawScript.trim();
+      let command;
+      const rawCommand = rawEvaluator.command ?? rawEvaluator.script;
+      if (typeof rawCommand === "string") {
+        const trimmed = rawCommand.trim();
         if (trimmed.length === 0) {
           throw new Error(
-            `Invalid code_judge script for evaluator '${name}' in '${evalId}': script cannot be empty`
+            `Invalid code_judge command for evaluator '${name}' in '${evalId}': command cannot be empty`
           );
         }
-        script = parseCommandToArgv(trimmed);
+        command = parseCommandToArgv(trimmed);
       } else {
-        script = asStringArray(
-          rawScript,
-          `code_judge script for evaluator '${name}' in '${evalId}'`
+        command = asStringArray(
+          rawCommand,
+          `code_judge command for evaluator '${name}' in '${evalId}'`
         );
       }
-      if (!script) {
-        logWarning2(`Skipping code_judge evaluator '${name}' in '${evalId}': missing script`);
+      if (!command) {
+        logWarning2(`Skipping code_judge evaluator '${name}' in '${evalId}': missing command`);
         continue;
       }
       const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
@@ -732,6 +750,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       const knownProps2 = /* @__PURE__ */ new Set([
         "name",
         "type",
+        "command",
         "script",
         "cwd",
         "weight",
@@ -748,7 +767,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
       evaluators.push({
         name,
         type: "code",
-        script,
+        command,
         cwd,
         resolvedCwd,
         ...weight2 !== void 0 ? { weight: weight2 } : {},
@@ -1350,20 +1369,20 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
     let resolvedPromptScript;
     let promptScriptConfig;
     if (isJsonObject2(rawPrompt)) {
-      const scriptArray = asStringArray(
-        rawPrompt.script,
-        `prompt.script for evaluator '${name}' in '${evalId}'`
+      const commandArray = asStringArray(
+        rawPrompt.command ?? rawPrompt.script,
+        `prompt.command for evaluator '${name}' in '${evalId}'`
       );
-      if (!scriptArray) {
-        throw new Error(`Evaluator '${name}' in '${evalId}': prompt object requires script array`);
+      if (!commandArray) {
+        throw new Error(`Evaluator '${name}' in '${evalId}': prompt object requires command array`);
       }
-      const scriptPath = scriptArray[scriptArray.length - 1];
-      const resolved = await resolveFileReference3(scriptPath, searchRoots);
+      const commandPath = commandArray[commandArray.length - 1];
+      const resolved = await resolveFileReference3(commandPath, searchRoots);
       if (resolved.resolvedPath) {
-        resolvedPromptScript = [...scriptArray.slice(0, -1), path4.resolve(resolved.resolvedPath)];
+        resolvedPromptScript = [...commandArray.slice(0, -1), path4.resolve(resolved.resolvedPath)];
       } else {
         throw new Error(
-          `Evaluator '${name}' in '${evalId}': prompt script file not found: ${resolved.displayPath}`
+          `Evaluator '${name}' in '${evalId}': prompt command file not found: ${resolved.displayPath}`
         );
       }
       if (isJsonObject2(rawPrompt.config)) {
@@ -2594,6 +2613,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     trials: extractTrialsConfig(parsed),
     targets: extractTargetsFromSuite(parsed),
     cacheConfig: extractCacheConfig(parsed),
+    totalBudgetUsd: extractTotalBudgetUsd(parsed),
     ...metadata !== void 0 && { metadata }
   };
 }
@@ -2784,16 +2804,16 @@ var loadEvalCaseById = loadTestById;
 function parseWorkspaceScriptConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
-  const script = obj.script;
-  if (!Array.isArray(script) || script.length === 0) return void 0;
-  const scriptArr = script.filter((s) => typeof s === "string");
-  if (scriptArr.length === 0) return void 0;
+  const commandSource = obj.command ?? obj.script;
+  if (!Array.isArray(commandSource) || commandSource.length === 0) return void 0;
+  const commandArr = commandSource.filter((s) => typeof s === "string");
+  if (commandArr.length === 0) return void 0;
   const timeoutMs = typeof obj.timeout_ms === "number" ? obj.timeout_ms : void 0;
   let cwd = typeof obj.cwd === "string" ? obj.cwd : void 0;
   if (cwd && !path8.isAbsolute(cwd)) {
     cwd = path8.resolve(evalFileDir, cwd);
   }
-  const config = { script: scriptArr };
+  const config = { command: commandArr };
   if (timeoutMs !== void 0) {
     return { ...config, timeout_ms: timeoutMs, ...cwd !== void 0 && { cwd } };
   }
@@ -3871,50 +3891,58 @@ var CliProvider = class {
     await this.ensureHealthy(request.signal);
     const effectiveCwd = request.cwd ?? this.config.cwd;
     const outputFilePath = generateOutputFilePath(request.evalCaseId);
-    const templateValues = buildTemplateValues(request, this.config, outputFilePath);
-    const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    const { values: templateValues, promptFilePath } = await buildTemplateValues(
+      request,
+      this.config,
+      outputFilePath
+    );
+    const renderedCommand = renderTemplate(this.config.command, templateValues);
     if (this.verbose) {
       console.log(
         `[cli-provider:${this.targetName}] cwd=${effectiveCwd ?? ""} command=${renderedCommand}`
       );
     }
-    const startTime = Date.now();
-    const result = await this.runCommand(renderedCommand, {
-      cwd: effectiveCwd,
-      env: process.env,
-      timeoutMs: this.config.timeoutMs,
-      signal: request.signal
-    });
-    const measuredDurationMs = Date.now() - startTime;
-    if (result.failed || (result.exitCode ?? 0) !== 0) {
-      if (request.signal?.aborted) {
-        throw new Error("CLI provider request was aborted");
-      }
-      if (result.timedOut) {
-        throw new Error(
-          `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
-        );
-      }
-      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
-      const detail = result.stderr.trim() || result.stdout.trim();
-      const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
-      throw new Error(message);
-    }
-    const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
-    const parsed = this.parseOutputContent(responseContent);
-    return {
-      output: parsed.output,
-      tokenUsage: parsed.tokenUsage,
-      costUsd: parsed.costUsd,
-      durationMs: parsed.durationMs ?? measuredDurationMs,
-      raw: {
-        command: renderedCommand,
-        stderr: result.stderr,
-        exitCode: result.exitCode ?? 0,
+    try {
+      const startTime = Date.now();
+      const result = await this.runCommand(renderedCommand, {
         cwd: effectiveCwd,
-        outputFile: outputFilePath
+        env: process.env,
+        timeoutMs: this.config.timeoutMs,
+        signal: request.signal
+      });
+      const measuredDurationMs = Date.now() - startTime;
+      if (result.failed || (result.exitCode ?? 0) !== 0) {
+        if (request.signal?.aborted) {
+          throw new Error("CLI provider request was aborted");
+        }
+        if (result.timedOut) {
+          throw new Error(
+            `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
+          );
+        }
+        const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+        const detail = result.stderr.trim() || result.stdout.trim();
+        const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
+        throw new Error(message);
       }
-    };
+      const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
+      const parsed = this.parseOutputContent(responseContent);
+      return {
+        output: parsed.output,
+        tokenUsage: parsed.tokenUsage,
+        costUsd: parsed.costUsd,
+        durationMs: parsed.durationMs ?? measuredDurationMs,
+        raw: {
+          command: renderedCommand,
+          stderr: result.stderr,
+          exitCode: result.exitCode ?? 0,
+          cwd: effectiveCwd,
+          outputFile: outputFilePath
+        }
+      };
+    } finally {
+      await cleanupTempFile(promptFilePath, this.keepTempFiles);
+    }
   }
   async invokeBatch(requests) {
     if (requests.length === 0) {
@@ -3937,7 +3965,7 @@ var CliProvider = class {
         batchInputFiles.push(...request.inputFiles);
       }
     }
-    const templateValues = buildTemplateValues(
+    const { values: templateValues, promptFilePath } = await buildTemplateValues(
       {
         question: "",
         guidelines: "",
@@ -3948,87 +3976,91 @@ var CliProvider = class {
       this.config,
       outputFilePath
     );
-    const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    const renderedCommand = renderTemplate(this.config.command, templateValues);
     if (this.verbose) {
       console.log(
         `[cli-provider:${this.targetName}] (batch size=${requests.length}) cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
       );
     }
-    const startTime = Date.now();
-    const result = await this.runCommand(renderedCommand, {
-      cwd: this.config.cwd,
-      env: process.env,
-      timeoutMs: this.config.timeoutMs,
-      signal: controller.signal
-    });
-    const measuredDurationMs = Date.now() - startTime;
-    if (result.failed || (result.exitCode ?? 0) !== 0) {
-      if (controller.signal.aborted) {
-        throw new Error("CLI provider request was aborted");
-      }
-      if (result.timedOut) {
-        throw new Error(
-          `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
-        );
-      }
-      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
-      const detail = result.stderr.trim() || result.stdout.trim();
-      const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
-      throw new Error(message);
-    }
-    const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
-    const recordsById = this.parseJsonlBatchOutput(responseContent);
-    const perRequestFallbackMs = Math.round(measuredDurationMs / requests.length);
-    const responses = requests.map((request) => {
-      const evalCaseId = request.evalCaseId;
-      if (!evalCaseId) {
-        return {
-          output: [],
-          durationMs: perRequestFallbackMs,
-          raw: {
-            command: renderedCommand,
-            stderr: result.stderr,
-            exitCode: result.exitCode ?? 0,
-            cwd: this.config.cwd,
-            outputFile: outputFilePath
+    try {
+      const startTime = Date.now();
+      const result = await this.runCommand(renderedCommand, {
+        cwd: this.config.cwd,
+        env: process.env,
+        timeoutMs: this.config.timeoutMs,
+        signal: controller.signal
+      });
+      const measuredDurationMs = Date.now() - startTime;
+      if (result.failed || (result.exitCode ?? 0) !== 0) {
+        if (controller.signal.aborted) {
+          throw new Error("CLI provider request was aborted");
+        }
+        if (result.timedOut) {
+          throw new Error(
+            `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
+          );
+        }
+        const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+        const detail = result.stderr.trim() || result.stdout.trim();
+        const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
+        throw new Error(message);
+      }
+      const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
+      const recordsById = this.parseJsonlBatchOutput(responseContent);
+      const perRequestFallbackMs = Math.round(measuredDurationMs / requests.length);
+      const responses = requests.map((request) => {
+        const evalCaseId = request.evalCaseId;
+        if (!evalCaseId) {
+          return {
+            output: [],
+            durationMs: perRequestFallbackMs,
+            raw: {
+              command: renderedCommand,
+              stderr: result.stderr,
+              exitCode: result.exitCode ?? 0,
+              cwd: this.config.cwd,
+              outputFile: outputFilePath
+            }
+          };
+        }
+        const parsed = recordsById.get(evalCaseId);
+        if (!parsed) {
+          const errorMessage = `Batch output missing id '${evalCaseId}'`;
+          if (this.verbose) {
+            console.warn(`[cli-provider:${this.targetName}] ${errorMessage}`);
           }
-        };
-      }
-      const parsed = recordsById.get(evalCaseId);
-      if (!parsed) {
-        const errorMessage = `Batch output missing id '${evalCaseId}'`;
-        if (this.verbose) {
-          console.warn(`[cli-provider:${this.targetName}] ${errorMessage}`);
+          return {
+            output: [{ role: "assistant", content: `Error: ${errorMessage}` }],
+            durationMs: perRequestFallbackMs,
+            raw: {
+              command: renderedCommand,
+              stderr: result.stderr,
+              exitCode: result.exitCode ?? 0,
+              cwd: this.config.cwd,
+              outputFile: outputFilePath,
+              error: errorMessage
+            }
+          };
         }
         return {
-          output: [{ role: "assistant", content: `Error: ${errorMessage}` }],
-          durationMs: perRequestFallbackMs,
+          output: parsed.output,
+          tokenUsage: parsed.tokenUsage,
+          costUsd: parsed.costUsd,
+          durationMs: parsed.durationMs ?? perRequestFallbackMs,
           raw: {
             command: renderedCommand,
             stderr: result.stderr,
             exitCode: result.exitCode ?? 0,
             cwd: this.config.cwd,
             outputFile: outputFilePath,
-            error: errorMessage
+            recordId: evalCaseId
           }
         };
-      }
-      return {
-        output: parsed.output,
-        tokenUsage: parsed.tokenUsage,
-        costUsd: parsed.costUsd,
-        durationMs: parsed.durationMs ?? perRequestFallbackMs,
-        raw: {
-          command: renderedCommand,
-          stderr: result.stderr,
-          exitCode: result.exitCode ?? 0,
-          cwd: this.config.cwd,
-          outputFile: outputFilePath,
-          recordId: evalCaseId
-        }
-      };
-    });
-    return responses;
+      });
+      return responses;
+    } finally {
+      await cleanupTempFile(promptFilePath, this.keepTempFiles);
+    }
   }
   /**
    * Parse output content from CLI.
@@ -4143,7 +4175,7 @@ var CliProvider = class {
       return;
     }
     const timeoutMs = healthcheck.timeoutMs ?? this.config.timeoutMs;
-    if (healthcheck.type === "http") {
+    if ("url" in healthcheck && healthcheck.url) {
       const controller = new AbortController();
       const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
       signal?.addEventListener("abort", () => controller.abort(), { once: true });
@@ -4162,50 +4194,70 @@ var CliProvider = class {
       }
       return;
     }
-    const renderedCommand = renderTemplate(
-      healthcheck.commandTemplate,
-      buildTemplateValues(
-        {
-          question: "",
-          guidelines: "",
-          inputFiles: [],
-          evalCaseId: "healthcheck",
-          attempt: 0
-        },
-        this.config,
-        generateOutputFilePath("healthcheck")
-      )
+    const hcCommand = "command" in healthcheck ? healthcheck.command : void 0;
+    if (!hcCommand) {
+      throw new Error(`CLI healthcheck for '${this.targetName}': 'command' or 'url' is required`);
+    }
+    const { values: templateValues, promptFilePath } = await buildTemplateValues(
+      {
+        question: "",
+        guidelines: "",
+        inputFiles: [],
+        evalCaseId: "healthcheck",
+        attempt: 0
+      },
+      this.config,
+      generateOutputFilePath("healthcheck")
     );
+    const renderedCommand = renderTemplate(hcCommand, templateValues);
+    const hcCwd = "cwd" in healthcheck ? healthcheck.cwd : void 0;
     if (this.verbose) {
       console.log(
-        `[cli-provider:${this.targetName}] (healthcheck) cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
+        `[cli-provider:${this.targetName}] (healthcheck) cwd=${hcCwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
       );
     }
-    const result = await this.runCommand(renderedCommand, {
-      cwd: healthcheck.cwd ?? this.config.cwd,
-      env: process.env,
-      timeoutMs,
-      signal
-    });
-    if (result.failed || (result.exitCode ?? 0) !== 0) {
-      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
-      const detail = result.stderr.trim() || result.stdout.trim();
-      const message = detail ? `${detail} (exit code ${codeText})` : `CLI healthcheck command exited with code ${codeText}`;
-      throw new Error(`CLI healthcheck failed for '${this.targetName}': ${message}`);
+    try {
+      const result = await this.runCommand(renderedCommand, {
+        cwd: hcCwd ?? this.config.cwd,
+        env: process.env,
+        timeoutMs,
+        signal
+      });
+      if (result.failed || (result.exitCode ?? 0) !== 0) {
+        const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+        const detail = result.stderr.trim() || result.stdout.trim();
+        const message = detail ? `${detail} (exit code ${codeText})` : `CLI healthcheck command exited with code ${codeText}`;
+        throw new Error(`CLI healthcheck failed for '${this.targetName}': ${message}`);
+      }
+    } finally {
+      await cleanupTempFile(promptFilePath, this.keepTempFiles);
     }
   }
 };
-function buildTemplateValues(request, config, outputFilePath) {
+async function buildTemplateValues(request, config, outputFilePath) {
   const inputFiles = normalizeInputFiles2(request.inputFiles);
+  const promptFilePath = generateOutputFilePath(request.evalCaseId, ".prompt.txt");
+  await fs.writeFile(promptFilePath, request.question ?? "", "utf8");
   return {
-    PROMPT: shellEscape(request.question ?? ""),
-    GUIDELINES: shellEscape(request.guidelines ?? ""),
-    EVAL_ID: shellEscape(request.evalCaseId ?? ""),
-    ATTEMPT: shellEscape(String(request.attempt ?? 0)),
-    FILES: formatFileList(inputFiles, config.filesFormat),
-    OUTPUT_FILE: shellEscape(outputFilePath)
+    values: {
+      PROMPT: shellEscape(request.question ?? ""),
+      PROMPT_FILE: shellEscape(promptFilePath),
+      GUIDELINES: shellEscape(request.guidelines ?? ""),
+      EVAL_ID: shellEscape(request.evalCaseId ?? ""),
+      ATTEMPT: shellEscape(String(request.attempt ?? 0)),
+      FILES: formatFileList(inputFiles, config.filesFormat),
+      OUTPUT_FILE: shellEscape(outputFilePath)
+    },
+    promptFilePath
   };
 }
+async function cleanupTempFile(filePath, keepTempFiles) {
+  if (!filePath || keepTempFiles) {
+    return;
+  }
+  await fs.unlink(filePath).catch(() => {
+  });
+}
 function normalizeInputFiles2(inputFiles) {
   if (!inputFiles || inputFiles.length === 0) {
     return void 0;
@@ -8086,7 +8138,7 @@ async function discoverProviders(registry, baseDir) {
     }
     registry.register(kindName, (target) => {
       return new CliProvider(target.name, {
-        commandTemplate: `bun run ${filePath} {PROMPT}`
+        command: `bun run ${filePath} {PROMPT}`
       });
     });
     discoveredKinds.push(kindName);
@@ -8599,13 +8651,13 @@ function toCamelCaseDeep(obj) {
 var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
 var CodeEvaluator = class {
   kind = "code";
-  script;
+  command;
   cwd;
   agentTimeoutMs;
   config;
   target;
   constructor(options) {
-    this.script = options.script;
+    this.command = options.command ?? options.script ?? [];
     this.cwd = options.cwd;
     this.agentTimeoutMs = options.agentTimeoutMs;
     this.config = options.config;
@@ -8664,7 +8716,7 @@ var CodeEvaluator = class {
     const env = proxyEnv || workspaceEnv ? { ...proxyEnv, ...workspaceEnv } : void 0;
     try {
       const stdout = await executeScript(
-        this.script,
+        this.command,
         inputPayload,
         this.agentTimeoutMs,
         this.cwd,
@@ -8678,7 +8730,7 @@ var CodeEvaluator = class {
       const details = parsed?.details && typeof parsed.details === "object" && !Array.isArray(parsed.details) ? parsed.details : void 0;
       const proxyUsage = getProxyUsage?.();
       const evaluatorRawRequest = {
-        script: this.script,
+        command: this.command,
         ...this.cwd ? { cwd: this.cwd } : {},
         ...proxyUsage ? {
           target_proxy: {
@@ -8708,7 +8760,7 @@ var CodeEvaluator = class {
         expectedAspectCount: 1,
         reasoning: message,
         evaluatorRawRequest: {
-          script: this.script,
+          command: this.command,
           ...this.cwd ? { cwd: this.cwd } : {},
           ...proxyUsage ? {
             target_proxy: {
@@ -11638,7 +11690,7 @@ var llmJudgeFactory = (config, context) => {
 var codeFactory = (config, context) => {
   const c = config;
   return new CodeEvaluator({
-    script: c.script,
+    command: c.command ?? c.script ?? [],
     cwd: c.resolvedCwd ?? c.cwd,
     agentTimeoutMs: context.agentTimeoutMs,
     config: c.config,
@@ -11820,7 +11872,7 @@ async function discoverAssertions(registry, baseDir) {
     }
     const factory = (_config, context) => {
       return new CodeEvaluator({
-        script: ["bun", "run", filePath],
+        command: ["bun", "run", filePath],
         agentTimeoutMs: context.agentTimeoutMs
       });
     };
@@ -12174,7 +12226,8 @@ async function executeWorkspaceScript(config, context, failureMode = "fatal") {
   });
   const timeoutMs = config.timeout_ms ?? (failureMode === "fatal" ? 6e4 : 3e4);
   const cwd = config.cwd;
-  const result = await execFileWithStdin(config.script, stdin, {
+  const commandArray = config.command ?? config.script ?? [];
+  const result = await execFileWithStdin(commandArray, stdin, {
     timeoutMs,
     cwd
   });
@@ -12221,7 +12274,8 @@ async function runEvaluation(options) {
     keepWorkspaces,
     cleanupWorkspaces,
     trials,
-    streamCallbacks
+    streamCallbacks,
+    totalBudgetUsd
   } = options;
   let useCache = options.useCache;
   if (trials && trials.count > 1 && useCache) {
@@ -12394,10 +12448,39 @@ async function runEvaluation(options) {
   let nextWorkerId = 1;
   const workerIdByEvalId = /* @__PURE__ */ new Map();
   let beforeAllOutputAttached = false;
+  let cumulativeBudgetCost = 0;
+  let budgetExhausted = false;
   const promises = filteredEvalCases.map(
     (evalCase) => limit(async () => {
       const workerId = nextWorkerId++;
       workerIdByEvalId.set(evalCase.id, workerId);
+      if (totalBudgetUsd !== void 0 && budgetExhausted) {
+        const budgetResult = {
+          timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          dataset: evalCase.dataset,
+          score: 0,
+          hits: [],
+          misses: [],
+          answer: "",
+          target: target.name,
+          error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
+          budgetExceeded: true
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: budgetResult.error
+          });
+        }
+        if (onResult) {
+          await onResult(budgetResult);
+        }
+        return budgetResult;
+      }
       if (onProgress) {
         await onProgress({
           workerId,
@@ -12431,6 +12514,23 @@ async function runEvaluation(options) {
           typeRegistry
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
+        if (totalBudgetUsd !== void 0) {
+          let caseCost;
+          if (result.trials && result.trials.length > 0) {
+            const trialCostSum = result.trials.reduce((sum, t) => sum + (t.costUsd ?? 0), 0);
+            if (trialCostSum > 0) {
+              caseCost = trialCostSum;
+            }
+          } else {
+            caseCost = result.trace?.costUsd;
+          }
+          if (caseCost !== void 0) {
+            cumulativeBudgetCost += caseCost;
+            if (cumulativeBudgetCost >= totalBudgetUsd) {
+              budgetExhausted = true;
+            }
+          }
+        }
         if (beforeAllOutput && !beforeAllOutputAttached) {
           result = { ...result, beforeAllOutput };
           beforeAllOutputAttached = true;