@axlsdk/eval 0.23.3 → 0.24.1

package/dist/cli.js

@@ -1,12 +1,20 @@
 #!/usr/bin/env node
 import {
   aggregateRuns,
+  compareItemErrorRates,
+  describeItemErrorRate,
   evalCompare,
+  evaluateItemErrorRateGate,
   evaluateScorerErrorRateGate,
+  formatPercent,
+  isBudgetStopped,
+  isErrorRateLimit,
+  readAccounting,
   rescore,
   runEval,
-  scorerCounts
-} from "./chunk-AFUNZE7W.js";
+  scorerCounts,
+  serializeRequestRecords
+} from "./chunk-HKEAPM4B.js";
 
 // src/cli.ts
 import { readdirSync, statSync } from "fs";
@@ -71,6 +79,12 @@ function validateEvalConfig(cfg) {
       return `exports an invalid failOnScorerErrorRate (${typeof v === "number" ? v : typeof v}) \u2014 must be a number between 0 and 1.`;
     }
   }
+  {
+    const v = c.failOnItemErrorRate;
+    if (v != null && !isErrorRateLimit(v)) {
+      return `exports an invalid failOnItemErrorRate (${typeof v === "number" ? v : typeof v}) \u2014 must be a number between 0 and 1 (1 disables the gate).`;
+    }
+  }
   return void 0;
 }
 
@@ -83,8 +97,11 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--threshold",
   "--runs",
   "--capture-traces",
+  "--capture-requests",
   "--concurrency",
-  "--scorers"
+  "--scorers",
+  "--budget",
+  "--max-item-error-rate"
 ]);
 var VALUE_FLAGS = /* @__PURE__ */ new Set([
   "--output",
@@ -92,7 +109,9 @@ var VALUE_FLAGS = /* @__PURE__ */ new Set([
   "--conditions",
   "--runs",
   "--concurrency",
-  "--scorers"
+  "--scorers",
+  "--budget",
+  "--max-item-error-rate"
 ]);
 function envInt(name) {
   const raw = process.env[name];
@@ -101,14 +120,25 @@ function envInt(name) {
   if (!Number.isFinite(n) || n <= 0) return void 0;
   return n;
 }
+function parseErrorRateFlag(flag, raw) {
+  const n = /^\d*\.?\d+$/.test(raw.trim()) ? Number(raw) : NaN;
+  if (!Number.isFinite(n) || n < 0 || n > 1) {
+    console.error(`Error: ${flag} must be a number in [0, 1], got "${raw}"`);
+    process.exit(1);
+  }
+  return n;
+}
 function parseEvalArgs(args) {
   let outputPath;
   let configArg;
   let conditions = [];
   let runs = 1;
   let captureTraces = false;
+  let captureRequests = false;
   let concurrency;
   let scorerNames;
+  let budget;
+  let maxItemErrorRate;
   const paths = [];
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -129,6 +159,10 @@ function parseEvalArgs(args) {
         } else {
           concurrency = n;
         }
+      } else if (arg === "--budget") {
+        budget = value;
+      } else if (arg === "--max-item-error-rate") {
+        maxItemErrorRate = parseErrorRateFlag(arg, value);
       } else if (arg === "--scorers") {
         scorerNames = [
           ...new Set(
@@ -140,6 +174,8 @@ function parseEvalArgs(args) {
       }
     } else if (arg === "--capture-traces") {
       captureTraces = true;
+    } else if (arg === "--capture-requests") {
+      captureRequests = true;
     } else if (arg.startsWith("--")) {
       if (!KNOWN_FLAGS.has(arg)) {
         console.error(`Unknown flag: ${arg}`);
@@ -155,13 +191,90 @@ function parseEvalArgs(args) {
     conditions,
     runs,
     captureTraces,
+    captureRequests,
     concurrency,
     scorerNames,
+    budget,
+    maxItemErrorRate,
     paths
   };
 }
 
 // src/cli-format.ts
+function formatKnownSpend(accounting) {
+  const cost = `$${accounting.knownCost.toFixed(2)}`;
+  if (accounting.completeness === "complete") return cost;
+  if (accounting.completeness === "unverified") return `${cost} (unverified)`;
+  const reasons = Object.entries(accounting.reasons).map(([reason, count]) => `${count} ${reason}`).join(", ");
+  return `${cost} (incomplete: ${reasons || "unknown spend"})`;
+}
+function formatBudgetLine(accounting) {
+  const budget = accounting.budget;
+  if (!budget) return void 0;
+  const limit = `$${budget.limit.toFixed(2)}`;
+  const spent = `$${budget.knownSpend.toFixed(2)}`;
+  if (budget.status === "open") {
+    return `  Budget: ${spent} of ${limit} (open)`;
+  }
+  const by = budget.closedBy ? `, first observed by ${budget.closedBy}` : "";
+  return `  Budget: STOPPED \u2014 ${spent} known spend against a ${limit} limit, $${budget.knownOvershoot.toFixed(2)} over${by}`;
+}
+function formatCoverageLine(result) {
+  const items = result.summary.coverage?.items;
+  if (!items) return void 0;
+  const parts = [
+    ["failed", items.failed],
+    ["cancelled", items.cancelled],
+    ["budget-skipped", items.budget_skipped],
+    ["budget-interrupted", items.budget_interrupted]
+  ].filter(([, n]) => n > 0).map(([label, n]) => `${n} ${label}`);
+  if (parts.length === 0) return void 0;
+  const rate = result.summary.itemErrorRate;
+  const rateSuffix = rate ? ` \u2014 item error rate ${formatPercent(rate.rate)} (limit ${formatPercent(rate.limit)})` : "";
+  const line = `  Items: ${items.completed} completed, ${parts.join(", ")}${rateSuffix}`;
+  const causes = formatFailureCauses(result);
+  return causes ? `${line}
+${causes}` : line;
+}
+function formatFailureCauses(result) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const item of result.items) {
+    if (item.outcome !== "failed") continue;
+    const f = item.failure;
+    let label = "other";
+    if (f?.status !== void 0) {
+      const status = f.status === 0 ? "network" : String(f.status);
+      label = f.provider ? `${status} (${f.provider})` : status;
+    }
+    counts.set(label, (counts.get(label) ?? 0) + 1);
+  }
+  if (counts.size === 0) return void 0;
+  const groups = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([label, n]) => `${n} \xD7 ${label}`);
+  return `  Failure causes: ${groups.join(", ")}`;
+}
+function itemErrorRateMessage(result, label) {
+  const rate = result.summary.itemErrorRate;
+  if (!rate?.exceeded) return null;
+  return `[axl-eval] ITEM ERROR RATE EXCEEDED: ${label} \u2014 ${rate.failed} of ${rate.attempted} attempted item(s) failed in the workflow (item error rate ${formatPercent(rate.rate)}), over the ${formatPercent(rate.limit)} limit; the scores cover only the surviving items. Raise failOnItemErrorRate or pass --max-item-error-rate <0..1> to accept it (1 disables the gate).`;
+}
+function budgetStopMessage(result, label) {
+  const budget = readAccounting(result).budget;
+  if (!budget || !isBudgetStopped({ budget, coverage: result.summary.coverage })) return null;
+  const items = result.summary.coverage?.items;
+  const stopped = items ? ` ${items.budget_skipped} case(s) never started, ${items.budget_interrupted} stopped mid-flight, ${items.completed} completed.` : "";
+  return `[axl-eval] BUDGET STOPPED: ${label} \u2014 known spend $${budget.knownSpend.toFixed(2)} reached the $${budget.limit.toFixed(2)} limit (over by $${budget.knownOvershoot.toFixed(2)}).${stopped} The run is incomplete by design; this is NOT a model or scorer failure.`;
+}
+function isTotalWipeout(result) {
+  const { count, failures } = result.summary;
+  if (count === 0) return false;
+  const coverage = result.summary.coverage?.items;
+  if (coverage) {
+    const budgetStopped = coverage.budget_skipped + coverage.budget_interrupted;
+    if (coverage.completed > 0 || budgetStopped > 0) return false;
+    return coverage.failed === count;
+  }
+  return failures >= count;
+}
 function formatModelTimingLines(modelTiming, nameWidth) {
   if (!modelTiming) return [];
   return Object.entries(modelTiming).map(([model, t]) => {
@@ -171,6 +284,7 @@ function formatModelTimingLines(modelTiming, nameWidth) {
     }
     parts.push(`queued ${formatMs(t.queuedMs.mean)}`);
     parts.push(`retries ${formatMs(t.retryMs.mean)}`);
+    if (t.rateLimitRetries) parts.push(`rate-limited ${t.rateLimitRetries}\xD7`);
     const calls = `${t.calls} call${t.calls === 1 ? "" : "s"}`;
     return `    ${model.padEnd(Math.max(nameWidth - 2, 0))}  ${parts.join(" \xB7 ")}  (${calls}, mean/p95 per call)`;
   });
@@ -231,7 +345,16 @@ Usage:
                                           (success + failure). Adds memory
                                           overhead proportional to dataset size
                                           x turns x agents; off by default.
+  axl-eval <path> --budget <amount>       Stop admitting spend once known cost
+                                          reaches the limit (e.g. "$1", "0.50").
+                                          Overrides the eval file's budget and
+                                          applies PER RUN under --runs. A
+                                          budget-stopped run exits 1 with a
+                                          distinct BUDGET STOPPED line.
   axl-eval rescore <results> <eval-file>  Re-run scorers on saved outputs
+  axl-eval rescore <results> <eval-file> --budget <amount>  Limit the NEW judging
+                                          spend only; the source run's cost is
+                                          not counted against it.
   axl-eval compare <a> <b>                Compare two eval result files
   axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
   axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
@@ -246,7 +369,8 @@ Config auto-detection (when --config is not specified):
   ${CONFIG_CANDIDATES.join(" -> ")}
 
 When a config is found, the exported AxlRuntime is passed to executeWorkflow
-and cost is tracked automatically via runtime.trackCost().
+and spend is measured by the runner's accounting scope (a "cost" your callback
+returns is recorded under item.callerReport, never used as the measurement).
 When no config is found, a bare AxlRuntime is created (providers from env vars).
 `);
     process.exit(0);
@@ -296,17 +420,27 @@ function scorerFilteredScorers(r) {
   if (!meta || meta.scorerFiltered !== true) return void 0;
   return Array.isArray(meta.scorersRun) ? meta.scorersRun : [];
 }
+function parseMaxItemErrorRate(args) {
+  const idx = args.indexOf("--max-item-error-rate");
+  if (idx === -1) return void 0;
+  if (idx + 1 >= args.length) {
+    console.error("Error: --max-item-error-rate requires a value in [0, 1]");
+    process.exit(1);
+  }
+  return parseErrorRateFlag("--max-item-error-rate", args[idx + 1]);
+}
 async function runCompare(args) {
   const failOnRegression = args.includes("--fail-on-regression");
   const thresholds = parseThresholdArg(args);
   const maxScorerErrorRate = parseMaxScorerErrorRate(args);
-  const valueFlags = /* @__PURE__ */ new Set(["--threshold", "--max-scorer-error-rate"]);
+  const maxItemErrorRate = parseMaxItemErrorRate(args);
+  const valueFlags = /* @__PURE__ */ new Set(["--threshold", "--max-scorer-error-rate", "--max-item-error-rate"]);
   const files = args.filter(
     (a, i) => !a.startsWith("--") && !(i > 0 && valueFlags.has(args[i - 1]))
   );
   if (files.length !== 2) {
     console.error(
-      "Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>] [--fail-on-regression] [--max-scorer-error-rate <0..1>]"
+      "Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>] [--fail-on-regression] [--max-scorer-error-rate <0..1>] [--max-item-error-rate <0..1>]"
     );
     process.exit(1);
   }
@@ -371,9 +505,13 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   if (comparison.cost) {
     const c = comparison.cost;
     const sign = c.delta > 0 ? "+" : "";
+    const change = c.deltaPercent === null ? `${sign}$${Math.abs(c.delta).toFixed(2)}, no % from a $0 baseline` : `${sign}${c.deltaPercent.toFixed(1)}%`;
     console.log(
-      `  Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`
+      `  Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${change})`
     );
+    if (!c.certified) {
+      console.log(`        not certified: ${c.reason ?? "costs are not comparable"}`);
+    }
   }
   const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;
   const stable = Math.max(
@@ -418,6 +556,17 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
     const reason = evaluateScorerErrorRateGate(comparison, maxScorerErrorRate);
     if (reason) refuseToGate(reason);
   }
+  const itemRates = compareItemErrorRates(baseline, candidate, maxItemErrorRate);
+  const multiRunCompare = itemRates.some((r) => r.runIndex > 0);
+  for (const rate of itemRates) {
+    if (rate.failed > 0 && !rate.exceeded) {
+      console.error(
+        `[axl-eval] WARNING: ${describeItemErrorRate(rate, multiRunCompare)}, within the ${formatPercent(rate.limit)} limit; its scores cover only the surviving items.`
+      );
+    }
+  }
+  const coverageReason = evaluateItemErrorRateGate(baseline, candidate, maxItemErrorRate);
+  if (coverageReason) refuseToGate(coverageReason);
   if (failOnRegression && comparison.regressions.length > 0) {
     const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);
     if (hasSignificance) {
@@ -431,7 +580,16 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   }
 }
 async function runRescore(args, signal) {
-  const { outputPath, configArg, conditions, concurrency, scorerNames, paths } = parseEvalArgs(args);
+  const {
+    outputPath,
+    configArg,
+    conditions,
+    concurrency,
+    scorerNames,
+    budget,
+    maxItemErrorRate,
+    paths
+  } = parseEvalArgs(args);
   if (paths.length < 2) {
     console.error("Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]");
     process.exit(1);
@@ -442,11 +600,18 @@ async function runRescore(args, signal) {
     );
     process.exit(1);
   }
+  if (maxItemErrorRate != null) {
+    console.error(
+      "Error: --max-item-error-rate is not supported with rescore; item failures belong to the source run \u2014 gate them with `axl-eval compare`."
+    );
+    process.exit(1);
+  }
   const itemConcurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY");
   const [resultsPath, evalFilePath] = paths;
   const raw = JSON.parse(await readFileAsync(resultsPath, "utf-8"));
   const results = Array.isArray(raw) ? raw : [raw];
   const runtime = await getRuntime(configArg, conditions);
+  let exitCode = 0;
   try {
     const mod = await importModule(path.resolve(evalFilePath), import.meta.url);
     const evalConfig = pickDefault(mod);
@@ -461,29 +626,49 @@ async function runRescore(args, signal) {
       process.exit(1);
     }
     const rescored = [];
-    for (const resultData of results) {
-      if (signal.aborted) break;
-      rescored.push(
-        await rescore(resultData, evalConfig.scorers, runtime, {
-          signal,
-          ...itemConcurrency != null ? { concurrency: itemConcurrency } : {}
-        })
-      );
-    }
-    for (const r of rescored) {
-      console.log("\n" + formatTable(r) + "\n");
+    let budgetUsable = true;
+    try {
+      for (const resultData of results) {
+        if (signal.aborted) break;
+        rescored.push(
+          await rescore(resultData, evalConfig.scorers, runtime, {
+            signal,
+            ...itemConcurrency != null ? { concurrency: itemConcurrency } : {},
+            // Gates the NEW judging spend only — the source run's cost is history
+            // and is not counted against this limit.
+            ...budget != null ? { budget } : {}
+          })
+        );
+      }
+    } catch (err) {
+      if (err?.code === "INVALID_BUDGET") {
+        console.error(`[axl-eval] ${err instanceof Error ? err.message : String(err)}`);
+        exitCode = 1;
+        budgetUsable = false;
+      } else {
+        throw err;
+      }
     }
-    if (outputPath) {
-      const output = Array.isArray(raw) ? rescored : rescored[0];
-      const outputDir = path.dirname(path.resolve(outputPath));
-      await mkdir(outputDir, { recursive: true });
-      await writeFileAsync(outputPath, JSON.stringify(output, null, 2), "utf-8");
-      console.log(`Rescored results saved to ${outputPath}`);
+    if (budgetUsable) {
+      for (const r of rescored) {
+        console.log("\n" + formatTable(r) + "\n");
+      }
+      if (outputPath) {
+        const output = Array.isArray(raw) ? rescored : rescored[0];
+        const outputDir = path.dirname(path.resolve(outputPath));
+        await mkdir(outputDir, { recursive: true });
+        await writeFileAsync(outputPath, JSON.stringify(output, null, 2), "utf-8");
+        console.log(`Rescored results saved to ${outputPath}`);
+      }
+      for (const r of rescored) {
+        if (reportBudgetStop(r, `rescore ${resultsPath}`)) exitCode = 1;
+      }
     }
   } finally {
     await runtime.shutdown().catch(() => {
     });
   }
+  if (exitCode !== 0) process.exit(exitCode);
 }
 function collectEvalFiles(p) {
   if (p.includes("*")) return expandGlob(p, process.cwd());
@@ -541,11 +726,15 @@ function formatTable(result) {
   }
   lines.push(...formatModelTimingLines(result.summary.modelTiming, maxNameLen));
   const durationSec = (result.duration / 1e3).toFixed(1);
-  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : "$0.00";
+  const accounting = readAccounting(result);
   lines.push("");
   lines.push(
-    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`
+    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${formatKnownSpend(accounting)} | Duration: ${durationSec}s`
   );
+  const budgetLine = formatBudgetLine(accounting);
+  if (budgetLine) lines.push(budgetLine);
+  const coverageLine = formatCoverageLine(result);
+  if (coverageLine) lines.push(coverageLine);
   const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);
   if (itemsWithErrors.length > 0) {
     const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors))];
@@ -589,10 +778,11 @@ function formatMultiRunTable(summary) {
       `  ${"Timing".padEnd(maxNameLen)}  ${((summary.timing.mean / 1e3).toFixed(2) + " \xB1 " + (summary.timing.std / 1e3).toFixed(2) + "s").padStart(colWidth)}`
     );
   }
-  const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : "$0.00";
   const durationStr = (summary.totalDuration / 1e3).toFixed(1);
   lines.push("");
-  lines.push(`  Total Cost: ${costStr} | Total Duration: ${durationStr}s`);
+  lines.push(
+    `  Total Cost: ${formatKnownSpend(summary.accounting)} | Total Duration: ${durationStr}s`
+  );
   return lines.join("\n");
 }
 function reportDegraded(result, label) {
@@ -617,11 +807,22 @@ function reportFullySkippedScorers(result, label) {
     }
   }
 }
+function reportBudgetStop(result, label) {
+  const message = budgetStopMessage(result, label);
+  if (!message) return false;
+  console.error(message);
+  return true;
+}
+function reportItemErrorRate(result, label, wipeout) {
+  const message = itemErrorRateMessage(result, label);
+  if (!message) return false;
+  if (!wipeout) console.error(message);
+  return true;
+}
 function reportTotalWipeout(result, label) {
-  const { count, failures } = result.summary;
-  if (count === 0 || failures < count) return false;
+  if (!isTotalWipeout(result)) return false;
   console.error(
-    `[axl-eval] FAILED: ${label} \u2014 all ${count} item(s) errored in the workflow (0 succeeded); the eval produced no scorable output.`
+    `[axl-eval] FAILED: ${label} \u2014 all ${result.summary.count} item(s) errored in the workflow (0 succeeded); the eval produced no scorable output.`
   );
   return true;
 }
@@ -685,6 +886,27 @@ async function getRuntime(configArg, conditions) {
   const { AxlRuntime } = await import("@axlsdk/axl");
   return new AxlRuntime();
 }
+async function writeRequestSidecar(runtime, outputPath, results) {
+  const artifactIds = results.map((result) => result.diagnostics?.artifactId).filter((id) => typeof id === "string" && id !== "");
+  if (artifactIds.length === 0) return;
+  const store = runtime.getDiagnosticArtifactStore();
+  if (!store) return;
+  const records = [];
+  for (const artifactId of artifactIds) {
+    const opened = await store.open(artifactId).catch(() => void 0);
+    if (!opened) continue;
+    for await (const line of opened.lines) {
+      try {
+        records.push(JSON.parse(line));
+      } catch {
+      }
+    }
+  }
+  if (records.length === 0) return;
+  const sidecarPath = outputPath.replace(/\.json$/i, "") + ".requests.jsonl";
+  await writeFileAsync(sidecarPath, serializeRequestRecords(records), "utf-8");
+  console.log(`Captured requests saved to ${sidecarPath}`);
+}
 async function runEvalCommand(args, signal) {
   const {
     outputPath,
@@ -692,8 +914,11 @@ async function runEvalCommand(args, signal) {
     conditions,
     runs,
     captureTraces,
+    captureRequests,
     concurrency,
     scorerNames,
+    budget,
+    maxItemErrorRate,
     paths
   } = parseEvalArgs(args);
   if (paths.length === 0) {
@@ -728,6 +953,8 @@ async function runEvalCommand(args, signal) {
           continue;
         }
         evalConfig.concurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY") ?? evalConfig.concurrency ?? 5;
+        if (budget != null) evalConfig.budget = budget;
+        if (maxItemErrorRate != null) evalConfig.failOnItemErrorRate = maxItemErrorRate;
         if (scorerNames?.length) {
           const available = evalConfig.scorers.map((s) => s.name);
           const unknown = scorerNames.filter((n) => !available.includes(n));
@@ -756,27 +983,11 @@ async function runEvalCommand(args, signal) {
         }
         let executeWorkflow;
         if (customExecute) {
-          executeWorkflow = async (input, rt) => {
-            const {
-              result,
-              cost: trackedCost,
-              metadata
-            } = await runtime.trackExecution(async () => {
-              return customExecute(input, rt);
-            });
-            return {
-              output: result.output,
-              cost: result.cost ?? trackedCost,
-              metadata: result.metadata ?? metadata
-            };
-          };
+          executeWorkflow = async (input, rt) => customExecute(input, rt);
         } else if (runtime.getWorkflow(evalConfig.workflow)) {
-          executeWorkflow = async (input) => {
-            const { result, cost, metadata } = await runtime.trackExecution(async () => {
-              return runtime.execute(evalConfig.workflow, input);
-            });
-            return { output: result, cost, metadata };
-          };
+          executeWorkflow = async (input) => ({
+            output: await runtime.execute(evalConfig.workflow, input)
+          });
         } else {
           const exportShape = (() => {
             const top = Object.keys(mod).filter((k) => k !== "default");
@@ -798,7 +1009,11 @@ async function runEvalCommand(args, signal) {
           failedFiles++;
           continue;
         }
-        const runOptions = { signal, ...captureTraces ? { captureTraces: true } : {} };
+        const runOptions = {
+          signal,
+          ...captureTraces ? { captureTraces: true } : {},
+          ...captureRequests ? { captureRequests: true } : {}
+        };
         const scorersRun = scorerNames?.length ? evalConfig.scorers.map((s) => s.name) : void 0;
         const stampFiltered = (result) => {
           if (scorersRun) {
@@ -851,11 +1066,13 @@ async function runEvalCommand(args, signal) {
           console.log("\n" + formatMultiRunTable(summary) + "\n");
           for (const r of runResults) results.push(r);
           let anyFailing = false;
-          for (const r of runResults) {
+          for (const [i, r] of runResults.entries()) {
+            const budgetStopped = reportBudgetStop(r, filePath);
             const wipeout = reportTotalWipeout(r, filePath);
+            const thinned = reportItemErrorRate(r, `${filePath} run ${i + 1}/${runs}`, wipeout);
             const degraded = reportDegraded(r, filePath);
             reportFullySkippedScorers(r, filePath);
-            if (wipeout || degraded) anyFailing = true;
+            if (budgetStopped || wipeout || thinned || degraded) anyFailing = true;
           }
           if (anyFailing && !partial) failedFiles++;
         } else {
@@ -863,10 +1080,12 @@ async function runEvalCommand(args, signal) {
           stampFiltered(result);
           results.push(result);
           console.log("\n" + formatTable(result) + "\n");
+          const budgetStopped = reportBudgetStop(result, filePath);
           const wipeout = reportTotalWipeout(result, filePath);
+          const thinned = reportItemErrorRate(result, filePath, wipeout);
           const degraded = reportDegraded(result, filePath);
           reportFullySkippedScorers(result, filePath);
-          if (wipeout || degraded) failedFiles++;
+          if (budgetStopped || wipeout || thinned || degraded) failedFiles++;
         }
       } catch (err) {
         console.error(
@@ -881,6 +1100,7 @@ async function runEvalCommand(args, signal) {
       await mkdir(outputDir, { recursive: true });
       await writeFileAsync(outputPath, JSON.stringify(output, null, 2), "utf-8");
       console.log(`Results saved to ${outputPath}`);
+      await writeRequestSidecar(runtime, outputPath, results);
     }
   } finally {
     await runtime.shutdown().catch(() => {