@axlsdk/eval 0.23.2 → 0.24.0

package/dist/cli.cjs

@@ -108,15 +108,35 @@ function computeStats(scores) {
 function round2(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+function formatPercent(rate) {
+  return `${Number((rate * 100).toFixed(1))}%`;
+}
+var DEFAULT_ITEM_ERROR_RATE_LIMIT = 0.05;
+function isErrorRateLimit(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+function evaluateItemErrorRate(items, count, limit) {
+  const failed = items.failed;
+  const attempted = count - items.cancelled - items.budget_skipped - items.budget_interrupted;
+  const rate = attempted > 0 ? failed / attempted : 0;
+  return { failed, attempted, rate, limit, exceeded: attempted > 0 && rate > limit };
+}
 function scorerCounts(items, name) {
   let scored = 0;
   let failed = 0;
   let skipped = 0;
   for (const i of items) {
     if (i.error) continue;
+    const detail = i.scoreDetails?.[name];
+    if (detail?.outcome !== void 0) {
+      if (detail.outcome === "scored") scored++;
+      else if (detail.outcome === "failed") failed++;
+      else if (detail.outcome === "skipped") skipped++;
+      continue;
+    }
     if (i.scores[name] != null) scored++;
-    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;
-    else if (i.scoreDetails?.[name]?.duration != null) failed++;
+    else if (detail?.skipped === true) skipped++;
+    else if (detail?.duration != null) failed++;
   }
   return { scored, failed, skipped };
 }
@@ -137,11 +157,918 @@ async function mapWithConcurrency(items, concurrency, task) {
       results[i] = await task(items[i], i);
     }
   }
-  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
-  return results;
+  await Promise.all(Array.from({ length: workerCount }, () => runNext()));
+  return results;
+}
+
+// src/accounting.ts
+var import_axl = require("@axlsdk/axl");
+var USAGE_KEYS = [
+  "inputTokens",
+  "outputTokens",
+  "reasoningTokens",
+  "cachedTokens",
+  "cacheWriteTokens",
+  "audioSeconds"
+];
+function zeroUsage() {
+  return {
+    inputTokens: 0,
+    outputTokens: 0,
+    reasoningTokens: 0,
+    cachedTokens: 0,
+    cacheWriteTokens: 0,
+    audioSeconds: 0
+  };
+}
+function emptyAccounting() {
+  return {
+    version: 1,
+    currency: "USD",
+    knownCost: 0,
+    completeness: "complete",
+    reasons: {},
+    usage: zeroUsage(),
+    operations: { total: 0, settled: 0, unknown: 0, denied: 0, byKind: {} },
+    breakdown: { generation: 0, judging: 0, external: 0 },
+    provenance: {}
+  };
+}
+function uninstrumentedAccounting() {
+  return { ...emptyAccounting(), completeness: "incomplete", reasons: { uninstrumented: 1 } };
+}
+function isInstrumented(runtime) {
+  return typeof runtime?.trackOutcome === "function";
+}
+async function trackScope(runtime, fn, options) {
+  if (!isInstrumented(runtime)) {
+    try {
+      return {
+        status: "fulfilled",
+        value: await fn(),
+        accounting: uninstrumentedAccounting(),
+        instrumented: false
+      };
+    } catch (error) {
+      return {
+        status: "rejected",
+        error,
+        accounting: uninstrumentedAccounting(),
+        instrumented: false
+      };
+    }
+  }
+  const outcome = await runtime.trackOutcome(fn, options);
+  return { ...outcome, instrumented: true };
+}
+function parseBudget(budget) {
+  const match = typeof budget === "string" ? budget.match(/^\$?(\d+(?:\.\d+)?)$/) : null;
+  if (!match) {
+    throw new import_axl.AxlError(
+      "INVALID_BUDGET",
+      `Invalid budget "${String(budget)}": expected a non-negative USD amount like "$1" or "0.50".`
+    );
+  }
+  const limit = Number(match[1]);
+  if (!Number.isFinite(limit)) {
+    throw new import_axl.AxlError("INVALID_BUDGET", `Invalid budget "${budget}": not a finite amount.`);
+  }
+  return limit;
+}
+function usableCost(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
+}
+function isAdmissionDenied(err) {
+  return (0, import_axl.isAdmissionDeniedError)(err);
+}
+function readAccounting(result) {
+  if (result?.accounting) return result.accounting;
+  const rescored = result?.metadata?.rescored === true;
+  return {
+    ...emptyAccounting(),
+    knownCost: usableCost(result?.totalCost),
+    completeness: "unverified",
+    scope: rescored ? "rescore" : "run"
+  };
+}
+function refusedWork(coverage) {
+  if (!coverage) return 0;
+  return refusedIn(coverage.items) + Object.values(coverage.scorers ?? {}).reduce((n, s) => n + refusedIn(s), 0);
+}
+function refusedIn(counts) {
+  return atLeastZero(counts?.budget_skipped) + atLeastZero(counts?.budget_interrupted);
+}
+function atLeastZero(value) {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+function isBudgetStopped(summary) {
+  if (summary.budget?.status !== "closed") return false;
+  return refusedWork(summary.coverage) > 0;
+}
+function addUsage(target, source) {
+  if (!source) return;
+  for (const key of USAGE_KEYS) {
+    const value = source[key];
+    if (typeof value === "number" && Number.isFinite(value)) target[key] += value;
+  }
+}
+function addCounts(target, source) {
+  if (!source) return;
+  for (const [key, count] of Object.entries(source)) {
+    if (typeof count === "number" && Number.isFinite(count)) {
+      target[key] = (target[key] ?? 0) + count;
+    }
+  }
+}
+function aggregateAccounting(inputs) {
+  const out = emptyAccounting();
+  let anyUnverified = false;
+  let anyIncomplete = false;
+  for (const input of inputs) {
+    if (!input) continue;
+    if (input.completeness === "unverified") anyUnverified = true;
+    else if (input.completeness === "incomplete") anyIncomplete = true;
+    out.knownCost += usableCost(input.knownCost);
+    addUsage(out.usage, input.usage);
+    addCounts(out.reasons, input.reasons);
+    addCounts(out.operations.byKind, input.operations?.byKind);
+    addCounts(out.provenance, input.provenance);
+    out.operations.total += input.operations?.total ?? 0;
+    out.operations.settled += input.operations?.settled ?? 0;
+    out.operations.unknown += input.operations?.unknown ?? 0;
+    out.operations.denied += input.operations?.denied ?? 0;
+    out.breakdown.generation += usableCost(input.breakdown?.generation);
+    out.breakdown.judging += usableCost(input.breakdown?.judging);
+    out.breakdown.external += usableCost(input.breakdown?.external);
+  }
+  out.completeness = anyUnverified ? "unverified" : anyIncomplete ? "incomplete" : "complete";
+  return out;
+}
+
+// src/runner.ts
+var import_axl2 = require("@axlsdk/axl");
+
+// src/scorer.ts
+function normalizeScorerResult(result) {
+  return typeof result === "number" ? { score: result } : result;
+}
+function extractScorerErrorCost(err) {
+  const v = err && typeof err === "object" ? err.cost : void 0;
+  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
+}
+
+// src/score-item.ts
+function describeInput(input) {
+  let rendered;
+  try {
+    rendered = JSON.stringify(input) ?? String(input);
+  } catch {
+    rendered = "[unserializable input]";
+  }
+  return rendered.length > 200 ? `${rendered.slice(0, 200)}\u2026` : rendered;
+}
+async function scoreItem(item, scorers, context) {
+  const scoreDetails = item.scoreDetails ??= {};
+  for (const s of scorers) {
+    item.scores[s.name] = null;
+    scoreDetails[s.name] = { score: null, outcome: "cancelled" };
+  }
+  const { runtime, scorerContext, signal, admission } = context;
+  const scorerErrorsByName = {};
+  await mapWithConcurrency(scorers, context.scorerConcurrency, async (scorer) => {
+    const record = (detail, outcome2) => {
+      scoreDetails[scorer.name] = { ...detail, outcome: outcome2 };
+    };
+    if (signal?.aborted) {
+      record({ score: null }, "cancelled");
+      return;
+    }
+    if (admission?.closed && scorer.isLlm) {
+      record({ score: null }, "budget_skipped");
+      return;
+    }
+    const scorerStart = Date.now();
+    let applicable = true;
+    try {
+      applicable = !scorer.applies || scorer.applies(item.output, item.input, item.annotations);
+    } catch (err) {
+      scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record({ score: null, duration: Date.now() - scorerStart }, "failed");
+      return;
+    }
+    if (!applicable) {
+      record({ score: null, skipped: true }, "skipped");
+      return;
+    }
+    const outcome = await trackScope(
+      runtime,
+      async () => scorer.score(item.output, item.input, item.annotations, scorerContext),
+      // The scorer stamp is merged OVER the enclosing item's correlation, so a
+      // judge's records keep the case index and add the judge's name -- which is
+      // what lets `scoreDetails[name].diagnostics` point at this judge's own
+      // calls rather than at the whole item's.
+      { purpose: "judging", captureCorrelation: { scorer: scorer.name } }
+    );
+    const duration = Date.now() - scorerStart;
+    const { accounting } = outcome;
+    const measured = accounting.operations.total > 0 || accounting.operations.denied > 0;
+    const withCost = (detail, callerCost) => ({
+      ...detail,
+      ...measured ? { accounting, cost: accounting.knownCost } : {},
+      ...!measured && !outcome.instrumented && callerCost != null ? { cost: callerCost } : {}
+    });
+    if (outcome.status === "rejected") {
+      const err = outcome.error;
+      if (err?.name === "AbortError") {
+        record(withCost({ score: null, duration }, void 0), "cancelled");
+        return;
+      }
+      if (isAdmissionDenied(err)) {
+        context.onClosure?.("operation");
+        record(withCost({ score: null, duration }, void 0), "budget_interrupted");
+        return;
+      }
+      scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record(withCost({ score: null, duration }, extractScorerErrorCost(err)), "failed");
+      if (admission?.closed) context.onClosure?.("scorer");
+      return;
+    }
+    try {
+      const scorerResult = normalizeScorerResult(outcome.value);
+      const callerCost = typeof scorerResult.cost === "number" && Number.isFinite(scorerResult.cost) && scorerResult.cost >= 0 ? scorerResult.cost : void 0;
+      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
+        scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" returned out-of-range score ${scorerResult.score} for input ${describeInput(item.input)}`;
+        record(
+          withCost({ score: null, metadata: scorerResult.metadata, duration }, callerCost),
+          "failed"
+        );
+      } else {
+        item.scores[scorer.name] = round2(scorerResult.score);
+        record(
+          withCost(
+            {
+              score: round2(scorerResult.score),
+              metadata: scorerResult.metadata,
+              duration
+            },
+            callerCost
+          ),
+          "scored"
+        );
+      }
+    } catch (err) {
+      scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record(withCost({ score: null, duration }, void 0), "failed");
+    }
+    if (admission?.closed) context.onClosure?.("scorer");
+  });
+  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
+  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
+}
+
+// src/diagnostics.ts
+var DEFAULT_COPY_MAX_BYTES = 16 * 1024 * 1024;
+function resolveCaptureLimits(option) {
+  if (option === void 0 || option === false) return void 0;
+  if (option === true) return {};
+  return {
+    ...option.maxRecordBytes !== void 0 ? { maxRecordBytes: option.maxRecordBytes } : {},
+    ...option.maxRunBytes !== void 0 ? { maxRunBytes: option.maxRunBytes } : {},
+    ...option.maxQueueBytes !== void 0 ? { maxQueueBytes: option.maxQueueBytes } : {}
+  };
+}
+function toDiagnosticManifest(manifest) {
+  return {
+    version: 1,
+    artifactId: manifest.artifactId,
+    fidelity: "runtime_request",
+    status: manifest.status,
+    ...manifest.reason !== void 0 ? { reason: manifest.reason } : {},
+    records: manifest.records,
+    bytes: manifest.bytes,
+    redaction: manifest.redaction,
+    ...manifest.expiresAt !== void 0 ? { expiresAt: manifest.expiresAt } : {}
+  };
+}
+function unavailableManifest(artifactId, reason) {
+  return {
+    version: 1,
+    artifactId,
+    fidelity: "runtime_request",
+    status: "unavailable",
+    reason,
+    records: 0,
+    bytes: 0,
+    redaction: "none"
+  };
+}
+var DEFAULT_SIDECAR_MAX_BYTES = 16 * 1024 * 1024;
+var DEFAULT_SIDECAR_MAX_RECORD_BYTES = 4 * 1024 * 1024;
+function serializeRequestRecords(records) {
+  return records.map((record) => JSON.stringify(record)).join("\n") + (records.length ? "\n" : "");
+}
+
+// src/runner.ts
+var import_node_crypto = require("crypto");
+var RESERVED_METADATA_KEYS = [
+  "models",
+  "modelCallCounts",
+  "workflows",
+  "workflowCallCounts",
+  "tokens",
+  "agentCalls"
+];
+var ITEM_OUTCOMES = [
+  "completed",
+  "failed",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+var SCORER_OUTCOMES = [
+  "scored",
+  "failed",
+  "skipped",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+function extractUserCost(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const raw = result.cost;
+  if (raw === void 0) return void 0;
+  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
+  console.warn(
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Measured spend is unaffected.`
+  );
+  return void 0;
+}
+function extractUserMetadata(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const meta = result.metadata;
+  if (meta === void 0) return void 0;
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
+    );
+    return void 0;
+  }
+  const proto = Object.getPrototypeOf(meta);
+  if (proto !== Object.prototype && proto !== null) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
+    );
+    return void 0;
+  }
+  return meta;
+}
+function splitCallerMetadata(user) {
+  if (!user) return {};
+  let reserved;
+  const safe = {};
+  for (const [key, value] of Object.entries(user)) {
+    if (RESERVED_METADATA_KEYS.includes(key)) {
+      (reserved ??= {})[key] = value;
+    } else {
+      safe[key] = value;
+    }
+  }
+  return { reserved, ...Object.keys(safe).length > 0 ? { safe } : {} };
+}
+function isAbortError(err) {
+  return err?.name === "AbortError";
+}
+async function stageCapture(runtime, ownerId, option) {
+  const limits = resolveCaptureLimits(option);
+  if (!limits) return void 0;
+  const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+  return {
+    artifactId: staged.artifactId,
+    channel: new import_axl2.RequestCaptureChannel({
+      sink: staged.sink,
+      ...limits,
+      // Redaction is the RUNTIME's policy, not the caller's: a run cannot opt
+      // out of compliance mode by asking for diagnostics.
+      redact: runtime.isRedactEnabled()
+    })
+  };
+}
+async function finishCapture(runtime, capture) {
+  const status = await capture.channel.close();
+  try {
+    return await runtime.finalizeDiagnosticArtifact(
+      capture.artifactId,
+      status.status,
+      status.reason,
+      // What the channel ACTUALLY applied. The store sees only scrubbed bytes
+      // and cannot tell, so a manifest without this reports every compliance-
+      // mode artifact as unredacted.
+      status.redaction
+    );
+  } catch {
+    return void 0;
+  }
+}
+async function abandonCapture(runtime, capture) {
+  if (!capture) return;
+  await capture.channel.close().catch(() => void 0);
+  await runtime.rollbackDiagnosticArtifact(capture.artifactId).catch(() => void 0);
+}
+function attachOperationRefs(items, channel) {
+  const byItem = /* @__PURE__ */ new Map();
+  const byScorer = /* @__PURE__ */ new Map();
+  for (const entry of channel.operations()) {
+    if (entry.caseIndex === void 0) continue;
+    const ref = {
+      operationId: entry.operationId,
+      kind: entry.kind,
+      ...entry.turn !== void 0 ? { turn: entry.turn } : {},
+      ...entry.attempt !== void 0 ? { attempt: entry.attempt } : {},
+      status: entry.status
+    };
+    if (entry.scorer !== void 0) {
+      const forItem = byScorer.get(entry.caseIndex) ?? /* @__PURE__ */ new Map();
+      const list = forItem.get(entry.scorer) ?? [];
+      list.push(ref);
+      forItem.set(entry.scorer, list);
+      byScorer.set(entry.caseIndex, forItem);
+    } else {
+      const list = byItem.get(entry.caseIndex) ?? [];
+      list.push(ref);
+      byItem.set(entry.caseIndex, list);
+    }
+  }
+  for (const [caseIndex, operations] of byItem) {
+    const item = items[caseIndex];
+    if (!item) continue;
+    item.diagnostics = { operations: [...item.diagnostics?.operations ?? [], ...operations] };
+  }
+  for (const [caseIndex, forItem] of byScorer) {
+    for (const [scorer, operations] of forItem) {
+      const detail = items[caseIndex]?.scoreDetails?.[scorer];
+      if (detail) detail.diagnostics = { operations };
+    }
+  }
+}
+var MAX_CAUSE_DEPTH = 8;
+function isProviderErrorLike(value) {
+  if (value instanceof import_axl2.ProviderError) return true;
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value;
+  return candidate.code === "PROVIDER_ERROR" && candidate.name === "ProviderError";
+}
+function describeItemFailure(thrown) {
+  let current = thrown;
+  for (let depth = 0; depth <= MAX_CAUSE_DEPTH; depth++) {
+    if (typeof current !== "object" || current === null) break;
+    if (isProviderErrorLike(current)) {
+      const { name: name2, provider, status, retryable, requestId } = current;
+      return {
+        name: typeof name2 === "string" ? name2 : "ProviderError",
+        ...typeof provider === "string" ? { provider } : {},
+        ...typeof status === "number" && Number.isFinite(status) ? { status } : {},
+        ...typeof retryable === "boolean" ? { retryable } : {},
+        ...typeof requestId === "string" ? { requestId } : {}
+      };
+    }
+    current = current.cause;
+  }
+  const name = thrown?.name;
+  return typeof thrown === "object" && thrown !== null && typeof name === "string" && name ? { name } : void 0;
+}
+async function runEval(config, executeWorkflow, runtime, options) {
+  const startTime = Date.now();
+  const id = (0, import_node_crypto.randomUUID)();
+  const itemErrorRateLimit = config.failOnItemErrorRate ?? DEFAULT_ITEM_ERROR_RATE_LIMIT;
+  if (!isErrorRateLimit(itemErrorRateLimit)) {
+    throw new import_axl2.AxlError(
+      "INVALID_ITEM_ERROR_RATE",
+      `Invalid failOnItemErrorRate (${String(itemErrorRateLimit)}): expected a number in [0, 1]; 1 disables the gate.`
+    );
+  }
+  const admission = config.budget != null ? new import_axl2.AdmissionController({ limit: parseBudget(config.budget) }) : void 0;
+  const capture = await stageCapture(runtime, id, options?.captureRequests);
+  let items;
+  try {
+    items = await config.dataset.getItems();
+  } catch (error) {
+    await abandonCapture(runtime, capture);
+    throw error;
+  }
+  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
+  const concurrency = config.concurrency ?? 5;
+  const scorerConcurrency = config.scorerConcurrency ?? 5;
+  let closedBy;
+  const noteClosure = (source) => {
+    if (closedBy === void 0 && admission?.closed) closedBy = source;
+  };
+  const scorerContext = {
+    resolveProvider: (uri) => {
+      if (typeof runtime.resolveProvider !== "function") {
+        throw new Error(
+          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
+        );
+      }
+      return runtime.resolveProvider(uri);
+    },
+    signal: options?.signal
+  };
+  const evalItems = new Array(items.length);
+  const callerReported = { costItems: 0, costTotal: 0, metadataItems: 0 };
+  const runCallSamples = /* @__PURE__ */ new Map();
+  function absorbTiming(rollup, evalItem) {
+    if (!rollup) return;
+    const perItem = {};
+    for (const [model, bucket] of Object.entries(rollup)) {
+      const sums = {
+        calls: bucket.calls,
+        queuedMs: bucket.queuedMs,
+        retryMs: bucket.retryMs,
+        wireMs: bucket.wireMs,
+        ...bucket.firstTokenMs !== void 0 && bucket.firstTokenCalls !== void 0 ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls } : {}
+      };
+      perItem[model] = sums;
+      const samples = bucket.samples;
+      if (!samples) continue;
+      const pooled = runCallSamples.get(model) ?? [];
+      for (const sample of samples) pooled.push(sample);
+      runCallSamples.set(model, pooled);
+    }
+    evalItem.timing = perItem;
+  }
+  function stub(item, outcome, error) {
+    return {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      error,
+      outcome,
+      accounting: emptyAccounting(),
+      cost: 0,
+      scorerCost: 0,
+      scores: {}
+    };
+  }
+  async function processItem(item, itemIndex) {
+    if (options?.signal?.aborted) {
+      evalItems[itemIndex] = stub(item, "cancelled", "Cancelled");
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
+    if (admission?.closed) {
+      noteClosure("case");
+      evalItems[itemIndex] = stub(item, "budget_skipped", "Budget exceeded");
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
+    const evalItem = {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      scores: {}
+    };
+    const itemStart = Date.now();
+    const itemOutcome = await trackScope(
+      runtime,
+      async () => {
+        const generation = await trackScope(
+          runtime,
+          async () => executeWorkflow(item.input, runtime),
+          {
+            purpose: "generation",
+            captureTraces: options?.captureTraces === true,
+            captureTimingSamples: true
+          }
+        );
+        evalItem.duration = Date.now() - itemStart;
+        absorbTiming(generation.modelTiming, evalItem);
+        if (generation.traces && generation.traces.length > 0) {
+          evalItem.traces = generation.traces;
+        }
+        if (generation.status === "rejected") {
+          const err = generation.error;
+          if (isAdmissionDenied(err)) {
+            noteClosure("operation");
+            evalItem.outcome = "budget_interrupted";
+            evalItem.error = "Budget interrupted";
+          } else if (isAbortError(err)) {
+            evalItem.outcome = "cancelled";
+            evalItem.error = "Cancelled";
+          } else {
+            evalItem.outcome = "failed";
+            const failure = describeItemFailure(err);
+            if (failure) evalItem.failure = failure;
+            evalItem.error = err instanceof Error ? err.message : String(err);
+            if (admission?.closed) noteClosure("case");
+          }
+          return;
+        }
+        const result = generation.value;
+        evalItem.outcome = "completed";
+        evalItem.output = result.output;
+        const callerCost = extractUserCost(result);
+        if (callerCost !== void 0) {
+          (evalItem.callerReport ??= {}).cost = callerCost;
+          callerReported.costItems += 1;
+          callerReported.costTotal += callerCost;
+        }
+        const { reserved, safe } = splitCallerMetadata(extractUserMetadata(result));
+        if (reserved) {
+          (evalItem.callerReport ??= {}).metadata = reserved;
+          callerReported.metadataItems += 1;
+        }
+        if (generation.metadata || safe) {
+          evalItem.metadata = { ...generation.metadata, ...safe };
+        }
+        if (admission?.closed) noteClosure("case");
+        await scoreItem(evalItem, config.scorers, {
+          runtime,
+          scorerContext,
+          scorerConcurrency,
+          signal: options?.signal,
+          admission,
+          onClosure: noteClosure
+        });
+      },
+      {
+        purpose: "generation",
+        // Every record produced anywhere under this item — workflow turns,
+        // nested asks, tool continuations, judges — is stamped with the case
+        // index, which is how the result can point an item at its own requests.
+        ...capture ? { captureCorrelation: { caseIndex: itemIndex } } : {}
+      }
+    );
+    if (itemOutcome.status === "rejected") throw itemOutcome.error;
+    evalItem.accounting = itemOutcome.accounting;
+    evalItem.cost = itemOutcome.accounting.breakdown.generation;
+    evalItem.scorerCost = itemOutcome.accounting.breakdown.judging;
+    if (itemOutcome.accounting.completeness !== "complete") evalItem.unpriced = true;
+    evalItems[itemIndex] = evalItem;
+    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+  }
+  const runOutcome = await trackScope(
+    runtime,
+    async () => {
+      await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
+    },
+    // The capture channel is declared ONCE, at the run scope: its byte budget
+    // and pending queue are per RUN, and every nested item/scorer scope
+    // inherits it rather than opening a competing budget of its own.
+    { admission, ...capture ? { capture: capture.channel } : {} }
+  );
+  if (runOutcome.status === "rejected") {
+    await abandonCapture(runtime, capture);
+    throw runOutcome.error;
+  }
+  let diagnostics;
+  if (capture) {
+    try {
+      attachOperationRefs(evalItems, capture.channel);
+    } catch (error) {
+      await abandonCapture(runtime, capture);
+      throw error;
+    }
+    const manifest = await finishCapture(runtime, capture);
+    diagnostics = manifest ? toDiagnosticManifest(manifest) : (
+      // An artifact that was never sealed is never committed, so publishing
+      // its id would point a reader at bytes the sweeper is about to reclaim.
+      unavailableManifest("", "the diagnostic artifact could not be finalized")
+    );
+  }
+  const accounting = {
+    ...runOutcome.accounting,
+    scope: "run",
+    ...admission ? {
+      budget: {
+        ...admission.snapshot(),
+        ...closedBy ? { closedBy } : {}
+      }
+    } : {},
+    ...callerReported.costItems > 0 || callerReported.metadataItems > 0 ? { callerReported } : {}
+  };
+  const failures = evalItems.filter((i) => i.error).length;
+  const scorerNames = config.scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
+    const { scored, failed, skipped } = scorerCounts(evalItems, name);
+    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
+  }
+  const coverage = buildCoverage(evalItems, scorerNames);
+  const scorerTypes = {};
+  for (const s of config.scorers) {
+    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
+  }
+  let degraded;
+  const rawLimit = config.failOnScorerErrorRate;
+  if (rawLimit != null) {
+    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
+      console.warn(
+        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
+      );
+    } else {
+      for (const name of scorerNames) {
+        const stats = scorerStats[name];
+        const scored = stats.scored ?? 0;
+        const failed = stats.failed ?? 0;
+        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
+        const limit = type === "deterministic" ? 0 : rawLimit;
+        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+        if (verdict.exceeds) {
+          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
+        }
+      }
+    }
+  }
+  const itemErrorRate = evaluateItemErrorRate(coverage.items, items.length, itemErrorRateLimit);
+  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
+  const timing = durations.length > 0 ? computeStats(durations) : void 0;
+  const modelTiming = buildModelTiming(runCallSamples);
+  const totalModelCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.modelCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [m, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
+      }
+    } else {
+      const itemModels = item.metadata?.models;
+      if (Array.isArray(itemModels)) {
+        for (const m of itemModels) {
+          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const modelsMeta = {};
+  if (totalModelCalls.size > 0) {
+    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
+    modelsMeta.models = sorted.map(([m]) => m);
+    modelsMeta.modelCounts = Object.fromEntries(sorted);
+  }
+  const totalWorkflowCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.workflowCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [w, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);
+      }
+    } else {
+      const itemWorkflows = item.metadata?.workflows;
+      if (Array.isArray(itemWorkflows)) {
+        for (const w of itemWorkflows) {
+          if (typeof w === "string")
+            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const workflowsMeta = {};
+  if (totalWorkflowCalls.size > 0) {
+    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);
+    workflowsMeta.workflows = sorted.map(([w]) => w);
+    workflowsMeta.workflowCounts = Object.fromEntries(sorted);
+  } else if (config.workflow) {
+    workflowsMeta.workflows = [config.workflow];
+    workflowsMeta.workflowCounts = { [config.workflow]: items.length };
+  }
+  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
+  return {
+    id,
+    dataset: config.dataset.name,
+    metadata: {
+      ...config.metadata,
+      scorerTypes,
+      ...modelsMeta,
+      ...workflowsMeta,
+      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
+    },
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost: accounting.knownCost,
+    ...accounting.completeness !== "complete" ? { unpriced: true } : {},
+    accounting,
+    ...diagnostics ? { diagnostics } : {},
+    duration: Date.now() - startTime,
+    items: evalItems,
+    summary: {
+      count: items.length,
+      failures,
+      coverage,
+      scorers: scorerStats,
+      timing,
+      ...modelTiming ? { modelTiming } : {},
+      ...degraded ? { degraded } : {},
+      ...itemErrorRate.failed > 0 ? { itemErrorRate } : {}
+    }
+  };
+}
+function buildCoverage(items, scorerNames) {
+  const coverage = {
+    items: Object.fromEntries(ITEM_OUTCOMES.map((o) => [o, 0])),
+    scorers: {}
+  };
+  for (const name of scorerNames) {
+    coverage.scorers[name] = Object.fromEntries(SCORER_OUTCOMES.map((o) => [o, 0]));
+  }
+  for (const item of items) {
+    const outcome = item.outcome ?? (item.error ? "failed" : "completed");
+    if (coverage.items[outcome] !== void 0) coverage.items[outcome] += 1;
+    for (const name of scorerNames) {
+      const detail = item.scoreDetails?.[name];
+      if (!detail?.outcome) continue;
+      const bucket = coverage.scorers[name];
+      if (bucket[detail.outcome] !== void 0) bucket[detail.outcome] += 1;
+    }
+  }
+  return coverage;
+}
+function buildModelTiming(runCallSamples) {
+  if (runCallSamples.size === 0) return void 0;
+  return Object.fromEntries(
+    [...runCallSamples.entries()].map(([model, samples]) => {
+      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
+      return [
+        model,
+        {
+          calls: samples.length,
+          wireMs: computeStats(samples.map((s) => s.wireMs)),
+          queuedMs: computeStats(samples.map((s) => s.queuedMs)),
+          retryMs: computeStats(samples.map((s) => s.retryMs)),
+          // Custom providers may omit the optional field; that call adds 0.
+          rateLimitRetries: samples.reduce((sum, s) => sum + (s.rateLimitRetries ?? 0), 0),
+          ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
+        }
+      ];
+    })
+  );
+}
+
+// src/compare.ts
+function certifyCost(baselineRuns, candidateRuns, scorerNames) {
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (const run of runs) {
+      const accounting = readAccounting(run);
+      if (accounting.completeness === "unverified") {
+        return `${side} run ${run.id} carries no accounting (unverified) \u2014 its total is not a measured figure.`;
+      }
+      if (accounting.completeness === "incomplete") {
+        const reasons = Object.keys(accounting.reasons).join(", ") || "unknown spend";
+        return `${side} run ${run.id} has incomplete accounting (${reasons}) \u2014 its total is a lower bound.`;
+      }
+    }
+  }
+  const scopes = /* @__PURE__ */ new Map();
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (const run of runs) {
+      const scope = readAccounting(run).scope;
+      const seen = scopes.get(side);
+      if (seen !== void 0 && seen !== scope) {
+        return `${side} mixes accounting scopes ("${seen}" and "${scope}") \u2014 a rescore total covers judging only and cannot be averaged with a full run.`;
+      }
+      scopes.set(side, scope);
+    }
+  }
+  const baselineScope = scopes.get("baseline");
+  const candidateScope = scopes.get("candidate");
+  if (baselineScope !== candidateScope) {
+    return `accounting scope differs (baseline "${baselineScope}" vs candidate "${candidateScope}") \u2014 a rescore total covers judging only.`;
+  }
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    if (bRun.items.length !== cRun.items.length) {
+      return `item counts differ (baseline ${bRun.items.length} vs candidate ${cRun.items.length}) \u2014 the runs did different amounts of work.`;
+    }
+    for (let i = 0; i < bRun.items.length; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      const bRan = !bItem.error;
+      const cRan = !cItem.error;
+      if (bRan !== cRan) {
+        const stopped = bRan ? cItem : bItem;
+        return `case coverage differs at item ${i}: ${bRan ? "candidate" : "baseline"} did not complete it (${stopped.outcome ?? stopped.error}) \u2014 the cheaper side did less work.`;
+      }
+      if (!bRan) continue;
+      for (const name of scorerNames) {
+        const bScored = bItem.scores[name] != null;
+        const cScored = cItem.scores[name] != null;
+        if (bScored !== cScored) {
+          return `scorer coverage differs for "${name}" at item ${i}: only ${bScored ? "baseline" : "candidate"} produced a score.`;
+        }
+      }
+    }
+  }
+  return null;
 }
-
-// src/compare.ts
 var DEFAULT_LLM_THRESHOLD = 0.05;
 var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
 var LEGACY_THRESHOLD = 0.1;
@@ -306,17 +1233,21 @@ function evalCompare(baseline, candidate, options) {
     timing = { baselineMean: round2(bMean), candidateMean: round2(cMean), delta, deltaPercent };
   }
   let cost;
-  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
-  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
-  if (baselineAvgCost > 0 || candidateAvgCost > 0) {
+  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + readAccounting(r).knownCost, 0) / baselineRuns.length;
+  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + readAccounting(r).knownCost, 0) / candidateRuns.length;
+  const hasAccounting = [...baselineRuns, ...candidateRuns].some((r) => r.accounting != null);
+  if (baselineAvgCost > 0 || candidateAvgCost > 0 || hasAccounting) {
     const deltaRaw = candidateAvgCost - baselineAvgCost;
     const delta = round2(deltaRaw);
-    const deltaPercent = baselineAvgCost > 0 ? round2(deltaRaw / baselineAvgCost * 100) : 0;
+    const deltaPercent = baselineAvgCost > 0 ? round2(deltaRaw / baselineAvgCost * 100) : null;
+    const verdict = certifyCost(baselineRuns, candidateRuns, baselineScorerNames);
     cost = {
       baselineTotal: round2(baselineAvgCost),
       candidateTotal: round2(candidateAvgCost),
       delta,
-      deltaPercent
+      deltaPercent,
+      certified: verdict === null,
+      ...verdict !== null ? { reason: verdict } : {}
     };
   }
   const parts = [];
@@ -333,469 +1264,219 @@ function evalCompare(baseline, candidate, options) {
       );
     }
   }
-  if (timing && Math.abs(timing.deltaPercent) > 1) {
-    const dir = timing.delta > 0 ? "slower" : "faster";
-    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
-  }
-  if (cost && Math.abs(cost.deltaPercent) > 1) {
-    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
-    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
-  }
-  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
-  const baselinePartial = detectPartial(baselineRunsRaw);
-  const candidatePartial = detectPartial(candidateRunsRaw);
-  return {
-    baseline: {
-      id: baselineRef.id,
-      metadata: baselineRef.metadata,
-      runCount,
-      ...baselinePartial ? { partial: baselinePartial } : {}
-    },
-    candidate: {
-      id: candidateRef.id,
-      metadata: candidateRef.metadata,
-      runCount,
-      ...candidatePartial ? { partial: candidatePartial } : {}
-    },
-    scorers,
-    timing,
-    cost,
-    regressions,
-    improvements,
-    summary: summaryStr
-  };
-}
-function detectPartial(runs) {
-  if (runs.length === 0) return void 0;
-  let attempted;
-  for (const r of runs) {
-    const v = r.metadata?.batchAttempted;
-    if (typeof v === "number" && Number.isFinite(v)) {
-      attempted = v;
-      break;
-    }
-  }
-  if (attempted === void 0) return void 0;
-  if (runs.length >= attempted) return void 0;
-  return { completed: runs.length, attempted };
-}
-function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
-  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
-  for (const name of Object.keys(comparison.scorers)) {
-    const s = comparison.scorers[name];
-    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
-    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
-    for (const side of ["baseline", "candidate"]) {
-      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
-      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
-      const skipped = (side === "baseline" ? s.baselineSkipped : s.candidateSkipped) ?? 0;
-      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
-      if (verdict.zeroSample) {
-        const cause = skipped > 0 ? `every one of its items was N/A (the \`applies\` predicate skipped all ${skipped})` : `its items errored or produced no score`;
-        return `scorer "${name}" has no scored items on ${side} (${cause}) \u2014 cannot certify a mean over an empty sample.`;
-      }
-      if (verdict.exceeds) {
-        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
-        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
-      }
-    }
-  }
-  return null;
-}
-
-// src/scorer.ts
-function normalizeScorerResult(result) {
-  return typeof result === "number" ? { score: result } : result;
-}
-function extractScorerErrorCost(err) {
-  const v = err && typeof err === "object" ? err.cost : void 0;
-  return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : void 0;
-}
-
-// src/score-item.ts
-async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal) {
-  const scoreDetails = item.scoreDetails ??= {};
-  for (const s of scorers) {
-    item.scores[s.name] = null;
-    scoreDetails[s.name] = { score: null };
-  }
-  let itemScorerCost = 0;
-  const scorerErrorsByName = {};
-  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer) => {
-    if (signal?.aborted) return;
-    const scorerStart = Date.now();
-    try {
-      if (scorer.applies && !scorer.applies(item.output, item.input, item.annotations)) {
-        item.scores[scorer.name] = null;
-        scoreDetails[scorer.name] = { score: null, skipped: true };
-        return;
-      }
-      const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);
-      const scorerResult = normalizeScorerResult(raw);
-      if (scorerResult.cost != null) {
-        itemScorerCost += scorerResult.cost;
-      }
-      const scorerDuration = Date.now() - scorerStart;
-      if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-        scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;
-        item.scores[scorer.name] = null;
-        scoreDetails[scorer.name] = {
-          score: null,
-          metadata: scorerResult.metadata,
-          duration: scorerDuration,
-          cost: scorerResult.cost
-        };
-      } else {
-        item.scores[scorer.name] = round2(scorerResult.score);
-        scoreDetails[scorer.name] = {
-          score: round2(scorerResult.score),
-          metadata: scorerResult.metadata,
-          duration: scorerDuration,
-          cost: scorerResult.cost
-        };
-      }
-    } catch (err) {
-      if (err?.name === "AbortError") return;
-      const errCost = extractScorerErrorCost(err);
-      if (errCost != null) itemScorerCost += errCost;
-      scorerErrorsByName[scorer.name] = `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
-      item.scores[scorer.name] = null;
-      scoreDetails[scorer.name] = {
-        score: null,
-        duration: Date.now() - scorerStart,
-        cost: errCost
-      };
-    }
-  });
-  const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
-  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
-  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
-  return itemScorerCost;
-}
-
-// src/runner.ts
-var import_node_crypto = require("crypto");
-function parseCost(cost) {
-  const match = cost.match(/^\$?([\d.]+)$/);
-  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
-  return parseFloat(match[1]);
-}
-function extractUserCost(result, label = "executeWorkflow") {
-  if (result === null || typeof result !== "object") return void 0;
-  const raw = result.cost;
-  if (raw === void 0) return void 0;
-  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
-  console.warn(
-    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
-  );
-  return void 0;
-}
-function extractUserMetadata(result, label = "executeWorkflow") {
-  if (result === null || typeof result !== "object") return void 0;
-  const meta = result.metadata;
-  if (meta === void 0) return void 0;
-  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
-    console.warn(
-      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
-    );
-    return void 0;
-  }
-  const proto = Object.getPrototypeOf(meta);
-  if (proto !== Object.prototype && proto !== null) {
-    console.warn(
-      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
-    );
-    return void 0;
-  }
-  return meta;
-}
-function mergeMetadata(tracked, user) {
-  const merged = { ...tracked, ...user };
-  for (const [list, counts] of [
-    ["models", "modelCallCounts"],
-    ["workflows", "workflowCallCounts"]
-  ]) {
-    if (user && Object.hasOwn(user, list) && !Object.hasOwn(user, counts)) {
-      delete merged[counts];
-    }
-  }
-  return merged;
-}
-async function runEval(config, executeWorkflow, runtime, options) {
-  const startTime = Date.now();
-  const id = (0, import_node_crypto.randomUUID)();
-  const items = await config.dataset.getItems();
-  const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
-  const concurrency = config.concurrency ?? 5;
-  const scorerConcurrency = config.scorerConcurrency ?? 5;
-  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
-  const scorerContext = {
-    resolveProvider: (uri) => {
-      if (typeof runtime.resolveProvider !== "function") {
-        throw new Error(
-          `LLM scorers require a runtime with resolveProvider(). Ensure you are using a real AxlRuntime instance, not a mock.`
-        );
-      }
-      return runtime.resolveProvider(uri);
-    },
-    signal: options?.signal
-  };
-  const evalItems = new Array(items.length);
-  let totalCost = 0;
-  let budgetExceeded = false;
-  const runCallSamples = /* @__PURE__ */ new Map();
-  function absorbTiming(rollup, evalItem) {
-    if (!rollup) return;
-    const perItem = {};
-    for (const [model, bucket] of Object.entries(rollup)) {
-      const sums = {
-        calls: bucket.calls,
-        queuedMs: bucket.queuedMs,
-        retryMs: bucket.retryMs,
-        wireMs: bucket.wireMs,
-        ...bucket.firstTokenMs !== void 0 && bucket.firstTokenCalls !== void 0 ? { firstTokenMs: bucket.firstTokenMs, firstTokenCalls: bucket.firstTokenCalls } : {}
-      };
-      perItem[model] = sums;
-      const samples = bucket.samples;
-      if (!samples) continue;
-      const pooled = runCallSamples.get(model) ?? [];
-      for (const sample of samples) pooled.push(sample);
-      runCallSamples.set(model, pooled);
-    }
-    evalItem.timing = perItem;
-  }
-  async function processItem(item, itemIndex) {
-    if (options?.signal?.aborted) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Cancelled",
-        scores: {}
-      };
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    if (budgetExceeded) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Budget exceeded",
-        scores: {}
-      };
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    const evalItem = {
-      input: item.input,
-      annotations: item.annotations,
-      output: null,
-      scores: {}
-    };
-    const itemStart = Date.now();
-    try {
-      if (options?.captureTraces) {
-        const tracked = await runtime.trackExecution(
-          async () => executeWorkflow(item.input, runtime),
-          { captureTraces: true, captureTimingSamples: true }
-        );
-        evalItem.duration = Date.now() - itemStart;
-        evalItem.output = tracked.result.output;
-        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
-        if (tracked.unpriced) evalItem.unpriced = true;
-        evalItem.metadata = mergeMetadata(tracked.metadata, extractUserMetadata(tracked.result));
-        absorbTiming(tracked.modelTiming, evalItem);
-        if (tracked.traces && tracked.traces.length > 0) {
-          evalItem.traces = tracked.traces;
-        }
-        if (evalItem.cost != null) {
-          totalCost += evalItem.cost;
-        }
-      } else {
-        let trackedTiming;
-        let trackedMetadata;
-        let result;
-        if (typeof runtime.trackExecution === "function") {
-          const tracked = await runtime.trackExecution(
-            async () => executeWorkflow(item.input, runtime),
-            { captureTraces: false, captureTimingSamples: true }
-          );
-          result = tracked.result;
-          trackedTiming = tracked.modelTiming;
-          trackedMetadata = tracked.metadata;
-        } else {
-          result = await executeWorkflow(item.input, runtime);
-        }
-        evalItem.duration = Date.now() - itemStart;
-        evalItem.output = result.output;
-        evalItem.cost = extractUserCost(result);
-        const meta = extractUserMetadata(result);
-        if (trackedMetadata || meta) evalItem.metadata = mergeMetadata(trackedMetadata, meta);
-        absorbTiming(trackedTiming, evalItem);
-        if (evalItem.cost != null) {
-          totalCost += evalItem.cost;
-        }
-      }
-    } catch (err) {
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.error = err instanceof Error ? err.message : String(err);
-      if (options?.captureTraces) {
-        const captured = err.axlCapturedTraces;
-        if (Array.isArray(captured) && captured.length > 0) {
-          evalItem.traces = captured;
-        }
-      }
-      evalItems[itemIndex] = evalItem;
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    if (budgetLimit != null && totalCost > budgetLimit) {
-      budgetExceeded = true;
-    }
-    const itemScorerCost = await scoreItem(
-      evalItem,
-      config.scorers,
-      scorerConcurrency,
-      scorerContext,
-      options?.signal
-    );
-    totalCost += itemScorerCost;
-    evalItems[itemIndex] = evalItem;
-    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-  }
-  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
-  const failures = evalItems.filter((i) => i.error).length;
-  const scorerNames = config.scorers.map((s) => s.name);
-  const scorerStats = {};
-  for (const name of scorerNames) {
-    const scores = evalItems.filter((i) => !i.error && i.scores[name] != null).map((i) => i.scores[name]);
-    const { scored, failed, skipped } = scorerCounts(evalItems, name);
-    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
-  }
-  const scorerTypes = {};
-  for (const s of config.scorers) {
-    scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
-  }
-  let degraded;
-  const rawLimit = config.failOnScorerErrorRate;
-  if (rawLimit != null) {
-    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {
-      console.warn(
-        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`
-      );
-    } else {
-      for (const name of scorerNames) {
-        const stats = scorerStats[name];
-        const scored = stats.scored ?? 0;
-        const failed = stats.failed ?? 0;
-        const type = scorerTypes[name] === "llm" ? "llm" : "deterministic";
-        const limit = type === "deterministic" ? 0 : rawLimit;
-        const verdict = evaluateScorerTolerance(scored, failed, type, limit);
-        if (verdict.exceeds) {
-          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });
-        }
-      }
-    }
-  }
-  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
-  const timing = durations.length > 0 ? computeStats(durations) : void 0;
-  const modelTiming = runCallSamples.size > 0 ? Object.fromEntries(
-    [...runCallSamples.entries()].map(([model, samples]) => {
-      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
-      const stats = {
-        calls: samples.length,
-        wireMs: computeStats(samples.map((s) => s.wireMs)),
-        queuedMs: computeStats(samples.map((s) => s.queuedMs)),
-        retryMs: computeStats(samples.map((s) => s.retryMs)),
-        ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
-      };
-      return [model, stats];
-    })
-  ) : void 0;
-  const totalModelCalls = /* @__PURE__ */ new Map();
-  for (const item of evalItems) {
-    const itemCounts = item.metadata?.modelCallCounts;
-    if (itemCounts && typeof itemCounts === "object") {
-      for (const [m, count] of Object.entries(itemCounts)) {
-        if (typeof count === "number")
-          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);
-      }
-    } else {
-      const itemModels = item.metadata?.models;
-      if (Array.isArray(itemModels)) {
-        for (const m of itemModels) {
-          if (typeof m === "string") totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);
-        }
-      }
-    }
+  if (timing && Math.abs(timing.deltaPercent) > 1) {
+    const dir = timing.delta > 0 ? "slower" : "faster";
+    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
   }
-  const modelsMeta = {};
-  if (totalModelCalls.size > 0) {
-    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);
-    modelsMeta.models = sorted.map(([m]) => m);
-    modelsMeta.modelCounts = Object.fromEntries(sorted);
+  if (cost?.certified && cost.deltaPercent !== null && Math.abs(cost.deltaPercent) > 1) {
+    const dir = cost.delta > 0 ? "more expensive" : "cheaper";
+    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
   }
-  const totalWorkflowCalls = /* @__PURE__ */ new Map();
-  for (const item of evalItems) {
-    const itemCounts = item.metadata?.workflowCallCounts;
-    if (itemCounts && typeof itemCounts === "object") {
-      for (const [w, count] of Object.entries(itemCounts)) {
-        if (typeof count === "number")
-          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  const baselinePartial = detectPartial(baselineRunsRaw);
+  const candidatePartial = detectPartial(candidateRunsRaw);
+  return {
+    baseline: {
+      id: baselineRef.id,
+      metadata: baselineRef.metadata,
+      runCount,
+      ...baselinePartial ? { partial: baselinePartial } : {}
+    },
+    candidate: {
+      id: candidateRef.id,
+      metadata: candidateRef.metadata,
+      runCount,
+      ...candidatePartial ? { partial: candidatePartial } : {}
+    },
+    scorers,
+    timing,
+    cost,
+    regressions,
+    improvements,
+    summary: summaryStr
+  };
+}
+function detectPartial(runs) {
+  if (runs.length === 0) return void 0;
+  let attempted;
+  for (const r of runs) {
+    const v = r.metadata?.batchAttempted;
+    if (typeof v === "number" && Number.isFinite(v)) {
+      attempted = v;
+      break;
+    }
+  }
+  if (attempted === void 0) return void 0;
+  if (runs.length >= attempted) return void 0;
+  return { completed: runs.length, attempted };
+}
+function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
+  const scorerTypes = comparison.baseline.metadata?.scorerTypes ?? {};
+  for (const name of Object.keys(comparison.scorers)) {
+    const s = comparison.scorers[name];
+    const type = scorerTypes[name] === "deterministic" ? "deterministic" : "llm";
+    const limit = type === "deterministic" ? 0 : maxScorerErrorRate;
+    for (const side of ["baseline", "candidate"]) {
+      const scored = (side === "baseline" ? s.baselineScored : s.candidateScored) ?? 0;
+      const failed = (side === "baseline" ? s.baselineFailed : s.candidateFailed) ?? 0;
+      const skipped = (side === "baseline" ? s.baselineSkipped : s.candidateSkipped) ?? 0;
+      const verdict = evaluateScorerTolerance(scored, failed, type, limit);
+      if (verdict.zeroSample) {
+        const cause = skipped > 0 ? `every one of its items was N/A (the \`applies\` predicate skipped all ${skipped})` : `its items errored or produced no score`;
+        return `scorer "${name}" has no scored items on ${side} (${cause}) \u2014 cannot certify a mean over an empty sample.`;
       }
-    } else {
-      const itemWorkflows = item.metadata?.workflows;
-      if (Array.isArray(itemWorkflows)) {
-        for (const w of itemWorkflows) {
-          if (typeof w === "string")
-            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);
-        }
+      if (verdict.exceeds) {
+        const limitStr = type === "deterministic" ? "deterministic zero-tolerance" : `${(limit * 100).toFixed(1)}%`;
+        return `scorer "${name}" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;
       }
     }
   }
-  const workflowsMeta = {};
-  if (totalWorkflowCalls.size > 0) {
-    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);
-    workflowsMeta.workflows = sorted.map(([w]) => w);
-    workflowsMeta.workflowCounts = Object.fromEntries(sorted);
-  } else if (config.workflow) {
-    workflowsMeta.workflows = [config.workflow];
-    workflowsMeta.workflowCounts = { [config.workflow]: items.length };
+  return null;
+}
+function compareItemErrorRates(baseline, candidate, maxItemErrorRate = DEFAULT_ITEM_ERROR_RATE_LIMIT) {
+  if (!isErrorRateLimit(maxItemErrorRate)) {
+    throw new Error(
+      `Invalid maxItemErrorRate (${String(maxItemErrorRate)}): expected a number in [0, 1].`
+    );
   }
-  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
-  return {
-    id,
-    dataset: config.dataset.name,
-    metadata: {
-      ...config.metadata,
-      scorerTypes,
-      ...modelsMeta,
-      ...workflowsMeta,
-      ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
-    },
-    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
-    ...evalItems.some((item) => item.unpriced) ? { unpriced: true } : {},
-    duration: Date.now() - startTime,
-    items: evalItems,
-    summary: {
-      count: items.length,
-      failures,
-      scorers: scorerStats,
-      timing,
-      ...modelTiming ? { modelTiming } : {},
-      ...degraded ? { degraded } : {}
+  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  const rates = [];
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (let r = 0; r < runCount; r++) {
+      const run = runs[r];
+      const coverage = run.summary.coverage?.items;
+      const legacy = coverage === void 0;
+      const verdict = legacy ? evaluateItemErrorRate(
+        buildCoverage(run.items, []).items,
+        run.items.length,
+        maxItemErrorRate
+      ) : evaluateItemErrorRate(coverage, run.summary.count, maxItemErrorRate);
+      rates.push({ side, runIndex: r, runId: run.id, legacy, ...verdict });
     }
-  };
+  }
+  return rates;
+}
+function describeItemErrorRate(rate, multiRun) {
+  const which = multiRun ? ` run ${rate.runIndex + 1} (${rate.runId})` : ` (${rate.runId})`;
+  const legacy = rate.legacy ? ", derived from the items of a pre-0.24 artifact" : "";
+  return `${rate.side}${which} item error rate ${formatPercent(rate.rate)} (${rate.failed}/${rate.attempted} attempted items failed${legacy})`;
+}
+function evaluateItemErrorRateGate(baseline, candidate, maxItemErrorRate = DEFAULT_ITEM_ERROR_RATE_LIMIT) {
+  const rates = compareItemErrorRates(baseline, candidate, maxItemErrorRate);
+  const multiRun = rates.some((r) => r.runIndex > 0);
+  const over = rates.find((r) => r.exceeded);
+  if (!over) return null;
+  return `coverage: ${describeItemErrorRate(over, multiRun)} exceeds the ${formatPercent(maxItemErrorRate)} limit \u2014 its scores cover only the surviving items. Pass --max-item-error-rate <0..1> to accept a thinned side (1 disables the check).`;
 }
 
 // src/rescore.ts
+var import_axl3 = require("@axlsdk/axl");
 var import_node_crypto2 = require("crypto");
+var COPY_SHARE_OF_RUN_BOUND = 0.75;
+function describeFailure(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function isDiagnosticsUnavailable(error) {
+  return error?.code === "DIAGNOSTICS_UNAVAILABLE";
+}
+async function beginCapture(source, ownerId, runtime, options) {
+  const limits = resolveCaptureLimits(options?.captureRequests);
+  if (!limits) return {};
+  const open = (artifactId, sink, carriedBytes) => ({
+    artifactId,
+    channel: new import_axl3.RequestCaptureChannel({
+      sink,
+      ...limits,
+      carriedBytes,
+      // Redaction is the RUNTIME's policy, not the caller's: a rescore cannot
+      // opt out of compliance mode by asking for diagnostics.
+      redact: runtime.isRedactEnabled()
+    })
+  });
+  const sourceId = source.diagnostics?.artifactId;
+  let stagedId;
+  try {
+    if (!sourceId) {
+      const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+      stagedId = staged.artifactId;
+      return { capture: { ...open(staged.artifactId, staged.sink, 0), copied: false } };
+    }
+    const runBound = limits.maxRunBytes ?? DEFAULT_COPY_MAX_BYTES;
+    const maxBytes = Math.max(1, Math.floor(runBound * COPY_SHARE_OF_RUN_BOUND));
+    const copied = await runtime.copyDiagnosticArtifact(
+      sourceId,
+      { kind: "eval", id: ownerId },
+      { maxBytes }
+    );
+    if (!copied) {
+      return {
+        degraded: unavailableManifest(
+          "",
+          "the source run's captured requests are no longer available"
+        )
+      };
+    }
+    stagedId = copied.artifactId;
+    const capture = {
+      ...open(copied.artifactId, copied.sink, copied.bytes),
+      copied: true,
+      carriedRedaction: copied.redaction
+    };
+    if (copied.truncated) {
+      capture.copyTruncated = `the copied source records stopped at ${maxBytes} bytes, the share of the ${runBound} byte run limit reserved for them`;
+    }
+    return { capture };
+  } catch (error) {
+    if (stagedId !== void 0) {
+      await runtime.rollbackDiagnosticArtifact(stagedId).catch(() => void 0);
+    }
+    if (isDiagnosticsUnavailable(error)) throw error;
+    return {
+      degraded: unavailableManifest(
+        "",
+        `the captured requests could not be carried forward: ${describeFailure(error)}`
+      )
+    };
+  }
+}
+async function abandonCapture2(runtime, capture) {
+  if (!capture) return;
+  await capture.channel.close().catch(() => void 0);
+  await runtime.rollbackDiagnosticArtifact(capture.artifactId).catch(() => void 0);
+}
+async function finishCapture2(runtime, capture) {
+  const status = await capture.channel.close();
+  const reason = [capture.copyTruncated, status.reason].filter(Boolean).join("; ") || void 0;
+  const truncated = capture.copyTruncated !== void 0 || status.status === "truncated";
+  const finalStatus = status.status === "unavailable" ? "unavailable" : truncated ? "truncated" : status.status;
+  const redaction = status.redaction === "applied" && (capture.carriedRedaction ?? "applied") === "applied" ? "applied" : "none";
+  try {
+    const manifest = await runtime.finalizeDiagnosticArtifact(
+      capture.artifactId,
+      finalStatus,
+      reason,
+      redaction
+    );
+    return toDiagnosticManifest(manifest);
+  } catch {
+    return unavailableManifest("", "the diagnostic artifact could not be finalized");
+  }
+}
 async function rescore(result, scorers, runtime, options) {
   const startTime = Date.now();
+  const rescoredId = (0, import_node_crypto2.randomUUID)();
   const concurrency = options?.concurrency ?? 5;
   const scorerConcurrency = options?.scorerConcurrency ?? 5;
+  const admission = options?.budget != null ? new import_axl3.AdmissionController({ limit: parseBudget(options.budget) }) : void 0;
+  let closedBy;
+  const noteClosure = (source) => {
+    if (closedBy === void 0 && admission?.closed) closedBy = source;
+  };
   const scorerContext = {
     resolveProvider: (uri) => {
       if (typeof runtime.resolveProvider !== "function") {
@@ -808,7 +1489,8 @@ async function rescore(result, scorers, runtime, options) {
     signal: options?.signal
   };
   const rescored = new Array(result.items.length);
-  let totalCost = 0;
+  const { capture, degraded } = await beginCapture(result, rescoredId, runtime, options);
+  const carryDiagnostics = capture?.copied === true;
   async function rescoreItem(original, itemIndex) {
     if (options?.signal?.aborted) {
       rescored[itemIndex] = {
@@ -816,16 +1498,31 @@ async function rescore(result, scorers, runtime, options) {
         annotations: original.annotations,
         output: original.output,
         error: "cancelled",
+        outcome: "cancelled",
+        accounting: emptyAccounting(),
         scores: {}
       };
       return;
     }
     if (original.error) {
+      const accounting2 = emptyAccounting();
       rescored[itemIndex] = {
         input: original.input,
         annotations: original.annotations,
         output: original.output,
         error: original.error,
+        ...original.outcome ? { outcome: original.outcome } : {},
+        // The failure cause is a fact about the source run too.
+        ...original.failure ? { failure: original.failure } : {},
+        accounting: accounting2,
+        // Same rule as a scored item: an item carrying `accounting` always
+        // carries both compat views, so no reader falls through to the legacy
+        // "no accounting → trust the caller's number" branch and reports the
+        // SOURCE run's spend as this rescore's. A passthrough did no work, so
+        // both are a measured $0; the source's spend lives in the run-level
+        // `accounting.source` provenance and nowhere else.
+        cost: accounting2.breakdown.generation,
+        scorerCost: accounting2.breakdown.judging,
         scores: {}
       };
       return;
@@ -836,20 +1533,77 @@ async function rescore(result, scorers, runtime, options) {
       output: original.output,
       metadata: original.metadata,
       traces: original.traces,
+      outcome: "completed",
+      // Carried forward VERBATIM, original operation ids included. The rescore
+      // did not re-run the workflow, so these still describe the calls that
+      // produced this output; reminting the ids would break the only link back
+      // to the run that made them. Only carried when the source records were
+      // actually copied into this rescore's artifact — a dangling reference is
+      // worse than none.
+      ...carryDiagnostics && original.diagnostics ? { diagnostics: original.diagnostics } : {},
       scores: {},
       scoreDetails: {}
     };
-    const itemScorerCost = await scoreItem(
-      item,
-      scorers,
-      scorerConcurrency,
-      scorerContext,
-      options?.signal
+    const itemOutcome = await trackScope(
+      runtime,
+      async () => {
+        await scoreItem(item, scorers, {
+          runtime,
+          scorerContext,
+          scorerConcurrency,
+          signal: options?.signal,
+          admission,
+          onClosure: noteClosure
+        });
+      },
+      {
+        purpose: "judging",
+        // Every record produced under this item is stamped with the case index,
+        // which is how a `ScorerDetail` can point at its own judge calls.
+        ...capture ? { captureCorrelation: { caseIndex: itemIndex } } : {}
+      }
     );
-    totalCost += itemScorerCost;
+    if (itemOutcome.status === "rejected") throw itemOutcome.error;
+    item.accounting = itemOutcome.accounting;
+    item.cost = itemOutcome.accounting.breakdown.generation;
+    item.scorerCost = itemOutcome.accounting.breakdown.judging;
+    if (itemOutcome.accounting.completeness !== "complete") item.unpriced = true;
     rescored[itemIndex] = item;
   }
-  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
+  const runOutcome = await trackScope(
+    runtime,
+    async () => {
+      await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
+    },
+    // The channel is declared ONCE, at the run scope: its byte budget and
+    // pending queue are per rescore, and every nested item/scorer scope inherits
+    // it rather than opening a competing budget of its own.
+    { purpose: "judging", admission, ...capture ? { capture: capture.channel } : {} }
+  );
+  if (runOutcome.status === "rejected") {
+    await abandonCapture2(runtime, capture);
+    throw runOutcome.error;
+  }
+  const sourceAccounting = result.accounting;
+  const generation = sourceAccounting?.scope === "rescore" ? sourceAccounting.source?.generation ?? null : sourceAccounting ?? null;
+  let diagnostics = degraded;
+  if (capture) {
+    try {
+      attachOperationRefs(rescored, capture.channel);
+    } catch (error) {
+      await abandonCapture2(runtime, capture);
+      throw error;
+    }
+    diagnostics = await finishCapture2(runtime, capture);
+  }
+  const accounting = {
+    ...runOutcome.accounting,
+    scope: "rescore",
+    // `?? null` is load-bearing: a legacy source has no accounting and must read
+    // as "unknown generation", never as a synthesized complete record.
+    source: { runId: result.id, generation },
+    ...admission ? { budget: { ...admission.snapshot(), ...closedBy ? { closedBy } : {} } } : {}
+  };
   const failures = rescored.filter((i) => i.error).length;
   const scorerNames = scorers.map((s) => s.name);
   const scorerStats = {};
@@ -863,7 +1617,7 @@ async function rescore(result, scorers, runtime, options) {
     scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
   }
   return {
-    id: (0, import_node_crypto2.randomUUID)(),
+    id: rescoredId,
     dataset: result.dataset,
     metadata: (() => {
       const rest = { ...result.metadata };
@@ -887,10 +1641,24 @@ async function rescore(result, scorers, runtime, options) {
       return merged;
     })(),
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
+    totalCost: accounting.knownCost,
+    ...accounting.completeness !== "complete" ? { unpriced: true } : {},
+    accounting,
+    ...diagnostics ? { diagnostics } : {},
     duration: Date.now() - startTime,
     items: rescored,
-    summary: { count: result.items.length, failures, scorers: scorerStats }
+    summary: {
+      count: result.items.length,
+      failures,
+      coverage: buildCoverage(rescored, scorerNames),
+      scorers: scorerStats,
+      // A rescore makes no generation calls, so the source run's wall-clock
+      // and provider latency are still the only measurements of these items.
+      // `modelTiming` cannot be rebuilt from `item.timing` (sums, not per-call
+      // samples): carry both forward.
+      ...result.summary.timing ? { timing: { ...result.summary.timing } } : {},
+      ...result.summary.modelTiming ? { modelTiming: structuredClone(result.summary.modelTiming) } : {}
+    }
   };
 }
 
@@ -960,6 +1728,7 @@ function aggregateRuns(runs) {
     workflows,
     dataset,
     totalCost: round2(totalCost),
+    accounting: aggregateAccounting(runs.map((r) => readAccounting(r))),
     totalDuration: round2(totalDuration),
     scorers,
     timing
@@ -967,7 +1736,7 @@ function aggregateRuns(runs) {
 }
 
 // src/cli-utils.ts
-var import_axl = require("@axlsdk/axl");
+var import_axl4 = require("@axlsdk/axl");
 
 // src/cli-validate.ts
 function validateEvalConfig(cfg) {
@@ -1014,6 +1783,12 @@ function validateEvalConfig(cfg) {
       return `exports an invalid failOnScorerErrorRate (${typeof v === "number" ? v : typeof v}) \u2014 must be a number between 0 and 1.`;
     }
   }
+  {
+    const v = c.failOnItemErrorRate;
+    if (v != null && !isErrorRateLimit(v)) {
+      return `exports an invalid failOnItemErrorRate (${typeof v === "number" ? v : typeof v}) \u2014 must be a number between 0 and 1 (1 disables the gate).`;
+    }
+  }
   return void 0;
 }
 
@@ -1026,8 +1801,11 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
   "--threshold",
   "--runs",
   "--capture-traces",
+  "--capture-requests",
   "--concurrency",
-  "--scorers"
+  "--scorers",
+  "--budget",
+  "--max-item-error-rate"
 ]);
 var VALUE_FLAGS = /* @__PURE__ */ new Set([
   "--output",
@@ -1035,7 +1813,9 @@ var VALUE_FLAGS = /* @__PURE__ */ new Set([
   "--conditions",
   "--runs",
   "--concurrency",
-  "--scorers"
+  "--scorers",
+  "--budget",
+  "--max-item-error-rate"
 ]);
 function envInt(name) {
   const raw = process.env[name];
@@ -1044,14 +1824,25 @@ function envInt(name) {
   if (!Number.isFinite(n) || n <= 0) return void 0;
   return n;
 }
+function parseErrorRateFlag(flag, raw) {
+  const n = /^\d*\.?\d+$/.test(raw.trim()) ? Number(raw) : NaN;
+  if (!Number.isFinite(n) || n < 0 || n > 1) {
+    console.error(`Error: ${flag} must be a number in [0, 1], got "${raw}"`);
+    process.exit(1);
+  }
+  return n;
+}
 function parseEvalArgs(args) {
   let outputPath;
   let configArg;
   let conditions = [];
   let runs = 1;
   let captureTraces = false;
+  let captureRequests = false;
   let concurrency;
   let scorerNames;
+  let budget;
+  let maxItemErrorRate;
   const paths = [];
   for (let i = 0; i < args.length; i++) {
     const arg = args[i];
@@ -1072,6 +1863,10 @@ function parseEvalArgs(args) {
         } else {
           concurrency = n;
         }
+      } else if (arg === "--budget") {
+        budget = value;
+      } else if (arg === "--max-item-error-rate") {
+        maxItemErrorRate = parseErrorRateFlag(arg, value);
       } else if (arg === "--scorers") {
         scorerNames = [
           ...new Set(
@@ -1083,6 +1878,8 @@ function parseEvalArgs(args) {
       }
     } else if (arg === "--capture-traces") {
       captureTraces = true;
+    } else if (arg === "--capture-requests") {
+      captureRequests = true;
     } else if (arg.startsWith("--")) {
       if (!KNOWN_FLAGS.has(arg)) {
         console.error(`Unknown flag: ${arg}`);
@@ -1098,13 +1895,90 @@ function parseEvalArgs(args) {
     conditions,
     runs,
     captureTraces,
+    captureRequests,
     concurrency,
     scorerNames,
+    budget,
+    maxItemErrorRate,
     paths
   };
 }
 
 // src/cli-format.ts
+function formatKnownSpend(accounting) {
+  const cost = `$${accounting.knownCost.toFixed(2)}`;
+  if (accounting.completeness === "complete") return cost;
+  if (accounting.completeness === "unverified") return `${cost} (unverified)`;
+  const reasons = Object.entries(accounting.reasons).map(([reason, count]) => `${count} ${reason}`).join(", ");
+  return `${cost} (incomplete: ${reasons || "unknown spend"})`;
+}
+function formatBudgetLine(accounting) {
+  const budget = accounting.budget;
+  if (!budget) return void 0;
+  const limit = `$${budget.limit.toFixed(2)}`;
+  const spent = `$${budget.knownSpend.toFixed(2)}`;
+  if (budget.status === "open") {
+    return `  Budget: ${spent} of ${limit} (open)`;
+  }
+  const by = budget.closedBy ? `, first observed by ${budget.closedBy}` : "";
+  return `  Budget: STOPPED \u2014 ${spent} known spend against a ${limit} limit, $${budget.knownOvershoot.toFixed(2)} over${by}`;
+}
+function formatCoverageLine(result) {
+  const items = result.summary.coverage?.items;
+  if (!items) return void 0;
+  const parts = [
+    ["failed", items.failed],
+    ["cancelled", items.cancelled],
+    ["budget-skipped", items.budget_skipped],
+    ["budget-interrupted", items.budget_interrupted]
+  ].filter(([, n]) => n > 0).map(([label, n]) => `${n} ${label}`);
+  if (parts.length === 0) return void 0;
+  const rate = result.summary.itemErrorRate;
+  const rateSuffix = rate ? ` \u2014 item error rate ${formatPercent(rate.rate)} (limit ${formatPercent(rate.limit)})` : "";
+  const line = `  Items: ${items.completed} completed, ${parts.join(", ")}${rateSuffix}`;
+  const causes = formatFailureCauses(result);
+  return causes ? `${line}
+${causes}` : line;
+}
+function formatFailureCauses(result) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const item of result.items) {
+    if (item.outcome !== "failed") continue;
+    const f = item.failure;
+    let label = "other";
+    if (f?.status !== void 0) {
+      const status = f.status === 0 ? "network" : String(f.status);
+      label = f.provider ? `${status} (${f.provider})` : status;
+    }
+    counts.set(label, (counts.get(label) ?? 0) + 1);
+  }
+  if (counts.size === 0) return void 0;
+  const groups = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([label, n]) => `${n} \xD7 ${label}`);
+  return `  Failure causes: ${groups.join(", ")}`;
+}
+function itemErrorRateMessage(result, label) {
+  const rate = result.summary.itemErrorRate;
+  if (!rate?.exceeded) return null;
+  return `[axl-eval] ITEM ERROR RATE EXCEEDED: ${label} \u2014 ${rate.failed} of ${rate.attempted} attempted item(s) failed in the workflow (item error rate ${formatPercent(rate.rate)}), over the ${formatPercent(rate.limit)} limit; the scores cover only the surviving items. Raise failOnItemErrorRate or pass --max-item-error-rate <0..1> to accept it (1 disables the gate).`;
+}
+function budgetStopMessage(result, label) {
+  const budget = readAccounting(result).budget;
+  if (!budget || !isBudgetStopped({ budget, coverage: result.summary.coverage })) return null;
+  const items = result.summary.coverage?.items;
+  const stopped = items ? ` ${items.budget_skipped} case(s) never started, ${items.budget_interrupted} stopped mid-flight, ${items.completed} completed.` : "";
+  return `[axl-eval] BUDGET STOPPED: ${label} \u2014 known spend $${budget.knownSpend.toFixed(2)} reached the $${budget.limit.toFixed(2)} limit (over by $${budget.knownOvershoot.toFixed(2)}).${stopped} The run is incomplete by design; this is NOT a model or scorer failure.`;
+}
+function isTotalWipeout(result) {
+  const { count, failures } = result.summary;
+  if (count === 0) return false;
+  const coverage = result.summary.coverage?.items;
+  if (coverage) {
+    const budgetStopped = coverage.budget_skipped + coverage.budget_interrupted;
+    if (coverage.completed > 0 || budgetStopped > 0) return false;
+    return coverage.failed === count;
+  }
+  return failures >= count;
+}
 function formatModelTimingLines(modelTiming, nameWidth) {
   if (!modelTiming) return [];
   return Object.entries(modelTiming).map(([model, t]) => {
@@ -1114,6 +1988,7 @@ function formatModelTimingLines(modelTiming, nameWidth) {
     }
     parts.push(`queued ${formatMs(t.queuedMs.mean)}`);
     parts.push(`retries ${formatMs(t.retryMs.mean)}`);
+    if (t.rateLimitRetries) parts.push(`rate-limited ${t.rateLimitRetries}\xD7`);
     const calls = `${t.calls} call${t.calls === 1 ? "" : "s"}`;
     return `    ${model.padEnd(Math.max(nameWidth - 2, 0))}  ${parts.join(" \xB7 ")}  (${calls}, mean/p95 per call)`;
   });
@@ -1175,7 +2050,16 @@ Usage:
                                           (success + failure). Adds memory
                                           overhead proportional to dataset size
                                           x turns x agents; off by default.
+  axl-eval <path> --budget <amount>       Stop admitting spend once known cost
+                                          reaches the limit (e.g. "$1", "0.50").
+                                          Overrides the eval file's budget and
+                                          applies PER RUN under --runs. A
+                                          budget-stopped run exits 1 with a
+                                          distinct BUDGET STOPPED line.
   axl-eval rescore <results> <eval-file>  Re-run scorers on saved outputs
+  axl-eval rescore <results> <eval-file> --budget <amount>  Limit the NEW judging
+                                          spend only; the source run's cost is
+                                          not counted against it.
   axl-eval compare <a> <b>                Compare two eval result files
   axl-eval compare <a> <b> --threshold <v>  Set regression threshold (global or per-scorer)
   axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
@@ -1187,10 +2071,11 @@ Usage:
                                           sample, even without this flag.
 
 Config auto-detection (when --config is not specified):
-  ${import_axl.CONFIG_CANDIDATES.join(" -> ")}
+  ${import_axl4.CONFIG_CANDIDATES.join(" -> ")}
 
 When a config is found, the exported AxlRuntime is passed to executeWorkflow
-and cost is tracked automatically via runtime.trackCost().
+and spend is measured by the runner's accounting scope (a "cost" your callback
+returns is recorded under item.callerReport, never used as the measurement).
 When no config is found, a bare AxlRuntime is created (providers from env vars).
 `);
     process.exit(0);
@@ -1240,17 +2125,27 @@ function scorerFilteredScorers(r) {
   if (!meta || meta.scorerFiltered !== true) return void 0;
   return Array.isArray(meta.scorersRun) ? meta.scorersRun : [];
 }
+function parseMaxItemErrorRate(args) {
+  const idx = args.indexOf("--max-item-error-rate");
+  if (idx === -1) return void 0;
+  if (idx + 1 >= args.length) {
+    console.error("Error: --max-item-error-rate requires a value in [0, 1]");
+    process.exit(1);
+  }
+  return parseErrorRateFlag("--max-item-error-rate", args[idx + 1]);
+}
 async function runCompare(args) {
   const failOnRegression = args.includes("--fail-on-regression");
   const thresholds = parseThresholdArg(args);
   const maxScorerErrorRate = parseMaxScorerErrorRate(args);
-  const valueFlags = /* @__PURE__ */ new Set(["--threshold", "--max-scorer-error-rate"]);
+  const maxItemErrorRate = parseMaxItemErrorRate(args);
+  const valueFlags = /* @__PURE__ */ new Set(["--threshold", "--max-scorer-error-rate", "--max-item-error-rate"]);
   const files = args.filter(
     (a, i) => !a.startsWith("--") && !(i > 0 && valueFlags.has(args[i - 1]))
   );
   if (files.length !== 2) {
     console.error(
-      "Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>] [--fail-on-regression] [--max-scorer-error-rate <0..1>]"
+      "Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>] [--fail-on-regression] [--max-scorer-error-rate <0..1>] [--max-item-error-rate <0..1>]"
     );
     process.exit(1);
   }
@@ -1315,9 +2210,13 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   if (comparison.cost) {
     const c = comparison.cost;
     const sign = c.delta > 0 ? "+" : "";
+    const change = c.deltaPercent === null ? `${sign}$${Math.abs(c.delta).toFixed(2)}, no % from a $0 baseline` : `${sign}${c.deltaPercent.toFixed(1)}%`;
     console.log(
-      `  Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`
+      `  Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${change})`
     );
+    if (!c.certified) {
+      console.log(`        not certified: ${c.reason ?? "costs are not comparable"}`);
+    }
   }
   const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;
   const stable = Math.max(
@@ -1362,6 +2261,17 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
     const reason = evaluateScorerErrorRateGate(comparison, maxScorerErrorRate);
     if (reason) refuseToGate(reason);
   }
+  const itemRates = compareItemErrorRates(baseline, candidate, maxItemErrorRate);
+  const multiRunCompare = itemRates.some((r) => r.runIndex > 0);
+  for (const rate of itemRates) {
+    if (rate.failed > 0 && !rate.exceeded) {
+      console.error(
+        `[axl-eval] WARNING: ${describeItemErrorRate(rate, multiRunCompare)}, within the ${formatPercent(rate.limit)} limit; its scores cover only the surviving items.`
+      );
+    }
+  }
+  const coverageReason = evaluateItemErrorRateGate(baseline, candidate, maxItemErrorRate);
+  if (coverageReason) refuseToGate(coverageReason);
   if (failOnRegression && comparison.regressions.length > 0) {
     const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);
     if (hasSignificance) {
@@ -1375,7 +2285,16 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
   }
 }
 async function runRescore(args, signal) {
-  const { outputPath, configArg, conditions, concurrency, scorerNames, paths } = parseEvalArgs(args);
+  const {
+    outputPath,
+    configArg,
+    conditions,
+    concurrency,
+    scorerNames,
+    budget,
+    maxItemErrorRate,
+    paths
+  } = parseEvalArgs(args);
   if (paths.length < 2) {
     console.error("Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]");
     process.exit(1);
@@ -1386,14 +2305,21 @@ async function runRescore(args, signal) {
     );
     process.exit(1);
   }
+  if (maxItemErrorRate != null) {
+    console.error(
+      "Error: --max-item-error-rate is not supported with rescore; item failures belong to the source run \u2014 gate them with `axl-eval compare`."
+    );
+    process.exit(1);
+  }
   const itemConcurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY");
   const [resultsPath, evalFilePath] = paths;
   const raw = JSON.parse(await (0, import_promises.readFile)(resultsPath, "utf-8"));
   const results = Array.isArray(raw) ? raw : [raw];
   const runtime = await getRuntime(configArg, conditions);
+  let exitCode = 0;
   try {
-    const mod = await (0, import_axl.importModule)(path.resolve(evalFilePath), import_meta.url);
-    const evalConfig = (0, import_axl.pickDefault)(mod);
+    const mod = await (0, import_axl4.importModule)(path.resolve(evalFilePath), import_meta.url);
+    const evalConfig = (0, import_axl4.pickDefault)(mod);
     if (!evalConfig || typeof evalConfig !== "object") {
       console.error(
         `Error: ${evalFilePath} does not export an eval config object \u2014 got ${evalConfig === null ? "null" : typeof evalConfig}.`
@@ -1405,32 +2331,52 @@ async function runRescore(args, signal) {
       process.exit(1);
     }
     const rescored = [];
-    for (const resultData of results) {
-      if (signal.aborted) break;
-      rescored.push(
-        await rescore(resultData, evalConfig.scorers, runtime, {
-          signal,
-          ...itemConcurrency != null ? { concurrency: itemConcurrency } : {}
-        })
-      );
-    }
-    for (const r of rescored) {
-      console.log("\n" + formatTable(r) + "\n");
+    let budgetUsable = true;
+    try {
+      for (const resultData of results) {
+        if (signal.aborted) break;
+        rescored.push(
+          await rescore(resultData, evalConfig.scorers, runtime, {
+            signal,
+            ...itemConcurrency != null ? { concurrency: itemConcurrency } : {},
+            // Gates the NEW judging spend only — the source run's cost is history
+            // and is not counted against this limit.
+            ...budget != null ? { budget } : {}
+          })
+        );
+      }
+    } catch (err) {
+      if (err?.code === "INVALID_BUDGET") {
+        console.error(`[axl-eval] ${err instanceof Error ? err.message : String(err)}`);
+        exitCode = 1;
+        budgetUsable = false;
+      } else {
+        throw err;
+      }
     }
-    if (outputPath) {
-      const output = Array.isArray(raw) ? rescored : rescored[0];
-      const outputDir = path.dirname(path.resolve(outputPath));
-      await (0, import_promises.mkdir)(outputDir, { recursive: true });
-      await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
-      console.log(`Rescored results saved to ${outputPath}`);
+    if (budgetUsable) {
+      for (const r of rescored) {
+        console.log("\n" + formatTable(r) + "\n");
+      }
+      if (outputPath) {
+        const output = Array.isArray(raw) ? rescored : rescored[0];
+        const outputDir = path.dirname(path.resolve(outputPath));
+        await (0, import_promises.mkdir)(outputDir, { recursive: true });
+        await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
+        console.log(`Rescored results saved to ${outputPath}`);
+      }
+      for (const r of rescored) {
+        if (reportBudgetStop(r, `rescore ${resultsPath}`)) exitCode = 1;
+      }
     }
   } finally {
     await runtime.shutdown().catch(() => {
     });
   }
+  if (exitCode !== 0) process.exit(exitCode);
 }
 function collectEvalFiles(p) {
-  if (p.includes("*")) return (0, import_axl.expandGlob)(p, process.cwd());
+  if (p.includes("*")) return (0, import_axl4.expandGlob)(p, process.cwd());
   const resolved = path.resolve(p);
   try {
     const stat = (0, import_node_fs.statSync)(resolved);
@@ -1485,11 +2431,15 @@ function formatTable(result) {
   }
   lines.push(...formatModelTimingLines(result.summary.modelTiming, maxNameLen));
   const durationSec = (result.duration / 1e3).toFixed(1);
-  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : "$0.00";
+  const accounting = readAccounting(result);
   lines.push("");
   lines.push(
-    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`
+    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${formatKnownSpend(accounting)} | Duration: ${durationSec}s`
   );
+  const budgetLine = formatBudgetLine(accounting);
+  if (budgetLine) lines.push(budgetLine);
+  const coverageLine = formatCoverageLine(result);
+  if (coverageLine) lines.push(coverageLine);
   const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);
   if (itemsWithErrors.length > 0) {
     const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors))];
@@ -1533,10 +2483,11 @@ function formatMultiRunTable(summary) {
       `  ${"Timing".padEnd(maxNameLen)}  ${((summary.timing.mean / 1e3).toFixed(2) + " \xB1 " + (summary.timing.std / 1e3).toFixed(2) + "s").padStart(colWidth)}`
     );
   }
-  const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : "$0.00";
   const durationStr = (summary.totalDuration / 1e3).toFixed(1);
   lines.push("");
-  lines.push(`  Total Cost: ${costStr} | Total Duration: ${durationStr}s`);
+  lines.push(
+    `  Total Cost: ${formatKnownSpend(summary.accounting)} | Total Duration: ${durationStr}s`
+  );
   return lines.join("\n");
 }
 function reportDegraded(result, label) {
@@ -1561,18 +2512,29 @@ function reportFullySkippedScorers(result, label) {
     }
   }
 }
+function reportBudgetStop(result, label) {
+  const message = budgetStopMessage(result, label);
+  if (!message) return false;
+  console.error(message);
+  return true;
+}
+function reportItemErrorRate(result, label, wipeout) {
+  const message = itemErrorRateMessage(result, label);
+  if (!message) return false;
+  if (!wipeout) console.error(message);
+  return true;
+}
 function reportTotalWipeout(result, label) {
-  const { count, failures } = result.summary;
-  if (count === 0 || failures < count) return false;
+  if (!isTotalWipeout(result)) return false;
   console.error(
-    `[axl-eval] FAILED: ${label} \u2014 all ${count} item(s) errored in the workflow (0 succeeded); the eval produced no scorable output.`
+    `[axl-eval] FAILED: ${label} \u2014 all ${result.summary.count} item(s) errored in the workflow (0 succeeded); the eval produced no scorable output.`
   );
   return true;
 }
 async function resolveRuntimeFromConfig(configPath) {
   try {
-    const mod = await (0, import_axl.importModule)(configPath, import_meta.url);
-    const runtime = (0, import_axl.resolveRuntime)(mod);
+    const mod = await (0, import_axl4.importModule)(configPath, import_meta.url);
+    const runtime = (0, import_axl4.resolveRuntime)(mod);
     if (!runtime || typeof runtime.execute !== "function") {
       console.error(`Config must export a default AxlRuntime instance.`);
       if (runtime) {
@@ -1609,7 +2571,7 @@ async function resolveRuntimeFromConfig(configPath) {
 }
 async function getRuntime(configArg, conditions) {
   if (conditions && conditions.length > 0) {
-    await (0, import_axl.registerConditions)(conditions);
+    await (0, import_axl4.registerConditions)(conditions);
   }
   if (configArg) {
     const configPath = path.resolve(process.cwd(), configArg);
@@ -1621,7 +2583,7 @@ async function getRuntime(configArg, conditions) {
     console.error(`[axl-eval] Loading config from ${configPath}`);
     return resolveRuntimeFromConfig(configPath);
   }
-  const found = (0, import_axl.findConfig)(process.cwd());
+  const found = (0, import_axl4.findConfig)(process.cwd());
   if (found) {
     console.error(`[axl-eval] Auto-detected config: ${found}`);
     return resolveRuntimeFromConfig(found);
@@ -1629,6 +2591,27 @@ async function getRuntime(configArg, conditions) {
   const { AxlRuntime } = await import("@axlsdk/axl");
   return new AxlRuntime();
 }
+async function writeRequestSidecar(runtime, outputPath, results) {
+  const artifactIds = results.map((result) => result.diagnostics?.artifactId).filter((id) => typeof id === "string" && id !== "");
+  if (artifactIds.length === 0) return;
+  const store = runtime.getDiagnosticArtifactStore();
+  if (!store) return;
+  const records = [];
+  for (const artifactId of artifactIds) {
+    const opened = await store.open(artifactId).catch(() => void 0);
+    if (!opened) continue;
+    for await (const line of opened.lines) {
+      try {
+        records.push(JSON.parse(line));
+      } catch {
+      }
+    }
+  }
+  if (records.length === 0) return;
+  const sidecarPath = outputPath.replace(/\.json$/i, "") + ".requests.jsonl";
+  await (0, import_promises.writeFile)(sidecarPath, serializeRequestRecords(records), "utf-8");
+  console.log(`Captured requests saved to ${sidecarPath}`);
+}
 async function runEvalCommand(args, signal) {
   const {
     outputPath,
@@ -1636,8 +2619,11 @@ async function runEvalCommand(args, signal) {
     conditions,
     runs,
     captureTraces,
+    captureRequests,
     concurrency,
     scorerNames,
+    budget,
+    maxItemErrorRate,
     paths
   } = parseEvalArgs(args);
   if (paths.length === 0) {
@@ -1663,8 +2649,8 @@ async function runEvalCommand(args, signal) {
     for (const filePath of evalFiles) {
       if (signal.aborted) break;
       try {
-        const mod = await (0, import_axl.importModule)(path.resolve(filePath), import_meta.url);
-        const evalConfig = (0, import_axl.pickDefault)(mod);
+        const mod = await (0, import_axl4.importModule)(path.resolve(filePath), import_meta.url);
+        const evalConfig = (0, import_axl4.pickDefault)(mod);
         const validationError = validateEvalConfig(evalConfig);
         if (validationError) {
           console.error(`Error: ${filePath} ${validationError}`);
@@ -1672,6 +2658,8 @@ async function runEvalCommand(args, signal) {
           continue;
         }
         evalConfig.concurrency = concurrency ?? envInt("AXL_EVAL_CONCURRENCY") ?? evalConfig.concurrency ?? 5;
+        if (budget != null) evalConfig.budget = budget;
+        if (maxItemErrorRate != null) evalConfig.failOnItemErrorRate = maxItemErrorRate;
         if (scorerNames?.length) {
           const available = evalConfig.scorers.map((s) => s.name);
           const unknown = scorerNames.filter((n) => !available.includes(n));
@@ -1687,7 +2675,7 @@ async function runEvalCommand(args, signal) {
             process.exit(1);
           }
         }
-        const exported = (0, import_axl.pickExport)(mod, "executeWorkflow");
+        const exported = (0, import_axl4.pickExport)(mod, "executeWorkflow");
         let customExecute;
         if (typeof exported === "function") {
           customExecute = exported;
@@ -1700,27 +2688,11 @@ async function runEvalCommand(args, signal) {
         }
         let executeWorkflow;
         if (customExecute) {
-          executeWorkflow = async (input, rt) => {
-            const {
-              result,
-              cost: trackedCost,
-              metadata
-            } = await runtime.trackExecution(async () => {
-              return customExecute(input, rt);
-            });
-            return {
-              output: result.output,
-              cost: result.cost ?? trackedCost,
-              metadata: result.metadata ?? metadata
-            };
-          };
+          executeWorkflow = async (input, rt) => customExecute(input, rt);
         } else if (runtime.getWorkflow(evalConfig.workflow)) {
-          executeWorkflow = async (input) => {
-            const { result, cost, metadata } = await runtime.trackExecution(async () => {
-              return runtime.execute(evalConfig.workflow, input);
-            });
-            return { output: result, cost, metadata };
-          };
+          executeWorkflow = async (input) => ({
+            output: await runtime.execute(evalConfig.workflow, input)
+          });
         } else {
           const exportShape = (() => {
             const top = Object.keys(mod).filter((k) => k !== "default");
@@ -1742,7 +2714,11 @@ async function runEvalCommand(args, signal) {
           failedFiles++;
           continue;
         }
-        const runOptions = { signal, ...captureTraces ? { captureTraces: true } : {} };
+        const runOptions = {
+          signal,
+          ...captureTraces ? { captureTraces: true } : {},
+          ...captureRequests ? { captureRequests: true } : {}
+        };
         const scorersRun = scorerNames?.length ? evalConfig.scorers.map((s) => s.name) : void 0;
         const stampFiltered = (result) => {
           if (scorersRun) {
@@ -1795,11 +2771,13 @@ async function runEvalCommand(args, signal) {
           console.log("\n" + formatMultiRunTable(summary) + "\n");
           for (const r of runResults) results.push(r);
           let anyFailing = false;
-          for (const r of runResults) {
+          for (const [i, r] of runResults.entries()) {
+            const budgetStopped = reportBudgetStop(r, filePath);
             const wipeout = reportTotalWipeout(r, filePath);
+            const thinned = reportItemErrorRate(r, `${filePath} run ${i + 1}/${runs}`, wipeout);
             const degraded = reportDegraded(r, filePath);
             reportFullySkippedScorers(r, filePath);
-            if (wipeout || degraded) anyFailing = true;
+            if (budgetStopped || wipeout || thinned || degraded) anyFailing = true;
           }
           if (anyFailing && !partial) failedFiles++;
         } else {
@@ -1807,10 +2785,12 @@ async function runEvalCommand(args, signal) {
           stampFiltered(result);
           results.push(result);
           console.log("\n" + formatTable(result) + "\n");
+          const budgetStopped = reportBudgetStop(result, filePath);
           const wipeout = reportTotalWipeout(result, filePath);
+          const thinned = reportItemErrorRate(result, filePath, wipeout);
           const degraded = reportDegraded(result, filePath);
           reportFullySkippedScorers(result, filePath);
-          if (wipeout || degraded) failedFiles++;
+          if (budgetStopped || wipeout || thinned || degraded) failedFiles++;
         }
       } catch (err) {
         console.error(
@@ -1825,6 +2805,7 @@ async function runEvalCommand(args, signal) {
       await (0, import_promises.mkdir)(outputDir, { recursive: true });
       await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
       console.log(`Results saved to ${outputPath}`);
+      await writeRequestSidecar(runtime, outputPath, results);
     }
   } finally {
     await runtime.shutdown().catch(() => {