@axlsdk/eval 0.23.3 → 0.24.0

package/dist/index.cjs

@@ -30,18 +30,31 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  DEFAULT_COPY_MAX_BYTES: () => DEFAULT_COPY_MAX_BYTES,
+  DEFAULT_SIDECAR_MAX_BYTES: () => DEFAULT_SIDECAR_MAX_BYTES,
+  aggregateAccounting: () => aggregateAccounting,
   aggregateRuns: () => aggregateRuns,
   dataset: () => dataset,
   defineEval: () => defineEval,
   evalCompare: () => evalCompare,
+  evaluateItemErrorRateGate: () => evaluateItemErrorRateGate,
   evaluateScorerErrorRateGate: () => evaluateScorerErrorRateGate,
   evaluateScorerTolerance: () => evaluateScorerTolerance,
+  isBudgetStopped: () => isBudgetStopped,
   llmScorer: () => llmScorer,
   normalizeScorerResult: () => normalizeScorerResult,
   pairedBootstrapCI: () => pairedBootstrapCI,
+  parseRequestRecords: () => parseRequestRecords,
+  readAccounting: () => readAccounting,
+  refusedWork: () => refusedWork,
   rescore: () => rescore,
+  resolveCaptureLimits: () => resolveCaptureLimits,
   runEval: () => runEval,
-  scorer: () => scorer
+  scorer: () => scorer,
+  serializeRequestRecords: () => serializeRequestRecords,
+  toDiagnosticManifest: () => toDiagnosticManifest,
+  unavailableManifest: () => unavailableManifest,
+  validateRequestSidecar: () => validateRequestSidecar
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -249,6 +262,9 @@ function defineEval(config) {
   return config;
 }
 
+// src/runner.ts
+var import_axl3 = require("@axlsdk/axl");
+
 // src/utils.ts
 function computeStats(scores) {
   if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
@@ -263,15 +279,35 @@ function computeStats(scores) {
 function round(n) {
   return Math.round(n * 1e3) / 1e3;
 }
+function formatPercent(rate) {
+  return `${Number((rate * 100).toFixed(1))}%`;
+}
+var DEFAULT_ITEM_ERROR_RATE_LIMIT = 0.05;
+function isErrorRateLimit(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+function evaluateItemErrorRate(items, count, limit) {
+  const failed = items.failed;
+  const attempted = count - items.cancelled - items.budget_skipped - items.budget_interrupted;
+  const rate = attempted > 0 ? failed / attempted : 0;
+  return { failed, attempted, rate, limit, exceeded: attempted > 0 && rate > limit };
+}
 function scorerCounts(items, name) {
   let scored = 0;
   let failed = 0;
   let skipped = 0;
   for (const i of items) {
     if (i.error) continue;
+    const detail = i.scoreDetails?.[name];
+    if (detail?.outcome !== void 0) {
+      if (detail.outcome === "scored") scored++;
+      else if (detail.outcome === "failed") failed++;
+      else if (detail.outcome === "skipped") skipped++;
+      continue;
+    }
     if (i.scores[name] != null) scored++;
-    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;
-    else if (i.scoreDetails?.[name]?.duration != null) failed++;
+    else if (detail?.skipped === true) skipped++;
+    else if (detail?.duration != null) failed++;
   }
   return { scored, failed, skipped };
 }
@@ -296,81 +332,380 @@ async function mapWithConcurrency(items, concurrency, task) {
   return results;
 }
 
+// src/accounting.ts
+var import_axl2 = require("@axlsdk/axl");
+var USAGE_KEYS = [
+  "inputTokens",
+  "outputTokens",
+  "reasoningTokens",
+  "cachedTokens",
+  "cacheWriteTokens",
+  "audioSeconds"
+];
+function zeroUsage() {
+  return {
+    inputTokens: 0,
+    outputTokens: 0,
+    reasoningTokens: 0,
+    cachedTokens: 0,
+    cacheWriteTokens: 0,
+    audioSeconds: 0
+  };
+}
+function emptyAccounting() {
+  return {
+    version: 1,
+    currency: "USD",
+    knownCost: 0,
+    completeness: "complete",
+    reasons: {},
+    usage: zeroUsage(),
+    operations: { total: 0, settled: 0, unknown: 0, denied: 0, byKind: {} },
+    breakdown: { generation: 0, judging: 0, external: 0 },
+    provenance: {}
+  };
+}
+function uninstrumentedAccounting() {
+  return { ...emptyAccounting(), completeness: "incomplete", reasons: { uninstrumented: 1 } };
+}
+function isInstrumented(runtime) {
+  return typeof runtime?.trackOutcome === "function";
+}
+async function trackScope(runtime, fn, options) {
+  if (!isInstrumented(runtime)) {
+    try {
+      return {
+        status: "fulfilled",
+        value: await fn(),
+        accounting: uninstrumentedAccounting(),
+        instrumented: false
+      };
+    } catch (error) {
+      return {
+        status: "rejected",
+        error,
+        accounting: uninstrumentedAccounting(),
+        instrumented: false
+      };
+    }
+  }
+  const outcome = await runtime.trackOutcome(fn, options);
+  return { ...outcome, instrumented: true };
+}
+function parseBudget(budget) {
+  const match = typeof budget === "string" ? budget.match(/^\$?(\d+(?:\.\d+)?)$/) : null;
+  if (!match) {
+    throw new import_axl2.AxlError(
+      "INVALID_BUDGET",
+      `Invalid budget "${String(budget)}": expected a non-negative USD amount like "$1" or "0.50".`
+    );
+  }
+  const limit = Number(match[1]);
+  if (!Number.isFinite(limit)) {
+    throw new import_axl2.AxlError("INVALID_BUDGET", `Invalid budget "${budget}": not a finite amount.`);
+  }
+  return limit;
+}
+function usableCost(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
+}
+function isAdmissionDenied(err) {
+  return (0, import_axl2.isAdmissionDeniedError)(err);
+}
+function readAccounting(result) {
+  if (result?.accounting) return result.accounting;
+  const rescored = result?.metadata?.rescored === true;
+  return {
+    ...emptyAccounting(),
+    knownCost: usableCost(result?.totalCost),
+    completeness: "unverified",
+    scope: rescored ? "rescore" : "run"
+  };
+}
+function refusedWork(coverage) {
+  if (!coverage) return 0;
+  return refusedIn(coverage.items) + Object.values(coverage.scorers ?? {}).reduce((n, s) => n + refusedIn(s), 0);
+}
+function refusedIn(counts) {
+  return atLeastZero(counts?.budget_skipped) + atLeastZero(counts?.budget_interrupted);
+}
+function atLeastZero(value) {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+function isBudgetStopped(summary) {
+  if (summary.budget?.status !== "closed") return false;
+  return refusedWork(summary.coverage) > 0;
+}
+function addUsage(target, source) {
+  if (!source) return;
+  for (const key of USAGE_KEYS) {
+    const value = source[key];
+    if (typeof value === "number" && Number.isFinite(value)) target[key] += value;
+  }
+}
+function addCounts(target, source) {
+  if (!source) return;
+  for (const [key, count] of Object.entries(source)) {
+    if (typeof count === "number" && Number.isFinite(count)) {
+      target[key] = (target[key] ?? 0) + count;
+    }
+  }
+}
+function aggregateAccounting(inputs) {
+  const out = emptyAccounting();
+  let anyUnverified = false;
+  let anyIncomplete = false;
+  for (const input of inputs) {
+    if (!input) continue;
+    if (input.completeness === "unverified") anyUnverified = true;
+    else if (input.completeness === "incomplete") anyIncomplete = true;
+    out.knownCost += usableCost(input.knownCost);
+    addUsage(out.usage, input.usage);
+    addCounts(out.reasons, input.reasons);
+    addCounts(out.operations.byKind, input.operations?.byKind);
+    addCounts(out.provenance, input.provenance);
+    out.operations.total += input.operations?.total ?? 0;
+    out.operations.settled += input.operations?.settled ?? 0;
+    out.operations.unknown += input.operations?.unknown ?? 0;
+    out.operations.denied += input.operations?.denied ?? 0;
+    out.breakdown.generation += usableCost(input.breakdown?.generation);
+    out.breakdown.judging += usableCost(input.breakdown?.judging);
+    out.breakdown.external += usableCost(input.breakdown?.external);
+  }
+  out.completeness = anyUnverified ? "unverified" : anyIncomplete ? "incomplete" : "complete";
+  return out;
+}
+
 // src/score-item.ts
-async function scoreItem(item, scorers, scorerConcurrency, scorerContext, signal) {
+function describeInput(input) {
+  let rendered;
+  try {
+    rendered = JSON.stringify(input) ?? String(input);
+  } catch {
+    rendered = "[unserializable input]";
+  }
+  return rendered.length > 200 ? `${rendered.slice(0, 200)}\u2026` : rendered;
+}
+async function scoreItem(item, scorers, context) {
   const scoreDetails = item.scoreDetails ??= {};
   for (const s of scorers) {
     item.scores[s.name] = null;
-    scoreDetails[s.name] = { score: null };
+    scoreDetails[s.name] = { score: null, outcome: "cancelled" };
   }
-  let itemScorerCost = 0;
+  const { runtime, scorerContext, signal, admission } = context;
   const scorerErrorsByName = {};
-  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer2) => {
-    if (signal?.aborted) return;
+  await mapWithConcurrency(scorers, context.scorerConcurrency, async (scorer2) => {
+    const record = (detail, outcome2) => {
+      scoreDetails[scorer2.name] = { ...detail, outcome: outcome2 };
+    };
+    if (signal?.aborted) {
+      record({ score: null }, "cancelled");
+      return;
+    }
+    if (admission?.closed && scorer2.isLlm) {
+      record({ score: null }, "budget_skipped");
+      return;
+    }
     const scorerStart = Date.now();
+    let applicable = true;
     try {
-      if (scorer2.applies && !scorer2.applies(item.output, item.input, item.annotations)) {
-        item.scores[scorer2.name] = null;
-        scoreDetails[scorer2.name] = { score: null, skipped: true };
+      applicable = !scorer2.applies || scorer2.applies(item.output, item.input, item.annotations);
+    } catch (err) {
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record({ score: null, duration: Date.now() - scorerStart }, "failed");
+      return;
+    }
+    if (!applicable) {
+      record({ score: null, skipped: true }, "skipped");
+      return;
+    }
+    const outcome = await trackScope(
+      runtime,
+      async () => scorer2.score(item.output, item.input, item.annotations, scorerContext),
+      // The scorer stamp is merged OVER the enclosing item's correlation, so a
+      // judge's records keep the case index and add the judge's name -- which is
+      // what lets `scoreDetails[name].diagnostics` point at this judge's own
+      // calls rather than at the whole item's.
+      { purpose: "judging", captureCorrelation: { scorer: scorer2.name } }
+    );
+    const duration = Date.now() - scorerStart;
+    const { accounting } = outcome;
+    const measured = accounting.operations.total > 0 || accounting.operations.denied > 0;
+    const withCost = (detail, callerCost) => ({
+      ...detail,
+      ...measured ? { accounting, cost: accounting.knownCost } : {},
+      ...!measured && !outcome.instrumented && callerCost != null ? { cost: callerCost } : {}
+    });
+    if (outcome.status === "rejected") {
+      const err = outcome.error;
+      if (err?.name === "AbortError") {
+        record(withCost({ score: null, duration }, void 0), "cancelled");
         return;
       }
-      const raw = await scorer2.score(item.output, item.input, item.annotations, scorerContext);
-      const scorerResult = normalizeScorerResult(raw);
-      if (scorerResult.cost != null) {
-        itemScorerCost += scorerResult.cost;
+      if (isAdmissionDenied(err)) {
+        context.onClosure?.("operation");
+        record(withCost({ score: null, duration }, void 0), "budget_interrupted");
+        return;
       }
-      const scorerDuration = Date.now() - scorerStart;
+      scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
+      record(withCost({ score: null, duration }, extractScorerErrorCost(err)), "failed");
+      if (admission?.closed) context.onClosure?.("scorer");
+      return;
+    }
+    try {
+      const scorerResult = normalizeScorerResult(outcome.value);
+      const callerCost = typeof scorerResult.cost === "number" && Number.isFinite(scorerResult.cost) && scorerResult.cost >= 0 ? scorerResult.cost : void 0;
       if (!Number.isFinite(scorerResult.score) || scorerResult.score < 0 || scorerResult.score > 1) {
-        scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;
-        item.scores[scorer2.name] = null;
-        scoreDetails[scorer2.name] = {
-          score: null,
-          metadata: scorerResult.metadata,
-          duration: scorerDuration,
-          cost: scorerResult.cost
-        };
+        scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" returned out-of-range score ${scorerResult.score} for input ${describeInput(item.input)}`;
+        record(
+          withCost({ score: null, metadata: scorerResult.metadata, duration }, callerCost),
+          "failed"
+        );
       } else {
         item.scores[scorer2.name] = round(scorerResult.score);
-        scoreDetails[scorer2.name] = {
-          score: round(scorerResult.score),
-          metadata: scorerResult.metadata,
-          duration: scorerDuration,
-          cost: scorerResult.cost
-        };
+        record(
+          withCost(
+            {
+              score: round(scorerResult.score),
+              metadata: scorerResult.metadata,
+              duration
+            },
+            callerCost
+          ),
+          "scored"
+        );
       }
     } catch (err) {
-      if (err?.name === "AbortError") return;
-      const errCost = extractScorerErrorCost(err);
-      if (errCost != null) itemScorerCost += errCost;
       scorerErrorsByName[scorer2.name] = `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`;
-      item.scores[scorer2.name] = null;
-      scoreDetails[scorer2.name] = {
-        score: null,
-        duration: Date.now() - scorerStart,
-        cost: errCost
-      };
+      record(withCost({ score: null, duration }, void 0), "failed");
     }
+    if (admission?.closed) context.onClosure?.("scorer");
   });
   const orderedErrors = scorers.map((s) => scorerErrorsByName[s.name]).filter((e) => e != null);
   if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;
-  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
-  return itemScorerCost;
+}
+
+// src/diagnostics.ts
+var DEFAULT_COPY_MAX_BYTES = 16 * 1024 * 1024;
+function resolveCaptureLimits(option) {
+  if (option === void 0 || option === false) return void 0;
+  if (option === true) return {};
+  return {
+    ...option.maxRecordBytes !== void 0 ? { maxRecordBytes: option.maxRecordBytes } : {},
+    ...option.maxRunBytes !== void 0 ? { maxRunBytes: option.maxRunBytes } : {},
+    ...option.maxQueueBytes !== void 0 ? { maxQueueBytes: option.maxQueueBytes } : {}
+  };
+}
+function toDiagnosticManifest(manifest) {
+  return {
+    version: 1,
+    artifactId: manifest.artifactId,
+    fidelity: "runtime_request",
+    status: manifest.status,
+    ...manifest.reason !== void 0 ? { reason: manifest.reason } : {},
+    records: manifest.records,
+    bytes: manifest.bytes,
+    redaction: manifest.redaction,
+    ...manifest.expiresAt !== void 0 ? { expiresAt: manifest.expiresAt } : {}
+  };
+}
+function unavailableManifest(artifactId, reason) {
+  return {
+    version: 1,
+    artifactId,
+    fidelity: "runtime_request",
+    status: "unavailable",
+    reason,
+    records: 0,
+    bytes: 0,
+    redaction: "none"
+  };
+}
+var DEFAULT_SIDECAR_MAX_BYTES = 16 * 1024 * 1024;
+var DEFAULT_SIDECAR_MAX_RECORD_BYTES = 4 * 1024 * 1024;
+function validateRequestSidecar(text, options) {
+  if (typeof text !== "string") {
+    return { ok: false, reason: "requests sidecar must be a JSONL string" };
+  }
+  const maxBytes = options?.maxBytes ?? DEFAULT_SIDECAR_MAX_BYTES;
+  const maxRecordBytes = options?.maxRecordBytes ?? DEFAULT_SIDECAR_MAX_RECORD_BYTES;
+  const bytes = Buffer.byteLength(text, "utf-8");
+  if (bytes > maxBytes) {
+    return { ok: false, reason: `requests sidecar exceeds ${maxBytes} bytes` };
+  }
+  const lines = text.split("\n").filter((line) => line.trim() !== "");
+  for (let i = 0; i < lines.length; i++) {
+    const lineBytes = Buffer.byteLength(lines[i], "utf-8");
+    if (lineBytes > maxRecordBytes) {
+      return {
+        ok: false,
+        reason: `requests sidecar line ${i + 1} exceeds ${maxRecordBytes} bytes`
+      };
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(lines[i]);
+    } catch {
+      return { ok: false, reason: `requests sidecar line ${i + 1} is not valid JSON` };
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return { ok: false, reason: `requests sidecar line ${i + 1} is not an object` };
+    }
+    const record = parsed;
+    if (record.v !== 1) {
+      return {
+        ok: false,
+        reason: `requests sidecar line ${i + 1} declares unsupported codec version ${String(record.v)}`
+      };
+    }
+    if (typeof record.operationId !== "string" || record.operationId === "") {
+      return { ok: false, reason: `requests sidecar line ${i + 1} has no operationId` };
+    }
+    if (record.phase !== "start" && record.phase !== "attempt" && record.phase !== "end") {
+      return { ok: false, reason: `requests sidecar line ${i + 1} has an unknown phase` };
+    }
+  }
+  return { ok: true, lines, bytes };
+}
+function parseRequestRecords(lines) {
+  return lines.map((line) => JSON.parse(line));
+}
+function serializeRequestRecords(records) {
+  return records.map((record) => JSON.stringify(record)).join("\n") + (records.length ? "\n" : "");
 }
 
 // src/runner.ts
 var import_node_crypto = require("crypto");
-function parseCost(cost) {
-  const match = cost.match(/^\$?([\d.]+)$/);
-  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
-  return parseFloat(match[1]);
-}
+var RESERVED_METADATA_KEYS = [
+  "models",
+  "modelCallCounts",
+  "workflows",
+  "workflowCallCounts",
+  "tokens",
+  "agentCalls"
+];
+var ITEM_OUTCOMES = [
+  "completed",
+  "failed",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
+var SCORER_OUTCOMES = [
+  "scored",
+  "failed",
+  "skipped",
+  "cancelled",
+  "budget_skipped",
+  "budget_interrupted"
+];
 function extractUserCost(result, label = "executeWorkflow") {
   if (result === null || typeof result !== "object") return void 0;
   const raw = result.cost;
   if (raw === void 0) return void 0;
   if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
   console.warn(
-    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Measured spend is unaffected.`
   );
   return void 0;
 }
@@ -393,26 +728,146 @@ function extractUserMetadata(result, label = "executeWorkflow") {
   }
   return meta;
 }
-function mergeMetadata(tracked, user) {
-  const merged = { ...tracked, ...user };
-  for (const [list, counts] of [
-    ["models", "modelCallCounts"],
-    ["workflows", "workflowCallCounts"]
-  ]) {
-    if (user && Object.hasOwn(user, list) && !Object.hasOwn(user, counts)) {
-      delete merged[counts];
+function splitCallerMetadata(user) {
+  if (!user) return {};
+  let reserved;
+  const safe = {};
+  for (const [key, value] of Object.entries(user)) {
+    if (RESERVED_METADATA_KEYS.includes(key)) {
+      (reserved ??= {})[key] = value;
+    } else {
+      safe[key] = value;
+    }
+  }
+  return { reserved, ...Object.keys(safe).length > 0 ? { safe } : {} };
+}
+function isAbortError(err) {
+  return err?.name === "AbortError";
+}
+async function stageCapture(runtime, ownerId, option) {
+  const limits = resolveCaptureLimits(option);
+  if (!limits) return void 0;
+  const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+  return {
+    artifactId: staged.artifactId,
+    channel: new import_axl3.RequestCaptureChannel({
+      sink: staged.sink,
+      ...limits,
+      // Redaction is the RUNTIME's policy, not the caller's: a run cannot opt
+      // out of compliance mode by asking for diagnostics.
+      redact: runtime.isRedactEnabled()
+    })
+  };
+}
+async function finishCapture(runtime, capture) {
+  const status = await capture.channel.close();
+  try {
+    return await runtime.finalizeDiagnosticArtifact(
+      capture.artifactId,
+      status.status,
+      status.reason,
+      // What the channel ACTUALLY applied. The store sees only scrubbed bytes
+      // and cannot tell, so a manifest without this reports every compliance-
+      // mode artifact as unredacted.
+      status.redaction
+    );
+  } catch {
+    return void 0;
+  }
+}
+async function abandonCapture(runtime, capture) {
+  if (!capture) return;
+  await capture.channel.close().catch(() => void 0);
+  await runtime.rollbackDiagnosticArtifact(capture.artifactId).catch(() => void 0);
+}
+function attachOperationRefs(items, channel) {
+  const byItem = /* @__PURE__ */ new Map();
+  const byScorer = /* @__PURE__ */ new Map();
+  for (const entry of channel.operations()) {
+    if (entry.caseIndex === void 0) continue;
+    const ref = {
+      operationId: entry.operationId,
+      kind: entry.kind,
+      ...entry.turn !== void 0 ? { turn: entry.turn } : {},
+      ...entry.attempt !== void 0 ? { attempt: entry.attempt } : {},
+      status: entry.status
+    };
+    if (entry.scorer !== void 0) {
+      const forItem = byScorer.get(entry.caseIndex) ?? /* @__PURE__ */ new Map();
+      const list = forItem.get(entry.scorer) ?? [];
+      list.push(ref);
+      forItem.set(entry.scorer, list);
+      byScorer.set(entry.caseIndex, forItem);
+    } else {
+      const list = byItem.get(entry.caseIndex) ?? [];
+      list.push(ref);
+      byItem.set(entry.caseIndex, list);
+    }
+  }
+  for (const [caseIndex, operations] of byItem) {
+    const item = items[caseIndex];
+    if (!item) continue;
+    item.diagnostics = { operations: [...item.diagnostics?.operations ?? [], ...operations] };
+  }
+  for (const [caseIndex, forItem] of byScorer) {
+    for (const [scorer2, operations] of forItem) {
+      const detail = items[caseIndex]?.scoreDetails?.[scorer2];
+      if (detail) detail.diagnostics = { operations };
     }
   }
-  return merged;
+}
+var MAX_CAUSE_DEPTH = 8;
+function isProviderErrorLike(value) {
+  if (value instanceof import_axl3.ProviderError) return true;
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value;
+  return candidate.code === "PROVIDER_ERROR" && candidate.name === "ProviderError";
+}
+function describeItemFailure(thrown) {
+  let current = thrown;
+  for (let depth = 0; depth <= MAX_CAUSE_DEPTH; depth++) {
+    if (typeof current !== "object" || current === null) break;
+    if (isProviderErrorLike(current)) {
+      const { name: name2, provider, status, retryable, requestId } = current;
+      return {
+        name: typeof name2 === "string" ? name2 : "ProviderError",
+        ...typeof provider === "string" ? { provider } : {},
+        ...typeof status === "number" && Number.isFinite(status) ? { status } : {},
+        ...typeof retryable === "boolean" ? { retryable } : {},
+        ...typeof requestId === "string" ? { requestId } : {}
+      };
+    }
+    current = current.cause;
+  }
+  const name = thrown?.name;
+  return typeof thrown === "object" && thrown !== null && typeof name === "string" && name ? { name } : void 0;
 }
 async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
-  const items = await config.dataset.getItems();
+  const itemErrorRateLimit = config.failOnItemErrorRate ?? DEFAULT_ITEM_ERROR_RATE_LIMIT;
+  if (!isErrorRateLimit(itemErrorRateLimit)) {
+    throw new import_axl3.AxlError(
+      "INVALID_ITEM_ERROR_RATE",
+      `Invalid failOnItemErrorRate (${String(itemErrorRateLimit)}): expected a number in [0, 1]; 1 disables the gate.`
+    );
+  }
+  const admission = config.budget != null ? new import_axl3.AdmissionController({ limit: parseBudget(config.budget) }) : void 0;
+  const capture = await stageCapture(runtime, id, options?.captureRequests);
+  let items;
+  try {
+    items = await config.dataset.getItems();
+  } catch (error) {
+    await abandonCapture(runtime, capture);
+    throw error;
+  }
   const droppedAnnotationKeys = [...config.dataset.droppedAnnotationKeys ?? []];
   const concurrency = config.concurrency ?? 5;
   const scorerConcurrency = config.scorerConcurrency ?? 5;
-  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
+  let closedBy;
+  const noteClosure = (source) => {
+    if (closedBy === void 0 && admission?.closed) closedBy = source;
+  };
   const scorerContext = {
     resolveProvider: (uri) => {
       if (typeof runtime.resolveProvider !== "function") {
@@ -425,8 +880,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
     signal: options?.signal
   };
   const evalItems = new Array(items.length);
-  let totalCost = 0;
-  let budgetExceeded = false;
+  const callerReported = { costItems: 0, costTotal: 0, metadataItems: 0 };
   const runCallSamples = /* @__PURE__ */ new Map();
   function absorbTiming(rollup, evalItem) {
     if (!rollup) return;
@@ -448,26 +902,28 @@ async function runEval(config, executeWorkflow, runtime, options) {
     }
     evalItem.timing = perItem;
   }
+  function stub(item, outcome, error) {
+    return {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      error,
+      outcome,
+      accounting: emptyAccounting(),
+      cost: 0,
+      scorerCost: 0,
+      scores: {}
+    };
+  }
   async function processItem(item, itemIndex) {
     if (options?.signal?.aborted) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Cancelled",
-        scores: {}
-      };
+      evalItems[itemIndex] = stub(item, "cancelled", "Cancelled");
       options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
-    if (budgetExceeded) {
-      evalItems[itemIndex] = {
-        input: item.input,
-        annotations: item.annotations,
-        output: null,
-        error: "Budget exceeded",
-        scores: {}
-      };
+    if (admission?.closed) {
+      noteClosure("case");
+      evalItems[itemIndex] = stub(item, "budget_skipped", "Budget exceeded");
       options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
@@ -478,77 +934,124 @@ async function runEval(config, executeWorkflow, runtime, options) {
       scores: {}
     };
     const itemStart = Date.now();
-    try {
-      if (options?.captureTraces) {
-        const tracked = await runtime.trackExecution(
+    const itemOutcome = await trackScope(
+      runtime,
+      async () => {
+        const generation = await trackScope(
+          runtime,
           async () => executeWorkflow(item.input, runtime),
-          { captureTraces: true, captureTimingSamples: true }
+          {
+            purpose: "generation",
+            captureTraces: options?.captureTraces === true,
+            captureTimingSamples: true
+          }
         );
         evalItem.duration = Date.now() - itemStart;
-        evalItem.output = tracked.result.output;
-        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
-        if (tracked.unpriced) evalItem.unpriced = true;
-        evalItem.metadata = mergeMetadata(tracked.metadata, extractUserMetadata(tracked.result));
-        absorbTiming(tracked.modelTiming, evalItem);
-        if (tracked.traces && tracked.traces.length > 0) {
-          evalItem.traces = tracked.traces;
+        absorbTiming(generation.modelTiming, evalItem);
+        if (generation.traces && generation.traces.length > 0) {
+          evalItem.traces = generation.traces;
         }
-        if (evalItem.cost != null) {
-          totalCost += evalItem.cost;
+        if (generation.status === "rejected") {
+          const err = generation.error;
+          if (isAdmissionDenied(err)) {
+            noteClosure("operation");
+            evalItem.outcome = "budget_interrupted";
+            evalItem.error = "Budget interrupted";
+          } else if (isAbortError(err)) {
+            evalItem.outcome = "cancelled";
+            evalItem.error = "Cancelled";
+          } else {
+            evalItem.outcome = "failed";
+            const failure = describeItemFailure(err);
+            if (failure) evalItem.failure = failure;
+            evalItem.error = err instanceof Error ? err.message : String(err);
+            if (admission?.closed) noteClosure("case");
+          }
+          return;
         }
-      } else {
-        let trackedTiming;
-        let trackedMetadata;
-        let result;
-        if (typeof runtime.trackExecution === "function") {
-          const tracked = await runtime.trackExecution(
-            async () => executeWorkflow(item.input, runtime),
-            { captureTraces: false, captureTimingSamples: true }
-          );
-          result = tracked.result;
-          trackedTiming = tracked.modelTiming;
-          trackedMetadata = tracked.metadata;
-        } else {
-          result = await executeWorkflow(item.input, runtime);
-        }
-        evalItem.duration = Date.now() - itemStart;
+        const result = generation.value;
+        evalItem.outcome = "completed";
         evalItem.output = result.output;
-        evalItem.cost = extractUserCost(result);
-        const meta = extractUserMetadata(result);
-        if (trackedMetadata || meta) evalItem.metadata = mergeMetadata(trackedMetadata, meta);
-        absorbTiming(trackedTiming, evalItem);
-        if (evalItem.cost != null) {
-          totalCost += evalItem.cost;
+        const callerCost = extractUserCost(result);
+        if (callerCost !== void 0) {
+          (evalItem.callerReport ??= {}).cost = callerCost;
+          callerReported.costItems += 1;
+          callerReported.costTotal += callerCost;
         }
-      }
-    } catch (err) {
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.error = err instanceof Error ? err.message : String(err);
-      if (options?.captureTraces) {
-        const captured = err.axlCapturedTraces;
-        if (Array.isArray(captured) && captured.length > 0) {
-          evalItem.traces = captured;
+        const { reserved, safe } = splitCallerMetadata(extractUserMetadata(result));
+        if (reserved) {
+          (evalItem.callerReport ??= {}).metadata = reserved;
+          callerReported.metadataItems += 1;
+        }
+        if (generation.metadata || safe) {
+          evalItem.metadata = { ...generation.metadata, ...safe };
         }
+        if (admission?.closed) noteClosure("case");
+        await scoreItem(evalItem, config.scorers, {
+          runtime,
+          scorerContext,
+          scorerConcurrency,
+          signal: options?.signal,
+          admission,
+          onClosure: noteClosure
+        });
+      },
+      {
+        purpose: "generation",
+        // Every record produced anywhere under this item — workflow turns,
+        // nested asks, tool continuations, judges — is stamped with the case
+        // index, which is how the result can point an item at its own requests.
+        ...capture ? { captureCorrelation: { caseIndex: itemIndex } } : {}
       }
-      evalItems[itemIndex] = evalItem;
-      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
-      return;
-    }
-    if (budgetLimit != null && totalCost > budgetLimit) {
-      budgetExceeded = true;
-    }
-    const itemScorerCost = await scoreItem(
-      evalItem,
-      config.scorers,
-      scorerConcurrency,
-      scorerContext,
-      options?.signal
     );
-    totalCost += itemScorerCost;
+    if (itemOutcome.status === "rejected") throw itemOutcome.error;
+    evalItem.accounting = itemOutcome.accounting;
+    evalItem.cost = itemOutcome.accounting.breakdown.generation;
+    evalItem.scorerCost = itemOutcome.accounting.breakdown.judging;
+    if (itemOutcome.accounting.completeness !== "complete") evalItem.unpriced = true;
     evalItems[itemIndex] = evalItem;
     options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
-  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
+  const runOutcome = await trackScope(
+    runtime,
+    async () => {
+      await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));
+    },
+    // The capture channel is declared ONCE, at the run scope: its byte budget
+    // and pending queue are per RUN, and every nested item/scorer scope
+    // inherits it rather than opening a competing budget of its own.
+    { admission, ...capture ? { capture: capture.channel } : {} }
+  );
+  if (runOutcome.status === "rejected") {
+    await abandonCapture(runtime, capture);
+    throw runOutcome.error;
+  }
+  let diagnostics;
+  if (capture) {
+    try {
+      attachOperationRefs(evalItems, capture.channel);
+    } catch (error) {
+      await abandonCapture(runtime, capture);
+      throw error;
+    }
+    const manifest = await finishCapture(runtime, capture);
+    diagnostics = manifest ? toDiagnosticManifest(manifest) : (
+      // An artifact that was never sealed is never committed, so publishing
+      // its id would point a reader at bytes the sweeper is about to reclaim.
+      unavailableManifest("", "the diagnostic artifact could not be finalized")
+    );
+  }
+  const accounting = {
+    ...runOutcome.accounting,
+    scope: "run",
+    ...admission ? {
+      budget: {
+        ...admission.snapshot(),
+        ...closedBy ? { closedBy } : {}
+      }
+    } : {},
+    ...callerReported.costItems > 0 || callerReported.metadataItems > 0 ? { callerReported } : {}
+  };
   const failures = evalItems.filter((i) => i.error).length;
   const scorerNames = config.scorers.map((s) => s.name);
   const scorerStats = {};
@@ -557,6 +1060,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
     const { scored, failed, skipped } = scorerCounts(evalItems, name);
     scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };
   }
+  const coverage = buildCoverage(evalItems, scorerNames);
   const scorerTypes = {};
   for (const s of config.scorers) {
     scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
@@ -582,21 +1086,10 @@ async function runEval(config, executeWorkflow, runtime, options) {
       }
     }
   }
+  const itemErrorRate = evaluateItemErrorRate(coverage.items, items.length, itemErrorRateLimit);
   const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration);
   const timing = durations.length > 0 ? computeStats(durations) : void 0;
-  const modelTiming = runCallSamples.size > 0 ? Object.fromEntries(
-    [...runCallSamples.entries()].map(([model, samples]) => {
-      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
-      const stats = {
-        calls: samples.length,
-        wireMs: computeStats(samples.map((s) => s.wireMs)),
-        queuedMs: computeStats(samples.map((s) => s.queuedMs)),
-        retryMs: computeStats(samples.map((s) => s.retryMs)),
-        ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
-      };
-      return [model, stats];
-    })
-  ) : void 0;
+  const modelTiming = buildModelTiming(runCallSamples);
   const totalModelCalls = /* @__PURE__ */ new Map();
   for (const item of evalItems) {
     const itemCounts = item.metadata?.modelCallCounts;
@@ -659,20 +1152,64 @@ async function runEval(config, executeWorkflow, runtime, options) {
       ...droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}
     },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
-    ...evalItems.some((item) => item.unpriced) ? { unpriced: true } : {},
+    totalCost: accounting.knownCost,
+    ...accounting.completeness !== "complete" ? { unpriced: true } : {},
+    accounting,
+    ...diagnostics ? { diagnostics } : {},
     duration: Date.now() - startTime,
     items: evalItems,
     summary: {
       count: items.length,
       failures,
+      coverage,
       scorers: scorerStats,
       timing,
       ...modelTiming ? { modelTiming } : {},
-      ...degraded ? { degraded } : {}
+      ...degraded ? { degraded } : {},
+      ...itemErrorRate.failed > 0 ? { itemErrorRate } : {}
     }
   };
 }
+function buildCoverage(items, scorerNames) {
+  const coverage = {
+    items: Object.fromEntries(ITEM_OUTCOMES.map((o) => [o, 0])),
+    scorers: {}
+  };
+  for (const name of scorerNames) {
+    coverage.scorers[name] = Object.fromEntries(SCORER_OUTCOMES.map((o) => [o, 0]));
+  }
+  for (const item of items) {
+    const outcome = item.outcome ?? (item.error ? "failed" : "completed");
+    if (coverage.items[outcome] !== void 0) coverage.items[outcome] += 1;
+    for (const name of scorerNames) {
+      const detail = item.scoreDetails?.[name];
+      if (!detail?.outcome) continue;
+      const bucket = coverage.scorers[name];
+      if (bucket[detail.outcome] !== void 0) bucket[detail.outcome] += 1;
+    }
+  }
+  return coverage;
+}
+function buildModelTiming(runCallSamples) {
+  if (runCallSamples.size === 0) return void 0;
+  return Object.fromEntries(
+    [...runCallSamples.entries()].map(([model, samples]) => {
+      const firstToken = samples.map((s) => s.firstTokenMs).filter((v) => v != null);
+      return [
+        model,
+        {
+          calls: samples.length,
+          wireMs: computeStats(samples.map((s) => s.wireMs)),
+          queuedMs: computeStats(samples.map((s) => s.queuedMs)),
+          retryMs: computeStats(samples.map((s) => s.retryMs)),
+          // Custom providers may omit the optional field; that call adds 0.
+          rateLimitRetries: samples.reduce((sum, s) => sum + (s.rateLimitRetries ?? 0), 0),
+          ...firstToken.length > 0 ? { firstTokenMs: computeStats(firstToken), firstTokenCalls: firstToken.length } : {}
+        }
+      ];
+    })
+  );
+}
 
 // src/bootstrap.ts
 function pairedBootstrapCI(differences, options) {
@@ -741,6 +1278,69 @@ function xorshift32(seed) {
 }
 
 // src/compare.ts
+function certifyCost(baselineRuns, candidateRuns, scorerNames) {
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (const run of runs) {
+      const accounting = readAccounting(run);
+      if (accounting.completeness === "unverified") {
+        return `${side} run ${run.id} carries no accounting (unverified) \u2014 its total is not a measured figure.`;
+      }
+      if (accounting.completeness === "incomplete") {
+        const reasons = Object.keys(accounting.reasons).join(", ") || "unknown spend";
+        return `${side} run ${run.id} has incomplete accounting (${reasons}) \u2014 its total is a lower bound.`;
+      }
+    }
+  }
+  const scopes = /* @__PURE__ */ new Map();
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (const run of runs) {
+      const scope = readAccounting(run).scope;
+      const seen = scopes.get(side);
+      if (seen !== void 0 && seen !== scope) {
+        return `${side} mixes accounting scopes ("${seen}" and "${scope}") \u2014 a rescore total covers judging only and cannot be averaged with a full run.`;
+      }
+      scopes.set(side, scope);
+    }
+  }
+  const baselineScope = scopes.get("baseline");
+  const candidateScope = scopes.get("candidate");
+  if (baselineScope !== candidateScope) {
+    return `accounting scope differs (baseline "${baselineScope}" vs candidate "${candidateScope}") \u2014 a rescore total covers judging only.`;
+  }
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  for (let r = 0; r < runCount; r++) {
+    const bRun = baselineRuns[r];
+    const cRun = candidateRuns[r];
+    if (bRun.items.length !== cRun.items.length) {
+      return `item counts differ (baseline ${bRun.items.length} vs candidate ${cRun.items.length}) \u2014 the runs did different amounts of work.`;
+    }
+    for (let i = 0; i < bRun.items.length; i++) {
+      const bItem = bRun.items[i];
+      const cItem = cRun.items[i];
+      const bRan = !bItem.error;
+      const cRan = !cItem.error;
+      if (bRan !== cRan) {
+        const stopped = bRan ? cItem : bItem;
+        return `case coverage differs at item ${i}: ${bRan ? "candidate" : "baseline"} did not complete it (${stopped.outcome ?? stopped.error}) \u2014 the cheaper side did less work.`;
+      }
+      if (!bRan) continue;
+      for (const name of scorerNames) {
+        const bScored = bItem.scores[name] != null;
+        const cScored = cItem.scores[name] != null;
+        if (bScored !== cScored) {
+          return `scorer coverage differs for "${name}" at item ${i}: only ${bScored ? "baseline" : "candidate"} produced a score.`;
+        }
+      }
+    }
+  }
+  return null;
+}
 var DEFAULT_LLM_THRESHOLD = 0.05;
 var DEFAULT_DETERMINISTIC_THRESHOLD = 0;
 var LEGACY_THRESHOLD = 0.1;
@@ -905,17 +1505,21 @@ function evalCompare(baseline, candidate, options) {
     timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };
   }
   let cost;
-  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;
-  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;
-  if (baselineAvgCost > 0 || candidateAvgCost > 0) {
+  const baselineAvgCost = baselineRuns.reduce((sum, r) => sum + readAccounting(r).knownCost, 0) / baselineRuns.length;
+  const candidateAvgCost = candidateRuns.reduce((sum, r) => sum + readAccounting(r).knownCost, 0) / candidateRuns.length;
+  const hasAccounting = [...baselineRuns, ...candidateRuns].some((r) => r.accounting != null);
+  if (baselineAvgCost > 0 || candidateAvgCost > 0 || hasAccounting) {
     const deltaRaw = candidateAvgCost - baselineAvgCost;
     const delta = round(deltaRaw);
-    const deltaPercent = baselineAvgCost > 0 ? round(deltaRaw / baselineAvgCost * 100) : 0;
+    const deltaPercent = baselineAvgCost > 0 ? round(deltaRaw / baselineAvgCost * 100) : null;
+    const verdict = certifyCost(baselineRuns, candidateRuns, baselineScorerNames);
     cost = {
       baselineTotal: round(baselineAvgCost),
       candidateTotal: round(candidateAvgCost),
       delta,
-      deltaPercent
+      deltaPercent,
+      certified: verdict === null,
+      ...verdict !== null ? { reason: verdict } : {}
     };
   }
   const parts = [];
@@ -936,7 +1540,7 @@ function evalCompare(baseline, candidate, options) {
     const dir = timing.delta > 0 ? "slower" : "faster";
     parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);
   }
-  if (cost && Math.abs(cost.deltaPercent) > 1) {
+  if (cost?.certified && cost.deltaPercent !== null && Math.abs(cost.deltaPercent) > 1) {
     const dir = cost.delta > 0 ? "more expensive" : "cheaper";
     parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
   }
@@ -1001,13 +1605,150 @@ function evaluateScorerErrorRateGate(comparison, maxScorerErrorRate) {
   }
   return null;
 }
+function compareItemErrorRates(baseline, candidate, maxItemErrorRate = DEFAULT_ITEM_ERROR_RATE_LIMIT) {
+  if (!isErrorRateLimit(maxItemErrorRate)) {
+    throw new Error(
+      `Invalid maxItemErrorRate (${String(maxItemErrorRate)}): expected a number in [0, 1].`
+    );
+  }
+  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
+  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
+  const rates = [];
+  for (const [side, runs] of [
+    ["baseline", baselineRuns],
+    ["candidate", candidateRuns]
+  ]) {
+    for (let r = 0; r < runCount; r++) {
+      const run = runs[r];
+      const coverage = run.summary.coverage?.items;
+      const legacy = coverage === void 0;
+      const verdict = legacy ? evaluateItemErrorRate(
+        buildCoverage(run.items, []).items,
+        run.items.length,
+        maxItemErrorRate
+      ) : evaluateItemErrorRate(coverage, run.summary.count, maxItemErrorRate);
+      rates.push({ side, runIndex: r, runId: run.id, legacy, ...verdict });
+    }
+  }
+  return rates;
+}
+function describeItemErrorRate(rate, multiRun) {
+  const which = multiRun ? ` run ${rate.runIndex + 1} (${rate.runId})` : ` (${rate.runId})`;
+  const legacy = rate.legacy ? ", derived from the items of a pre-0.24 artifact" : "";
+  return `${rate.side}${which} item error rate ${formatPercent(rate.rate)} (${rate.failed}/${rate.attempted} attempted items failed${legacy})`;
+}
+function evaluateItemErrorRateGate(baseline, candidate, maxItemErrorRate = DEFAULT_ITEM_ERROR_RATE_LIMIT) {
+  const rates = compareItemErrorRates(baseline, candidate, maxItemErrorRate);
+  const multiRun = rates.some((r) => r.runIndex > 0);
+  const over = rates.find((r) => r.exceeded);
+  if (!over) return null;
+  return `coverage: ${describeItemErrorRate(over, multiRun)} exceeds the ${formatPercent(maxItemErrorRate)} limit \u2014 its scores cover only the surviving items. Pass --max-item-error-rate <0..1> to accept a thinned side (1 disables the check).`;
+}
 
 // src/rescore.ts
+var import_axl4 = require("@axlsdk/axl");
 var import_node_crypto2 = require("crypto");
+var COPY_SHARE_OF_RUN_BOUND = 0.75;
+function describeFailure(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function isDiagnosticsUnavailable(error) {
+  return error?.code === "DIAGNOSTICS_UNAVAILABLE";
+}
+async function beginCapture(source, ownerId, runtime, options) {
+  const limits = resolveCaptureLimits(options?.captureRequests);
+  if (!limits) return {};
+  const open = (artifactId, sink, carriedBytes) => ({
+    artifactId,
+    channel: new import_axl4.RequestCaptureChannel({
+      sink,
+      ...limits,
+      carriedBytes,
+      // Redaction is the RUNTIME's policy, not the caller's: a rescore cannot
+      // opt out of compliance mode by asking for diagnostics.
+      redact: runtime.isRedactEnabled()
+    })
+  });
+  const sourceId = source.diagnostics?.artifactId;
+  let stagedId;
+  try {
+    if (!sourceId) {
+      const staged = await runtime.stageDiagnosticArtifact({ kind: "eval", id: ownerId });
+      stagedId = staged.artifactId;
+      return { capture: { ...open(staged.artifactId, staged.sink, 0), copied: false } };
+    }
+    const runBound = limits.maxRunBytes ?? DEFAULT_COPY_MAX_BYTES;
+    const maxBytes = Math.max(1, Math.floor(runBound * COPY_SHARE_OF_RUN_BOUND));
+    const copied = await runtime.copyDiagnosticArtifact(
+      sourceId,
+      { kind: "eval", id: ownerId },
+      { maxBytes }
+    );
+    if (!copied) {
+      return {
+        degraded: unavailableManifest(
+          "",
+          "the source run's captured requests are no longer available"
+        )
+      };
+    }
+    stagedId = copied.artifactId;
+    const capture = {
+      ...open(copied.artifactId, copied.sink, copied.bytes),
+      copied: true,
+      carriedRedaction: copied.redaction
+    };
+    if (copied.truncated) {
+      capture.copyTruncated = `the copied source records stopped at ${maxBytes} bytes, the share of the ${runBound} byte run limit reserved for them`;
+    }
+    return { capture };
+  } catch (error) {
+    if (stagedId !== void 0) {
+      await runtime.rollbackDiagnosticArtifact(stagedId).catch(() => void 0);
+    }
+    if (isDiagnosticsUnavailable(error)) throw error;
+    return {
+      degraded: unavailableManifest(
+        "",
+        `the captured requests could not be carried forward: ${describeFailure(error)}`
+      )
+    };
+  }
+}
+async function abandonCapture2(runtime, capture) {
+  if (!capture) return;
+  await capture.channel.close().catch(() => void 0);
+  await runtime.rollbackDiagnosticArtifact(capture.artifactId).catch(() => void 0);
+}
+async function finishCapture2(runtime, capture) {
+  const status = await capture.channel.close();
+  const reason = [capture.copyTruncated, status.reason].filter(Boolean).join("; ") || void 0;
+  const truncated = capture.copyTruncated !== void 0 || status.status === "truncated";
+  const finalStatus = status.status === "unavailable" ? "unavailable" : truncated ? "truncated" : status.status;
+  const redaction = status.redaction === "applied" && (capture.carriedRedaction ?? "applied") === "applied" ? "applied" : "none";
+  try {
+    const manifest = await runtime.finalizeDiagnosticArtifact(
+      capture.artifactId,
+      finalStatus,
+      reason,
+      redaction
+    );
+    return toDiagnosticManifest(manifest);
+  } catch {
+    return unavailableManifest("", "the diagnostic artifact could not be finalized");
+  }
+}
 async function rescore(result, scorers, runtime, options) {
   const startTime = Date.now();
+  const rescoredId = (0, import_node_crypto2.randomUUID)();
   const concurrency = options?.concurrency ?? 5;
   const scorerConcurrency = options?.scorerConcurrency ?? 5;
+  const admission = options?.budget != null ? new import_axl4.AdmissionController({ limit: parseBudget(options.budget) }) : void 0;
+  let closedBy;
+  const noteClosure = (source) => {
+    if (closedBy === void 0 && admission?.closed) closedBy = source;
+  };
   const scorerContext = {
     resolveProvider: (uri) => {
       if (typeof runtime.resolveProvider !== "function") {
@@ -1020,7 +1761,8 @@ async function rescore(result, scorers, runtime, options) {
     signal: options?.signal
   };
   const rescored = new Array(result.items.length);
-  let totalCost = 0;
+  const { capture, degraded } = await beginCapture(result, rescoredId, runtime, options);
+  const carryDiagnostics = capture?.copied === true;
   async function rescoreItem(original, itemIndex) {
     if (options?.signal?.aborted) {
       rescored[itemIndex] = {
@@ -1028,16 +1770,31 @@ async function rescore(result, scorers, runtime, options) {
         annotations: original.annotations,
         output: original.output,
         error: "cancelled",
+        outcome: "cancelled",
+        accounting: emptyAccounting(),
         scores: {}
       };
       return;
     }
     if (original.error) {
+      const accounting2 = emptyAccounting();
       rescored[itemIndex] = {
         input: original.input,
         annotations: original.annotations,
         output: original.output,
         error: original.error,
+        ...original.outcome ? { outcome: original.outcome } : {},
+        // The failure cause is a fact about the source run too.
+        ...original.failure ? { failure: original.failure } : {},
+        accounting: accounting2,
+        // Same rule as a scored item: an item carrying `accounting` always
+        // carries both compat views, so no reader falls through to the legacy
+        // "no accounting → trust the caller's number" branch and reports the
+        // SOURCE run's spend as this rescore's. A passthrough did no work, so
+        // both are a measured $0; the source's spend lives in the run-level
+        // `accounting.source` provenance and nowhere else.
+        cost: accounting2.breakdown.generation,
+        scorerCost: accounting2.breakdown.judging,
         scores: {}
       };
       return;
@@ -1048,20 +1805,77 @@ async function rescore(result, scorers, runtime, options) {
       output: original.output,
       metadata: original.metadata,
       traces: original.traces,
+      outcome: "completed",
+      // Carried forward VERBATIM, original operation ids included. The rescore
+      // did not re-run the workflow, so these still describe the calls that
+      // produced this output; reminting the ids would break the only link back
+      // to the run that made them. Only carried when the source records were
+      // actually copied into this rescore's artifact — a dangling reference is
+      // worse than none.
+      ...carryDiagnostics && original.diagnostics ? { diagnostics: original.diagnostics } : {},
       scores: {},
       scoreDetails: {}
     };
-    const itemScorerCost = await scoreItem(
-      item,
-      scorers,
-      scorerConcurrency,
-      scorerContext,
-      options?.signal
+    const itemOutcome = await trackScope(
+      runtime,
+      async () => {
+        await scoreItem(item, scorers, {
+          runtime,
+          scorerContext,
+          scorerConcurrency,
+          signal: options?.signal,
+          admission,
+          onClosure: noteClosure
+        });
+      },
+      {
+        purpose: "judging",
+        // Every record produced under this item is stamped with the case index,
+        // which is how a `ScorerDetail` can point at its own judge calls.
+        ...capture ? { captureCorrelation: { caseIndex: itemIndex } } : {}
+      }
     );
-    totalCost += itemScorerCost;
+    if (itemOutcome.status === "rejected") throw itemOutcome.error;
+    item.accounting = itemOutcome.accounting;
+    item.cost = itemOutcome.accounting.breakdown.generation;
+    item.scorerCost = itemOutcome.accounting.breakdown.judging;
+    if (itemOutcome.accounting.completeness !== "complete") item.unpriced = true;
     rescored[itemIndex] = item;
   }
-  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
+  const runOutcome = await trackScope(
+    runtime,
+    async () => {
+      await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));
+    },
+    // The channel is declared ONCE, at the run scope: its byte budget and
+    // pending queue are per rescore, and every nested item/scorer scope inherits
+    // it rather than opening a competing budget of its own.
+    { purpose: "judging", admission, ...capture ? { capture: capture.channel } : {} }
+  );
+  if (runOutcome.status === "rejected") {
+    await abandonCapture2(runtime, capture);
+    throw runOutcome.error;
+  }
+  const sourceAccounting = result.accounting;
+  const generation = sourceAccounting?.scope === "rescore" ? sourceAccounting.source?.generation ?? null : sourceAccounting ?? null;
+  let diagnostics = degraded;
+  if (capture) {
+    try {
+      attachOperationRefs(rescored, capture.channel);
+    } catch (error) {
+      await abandonCapture2(runtime, capture);
+      throw error;
+    }
+    diagnostics = await finishCapture2(runtime, capture);
+  }
+  const accounting = {
+    ...runOutcome.accounting,
+    scope: "rescore",
+    // `?? null` is load-bearing: a legacy source has no accounting and must read
+    // as "unknown generation", never as a synthesized complete record.
+    source: { runId: result.id, generation },
+    ...admission ? { budget: { ...admission.snapshot(), ...closedBy ? { closedBy } : {} } } : {}
+  };
   const failures = rescored.filter((i) => i.error).length;
   const scorerNames = scorers.map((s) => s.name);
   const scorerStats = {};
@@ -1075,7 +1889,7 @@ async function rescore(result, scorers, runtime, options) {
     scorerTypes[s.name] = s.isLlm ? "llm" : "deterministic";
   }
   return {
-    id: (0, import_node_crypto2.randomUUID)(),
+    id: rescoredId,
     dataset: result.dataset,
     metadata: (() => {
       const rest = { ...result.metadata };
@@ -1099,10 +1913,24 @@ async function rescore(result, scorers, runtime, options) {
       return merged;
     })(),
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-    totalCost,
+    totalCost: accounting.knownCost,
+    ...accounting.completeness !== "complete" ? { unpriced: true } : {},
+    accounting,
+    ...diagnostics ? { diagnostics } : {},
     duration: Date.now() - startTime,
     items: rescored,
-    summary: { count: result.items.length, failures, scorers: scorerStats }
+    summary: {
+      count: result.items.length,
+      failures,
+      coverage: buildCoverage(rescored, scorerNames),
+      scorers: scorerStats,
+      // A rescore makes no generation calls, so the source run's wall-clock
+      // and provider latency are still the only measurements of these items.
+      // `modelTiming` cannot be rebuilt from `item.timing` (sums, not per-call
+      // samples): carry both forward.
+      ...result.summary.timing ? { timing: { ...result.summary.timing } } : {},
+      ...result.summary.modelTiming ? { modelTiming: structuredClone(result.summary.modelTiming) } : {}
+    }
   };
 }
 
@@ -1172,6 +2000,7 @@ function aggregateRuns(runs) {
     workflows,
     dataset: dataset2,
     totalCost: round(totalCost),
+    accounting: aggregateAccounting(runs.map((r) => readAccounting(r))),
     totalDuration: round(totalDuration),
     scorers,
     timing
@@ -1179,17 +2008,30 @@ function aggregateRuns(runs) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  DEFAULT_COPY_MAX_BYTES,
+  DEFAULT_SIDECAR_MAX_BYTES,
+  aggregateAccounting,
   aggregateRuns,
   dataset,
   defineEval,
   evalCompare,
+  evaluateItemErrorRateGate,
   evaluateScorerErrorRateGate,
   evaluateScorerTolerance,
+  isBudgetStopped,
   llmScorer,
   normalizeScorerResult,
   pairedBootstrapCI,
+  parseRequestRecords,
+  readAccounting,
+  refusedWork,
   rescore,
+  resolveCaptureLimits,
   runEval,
-  scorer
+  scorer,
+  serializeRequestRecords,
+  toDiagnosticManifest,
+  unavailableManifest,
+  validateRequestSidecar
 });
 //# sourceMappingURL=index.cjs.map