@axlsdk/eval 0.23.3 → 0.24.1

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { Provider, Effort, AxlEvent, AxlRuntime } from '@axlsdk/axl';
+import { Provider, Effort, ArtifactManifest, CapturedRequestRecord, Accounting, AxlEvent, AxlRuntime } from '@axlsdk/axl';
 
 type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
@@ -173,6 +173,193 @@ type LlmScorerConfig = {
 };
 declare function llmScorer(config: LlmScorerConfig): Scorer;
 
+/**
+ * The eval side of opt-in request capture: the JSONL codec, the manifest an
+ * `EvalResult` carries, and the bundle validation that keeps an imported
+ * artifact from naming anything but an artifact id.
+ *
+ * The division of labour with the core is deliberate. `@axlsdk/axl` owns the
+ * measurement-adjacent parts — snapshotting the request before dispatch,
+ * redaction, byte bounds, the non-blocking queue, and the artifact lifecycle —
+ * because all of those must hold for ANY consumer of `trackOutcome`, not just
+ * evals. This module owns what is eval-shaped: which run/case/scorer a record
+ * belongs to, how a run reports what it managed to capture, and what an
+ * exported bundle is allowed to contain.
+ *
+ * `RequestRecord` is the core record type by alias rather than by copy. Two
+ * hand-maintained declarations of a wire format drift, and the drift is
+ * invisible until someone tries to read last month's artifact.
+ */
+
+/** One JSONL line of a captured-request artifact. Codec version 1. */
+type RequestRecord = CapturedRequestRecord;
+/**
+ * A pointer from an item or a scorer to one captured operation.
+ *
+ * `status` is the honest part: `'recorded'` is a complete request/response
+ * pair, `'start_only'` is a call that never came back (the one you most want
+ * to see), `'truncated'` is a record replaced by a stub because it exceeded the
+ * per-record byte bound, and `'omitted'` is an operation capture never reached.
+ */
+type OperationRef = {
+    operationId: string;
+    kind: 'chat' | 'stream';
+    turn?: number;
+    attempt?: number;
+    status: 'recorded' | 'start_only' | 'truncated' | 'omitted';
+};
+/** What an `EvalResult` says about its captured requests. */
+type DiagnosticManifest = {
+    version: 1;
+    artifactId: string;
+    fidelity: 'runtime_request';
+    status: ArtifactManifest['status'];
+    reason?: string;
+    records: number;
+    bytes: number;
+    redaction: 'applied' | 'none';
+    expiresAt?: number;
+};
+/** Per-run capture limits, as `runEval`/`rescore` accept them. */
+type CaptureRequestsOption = boolean | {
+    maxRecordBytes?: number;
+    maxRunBytes?: number;
+    maxQueueBytes?: number;
+};
+/**
+ * Default bytes a rescore may copy from the source run's artifact.
+ *
+ * The same ceiling as a run's own capture budget: a rescore's copied evidence
+ * should never be able to dwarf the evidence a fresh run is allowed to produce.
+ */
+declare const DEFAULT_COPY_MAX_BYTES: number;
+/** Normalize the option into explicit limits, or `undefined` when capture is off. */
+declare function resolveCaptureLimits(option: CaptureRequestsOption | undefined): {
+    maxRecordBytes?: number;
+    maxRunBytes?: number;
+    maxQueueBytes?: number;
+} | undefined;
+/** Build the result-level manifest from the store's finalized manifest. */
+declare function toDiagnosticManifest(manifest: ArtifactManifest): DiagnosticManifest;
+/** A manifest for a run whose capture could not be produced at all. */
+declare function unavailableManifest(artifactId: string, reason: string): DiagnosticManifest;
+/** Default ceiling on an imported `.requests.jsonl` sidecar. */
+declare const DEFAULT_SIDECAR_MAX_BYTES: number;
+type SidecarValidation = {
+    ok: true;
+    lines: string[];
+    bytes: number;
+} | {
+    ok: false;
+    reason: string;
+};
+/**
+ * Validate an imported JSONL sidecar before a single byte of it is stored.
+ *
+ * Imports are the one place where a stranger's JSON reaches this subsystem, so
+ * the rules are deliberately narrow: every line must parse, must declare `v: 1`,
+ * and must carry the identity fields a reader indexes on. Anything else — a
+ * `file://` reference, a `path` field, a record for a codec version we cannot
+ * interpret — is refused rather than stored and puzzled over later.
+ *
+ * Note what is NOT here: nothing in a record can name a filesystem path or a
+ * URL to fetch, because the reader never dereferences anything. Records are
+ * re-staged under a NEW artifact id owned by the NEW history row, so an
+ * imported reference can only ever point at bytes this runtime wrote itself.
+ */
+declare function validateRequestSidecar(text: unknown, options?: {
+    maxBytes?: number;
+    maxRecordBytes?: number;
+}): SidecarValidation;
+/** Parse a validated sidecar into records. */
+declare function parseRequestRecords(lines: readonly string[]): RequestRecord[];
+/** Encode records back into a sidecar body. */
+declare function serializeRequestRecords(records: readonly RequestRecord[]): string;
+
+/**
+ * How one dataset item ended. Absent only on artifacts written before 0.24.
+ *
+ * The four non-`completed` values are deliberately NOT interchangeable: a
+ * model/workflow failure, a caller cancellation, a case never started because
+ * the budget had closed, and a case stopped mid-flight by a denied operation
+ * are four different facts about the run, and collapsing them makes a
+ * budget-truncated run look like a broken model.
+ */
+type EvalItemOutcome = 
+/** Ran to completion (its scorers may still have been stopped). */
+'completed'
+/** The workflow threw — including a user `ctx.budget` block, which is user logic. */
+ | 'failed'
+/** The caller's `AbortSignal` fired. */
+ | 'cancelled'
+/** Never started: the run budget had already closed. No operations, no spend. */
+ | 'budget_skipped'
+/** Started and charged, then a further operation was denied by the budget. */
+ | 'budget_interrupted';
+/** How one scorer ended for one item. See {@link EvalItemOutcome} for the rationale. */
+type ScorerOutcome = 
+/** Produced a valid numeric score. */
+'scored'
+/** Ran and threw, or returned an out-of-range score. */
+ | 'failed'
+/** Its `applies` predicate returned `false` — deliberately not run. */
+ | 'skipped'
+/** The caller's `AbortSignal` fired before or during the call. */
+ | 'cancelled'
+/** A paid judge that was never started because the budget had closed. */
+ | 'budget_skipped'
+/** Started, then hit a denied operation. */
+ | 'budget_interrupted';
+/** The run budget's terminal state, persisted on {@link EvalAccounting}. */
+type EvalBudgetStatus = {
+    /** The configured USD threshold. */
+    limit: number;
+    status: 'open' | 'closed';
+    /** Known spend observed under the controller — `accounting.knownCost`. */
+    knownSpend: number;
+    /**
+     * `max(0, knownSpend - limit)`. A threshold is not a reservation: work
+     * already dispatched when the limit was crossed still settles, so a run can
+     * legitimately end above its limit. This says by how much rather than
+     * clamping the reported total.
+     */
+    knownOvershoot: number;
+    /** Which scheduling decision first observed the closure. */
+    closedBy?: 'case' | 'scorer' | 'operation';
+};
+/**
+ * A run's authoritative accounting: the core {@link Accounting} record plus the
+ * eval-specific context needed to read it honestly.
+ */
+type EvalAccounting = Accounting & {
+    /** `'run'` for `runEval`, `'rescore'` for `rescore` (judging-only spend). */
+    scope: 'run' | 'rescore';
+    /** Present exactly when a budget was configured. */
+    budget?: EvalBudgetStatus;
+    /**
+     * Rescore provenance: the source run and its UNMODIFIED generation
+     * accounting, or `null` when the source was a legacy artifact that carried
+     * none. The original spend is never added to the rescore's own total.
+     */
+    source?: {
+        runId: string;
+        generation: Accounting | null;
+    };
+    /**
+     * Legacy caller-reported values observed during the run. Inspection only —
+     * never summed into `knownCost`, which measures Axl-observed operations.
+     */
+    callerReported?: {
+        costItems: number;
+        costTotal: number;
+        metadataItems: number;
+    };
+};
+/** Per-outcome counts for a run. Every key is present, including zeros. */
+type EvalCoverage = {
+    items: Record<EvalItemOutcome, number>;
+    scorers: Record<string, Record<ScorerOutcome, number>>;
+};
 type EvalConfig = {
     workflow: string;
     dataset: Dataset<unknown, unknown>;
@@ -186,6 +373,17 @@ type EvalConfig = {
      * so lower `concurrency` if a rate-limited judge model needs a tighter ceiling.
      */
     scorerConcurrency?: number;
+    /**
+     * Known-spend threshold for the whole run, e.g. `'$1'`, `'1'`, `'0.50'`.
+     *
+     * Validated BEFORE the dataset is loaded — a malformed value throws
+     * `AxlError('INVALID_BUDGET')` before any item runs or any provider is
+     * called. Admission closes at `knownSpend >= limit` (so `'$0'` admits
+     * nothing), which stops new cases, new paid judges, and new instrumented
+     * operations inside cases that are already running. Work already dispatched
+     * settles and is still counted, so the final total can exceed the limit —
+     * `accounting.budget.knownOvershoot` says by how much.
+     */
     budget?: string;
     /**
      * Opt-in source-side trust gate. When set (0–1), `runEval` flags the run as
@@ -205,8 +403,45 @@ type EvalConfig = {
      * other at consume/gate time (refuse to certify a thinned baseline/candidate).
      */
     failOnScorerErrorRate?: number;
+    /**
+     * Source-side coverage gate, **on by default** at `0.05`. `runEval` records
+     * `summary.itemErrorRate` whenever an item failed, and marks it `exceeded`
+     * when more than this fraction of the attempted items failed; the CLI then
+     * exits non-zero. It exists because a run that lost most of its items to
+     * throttling or an incident otherwise exits clean and is scored over the
+     * survivors.
+     *
+     * Rate = `coverage.items.failed / (count − cancelled − budget_skipped −
+     * budget_interrupted)`: cancelled and budget-stopped items are reported by
+     * their own gates and never count twice. Fires on strictly `>`, so `1`
+     * disables it (the CLI flag `--max-item-error-rate` overrides this value).
+     * A run with nothing attempted never fires.
+     *
+     * Unlike `failOnScorerErrorRate`, an invalid value (non-number, non-finite,
+     * `< 0` or `> 1`) makes `runEval` THROW `AxlError('INVALID_ITEM_ERROR_RATE')`
+     * before the dataset loads: warning and skipping would silently switch off a
+     * default-on gate.
+     */
+    failOnItemErrorRate?: number;
     metadata?: Record<string, unknown>;
 };
+/**
+ * A run's item error rate against its `failOnItemErrorRate` limit, on
+ * `EvalSummary.itemErrorRate`. See {@link EvalConfig.failOnItemErrorRate} for
+ * the definition.
+ */
+type ItemErrorRate = {
+    /** Items whose workflow threw — `coverage.items.failed`. */
+    failed: number;
+    /** `count − cancelled − budget_skipped − budget_interrupted`. */
+    attempted: number;
+    /** `failed / attempted`; `0` when nothing was attempted. */
+    rate: number;
+    /** The limit in force for this run (config or CLI flag, default `0.05`). */
+    limit: number;
+    /** `rate > limit` with at least one attempted item. */
+    exceeded: boolean;
+};
 /**
  * One scorer whose failure rate tripped `EvalConfig.failOnScorerErrorRate`.
  * Surfaced on `EvalSummary.degraded` so consumers (CLI exit code, Studio
@@ -252,9 +487,32 @@ type EvalResult = {
      */
     metadata: Record<string, unknown>;
     timestamp: string;
+    /**
+     * Compatibility view of `accounting.knownCost` — the sum of settled, disjoint
+     * charges Axl observed. A caller-reported `cost` is NEVER part of it (read
+     * `accounting.callerReported` for those). Lower bound when `unpriced` is set.
+     */
     totalCost: number;
-    /** True when `totalCost` is a lower bound because at least one item used unpriced work. */
+    /** True when `totalCost` is a lower bound — present iff `accounting.completeness !== 'complete'`. */
     unpriced?: boolean;
+    /**
+     * The authoritative record `totalCost` / `unpriced` are derived from.
+     * Required on results this version produces; optional so a pre-0.24 artifact
+     * still types. Read it through `readAccounting()`, which reports an absent
+     * record as `'unverified'` instead of silently treating it as complete.
+     */
+    accounting?: EvalAccounting;
+    /**
+     * What this run captured of the requests it submitted, when
+     * `captureRequests` was on. Present only for a capturing run — the default
+     * artifact carries no `diagnostics` key at all.
+     *
+     * It is a POINTER plus a self-assessment, never content: `artifactId` names
+     * bytes owned by the runtime that saved this result, and `status`/`reason`
+     * say whether those bytes are complete, truncated, interrupted or
+     * unavailable. Numeric results stay readable when the artifact is gone.
+     */
+    diagnostics?: DiagnosticManifest;
     duration: number;
     items: EvalItem[];
     summary: EvalSummary;
@@ -263,7 +521,24 @@ type ScorerDetail = {
     score: number | null;
     metadata?: Record<string, unknown>;
     duration?: number;
+    /**
+     * Compatibility view of `accounting.knownCost` for this scorer. Falls back to
+     * a scorer-returned `cost` ONLY when the runtime measured nothing at all
+     * (no `trackOutcome`); a caller value never overrides a measured one and is
+     * never summed into any total.
+     */
     cost?: number;
+    /** How this scorer ended for this item. Absent only on pre-0.24 artifacts. */
+    outcome?: ScorerOutcome;
+    /** This scorer's own operations for this item (judging spend, plus any
+     *  `externalOperation` it declared). Absent on pre-0.24 artifacts. */
+    accounting?: Accounting;
+    /** The captured operations this judge performed for this item. Present only
+     *  under `captureRequests`; references only, resolved through the run's
+     *  artifact. */
+    diagnostics?: {
+        operations: OperationRef[];
+    };
     /**
      * `true` when the scorer's `applies` predicate returned `false` for this item,
      * so the scorer was deliberately skipped (NOT run). Distinct from a `null`
@@ -320,11 +595,15 @@ type TimingStats = {
 /**
  * Per-model latency stats across a run, on `EvalSummary.modelTiming`.
  *
- * Every field here is a distribution over **per-call** values, pooled across all
- * of the run's successful items — the same population `calls` counts. One
- * provider call is one sample, so an item that makes ten calls weighs ten times
- * an item that makes one. That is the right weighting for a model comparison:
- * these numbers describe the model's latency, not the item's.
+ * Every field here is a distribution over **per-call** values, pooled across
+ * every successful provider call the run made — the same population `calls`
+ * counts. Since 0.24 that includes the calls made by items that later FAILED or
+ * were stopped on budget: those calls really happened and really took that long,
+ * and dropping them would bias the latency of exactly the models whose slowness
+ * caused the timeouts. One provider call is one sample, so an item that makes
+ * ten calls weighs ten times an item that makes one. That is the right
+ * weighting for a model comparison: these numbers describe the model's latency,
+ * not the item's.
  *
  * This is deliberately a different weighting from the wall-clock
  * `EvalSummary.timing`, which samples once per item because it describes the
@@ -347,6 +626,14 @@ type ModelTimingStats = {
      *  throttled hard on the day of the run shows it here, which is what keeps
      *  `wireMs` an honest comparison. */
     retryMs: TimingStats;
+    /** Total `CallTiming.rateLimitRetries` over the same calls `calls` counts:
+     *  how many rate-limit 429s this model's calls absorbed and retried. A plain
+     *  sum, not a distribution; a call that did not report the field adds `0`.
+     *  Nonzero means the provider throttled the run, so lower `concurrency`
+     *  before calls start failing. `runEval` always sets it; optional because
+     *  artifacts written before it existed do not carry it, so treat absence as
+     *  "not recorded", not as `0`. */
+    rateLimitRetries?: number;
     /** Per-call time to first content delta (ms), over the STREAMING calls that
      *  reported one — non-streaming calls are excluded from the sample rather
      *  than entered as `0`. Absent when no call reported one. This is the
@@ -360,20 +647,72 @@ type ModelTimingStats = {
      *  them. Present exactly when `firstTokenMs` is. */
     firstTokenCalls?: number;
 };
+/**
+ * Why an item's workflow failed, captured from the thrown value before it is
+ * flattened to `EvalItem.error`. Populated on `failed` items only.
+ *
+ * When a `ProviderError` is found — the thrown value itself or the first one
+ * down its `cause` chain — every field comes from it. Otherwise only the thrown
+ * value's `name` is recorded. This record never copies `ProviderError.body` or
+ * the message: the body can echo prompt text and is redaction-eligible.
+ * `EvalItem.error` keeps the error message as before, which for some providers
+ * (a non-JSON error response) embeds the response text — that is outside this
+ * record's guarantee.
+ */
+type EvalItemFailure = {
+    /** `'ProviderError'` when one was found, else the thrown value's own `name`. */
+    name: string;
+    /** Adapter/profile name, e.g. `'openai'`. */
+    provider?: string;
+    /** HTTP status; `0` for a network-level failure. */
+    status?: number;
+    /** `ProviderError.retryable` — the semantic failover hint. */
+    retryable?: boolean;
+    /** Provider request id, when the response carried one. */
+    requestId?: string;
+};
 type EvalItem = {
     input: unknown;
     annotations?: unknown;
     output: unknown;
     error?: string;
+    /**
+     * Structured cause of a `failed` item — see {@link EvalItemFailure}. Absent
+     * on every other outcome, on pre-0.24 artifacts, and when the thrown value
+     * carried no `name` (a thrown string, say).
+     */
+    failure?: EvalItemFailure;
     scorerErrors?: string[];
     scores: Record<string, number | null>;
     duration?: number;
+    /** Compatibility view of `accounting.breakdown.generation` — this item's
+     *  measured workflow spend. On a pre-0.24 artifact this is whatever the
+     *  caller reported instead. */
     cost?: number;
     /** True when `cost` is a lower bound because this item used unpriced work. */
     unpriced?: boolean;
+    /** Compatibility view of `accounting.breakdown.judging` — this item's judges. */
     scorerCost?: number;
     scoreDetails?: Record<string, ScorerDetail>;
-    /** Tracked metadata merged with user keys (user wins; list-only overrides clear paired counts). */
+    /** How this item ended. Absent only on pre-0.24 artifacts. */
+    outcome?: EvalItemOutcome;
+    /** Generation AND judging spend for this item; `breakdown` splits them.
+     *  Absent on pre-0.24 artifacts. */
+    accounting?: Accounting;
+    /**
+     * What the `executeWorkflow` callback claimed, kept for inspection and never
+     * used as a measurement. `cost` is the callback's own `cost` return;
+     * `metadata` holds the reserved keys (`models`, `modelCallCounts`,
+     * `workflows`, `workflowCallCounts`, `tokens`, `agentCalls`) that used to
+     * overwrite the measured ones. Non-reserved caller keys still merge into
+     * `metadata` below.
+     */
+    callerReport?: {
+        cost?: number;
+        metadata?: Record<string, unknown>;
+    };
+    /** Tracked metadata merged with the caller's NON-reserved keys (caller wins
+     *  on those). Reserved keys stay measured — see {@link EvalItem.callerReport}. */
     metadata?: Record<string, unknown>;
     /** Per-model provider-call latency for this item, rolled up from
      *  `agent_call_end.timing`. Absent when the item made no timed provider call
@@ -389,6 +728,11 @@ type EvalItem = {
      *  sums and per-model totals from `item.timing`, but not the distribution.
      *  (`rescore` already reports no timing stats at all, so nothing regresses.) */
     timing?: Record<string, ItemModelTiming>;
+    /** The captured operations this item's workflow performed. Present only under
+     *  `captureRequests`; references only, resolved through the run's artifact. */
+    diagnostics?: {
+        operations: OperationRef[];
+    };
     /** Trace events captured during this item's execution. Only populated when
      *  `runEval` was called with `{ captureTraces: true }`. Verbose-mode
      *  `agent_call_start.data.messages` snapshots are stripped to keep memory bounded;
@@ -397,7 +741,18 @@ type EvalItem = {
 };
 type EvalSummary = {
     count: number;
+    /**
+     * Items carrying an `error` string. UNCHANGED legacy meaning — it therefore
+     * still counts budget-stopped and cancelled items, which also carry one. Read
+     * {@link EvalSummary.coverage} to tell a model failure from a budget stop.
+     */
     failures: number;
+    /**
+     * Per-outcome counts for items and for each scorer. Every key of the outcome
+     * unions is present, including zeros, so a consumer can render "0 skipped"
+     * without inferring it from an absent key. Absent on pre-0.24 artifacts.
+     */
+    coverage?: EvalCoverage;
     scorers: Record<string, {
         mean: number;
         min: number;
@@ -448,6 +803,15 @@ type EvalSummary = {
      * tripped (either not configured or all scorers within tolerance).
      */
     degraded?: DegradedScorer[];
+    /**
+     * The item error rate against the run's `failOnItemErrorRate` limit.
+     * Present exactly when at least one item `failed` (so a clean run's summary
+     * is unchanged); `exceeded` says whether the gate tripped. Like `degraded`,
+     * `runEval` never throws on it — the CLI and consumers decide the exit code.
+     * Absent on rescores (the source run owns item failures) and on pre-0.24
+     * artifacts.
+     */
+    itemErrorRate?: ItemErrorRate;
 };
 /**
  * Sample-size context for one side of a comparison.
@@ -548,7 +912,20 @@ type EvalComparison = {
         baselineTotal: number;
         candidateTotal: number;
         delta: number;
-        deltaPercent: number;
+        /** `null` when the baseline total is 0 — a percentage change from zero is
+         *  not a number, and reporting `0` or `Infinity` misleads. */
+        deltaPercent: number | null;
+        /**
+         * `true` only when the two sides are comparable as SPEND: both accountings
+         * `complete`, the same `scope`, and the same case/scorer coverage. A
+         * partial or budget-truncated run is cheaper because it did less work, so
+         * without this an incomplete candidate reads as a saving.
+         *
+         * Raw totals are reported either way, and quality comparison is unaffected.
+         */
+        certified: boolean;
+        /** Why certification was refused. Present exactly when `certified` is false. */
+        reason?: string;
     };
     regressions: EvalRegression[];
     improvements: EvalImprovement[];
@@ -607,6 +984,22 @@ type RunEvalOptions = {
      * payload, subscribe to `runtime.on('trace', ...)` directly.
      */
     captureTraces?: boolean;
+    /**
+     * Capture the provider-neutral REQUESTS this run submits into a diagnostic
+     * artifact, so a failed case can be inspected after the process exits.
+     *
+     * Off by default and deliberately explicit to turn on: it requires
+     * `config.diagnostics.artifacts` on the runtime, and without it the run fails
+     * fast with `AxlError('DIAGNOSTICS_UNAVAILABLE')` BEFORE the dataset is
+     * loaded — a capture misconfiguration must never be discovered after the run
+     * has already spent money.
+     *
+     * Capture is diagnostics, never measurement: `accounting` is byte-identical
+     * with capture on, off, truncated, or failing outright. Pass an object to
+     * override the byte bounds (defaults 256 KiB per record, 16 MiB per run,
+     * 1 MiB pending queue).
+     */
+    captureRequests?: CaptureRequestsOption;
 };
 
 declare function defineEval(config: EvalConfig): EvalConfig;
@@ -638,6 +1031,17 @@ declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: Eva
  * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).
  */
 declare function evaluateScorerErrorRateGate(comparison: EvalComparison, maxScorerErrorRate: number): string | null;
+/**
+ * Decide whether a comparison must be REFUSED because a side lost too many
+ * items (`axl-eval compare`, default-on at `0.05`, overridden by
+ * `--max-item-error-rate`; `1` disables it). Pure and testable — returns a
+ * human-readable refusal reason naming coverage and the side, or `null` to
+ * allow. See {@link compareItemErrorRates} for the per-run rule.
+ *
+ * Throws on an invalid limit: the gate is default-on, so a bad value must not
+ * quietly disable it.
+ */
+declare function evaluateItemErrorRateGate(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], maxItemErrorRate?: number): string | null;
 
 /** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */
 type ScorerToleranceVerdict = {
@@ -696,10 +1100,50 @@ type RescoreOptions = {
      *  be cancelled mid-flight. Also checked between items to short-circuit
      *  remaining work. Mirrors `RunEvalOptions.signal`. */
     signal?: AbortSignal;
+    /**
+     * Known-spend threshold for the NEW judging work only, e.g. `'$0.10'`.
+     *
+     * The source run's spend is history: it is recorded under
+     * `accounting.source.generation` and is NOT counted toward this limit, so a
+     * rescore budget means "spend at most this much re-judging", which is the
+     * only question a rescore can answer. Same validation and `>=` semantics as
+     * `EvalConfig.budget`.
+     */
+    budget?: string;
+    /**
+     * Carry the source run's captured requests forward into this rescore, and
+     * capture the judge calls this rescore makes.
+     *
+     * The source artifact is COPIED rather than referenced. Reference counting
+     * would make deleting the original run either impossible or silently
+     * destructive to every rescore of it; copying costs bytes once and makes each
+     * result independently deletable. The copy preserves the ORIGINAL operation
+     * ids as provenance, so a reader can still line a record up against the run
+     * that produced it — even after that run is gone.
+     *
+     * Both halves land in ONE artifact and share ONE `maxRunBytes` budget: the
+     * copied bytes count against it, so the artifact never grows to twice the
+     * bound the caller asked for. A copy that would exceed it is partial and the
+     * manifest says `truncated`; a source artifact that no longer exists, or a
+     * copy that fails, yields `unavailable` with `artifactId: ''` — never the
+     * source's id, which this result does not own. None of it prevents the
+     * numeric rescore results from being read.
+     */
+    captureRequests?: CaptureRequestsOption;
 };
 /**
  * Re-run scorers on the saved outputs of an existing eval result.
  * Preserves original input/output/annotations. Only re-runs scoring.
+ *
+ * ## Accounting
+ *
+ * A rescore opens its OWN scope (`accounting.scope === 'rescore'`) covering
+ * only the judging it performs. The original run's generation spend is never
+ * added to the new total — a rescore that reported the sum would make a $0.20
+ * re-judge of a $0.75 run look like $0.95 of new spend. Instead the source is
+ * recorded verbatim under `accounting.source`, so a reader can add them up
+ * deliberately, and `source.generation` is `null` when the source artifact
+ * predates accounting (never synthesized from `totalCost`).
  */
 declare function rescore(result: EvalResult, scorers: Scorer[], runtime: AxlRuntime, options?: RescoreOptions): Promise<EvalResult>;
 
@@ -714,7 +1158,16 @@ type MultiRunSummary = {
      */
     workflows: string[];
     dataset: string;
+    /** Compatibility view: the sum of each run's `totalCost`. */
     totalCost: number;
+    /**
+     * The union of every run's accounting, folded conservatively: known spend and
+     * operation counts sum, incompleteness reasons union, and the group takes the
+     * WORST completeness in it — one legacy run makes the whole group
+     * `'unverified'`, one incomplete run makes it `'incomplete'`. Inheriting
+     * run[0]'s flags would report a contaminated group as clean.
+     */
+    accounting: Accounting;
     totalDuration: number;
     scorers: Record<string, {
         mean: number;
@@ -739,4 +1192,87 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemModelTiming, type LlmScorerConfig, type ModelTimingStats, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+/**
+ * Eval-side accounting helpers.
+ *
+ * The core owns measurement (`@axlsdk/axl`'s `Accounting`, `trackOutcome`,
+ * `AdmissionController`). This module owns the three things eval adds on top:
+ *
+ * - **Scope plumbing** — {@link trackScope} opens a core accounting scope when
+ *   the runtime has one, and produces an explicitly `uninstrumented` record
+ *   when it does not. An eval driven by a hand-rolled runtime therefore reports
+ *   "we measured nothing", never a confident `$0`.
+ * - **Reading** — {@link readAccounting} turns any `EvalResult`, including a
+ *   pre-0.24 artifact with no `accounting` at all, into an `EvalAccounting`.
+ *   An absent record reads `'unverified'` and is NEVER upgraded to `'complete'`.
+ * - **Union** — {@link aggregateAccounting} folds several records
+ *   conservatively: sums of known spend, unions of reasons, and the WORST
+ *   completeness of any input.
+ *
+ * Everything here is pure except `trackScope`, which only delegates.
+ */
+
+/**
+ * Read the accounting of any `EvalResult`, including one written before this
+ * field existed.
+ *
+ * A legacy artifact reads `'unverified'`: its `totalCost` is repeated as
+ * `knownCost` for continuity, but no operation counts, no usage and no
+ * generation/judging split are invented, and it is never reported as
+ * `'complete'`. Turning an absent completeness into `'complete'` is exactly the
+ * bug that would let a legacy run certify a cost comparison.
+ *
+ * **The structural identities do not hold on a synthesized record.** For a live
+ * record (`'complete'` or `'incomplete'`) `provenance` sums to `knownCost` and
+ * `breakdown` splits it. A legacy artifact reports only a single total, so the
+ * synthesized record carries `knownCost` with an all-zero `breakdown` and
+ * `provenance` — inventing a generation/judging split from a number of unknown
+ * origin would be a fabrication. Consumers that render a breakdown must treat
+ * `'unverified'` as "no split available", not as "the split is zero"; the
+ * completeness field is what tells them which case they are in.
+ */
+declare function readAccounting(result: EvalResult): EvalAccounting;
+/**
+ * How much work the budget actually refused: cases never started or stopped
+ * mid-flight, plus judges that never ran for the same reason.
+ *
+ * This is the quantity that makes a run short. Spend alone does not: a run can
+ * settle exactly on its limit having refused nothing. Missing coverage (a
+ * pre-0.24 artifact) reads `0` — an artifact that never recorded outcomes
+ * cannot be used to assert that work was refused — and so do missing or
+ * negative counts inside a malformed block.
+ */
+declare function refusedWork(coverage: EvalCoverage | undefined): number;
+/**
+ * `true` when a run's budget both closed AND refused work — the one reading
+ * that entitles a consumer to call the run truncated.
+ *
+ * A closed controller alone is the NORMAL end state of a run whose final
+ * settlement lands exactly on its limit, which is exactly how `--budget` is
+ * used as a CI threshold. Reporting that as a stop fails a perfect run and
+ * prints a self-contradicting message ("0 never started, 0 stopped mid-flight,
+ * 2 completed … incomplete by design").
+ *
+ * The argument is structural rather than an `EvalResult` because the three
+ * consumers hold different things: the CLI has the result, the Studio server
+ * has a persisted blob it parses itself, and the Studio browser mirror has a
+ * client-side type. All three can produce `{ budget, coverage }`.
+ */
+declare function isBudgetStopped(summary: {
+    budget?: {
+        status?: string;
+    } | undefined;
+    coverage?: EvalCoverage | undefined;
+}): boolean;
+/**
+ * Fold several accounting records into one, conservatively.
+ *
+ * Numbers sum; reasons union with their counts; completeness takes the WORST
+ * of the inputs (`unverified` > `incomplete` > `complete`), so a group
+ * containing one legacy run is `unverified` as a whole and one incomplete run
+ * makes the group incomplete. Inheriting the first input's flags — the bug this
+ * exists to prevent — would report a group as complete because run 1 was.
+ */
+declare function aggregateAccounting(inputs: readonly Accounting[]): Accounting;
+
+export { type BootstrapCIResult, type CaptureRequestsOption, DEFAULT_COPY_MAX_BYTES, DEFAULT_SIDECAR_MAX_BYTES, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type DiagnosticManifest, type EvalAccounting, type EvalBudgetStatus, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalCoverage, type EvalImprovement, type EvalItem, type EvalItemFailure, type EvalItemOutcome, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemErrorRate, type ItemModelTiming, type LlmScorerConfig, type ModelTimingStats, type MultiRunSummary, type OperationRef, type RequestRecord, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerOutcome, type ScorerResult, type ScorerToleranceVerdict, type SidecarValidation, aggregateAccounting, aggregateRuns, dataset, defineEval, evalCompare, evaluateItemErrorRateGate, evaluateScorerErrorRateGate, evaluateScorerTolerance, isBudgetStopped, llmScorer, normalizeScorerResult, pairedBootstrapCI, parseRequestRecords, readAccounting, refusedWork, rescore, resolveCaptureLimits, runEval, scorer, serializeRequestRecords, toDiagnosticManifest, unavailableManifest, validateRequestSidecar };