modelrig 0.0.1 → 0.2.0

package/dist/bakeoff/engine.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Bake-off engine — Phase 3 spec WS5 (closed core). Replays the last N
+ * captures through each named variant, aggregates per-variant metrics
+ * (stats.ts), and assembles the BakeoffArtifact with the proposal verdict.
+ *
+ * Proposal gate (builder decision, flagged): the incumbent is the "default"
+ * variant when compared, else the FIRST listed variant. A challenger
+ * proposes when (a) its conformance CI lower bound ≥ the incumbent's CI
+ * lower bound, and (b) its effective $/1K-conformant is strictly lower.
+ * Value gates beyond mechanical conformance arrive with Phase 4 judges —
+ * the artifact records value_accuracy where the hook supplies it.
+ */
+import type { CaptureStore } from "../telemetry/capture";
+import type { RunDeps } from "../run/run";
+import type { ArmSamples, KeepOutputsMode } from "./samples";
+import type { BakeoffArtifact, VariantMetrics } from "./types";
+/** Sink for retained review samples (addendum A3). The engine assembles the
+ * arms to persist (incumbent + winner, or all); the caller — createRig with
+ * the service-role config — does the truncation + PostgREST insert. Keeps
+ * payloads out of the engine and out of the (anon-readable) BakeoffArtifact. */
+export type BakeoffSamplesSink = (input: {
+    readonly bakeoffId: string;
+    readonly route: string;
+    readonly arms: readonly ArmSamples[];
+}) => Promise<void>;
+export interface BakeoffRequest {
+    readonly route: string;
+    /** Variant names to compare; "default" = the route's base config. */
+    readonly variants: readonly string[];
+    readonly replayLastN: number;
+    /** Hard cap PER VARIANT replay pass (spec: replay is envelope-guarded). */
+    readonly envelopeUsdPerVariant: number;
+    /** Addendum A3: retain outputs for a value review. "winner" = incumbent +
+     * winning challenger; "all" = every arm. Undefined = keep nothing (the
+     * default pure-metrics path). Requires a samplesSink to persist. */
+    readonly keepOutputs?: KeepOutputsMode;
+}
+export declare function runBakeoff(deps: RunDeps, captures: CaptureStore, req: BakeoffRequest, samplesSink?: BakeoffSamplesSink): Promise<BakeoffArtifact>;
+/** Winner among gate-passing challengers; null when nobody beats the
+ * incumbent. Exported for direct unit testing. */
+export declare function computeProposal(requested: readonly string[], metrics: readonly VariantMetrics[]): BakeoffArtifact["proposal"];
+export interface BakeoffMirrorConfig {
+    readonly supabaseUrl: string;
+    readonly serviceRoleKey: string;
+    readonly sourceId: string;
+    readonly fetchFn?: typeof fetch;
+}
+/** Upsert the artifact into the `bakeoffs` mirror table (console read path).
+ * Same PostgREST transport pattern as the telemetry exporter; captures have
+ * no cloud surface — ONLY aggregate metrics travel. */
+export declare function mirrorBakeoff(artifact: BakeoffArtifact, config: BakeoffMirrorConfig): Promise<void>;
+//# sourceMappingURL=engine.d.ts.map

package/dist/bakeoff/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAO1C,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE/D;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;CACxC;AAED,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CA2E1B;AAED;kDACkD;AAClD,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,GACjC,eAAe,CAAC,UAAU,CAAC,CA4B7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf"}

package/dist/bakeoff/engine.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Bake-off engine — Phase 3 spec WS5 (closed core). Replays the last N
+ * captures through each named variant, aggregates per-variant metrics
+ * (stats.ts), and assembles the BakeoffArtifact with the proposal verdict.
+ *
+ * Proposal gate (builder decision, flagged): the incumbent is the "default"
+ * variant when compared, else the FIRST listed variant. A challenger
+ * proposes when (a) its conformance CI lower bound ≥ the incumbent's CI
+ * lower bound, and (b) its effective $/1K-conformant is strictly lower.
+ * Value gates beyond mechanical conformance arrive with Phase 4 judges —
+ * the artifact records value_accuracy where the hook supplies it.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runBakeoff = runBakeoff;
+exports.computeProposal = computeProposal;
+exports.mirrorBakeoff = mirrorBakeoff;
+const node_crypto_1 = require("node:crypto");
+const loader_1 = require("../routes/loader");
+const replay_1 = require("./replay");
+const stats_1 = require("./stats");
+const samples_1 = require("./samples");
+/** Arm metadata for the reviewer: which model/rung/scaffold served it. */
+function deriveArmMeta(route, arm) {
+    if (arm === "default") {
+        return { model: route.candidates[0]?.model ?? null, rung: route.policy.json };
+    }
+    const variant = route.variants.find((v) => v.name === arm);
+    const model = (variant?.candidates?.[0] ?? route.candidates[0])?.model ?? null;
+    return {
+        model,
+        rung: variant?.json ?? route.policy.json,
+        ...(variant?.scaffold !== undefined ? { scaffold: true } : {}),
+    };
+}
+async function runBakeoff(deps, captures, req, samplesSink) {
+    if (req.variants.length === 0) {
+        throw new loader_1.RouteConfigError("bakeoff needs at least one variant (use \"default\" for the base config)");
+    }
+    const route = deps.routes.get(req.route);
+    for (const name of req.variants) {
+        if (name !== "default" && !route.variants.some((v) => v.name === name)) {
+            throw new loader_1.RouteConfigError(`route "${req.route}" has no variant "${name}" — defined: [default, ${route.variants
+                .map((v) => v.name)
+                .join(", ")}]`);
+        }
+    }
+    const keepOutputs = req.keepOutputs !== undefined && samplesSink !== undefined;
+    const bakeoffId = (0, node_crypto_1.randomUUID)();
+    const metrics = [];
+    const rawByArm = new Map();
+    let replayN = 0;
+    for (const variant of req.variants) {
+        const samples = await (0, replay_1.replayVariant)(deps, captures, {
+            route: req.route,
+            variant,
+            lastN: req.replayLastN,
+            envelopeUsd: req.envelopeUsdPerVariant,
+            ...(keepOutputs ? { keepOutputs: true } : {}),
+        });
+        replayN = Math.max(replayN, samples.length);
+        metrics.push((0, stats_1.aggregateVariantMetrics)(variant, samples));
+        if (keepOutputs)
+            rawByArm.set(variant, samples);
+    }
+    const proposal = computeProposal(req.variants, metrics);
+    // Persist review samples (A3) — AFTER the winner is known, for the selected
+    // arms only. Never touches the artifact (payloads stay off the anon table).
+    if (keepOutputs && samplesSink !== undefined) {
+        const incumbentArm = req.variants.includes("default") ? "default" : req.variants[0];
+        const keptArms = (0, samples_1.selectArmsToKeep)(req.keepOutputs, incumbentArm, proposal?.variant ?? null, req.variants);
+        const arms = keptArms.map((arm) => ({
+            arm,
+            armMeta: deriveArmMeta(route, arm),
+            samples: (rawByArm.get(arm) ?? []).map((s) => ({
+                inputRef: s.captureId,
+                inputText: s.inputText ?? "",
+                outputText: s.outputText ?? null,
+                conform: s.conform,
+                costUsd: s.costUsd,
+                latencyMs: s.latencyMs,
+            })),
+        }));
+        // Best-effort: a hiccup persisting the SECONDARY review samples must not
+        // lose the PRIMARY metrics artifact of an expensive bake-off. Warn and
+        // continue; the artifact still returns, mirrors, and writes to disk.
+        try {
+            await samplesSink({ bakeoffId, route: route.route, arms });
+        }
+        catch (err) {
+            console.warn(`[modelrig] bake-off ${bakeoffId}: review samples not persisted ` +
+                `(${err instanceof Error ? err.message : String(err)}) — metrics unaffected; ` +
+                "re-run with --keep-outputs to retry the samples.");
+        }
+    }
+    return {
+        id: bakeoffId,
+        route: route.route,
+        routeVersion: route.version,
+        ranAt: new Date().toISOString(),
+        replayN,
+        variants: metrics,
+        proposal,
+    };
+}
+/** Winner among gate-passing challengers; null when nobody beats the
+ * incumbent. Exported for direct unit testing. */
+function computeProposal(requested, metrics) {
+    const incumbentName = requested.includes("default") ? "default" : requested[0];
+    const incumbent = metrics.find((m) => m.variant === incumbentName);
+    // A1: unpriced cost columns under-count — an unpriced incumbent can't be
+    // compared against, and an unpriced challenger must never look cheap.
+    if (incumbent === undefined || incumbent.samples === 0 || incumbent.pricingMissing)
+        return null;
+    const challengers = metrics.filter((m) => m.variant !== incumbentName &&
+        m.samples > 0 &&
+        !m.pricingMissing &&
+        m.conformCi95[0] >= incumbent.conformCi95[0] &&
+        m.effectiveUsdPer1kConformant < incumbent.effectiveUsdPer1kConformant);
+    if (challengers.length === 0)
+        return null;
+    const winner = [...challengers].sort((a, b) => a.effectiveUsdPer1kConformant - b.effectiveUsdPer1kConformant)[0];
+    const savings = incumbent.effectiveUsdPer1kConformant === 0 ||
+        !Number.isFinite(incumbent.effectiveUsdPer1kConformant)
+        ? 0
+        : ((incumbent.effectiveUsdPer1kConformant - winner.effectiveUsdPer1kConformant) /
+            incumbent.effectiveUsdPer1kConformant) *
+            100;
+    return { variant: winner.variant, vsIncumbentSavingsPct: savings };
+}
+/** Upsert the artifact into the `bakeoffs` mirror table (console read path).
+ * Same PostgREST transport pattern as the telemetry exporter; captures have
+ * no cloud surface — ONLY aggregate metrics travel. */
+async function mirrorBakeoff(artifact, config) {
+    const fetchFn = config.fetchFn ?? fetch;
+    const base = config.supabaseUrl.replace(/\/$/, "");
+    const response = await fetchFn(`${base}/rest/v1/bakeoffs`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/json",
+            apikey: config.serviceRoleKey,
+            authorization: `Bearer ${config.serviceRoleKey}`,
+            prefer: "resolution=merge-duplicates,return=minimal",
+        },
+        body: JSON.stringify([
+            {
+                id: artifact.id,
+                route: artifact.route,
+                route_version: artifact.routeVersion,
+                ran_at: artifact.ranAt,
+                variants: artifact.variants.map((m) => m.variant),
+                // JSON has no Infinity: a variant with zero conformant outputs
+                // serializes effectiveUsdPer1kConformant as null (console renders ∞).
+                matrix: JSON.parse(JSON.stringify(artifact.variants)),
+                replay_n: artifact.replayN,
+                source_id: config.sourceId,
+            },
+        ]),
+    });
+    if (!response.ok) {
+        throw new Error(`bakeoff mirror upsert → ${response.status} ${await response.text().catch(() => "")}`);
+    }
+}

package/dist/bakeoff/eval-cases.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Bake-off input selection from eval cases — Expansion E4 chunk A (spec §3).
+ *
+ * `bakeoff --from-eval-cases <task>` replays the promoted regression/golden cases
+ * for a task instead of the last-N captures. A case only holds METADATA (a source
+ * artifact id); the INPUT to replay lives in a LOCAL capture, reachable through
+ * the chain:
+ *
+ *   eval_case → source artifact (run_id, step_key) → step's inference_id → capture
+ *
+ * Any link may be missing — the source artifact predates local retention, the
+ * route never had `capture: true`, the capture was evicted. When it is, the case
+ * is UNREHYDRATABLE. The load-bearing doctrine (spec §3.3): the bake-off runs on
+ * the rehydratable subset and NAMES every refused case with a reason — NEVER a
+ * silent subset. `selectEvalCaseCaptures` returns both halves so the CLI can
+ * print the honest-refusal report before it spends a cent.
+ */
+import type { CaptureRow, CaptureStore } from "../telemetry/capture";
+import type { ArtifactStore } from "../telemetry/artifact-store";
+import type { ExportableTelemetryStore } from "../telemetry/store";
+import type { EvalCaseRow } from "../types/telemetry";
+import type { RunDeps } from "../run/run";
+import { type BakeoffSamplesSink } from "./engine";
+import type { KeepOutputsMode } from "./samples";
+import type { BakeoffArtifact } from "./types";
+/** The narrow store surface the selection reads — kept minimal so this module
+ * depends on four reads, not three whole stores. */
+export interface EvalCaseSelectionDeps {
+    readonly readEvalCasesByTask: (task: string) => EvalCaseRow[];
+    readonly getArtifactById: (id: string) => {
+        runId: string | null;
+        stepKey: string | null;
+    } | null;
+    readonly getStepInferenceId: (runId: string, stepKey: string) => string | null;
+    readonly getCaptureByInferenceId: (inferenceId: string) => CaptureRow | null;
+}
+export interface RehydratedCase {
+    readonly caseId: string;
+    readonly capture: CaptureRow;
+}
+export interface EvalCaseRefusal {
+    readonly caseId: string;
+    readonly sourceArtifactId: string;
+    /** Which link of the chain broke — the honest, actionable reason. */
+    readonly reason: string;
+}
+export interface EvalCaseSelection {
+    readonly task: string;
+    /** Total eval cases found for the task (rehydratable + refused). */
+    readonly total: number;
+    readonly rehydratable: readonly RehydratedCase[];
+    readonly refusals: readonly EvalCaseRefusal[];
+}
+/**
+ * Partition a task's eval cases into rehydratable (with their captured input) and
+ * refused (named, with a reason). Pure over the injected reads — no network, no
+ * spend; the CLI runs this first and reports before replaying.
+ */
+export declare function selectEvalCaseCaptures(deps: EvalCaseSelectionDeps, task: string): EvalCaseSelection;
+/**
+ * A read-only in-memory CaptureStore over a fixed set of rehydrated captures, so
+ * the EXISTING bake-off engine (which reads `captures.readLast(route, n)`) runs
+ * over the eval-case selection with NO second actuation path (spec §3.3). Only
+ * `readLast` is meaningful; `getByInferenceId` is supported for symmetry; writes
+ * throw (this store is a replay lens, never a sink).
+ */
+export declare function evalCaseCaptureStore(rows: readonly RehydratedCase[]): CaptureStore;
+export interface EvalCaseBakeoffResult {
+    readonly selection: EvalCaseSelection;
+    /** The bake-off artifact over the rehydratable subset, or null when NOTHING
+     * was rehydratable (the CLI then reports the refusals and exits without spend). */
+    readonly artifact: BakeoffArtifact | null;
+}
+/**
+ * `bakeoff --from-eval-cases <task>` end to end: select rehydratable cases, then
+ * run the EXISTING bake-off engine over exactly those captured inputs (no second
+ * actuation path — spec §3.3). Replays every rehydratable case (replayLastN =
+ * the rehydratable count). Returns the selection (so the caller reports refusals)
+ * alongside the artifact.
+ */
+/** The rig-side store bundle the orchestrator adapts into `EvalCaseSelectionDeps`
+ * — passed straight from `createRig` so `index.ts` stays a one-line delegate.
+ * `artifacts` is null when the namespace is off (nothing to rehydrate); `captures`
+ * is a thunk because the rig opens that handle lazily. */
+export interface EvalCaseStores {
+    readonly artifacts: Pick<ArtifactStore, "readEvalCasesByTask" | "getArtifactById"> | null;
+    readonly telemetry: Pick<ExportableTelemetryStore, "getStepInferenceId">;
+    readonly captures: () => Pick<CaptureStore, "getByInferenceId">;
+}
+export declare function runBakeoffFromEvalCases(deps: RunDeps, stores: EvalCaseStores, req: {
+    readonly task: string;
+    readonly variants: readonly string[];
+    readonly envelopeUsdPerVariant: number;
+    readonly keepOutputs?: KeepOutputsMode;
+}, samplesSink?: BakeoffSamplesSink): Promise<EvalCaseBakeoffResult>;
+//# sourceMappingURL=eval-cases.d.ts.map

package/dist/bakeoff/eval-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAgCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;CACjE;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;CACxC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CA6BhC"}

package/dist/bakeoff/eval-cases.js

@@ -0,0 +1,105 @@
+"use strict";
+/**
+ * Bake-off input selection from eval cases — Expansion E4 chunk A (spec §3).
+ *
+ * `bakeoff --from-eval-cases <task>` replays the promoted regression/golden cases
+ * for a task instead of the last-N captures. A case only holds METADATA (a source
+ * artifact id); the INPUT to replay lives in a LOCAL capture, reachable through
+ * the chain:
+ *
+ *   eval_case → source artifact (run_id, step_key) → step's inference_id → capture
+ *
+ * Any link may be missing — the source artifact predates local retention, the
+ * route never had `capture: true`, the capture was evicted. When it is, the case
+ * is UNREHYDRATABLE. The load-bearing doctrine (spec §3.3): the bake-off runs on
+ * the rehydratable subset and NAMES every refused case with a reason — NEVER a
+ * silent subset. `selectEvalCaseCaptures` returns both halves so the CLI can
+ * print the honest-refusal report before it spends a cent.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.selectEvalCaseCaptures = selectEvalCaseCaptures;
+exports.evalCaseCaptureStore = evalCaseCaptureStore;
+exports.runBakeoffFromEvalCases = runBakeoffFromEvalCases;
+const engine_1 = require("./engine");
+/**
+ * Partition a task's eval cases into rehydratable (with their captured input) and
+ * refused (named, with a reason). Pure over the injected reads — no network, no
+ * spend; the CLI runs this first and reports before replaying.
+ */
+function selectEvalCaseCaptures(deps, task) {
+    const cases = deps.readEvalCasesByTask(task);
+    const rehydratable = [];
+    const refusals = [];
+    for (const c of cases) {
+        const refuse = (reason) => {
+            refusals.push({ caseId: c.id, sourceArtifactId: c.sourceArtifactId, reason });
+        };
+        const art = deps.getArtifactById(c.sourceArtifactId);
+        if (art === null) {
+            refuse("source artifact is not in the local store (promoted on another deployment, or evicted)");
+            continue;
+        }
+        if (art.runId === null || art.stepKey === null) {
+            refuse("source artifact was saved outside a step — no inference to key its input");
+            continue;
+        }
+        const inferenceId = deps.getStepInferenceId(art.runId, art.stepKey);
+        if (inferenceId === null) {
+            refuse("no ground-truth inference recorded for the source step");
+            continue;
+        }
+        const capture = deps.getCaptureByInferenceId(inferenceId);
+        if (capture === null) {
+            refuse("input not captured locally (route `capture` was off, or the capture was evicted)");
+            continue;
+        }
+        rehydratable.push({ caseId: c.id, capture });
+    }
+    return { task, total: cases.length, rehydratable, refusals };
+}
+/**
+ * A read-only in-memory CaptureStore over a fixed set of rehydrated captures, so
+ * the EXISTING bake-off engine (which reads `captures.readLast(route, n)`) runs
+ * over the eval-case selection with NO second actuation path (spec §3.3). Only
+ * `readLast` is meaningful; `getByInferenceId` is supported for symmetry; writes
+ * throw (this store is a replay lens, never a sink).
+ */
+function evalCaseCaptureStore(rows) {
+    const captures = rows.map((r) => r.capture);
+    return {
+        record() {
+            throw new Error("evalCaseCaptureStore is read-only (a replay lens, not a sink)");
+        },
+        readLast(route, n) {
+            return captures.filter((c) => c.route === route).slice(0, n);
+        },
+        getByInferenceId(inferenceId) {
+            return captures.find((c) => c.inferenceId === inferenceId) ?? null;
+        },
+        close() { },
+    };
+}
+async function runBakeoffFromEvalCases(deps, stores, req, samplesSink) {
+    const empty = { task: req.task, total: 0, rehydratable: [], refusals: [] };
+    // No artifact namespace ⇒ nothing to rehydrate; honest empty selection.
+    if (stores.artifacts === null)
+        return { selection: empty, artifact: null };
+    const arts = stores.artifacts;
+    const caps = stores.captures();
+    const selection = selectEvalCaseCaptures({
+        readEvalCasesByTask: (t) => arts.readEvalCasesByTask(t),
+        getArtifactById: (id) => arts.getArtifactById(id),
+        getStepInferenceId: (r, s) => stores.telemetry.getStepInferenceId(r, s),
+        getCaptureByInferenceId: (i) => caps.getByInferenceId(i),
+    }, req.task);
+    if (selection.rehydratable.length === 0)
+        return { selection, artifact: null };
+    const artifact = await (0, engine_1.runBakeoff)(deps, evalCaseCaptureStore(selection.rehydratable), {
+        route: req.task,
+        variants: req.variants,
+        replayLastN: selection.rehydratable.length, // replay every rehydratable case
+        envelopeUsdPerVariant: req.envelopeUsdPerVariant,
+        ...(req.keepOutputs !== undefined ? { keepOutputs: req.keepOutputs } : {}),
+    }, samplesSink);
+    return { selection, artifact };
+}

package/dist/bakeoff/replay.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Offline shadow replay — Phase 3 spec WS4/WS5. Re-runs locally captured
+ * inputs against candidate variants with ZERO production exposure. Replay
+ * dispatches are real (billed) adapter calls, but: envelope-guarded, tagged
+ * as replay (distinct from production telemetry), and never written to
+ * captures (no replay-of-replay).
+ *
+ * Builder decisions (flagged for architect review):
+ *  - Replay rows go into the SAME local store under the `replay:<id>` TAG
+ *    NAMESPACE (the scaffold's sanctioned option A): every replay attempt row
+ *    carries tags {replay, replay_capture, variant} — production queries
+ *    filter them out by tag, and per-sample cost is reconstructed from the
+ *    same rows (base + retries + repair + search — F1-correct).
+ *  - ReplaySampleResult gained `repairCostUsd` (additive) so the engine can
+ *    report repair cost separately from total cost.
+ *  - Envelope breach stops the pass typed-ly: the breaching sample is
+ *    recorded with failureClass budget_exhausted and the PARTIAL result set
+ *    is returned (honest Ns beat discarded spend).
+ */
+import type { CaptureStore } from "../telemetry/capture";
+import type { RunDeps } from "../run/run";
+export interface ReplayRequest {
+    readonly route: string;
+    readonly variant: string;
+    readonly lastN: number;
+    readonly envelopeUsd: number;
+    /** Addendum A3: retain the rendered input + this arm's output on each
+     * sample for the human value review. Off by default (bounds memory + keeps
+     * the pure-metrics path clean). */
+    readonly keepOutputs?: boolean;
+}
+export interface ReplaySampleResult {
+    readonly captureId: string;
+    readonly conform: boolean;
+    readonly repaired: boolean;
+    readonly valueAccuracy: number | null;
+    readonly costUsd: number;
+    /** Portion of costUsd spent on repair calls (repaired_by rows). */
+    readonly repairCostUsd: number;
+    /** Addendum A1: any attempt row lacked pricing — costUsd under-counts and
+     * this sample must never rank a variant as cheap. */
+    readonly pricingMissing: boolean;
+    readonly latencyMs: number;
+    readonly failureClass: string | null;
+    /** Addendum A3 (keepOutputs only): the replayed input, verbatim. */
+    readonly inputText?: string;
+    /** Addendum A3 (keepOutputs only): this arm's output for the review, or
+     * null when the arm produced none (a failure). */
+    readonly outputText?: string | null;
+}
+export declare function replayVariant(deps: RunDeps, captures: CaptureStore, req: ReplayRequest): Promise<readonly ReplaySampleResult[]>;
+//# sourceMappingURL=replay.d.ts.map

package/dist/bakeoff/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAsExC"}

package/dist/bakeoff/replay.js

@@ -0,0 +1,94 @@
+"use strict";
+/**
+ * Offline shadow replay — Phase 3 spec WS4/WS5. Re-runs locally captured
+ * inputs against candidate variants with ZERO production exposure. Replay
+ * dispatches are real (billed) adapter calls, but: envelope-guarded, tagged
+ * as replay (distinct from production telemetry), and never written to
+ * captures (no replay-of-replay).
+ *
+ * Builder decisions (flagged for architect review):
+ *  - Replay rows go into the SAME local store under the `replay:<id>` TAG
+ *    NAMESPACE (the scaffold's sanctioned option A): every replay attempt row
+ *    carries tags {replay, replay_capture, variant} — production queries
+ *    filter them out by tag, and per-sample cost is reconstructed from the
+ *    same rows (base + retries + repair + search — F1-correct).
+ *  - ReplaySampleResult gained `repairCostUsd` (additive) so the engine can
+ *    report repair cost separately from total cost.
+ *  - Envelope breach stops the pass typed-ly: the breaching sample is
+ *    recorded with failureClass budget_exhausted and the PARTIAL result set
+ *    is returned (honest Ns beat discarded spend).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayVariant = replayVariant;
+const node_crypto_1 = require("node:crypto");
+const failures_1 = require("../types/failures");
+const run_1 = require("../run/run");
+async function replayVariant(deps, captures, req) {
+    const rows = captures.readLast(req.route, req.lastN);
+    const replayId = (0, node_crypto_1.randomUUID)();
+    const envelopeKey = `replay:${replayId}`;
+    // Zero production exposure knobs: no capture writes (no replay-of-replay),
+    // and the whole pass shares one hard envelope.
+    const replayDeps = {
+        ...deps,
+        captures: null,
+        envelopeBudgetUsd: req.envelopeUsd,
+    };
+    const results = [];
+    for (const capture of rows) {
+        const sampleTag = `${replayId}:${capture.inferenceId}`;
+        const started = Date.now();
+        let conform = false;
+        let repaired = false;
+        let failureClass = null;
+        let outputText = null;
+        try {
+            // Through the REAL run loop: the variant's template re-renders against
+            // the replay candidate's capability flags (never the captured prompt
+            // text — plan §10 replay-validity risk), repair/grounding rungs and
+            // retry budgets all behave exactly as production serving would.
+            const result = await (0, run_1.run)(replayDeps, req.route, {
+                input: JSON.parse(capture.variablesJson),
+                tags: { replay: replayId, replay_capture: sampleTag, variant: req.variant },
+                ...(req.variant !== "default" ? { variant: req.variant } : {}),
+                budget: { envelope: envelopeKey },
+            });
+            conform = result.meta.validated;
+            repaired = result.meta.repairedBy !== null;
+            if (req.keepOutputs)
+                outputText = JSON.stringify(result.output, null, 2);
+        }
+        catch (err) {
+            if (!(err instanceof failures_1.RigFailureError))
+                throw err;
+            failureClass = err.failure.class;
+        }
+        const latencyMs = Date.now() - started;
+        // Per-sample cost from the tagged attempt rows — base + retries + repair
+        // + search, exactly what telemetry billed (F1-corrected).
+        const attemptRows = deps.telemetry.queryByTag("replay_capture", sampleTag);
+        const costUsd = attemptRows.reduce((a, r) => a + r.costEstimateUsd, 0);
+        const repairCostUsd = attemptRows
+            .filter((r) => r.repairedBy !== null)
+            .reduce((a, r) => a + r.costEstimateUsd, 0);
+        const pricingMissing = attemptRows.some((r) => r.pricingMissing);
+        results.push({
+            captureId: capture.inferenceId,
+            conform,
+            repaired,
+            valueAccuracy: null, // golden-set hook — expected values are Phase 4's judge/goldens work
+            costUsd,
+            repairCostUsd,
+            pricingMissing,
+            latencyMs,
+            failureClass,
+            ...(req.keepOutputs
+                ? { inputText: capture.variablesJson, outputText }
+                : {}),
+        });
+        if (failureClass === "budget_exhausted") {
+            break; // typed stop at the envelope; partial results stand
+        }
+    }
+    return results;
+}

package/dist/bakeoff/report.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Bake-off terminal reporting — the matrix printer (moved here from cli.ts for
+ * the LOC cap, P1) and the `--from-eval-cases` runner (E4 chunk A) with its
+ * HONEST-REFUSAL report: every eval case that could not be rehydrated is NAMED
+ * with a reason before any bake-off runs — never a silent subset (spec §3.3).
+ */
+import type { RigConfig } from "../config";
+import type { CacheSimAssumptions } from "./cache-sim";
+import type { KeepOutputsMode } from "./samples";
+import type { BakeoffArtifact } from "./types";
+/**
+ * One variant's measured-cold vs cache-adjusted input cost (cache-aware-routing
+ * spec §5.4). Built by the caller from `simulateCache` + the candidate's rates;
+ * this module only RENDERS it, so the report stays pure of pricing lookups.
+ */
+export interface CacheAdjustedRow {
+    readonly variant: string;
+    /** Input cost the way replay actually paid it — every token at list (§5.4). */
+    readonly measuredColdUsd: number;
+    /** Input cost at production cadence — reads/writes priced per policy (§5.4). */
+    readonly cacheAdjustedUsd: number;
+    readonly assumptions: CacheSimAssumptions;
+}
+/**
+ * Render the cache-adjusted section (spec §5.4) as lines — measured-cold AND
+ * cache-adjusted input cost side by side, with the simulator's assumptions
+ * PRINTED so the numbers are never mistaken for measured savings. Pure (returns
+ * strings; `printMatrix` does the console I/O) so it is unit-tested directly.
+ * Empty input ⇒ no lines (the section is omitted entirely — back-compat).
+ */
+export declare function formatCacheAdjustedSection(rows: readonly CacheAdjustedRow[]): string[];
+export declare function printMatrix(artifact: BakeoffArtifact, cacheAdjusted?: readonly CacheAdjustedRow[]): void;
+/**
+ * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's
+ * promoted eval cases through each variant. Prints the honest-refusal report,
+ * then (if anything rehydrated) runs the bake-off and persists/mirrors/prints it.
+ * Returns the process exit code: 0 on a run (or an empty suite), 1 when cases
+ * existed but NONE could be rehydrated (an actionable non-zero, not a crash).
+ */
+export declare function runEvalCaseBakeoffCli(config: RigConfig, task: string, variants: readonly string[], envelope: number | undefined, keep: KeepOutputsMode | undefined, asJson: boolean): Promise<number>;
+//# sourceMappingURL=report.d.ts.map

package/dist/bakeoff/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAM/D;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAgC5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,MAAM,CAAC,CA8CjB"}