modelrig 0.1.0 → 0.3.0

package/dist/bakeoff/report.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Bake-off terminal reporting — the matrix printer (moved here from cli.ts for
+ * the LOC cap, P1) and the `--from-eval-cases` runner (E4 chunk A) with its
+ * HONEST-REFUSAL report: every eval case that could not be rehydrated is NAMED
+ * with a reason before any bake-off runs — never a silent subset (spec §3.3).
+ */
+import type { RigConfig } from "../config";
+import type { CacheSimAssumptions } from "./cache-sim";
+import type { KeepOutputsMode } from "./samples";
+import type { BakeoffArtifact } from "./types";
+/**
+ * One variant's measured-cold vs cache-adjusted input cost (cache-aware-routing
+ * spec §5.4). Built by the caller from `simulateCache` + the candidate's rates;
+ * this module only RENDERS it, so the report stays pure of pricing lookups.
+ */
+export interface CacheAdjustedRow {
+    readonly variant: string;
+    /** Input cost the way replay actually paid it — every token at list (§5.4). */
+    readonly measuredColdUsd: number;
+    /** Input cost at production cadence — reads/writes priced per policy (§5.4). */
+    readonly cacheAdjustedUsd: number;
+    readonly assumptions: CacheSimAssumptions;
+}
+/**
+ * Render the cache-adjusted section (spec §5.4) as lines — measured-cold AND
+ * cache-adjusted input cost side by side, with the simulator's assumptions
+ * PRINTED so the numbers are never mistaken for measured savings. Pure (returns
+ * strings; `printMatrix` does the console I/O) so it is unit-tested directly.
+ * Empty input ⇒ no lines (the section is omitted entirely — back-compat).
+ */
+export declare function formatCacheAdjustedSection(rows: readonly CacheAdjustedRow[]): string[];
+export declare function printMatrix(artifact: BakeoffArtifact, cacheAdjusted?: readonly CacheAdjustedRow[]): void;
+/**
+ * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's
+ * promoted eval cases through each variant. Prints the honest-refusal report,
+ * then (if anything rehydrated) runs the bake-off and persists/mirrors/prints it.
+ * Returns the process exit code: 0 on a run (or an empty suite), 1 when cases
+ * existed but NONE could be rehydrated (an actionable non-zero, not a crash).
+ */
+export declare function runEvalCaseBakeoffCli(config: RigConfig, task: string, variants: readonly string[], envelope: number | undefined, keep: KeepOutputsMode | undefined, asJson: boolean): Promise<number>;
+//# sourceMappingURL=report.d.ts.map

package/dist/bakeoff/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAM/D;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAgC5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,MAAM,CAAC,CA8CjB"}

package/dist/bakeoff/report.js

@@ -0,0 +1,142 @@
+"use strict";
+/**
+ * Bake-off terminal reporting — the matrix printer (moved here from cli.ts for
+ * the LOC cap, P1) and the `--from-eval-cases` runner (E4 chunk A) with its
+ * HONEST-REFUSAL report: every eval case that could not be rehydrated is NAMED
+ * with a reason before any bake-off runs — never a silent subset (spec §3.3).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatCacheAdjustedSection = formatCacheAdjustedSection;
+exports.printMatrix = printMatrix;
+exports.runEvalCaseBakeoffCli = runEvalCaseBakeoffCli;
+const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
+const node_path_1 = require("node:path");
+const index_1 = require("../index");
+const engine_1 = require("./engine");
+function fmtEff(usd) {
+    return Number.isFinite(usd) ? `$${usd.toFixed(4)}` : "∞ (0 conformant)";
+}
+/**
+ * Render the cache-adjusted section (spec §5.4) as lines — measured-cold AND
+ * cache-adjusted input cost side by side, with the simulator's assumptions
+ * PRINTED so the numbers are never mistaken for measured savings. Pure (returns
+ * strings; `printMatrix` does the console I/O) so it is unit-tested directly.
+ * Empty input ⇒ no lines (the section is omitted entirely — back-compat).
+ */
+function formatCacheAdjustedSection(rows) {
+    if (rows.length === 0)
+        return [];
+    const lines = [
+        "cache-adjusted input cost (SIMULATED — not measured savings; the odometer stays measured-only):",
+    ];
+    const header = ["variant", "measured-cold", "cache-adjusted", "sim-savings"];
+    const body = rows.map((r) => {
+        const savings = r.measuredColdUsd > 0
+            ? `${(((r.measuredColdUsd - r.cacheAdjustedUsd) / r.measuredColdUsd) * 100).toFixed(1)}%`
+            : "n/a";
+        return [r.variant, `$${r.measuredColdUsd.toFixed(4)}`, `$${r.cacheAdjustedUsd.toFixed(4)}`, savings];
+    });
+    const widths = header.map((h, c) => Math.max(h.length, ...body.map((row) => row[c].length)));
+    const line = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join("  ");
+    lines.push(line(header));
+    for (const row of body)
+        lines.push(line(row));
+    // Assumptions basis — one line per distinct policy so a mixed-provider matrix
+    // is honest about which TTL/min-cacheable each arm assumed.
+    const seen = new Set();
+    for (const r of rows) {
+        const a = r.assumptions;
+        const key = `${a.provider}/${a.model}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        lines.push(`  assumptions ${key}: ${a.cacheSupported ? "" : "cache unsupported; "}ttl=${a.ttlSeconds}s` +
+            `${a.ttlSourced ? "" : " (fallback — undocumented)"}, min-cacheable=${a.minCacheableTokens}tok, ` +
+            `refresh-on-read=${a.refreshOnRead}`);
+    }
+    return lines;
+}
+function printMatrix(artifact, cacheAdjusted = []) {
+    console.log(`bakeoff ${artifact.id}\nroute ${artifact.route}@${artifact.routeVersion} · replayed ${artifact.replayN} captured inputs · ${artifact.ranAt}`);
+    const header = ["variant", "n", "conform", "ci95", "repair", "repair$", "lat(ms)", "eff-$/1K-conf"];
+    const rows = artifact.variants.map((m) => [
+        m.variant,
+        String(m.samples),
+        `${(m.conformRate * 100).toFixed(0)}%`,
+        `[${(m.conformCi95[0] * 100).toFixed(0)},${(m.conformCi95[1] * 100).toFixed(0)}]`,
+        `${(m.repairRate * 100).toFixed(0)}%`,
+        `$${m.meanRepairCostUsd.toFixed(4)}`,
+        m.meanLatencyMs.toFixed(0),
+        m.pricingMissing ? "UNPRICED (no ranking)" : fmtEff(m.effectiveUsdPer1kConformant),
+    ]);
+    const widths = header.map((h, c) => Math.max(h.length, ...rows.map((r) => r[c].length)));
+    const line = (cells) => cells.map((cell, c) => cell.padEnd(widths[c])).join("  ");
+    console.log(line(header));
+    for (const row of rows)
+        console.log(line(row));
+    if (artifact.proposal !== null) {
+        console.log(`PROPOSAL: variant "${artifact.proposal.variant}" beats the incumbent by ` +
+            `${artifact.proposal.vsIncumbentSavingsPct.toFixed(1)}% effective cost — human decision required; ` +
+            "nothing is executed.");
+    }
+    else {
+        console.log("proposal: none (no variant beats the incumbent's gates)");
+    }
+    // Cache-adjusted section (spec §5.4) — printed only when the caller supplied
+    // simulator rows; absent ⇒ the matrix output is byte-identical to pre-WS-B.
+    for (const l of formatCacheAdjustedSection(cacheAdjusted))
+        console.log(l);
+}
+/**
+ * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's
+ * promoted eval cases through each variant. Prints the honest-refusal report,
+ * then (if anything rehydrated) runs the bake-off and persists/mirrors/prints it.
+ * Returns the process exit code: 0 on a run (or an empty suite), 1 when cases
+ * existed but NONE could be rehydrated (an actionable non-zero, not a crash).
+ */
+async function runEvalCaseBakeoffCli(config, task, variants, envelope, keep, asJson) {
+    const mirrorTarget = config.export?.mode === "direct" ? config.export : undefined;
+    const rig = (0, index_1.createRig)(config);
+    try {
+        const { selection, artifact } = await rig.bakeoffFromEvalCases(task, {
+            variants,
+            ...(envelope !== undefined ? { envelopeUsdPerVariant: envelope } : {}),
+            ...(keep !== undefined ? { keepOutputs: keep } : {}),
+        });
+        console.log(`eval cases for "${task}": ${selection.total} total · ` +
+            `${selection.rehydratable.length} rehydratable · ${selection.refusals.length} refused`);
+        // NEVER a silent subset: name every refused case with its reason.
+        for (const r of selection.refusals) {
+            console.log(`  refused ${r.caseId} (source artifact ${r.sourceArtifactId}): ${r.reason}`);
+        }
+        if (artifact === null) {
+            console.log(selection.total === 0
+                ? `no eval cases promoted for "${task}" yet — promote a failure first (artifact.promoteToEval).`
+                : "nothing rehydratable — no bake-off run (see the refusals above).");
+            return selection.total === 0 ? 0 : 1;
+        }
+        const outDir = (0, node_path_1.join)((0, node_path_1.dirname)(config.telemetryDbPath), "bakeoffs");
+        (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
+        const artifactPath = (0, node_path_1.join)(outDir, `${artifact.id}.json`);
+        (0, node_fs_1.writeFileSync)(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`);
+        let mirrored = false;
+        if (mirrorTarget !== undefined) {
+            await (0, engine_1.mirrorBakeoff)(artifact, {
+                supabaseUrl: mirrorTarget.supabaseUrl,
+                serviceRoleKey: mirrorTarget.serviceRoleKey,
+                sourceId: mirrorTarget.sourceId ?? (0, node_os_1.hostname)(),
+            });
+            mirrored = true;
+        }
+        if (asJson)
+            console.log(JSON.stringify(artifact, null, 2));
+        else
+            printMatrix(artifact);
+        console.log(`artifact → ${artifactPath}${mirrored ? " · mirrored to bakeoffs" : ""}`);
+        return 0;
+    }
+    finally {
+        await rig.close();
+    }
+}

package/dist/bakeoff/samples.d.ts

@@ -9,6 +9,7 @@
  *
  * Transport is raw PostgREST over fetch (same pattern as the exporter).
  */
+import type { BakeoffSamplesSink } from "./engine";
 /** 256KB per output, per the architect's caps; TOAST compresses at rest. */
 export declare const OUTPUT_MAX_BYTES: number;
 export type KeepOutputsMode = "winner" | "all";
@@ -59,4 +60,18 @@ export interface BakeoffSamplesClient {
     }): Promise<number>;
 }
 export declare function createBakeoffSamplesClient(config: BakeoffSamplesClientConfig): BakeoffSamplesClient;
+/**
+ * Build the direct-mode bake-off review-sample sink (extracted from index.ts for
+ * the LOC cap, P1). Persists retained outputs to the SEPARATE `bakeoff_samples`
+ * store (service role; deny-anon RLS); captures never touch it.
+ *
+ * DIRECT MODE ONLY, and that is a real limitation, not an oversight:
+ * `bakeoff_samples` has no ingest route (telemetry-ingestion-boundary.md §2
+ * covers telemetry, not review samples), so an api-mode rig keeps its retained
+ * outputs local. Callers pass `undefined` for a non-direct rig.
+ */
+export declare function buildDirectSamplesSink(cfg: {
+    readonly supabaseUrl: string;
+    readonly serviceRoleKey: string;
+}): BakeoffSamplesSink;
 //# sourceMappingURL=samples.d.ts.map

package/dist/bakeoff/samples.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"samples.d.ts","sourceRoot":"","sources":["../../src/bakeoff/samples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,QAAa,CAAC;AAE3C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,EAAE,CAKV;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOjE;AAED;+DAC+D;AAC/D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,MAAM,CAEhE;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,UAAU,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,aAAa,EAAE,CAoBjB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,0BAA0B,GAAG,oBAAoB,CAoEnG"}
+{"version":3,"file":"samples.d.ts","sourceRoot":"","sources":["../../src/bakeoff/samples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEnD,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,QAAa,CAAC;AAE3C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,EAAE,CAKV;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOjE;AAED;+DAC+D;AAC/D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,MAAM,CAEhE;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,UAAU,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,aAAa,EAAE,CAoBjB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,0BAA0B,GAAG,oBAAoB,CAoEnG;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC,GAAG,kBAAkB,CAQrB"}

package/dist/bakeoff/samples.js

@@ -17,6 +17,7 @@ exports.truncateOutput = truncateOutput;
 exports.retentionExpiry = retentionExpiry;
 exports.buildBakeoffSamples = buildBakeoffSamples;
 exports.createBakeoffSamplesClient = createBakeoffSamplesClient;
+exports.buildDirectSamplesSink = buildDirectSamplesSink;
 /** 256KB per output, per the architect's caps; TOAST compresses at rest. */
 exports.OUTPUT_MAX_BYTES = 256 * 1024;
 /** Default persists incumbent + winner only; "all" persists every arm. */
@@ -126,3 +127,22 @@ function createBakeoffSamplesClient(config) {
         },
     };
 }
+/**
+ * Build the direct-mode bake-off review-sample sink (extracted from index.ts for
+ * the LOC cap, P1). Persists retained outputs to the SEPARATE `bakeoff_samples`
+ * store (service role; deny-anon RLS); captures never touch it.
+ *
+ * DIRECT MODE ONLY, and that is a real limitation, not an oversight:
+ * `bakeoff_samples` has no ingest route (telemetry-ingestion-boundary.md §2
+ * covers telemetry, not review samples), so an api-mode rig keeps its retained
+ * outputs local. Callers pass `undefined` for a non-direct rig.
+ */
+function buildDirectSamplesSink(cfg) {
+    return async ({ bakeoffId, route, arms }) => {
+        const rows = buildBakeoffSamples(bakeoffId, route, arms, retentionExpiry(Date.now()));
+        await createBakeoffSamplesClient({
+            supabaseUrl: cfg.supabaseUrl,
+            serviceRoleKey: cfg.serviceRoleKey,
+        }).insert(rows);
+    };
+}

package/dist/cli.d.ts

@@ -2,7 +2,7 @@
 /**
  * modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
  *   modelrig bakeoff --route X --variants a,b,c --replay-last N
- *     [--envelope usd] [--json] [--no-mirror]
+ *     [--budget-usd usd] [--json] [--no-mirror]   (--envelope is a kept alias)
  *   modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
  *   modelrig watch run [--events fixtures.json] [--json]
  *
@@ -15,13 +15,11 @@
  * executes a swap — that is `modelrig swap execute --approve`, a separate,
  * human command.
  */
-import type { BakeoffArtifact } from "./bakeoff/types";
 export interface Flags {
     readonly positional: readonly string[];
     readonly named: ReadonlyMap<string, string>;
     readonly bare: ReadonlySet<string>;
 }
 export declare function parseArgs(argv: readonly string[]): Flags;
-export declare function printMatrix(artifact: BakeoffArtifact): void;
 export declare function main(argv: readonly string[]): Promise<number>;
 //# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAgBH,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,iBAAiB,CAAC;AAcvE,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAaD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AAMD,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CA6B3D;AAqfD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAgCnE"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAiCH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AA0gBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAqCnE"}

package/dist/cli.js

@@ -3,7 +3,7 @@
 /**
  * modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
  *   modelrig bakeoff --route X --variants a,b,c --replay-last N
- *     [--envelope usd] [--json] [--no-mirror]
+ *     [--budget-usd usd] [--json] [--no-mirror]   (--envelope is a kept alias)
  *   modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
  *   modelrig watch run [--events fixtures.json] [--json]
  *
@@ -18,21 +18,24 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseArgs = parseArgs;
-exports.printMatrix = printMatrix;
 exports.main = main;
 const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
 const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
-const run_1 = require("./init/run");
+const init_cli_1 = require("./init/init-cli");
+const observe_cli_1 = require("./init/observe-cli");
 const failures_1 = require("./types/failures");
 const failures_teaching_1 = require("./types/failures-teaching");
-const rows_1 = require("./registry/rows");
 const config_1 = require("./config");
 const index_1 = require("./index");
+const status_1 = require("./status");
 const engine_1 = require("./bakeoff/engine");
 const samples_1 = require("./bakeoff/samples");
+const report_1 = require("./bakeoff/report");
+const cli_run_1 = require("./verify/cli-run");
 const loader_1 = require("./routes/loader");
+const validate_cli_1 = require("./validate-cli");
 const store_1 = require("./telemetry/store");
 const budget_1 = require("./watch/budget");
 const proposals_1 = require("./watch/proposals");
@@ -50,6 +53,8 @@ const BARE_FLAGS = new Set([
     // modelrig init (Phase 5 WS2)
     "dry-run",
     "include-tests",
+    // modelrig observe / init --observe (T0/T1 scaffold)
+    "observe",
 ]);
 function parseArgs(argv) {
     const positional = [];
@@ -76,36 +81,6 @@ function parseArgs(argv) {
     }
     return { positional, named, bare };
 }
-function fmtEff(usd) {
-    return Number.isFinite(usd) ? `$${usd.toFixed(4)}` : "∞ (0 conformant)";
-}
-function printMatrix(artifact) {
-    console.log(`bakeoff ${artifact.id}\nroute ${artifact.route}@${artifact.routeVersion} · replayed ${artifact.replayN} captured inputs · ${artifact.ranAt}`);
-    const header = ["variant", "n", "conform", "ci95", "repair", "repair$", "lat(ms)", "eff-$/1K-conf"];
-    const rows = artifact.variants.map((m) => [
-        m.variant,
-        String(m.samples),
-        `${(m.conformRate * 100).toFixed(0)}%`,
-        `[${(m.conformCi95[0] * 100).toFixed(0)},${(m.conformCi95[1] * 100).toFixed(0)}]`,
-        `${(m.repairRate * 100).toFixed(0)}%`,
-        `$${m.meanRepairCostUsd.toFixed(4)}`,
-        m.meanLatencyMs.toFixed(0),
-        m.pricingMissing ? "UNPRICED (no ranking)" : fmtEff(m.effectiveUsdPer1kConformant),
-    ]);
-    const widths = header.map((h, c) => Math.max(h.length, ...rows.map((r) => r[c].length)));
-    const line = (cells) => cells.map((cell, c) => cell.padEnd(widths[c])).join("  ");
-    console.log(line(header));
-    for (const row of rows)
-        console.log(line(row));
-    if (artifact.proposal !== null) {
-        console.log(`PROPOSAL: variant "${artifact.proposal.variant}" beats the incumbent by ` +
-            `${artifact.proposal.vsIncumbentSavingsPct.toFixed(1)}% effective cost — human decision required; ` +
-            "nothing is executed.");
-    }
-    else {
-        console.log("proposal: none (no variant beats the incumbent's gates)");
-    }
-}
 /** Control-plane client from config.export — the watch/proposal surfaces
  * need it; a clear error beats a silent no-op. */
 function proposalsClientFrom(config) {
@@ -153,7 +128,7 @@ async function runAndReportBakeoff(config, flags, route, variants, replayLastN,
             console.log(JSON.stringify(artifact, null, 2));
         }
         else {
-            printMatrix(artifact);
+            (0, report_1.printMatrix)(artifact);
         }
         console.log(`artifact → ${artifactPath}${mirrored ? " · mirrored to bakeoffs" : ""}` +
             (keep !== undefined
@@ -162,7 +137,9 @@ async function runAndReportBakeoff(config, flags, route, variants, replayLastN,
         return artifact;
     }
     finally {
-        rig.close();
+        // One-shot CLI: await the settled drain so the exporter's final batch is
+        // guaranteed to land before the process exits (H1).
+        await rig.close();
     }
 }
 async function cmdBakeoff(flags) {
@@ -225,6 +202,19 @@ async function cmdBakeoff(flags) {
             budgetStore.close();
         }
     }
+    // Replay a task's promoted eval cases instead of the last-N captures (E4 chunk A).
+    const evalTask = flags.named.get("from-eval-cases");
+    if (evalTask !== undefined) {
+        const vRaw = flags.named.get("variants");
+        if (vRaw === undefined)
+            throw new Error("bakeoff --from-eval-cases <task> requires --variants a,b,c");
+        const vs = vRaw.split(",").map((v) => v.trim()).filter((v) => v !== "");
+        const envRaw = flags.named.get("budget-usd") ?? flags.named.get("envelope");
+        const env = envRaw !== undefined ? Number(envRaw) : undefined;
+        if (env !== undefined && (!Number.isFinite(env) || env <= 0))
+            throw new Error("--budget-usd must be positive");
+        return (0, report_1.runEvalCaseBakeoffCli)(config, evalTask, vs, env, keepOutputsMode(flags), flags.bare.has("json"));
+    }
     const route = flags.named.get("route");
     const variantsRaw = flags.named.get("variants");
     const lastRaw = flags.named.get("replay-last");
@@ -236,10 +226,11 @@ async function cmdBakeoff(flags) {
     if (!Number.isInteger(replayLastN) || replayLastN < 1) {
         throw new Error("--replay-last must be a positive integer");
     }
-    const envelopeRaw = flags.named.get("envelope");
+    // §4: --budget-usd is the named envelope knob; --envelope is kept as an alias.
+    const envelopeRaw = flags.named.get("budget-usd") ?? flags.named.get("envelope");
     const envelope = envelopeRaw !== undefined ? Number(envelopeRaw) : undefined;
     if (envelope !== undefined && (!Number.isFinite(envelope) || envelope <= 0)) {
-        throw new Error("--envelope must be positive");
+        throw new Error("--budget-usd must be a positive number of dollars (per replay variant)");
     }
     await runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope);
     return 0;
@@ -413,75 +404,30 @@ async function cmdSamples(flags) {
     return 0;
 }
 /**
- * `modelrig init [path]` — Phase 5 WS2. Scans a repo for provider call
- * sites and drafts a route bundle per site, plus a MIGRATION.md map.
- *
- * It writes ONLY under <routes-dir>/drafts/. Your source is never edited;
- * applying the migration is a human's diff to make.
- */
-async function cmdInit(flags) {
-    const root = (0, node_path_1.resolve)(flags.positional[0] ?? ".");
-    const config = (0, config_1.loadConfigFromEnv)();
-    const draftsDir = flags.named.get("out") ?? (0, node_path_1.join)(config.routesDir, "drafts");
-    // Registry rows seed the challenger candidates. A missing/unbuilt registry
-    // is not fatal — drafts then carry the incumbent alone, and say so.
-    let registryRows = [];
-    try {
-        registryRows = (0, rows_1.loadRegistryRows)(config.registryPath);
+ * `modelrig verify-swap` — the autonomy-ladder Tier-2 verify step (Expansion
+ * §9). Replays a LOCAL capture fixture through the drafted SAME-MODEL route,
+ * diffs each output against the original, and emits the report the Tier-2 PR
+ * embeds. A plumbing proof; refuses uncapturable (streaming/dynamic) sites. */
+async function cmdVerifySwap(flags) {
+    const route = flags.named.get("route");
+    const fixturePath = flags.named.get("captures");
+    if (route === undefined || fixturePath === undefined) {
+        throw new Error("verify-swap requires --route <route> and --captures <fixture.json> " +
+            "(the local capture of the original call's inputs/outputs)");
     }
-    catch {
-        registryRows = [];
+    const envRaw = flags.named.get("budget-usd") ?? flags.named.get("envelope");
+    const envelopeUsd = envRaw !== undefined ? Number(envRaw) : undefined;
+    if (envelopeUsd !== undefined && (!Number.isFinite(envelopeUsd) || envelopeUsd <= 0)) {
+        throw new Error("--budget-usd must be a positive number of dollars (the replay-pass envelope)");
     }
-    const result = (0, run_1.runInit)({
-        root,
-        draftsDir,
-        fs: { readdirSync: node_fs_1.readdirSync, readFileSync: node_fs_1.readFileSync, statSync: node_fs_1.statSync, existsSync: node_fs_1.existsSync, mkdirSync: node_fs_1.mkdirSync, writeFileSync: node_fs_1.writeFileSync },
-        path: { join: node_path_1.join, relative: node_path_1.relative, dirname: node_path_1.dirname, basename: node_path_1.basename },
-    }, {
-        registryRows,
-        ...(flags.bare.has("dry-run") ? { dryRun: true } : {}),
-        ...(flags.bare.has("include-tests") ? { includeTests: true } : {}),
+    const config = (0, config_1.loadConfigFromEnv)();
+    return (0, cli_run_1.runVerifySwapCli)(config, {
+        route,
+        fixturePath,
+        ...(envelopeUsd !== undefined ? { envelopeUsd } : {}),
+        ...(flags.bare.has("json") ? { asJson: true } : {}),
+        ...(flags.named.get("out") !== undefined ? { outPath: flags.named.get("out") } : {}),
     });
-    if (flags.bare.has("json")) {
-        console.log(JSON.stringify({
-            sitesFound: result.sitesFound,
-            filesScanned: result.filesScanned,
-            filesSkipped: result.filesSkipped,
-            routes: result.drafts.map((d) => ({
-                route: d.route,
-                file: d.site.file,
-                line: d.site.line,
-                provider: d.site.provider,
-                model: d.site.model,
-                notes: d.site.notes,
-            })),
-            written: result.written,
-            skipped: result.skipped,
-        }, null, 2));
-        return 0;
-    }
-    console.log(`modelrig init — scanned ${result.filesScanned} file(s) under ${root}\n` +
-        `found ${result.sitesFound} provider call site(s)`);
-    if (result.sitesFound === 0) {
-        console.log("\nNothing to draft. If you know there are model calls here, they may sit behind a helper\n" +
-            "that hides the SDK — point init at the file that actually calls the provider.");
-        return 0;
-    }
-    for (const draft of result.drafts) {
-        console.log(`  ${draft.site.file}:${draft.site.line} (${draft.site.provider}` +
-            `${draft.site.model !== null ? ` ${draft.site.model}` : ""}) → ${draft.route}`);
-    }
-    if (result.filesSkipped.length > 0) {
-        console.log(`\n${result.filesSkipped.length} file(s) skipped (too large or unreadable).`);
-    }
-    if (flags.bare.has("dry-run")) {
-        console.log("\n--dry-run: nothing written.");
-        return 0;
-    }
-    console.log(`\nwrote ${result.written.length} file(s) to ${draftsDir}` +
-        (result.skipped.length > 0 ? ` (${result.skipped.length} left alone — already there)` : "") +
-        `\nread ${(0, node_path_1.join)(draftsDir, "MIGRATION.md")} next. Your source was not touched.`);
-    return 0;
 }
 /**
  * `modelrig serve` — Phase 5 WS4's zero-config path. The HTTP surface lives
@@ -519,10 +465,86 @@ async function cmdServe() {
     // The server owns the process from here (it holds the listener open).
     return 0;
 }
+/** `modelrig status` — print the runtime posture (§2): export mode, artifact
+ * namespace on/off + WHY, content custody, OTLP. Reads config only (no rig, no
+ * routes dir), so it works anywhere the env is set. */
+function cmdStatus() {
+    console.log((0, status_1.formatStatusReport)((0, status_1.buildRigStatus)((0, config_1.loadConfigFromEnv)())));
+    return 0;
+}
+/** `modelrig judge <run>` — E5 V.4: explicitly ask the bounded model-judge to
+ * score a run. Goes through the API boundary (T12: the CLI never holds a DB
+ * client) — POST /v1/runs/:id/judge on the configured server, authed with a
+ * rig_sk key. The daily cap + metering are the SERVER's; this just triggers one. */
+async function cmdJudge(flags) {
+    const runId = flags.positional[0] ?? "";
+    // Config is the only env reader (package-invariants rule). The judge endpoint
+    // lives on the same server as the api-mode ingest sink, so the base is that
+    // URL's origin and the credential is the api key.
+    const exp = (0, config_1.loadConfigFromEnv)().export;
+    const api = exp !== undefined && exp.mode === "api" ? exp : undefined;
+    const base = api !== undefined ? new URL(api.ingestUrl).origin : "";
+    const key = api?.apiKey ?? "";
+    if (runId === "" || base === "" || key === "") {
+        console.error([
+            "usage: modelrig judge <runId>",
+            "",
+            "Needs api-mode telemetry configured (the judge is on the same server):",
+            "  MODELRIG_API_KEY    a rig_sk_ key with the `oracle-act` scope",
+            "  MODELRIG_INGEST_URL optional — the server base; defaults to the hosted API",
+            "",
+            "The judge is EXPLICIT-only and metered; scoring rides the gateway (a route),",
+            "so the org's daily cap and spend-stops apply. Nothing schedules it.",
+        ].join("\n"));
+        return 1;
+    }
+    const res = await fetch(`${base}/v1/runs/${encodeURIComponent(runId)}/judge`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+        body: "{}",
+    });
+    const body = (await res.json().catch(() => ({})));
+    if (!res.ok) {
+        console.error(`judge ${res.status}: ${typeof body.error === "string" ? body.error : "failed"}`);
+        return 1;
+    }
+    console.log(JSON.stringify(body, null, 2));
+    return 0;
+}
+/** The one usage block, printed to stdout on `--help` (exit 0) and to stderr on
+ * an unknown command (exit 1). Kept as a const so the two paths can never
+ * drift. */
+const USAGE = [
+    "usage:",
+    "  modelrig init [path] [--scope glob,glob] [--out dir] [--dry-run] [--include-tests] [--json]",
+    "  modelrig observe [path] [--scope glob,glob] [--out dir] [--dry-run] [--json]   (T0/T1: see your runs first — no migration)",
+    "  modelrig validate                    (structurally check rig.yaml + every route bundle — no provider key needed)",
+    "  modelrig status                      (print export/artifact/otlp posture)",
+    "  modelrig serve                       (needs the modelrig-server package)",
+    "  modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--keep-outputs[=all]] [--json] [--no-mirror]",
+    "  modelrig bakeoff --from-proposal <id> [--replay-last N] [--keep-outputs[=all]] [--json]",
+    "  modelrig watch run [--events fixtures.json] [--json]",
+    "  modelrig verify-swap --route <route> --captures <fixture.json> [--budget-usd usd] [--out report.md] [--json]",
+    "  modelrig swap execute <proposalId> --approve [--approved-by name]",
+    "  modelrig swap revert <swapId>",
+    "  modelrig judge <runId>               (E5: score a run with the bounded model-judge; explicit + metered)",
+    "  modelrig samples purge [--bakeoff <id>]",
+].join("\n");
 async function main(argv) {
     const [command, ...rest] = argv;
+    // `--help`/`-h`/`help` is a success: print usage to stdout and exit 0. (An
+    // unknown command still prints usage to stderr and exits 1, below.) The
+    // clean-room tarball test asserts the shipped bin exits 0 on --help.
+    if (command === "--help" || command === "-h" || command === "help") {
+        console.log(USAGE);
+        return 0;
+    }
     const flags = parseArgs(rest);
     switch (command) {
+        case "status":
+            return cmdStatus();
+        case "validate":
+            return (0, validate_cli_1.runValidateCli)();
         case "bakeoff":
             return cmdBakeoff(flags);
         case "watch":
@@ -532,21 +554,17 @@ async function main(argv) {
         case "samples":
             return cmdSamples(flags);
         case "init":
-            return cmdInit(flags);
+            return (0, init_cli_1.cmdInit)(flags);
+        case "observe":
+            return (0, observe_cli_1.cmdObserve)(flags);
+        case "verify-swap":
+            return cmdVerifySwap(flags);
+        case "judge":
+            return cmdJudge(flags);
         case "serve":
             return cmdServe();
         default:
-            console.error([
-                "usage:",
-                "  modelrig init [path] [--out dir] [--dry-run] [--include-tests] [--json]",
-                "  modelrig serve                       (needs the modelrig-server package)",
-                "  modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--envelope usd] [--keep-outputs[=all]] [--json] [--no-mirror]",
-                "  modelrig bakeoff --from-proposal <id> [--replay-last N] [--keep-outputs[=all]] [--json]",
-                "  modelrig watch run [--events fixtures.json] [--json]",
-                "  modelrig swap execute <proposalId> --approve [--approved-by name]",
-                "  modelrig swap revert <swapId>",
-                "  modelrig samples purge [--bakeoff <id>]",
-            ].join("\n"));
+            console.error(USAGE);
             return 1;
     }
 }