modelrig 0.0.1 → 0.2.0

package/dist/bakeoff/report.js

@@ -0,0 +1,142 @@
+"use strict";
+/**
+ * Bake-off terminal reporting — the matrix printer (moved here from cli.ts for
+ * the LOC cap, P1) and the `--from-eval-cases` runner (E4 chunk A) with its
+ * HONEST-REFUSAL report: every eval case that could not be rehydrated is NAMED
+ * with a reason before any bake-off runs — never a silent subset (spec §3.3).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatCacheAdjustedSection = formatCacheAdjustedSection;
+exports.printMatrix = printMatrix;
+exports.runEvalCaseBakeoffCli = runEvalCaseBakeoffCli;
+const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
+const node_path_1 = require("node:path");
+const index_1 = require("../index");
+const engine_1 = require("./engine");
+function fmtEff(usd) {
+    return Number.isFinite(usd) ? `$${usd.toFixed(4)}` : "∞ (0 conformant)";
+}
+/**
+ * Render the cache-adjusted section (spec §5.4) as lines — measured-cold AND
+ * cache-adjusted input cost side by side, with the simulator's assumptions
+ * PRINTED so the numbers are never mistaken for measured savings. Pure (returns
+ * strings; `printMatrix` does the console I/O) so it is unit-tested directly.
+ * Empty input ⇒ no lines (the section is omitted entirely — back-compat).
+ */
+function formatCacheAdjustedSection(rows) {
+    if (rows.length === 0)
+        return [];
+    const lines = [
+        "cache-adjusted input cost (SIMULATED — not measured savings; the odometer stays measured-only):",
+    ];
+    const header = ["variant", "measured-cold", "cache-adjusted", "sim-savings"];
+    const body = rows.map((r) => {
+        const savings = r.measuredColdUsd > 0
+            ? `${(((r.measuredColdUsd - r.cacheAdjustedUsd) / r.measuredColdUsd) * 100).toFixed(1)}%`
+            : "n/a";
+        return [r.variant, `$${r.measuredColdUsd.toFixed(4)}`, `$${r.cacheAdjustedUsd.toFixed(4)}`, savings];
+    });
+    const widths = header.map((h, c) => Math.max(h.length, ...body.map((row) => row[c].length)));
+    const line = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join("  ");
+    lines.push(line(header));
+    for (const row of body)
+        lines.push(line(row));
+    // Assumptions basis — one line per distinct policy so a mixed-provider matrix
+    // is honest about which TTL/min-cacheable each arm assumed.
+    const seen = new Set();
+    for (const r of rows) {
+        const a = r.assumptions;
+        const key = `${a.provider}/${a.model}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        lines.push(`  assumptions ${key}: ${a.cacheSupported ? "" : "cache unsupported; "}ttl=${a.ttlSeconds}s` +
+            `${a.ttlSourced ? "" : " (fallback — undocumented)"}, min-cacheable=${a.minCacheableTokens}tok, ` +
+            `refresh-on-read=${a.refreshOnRead}`);
+    }
+    return lines;
+}
+function printMatrix(artifact, cacheAdjusted = []) {
+    console.log(`bakeoff ${artifact.id}\nroute ${artifact.route}@${artifact.routeVersion} · replayed ${artifact.replayN} captured inputs · ${artifact.ranAt}`);
+    const header = ["variant", "n", "conform", "ci95", "repair", "repair$", "lat(ms)", "eff-$/1K-conf"];
+    const rows = artifact.variants.map((m) => [
+        m.variant,
+        String(m.samples),
+        `${(m.conformRate * 100).toFixed(0)}%`,
+        `[${(m.conformCi95[0] * 100).toFixed(0)},${(m.conformCi95[1] * 100).toFixed(0)}]`,
+        `${(m.repairRate * 100).toFixed(0)}%`,
+        `$${m.meanRepairCostUsd.toFixed(4)}`,
+        m.meanLatencyMs.toFixed(0),
+        m.pricingMissing ? "UNPRICED (no ranking)" : fmtEff(m.effectiveUsdPer1kConformant),
+    ]);
+    const widths = header.map((h, c) => Math.max(h.length, ...rows.map((r) => r[c].length)));
+    const line = (cells) => cells.map((cell, c) => cell.padEnd(widths[c])).join("  ");
+    console.log(line(header));
+    for (const row of rows)
+        console.log(line(row));
+    if (artifact.proposal !== null) {
+        console.log(`PROPOSAL: variant "${artifact.proposal.variant}" beats the incumbent by ` +
+            `${artifact.proposal.vsIncumbentSavingsPct.toFixed(1)}% effective cost — human decision required; ` +
+            "nothing is executed.");
+    }
+    else {
+        console.log("proposal: none (no variant beats the incumbent's gates)");
+    }
+    // Cache-adjusted section (spec §5.4) — printed only when the caller supplied
+    // simulator rows; absent ⇒ the matrix output is byte-identical to pre-WS-B.
+    for (const l of formatCacheAdjustedSection(cacheAdjusted))
+        console.log(l);
+}
+/**
+ * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's
+ * promoted eval cases through each variant. Prints the honest-refusal report,
+ * then (if anything rehydrated) runs the bake-off and persists/mirrors/prints it.
+ * Returns the process exit code: 0 on a run (or an empty suite), 1 when cases
+ * existed but NONE could be rehydrated (an actionable non-zero, not a crash).
+ */
+async function runEvalCaseBakeoffCli(config, task, variants, envelope, keep, asJson) {
+    const mirrorTarget = config.export?.mode === "direct" ? config.export : undefined;
+    const rig = (0, index_1.createRig)(config);
+    try {
+        const { selection, artifact } = await rig.bakeoffFromEvalCases(task, {
+            variants,
+            ...(envelope !== undefined ? { envelopeUsdPerVariant: envelope } : {}),
+            ...(keep !== undefined ? { keepOutputs: keep } : {}),
+        });
+        console.log(`eval cases for "${task}": ${selection.total} total · ` +
+            `${selection.rehydratable.length} rehydratable · ${selection.refusals.length} refused`);
+        // NEVER a silent subset: name every refused case with its reason.
+        for (const r of selection.refusals) {
+            console.log(`  refused ${r.caseId} (source artifact ${r.sourceArtifactId}): ${r.reason}`);
+        }
+        if (artifact === null) {
+            console.log(selection.total === 0
+                ? `no eval cases promoted for "${task}" yet — promote a failure first (artifact.promoteToEval).`
+                : "nothing rehydratable — no bake-off run (see the refusals above).");
+            return selection.total === 0 ? 0 : 1;
+        }
+        const outDir = (0, node_path_1.join)((0, node_path_1.dirname)(config.telemetryDbPath), "bakeoffs");
+        (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
+        const artifactPath = (0, node_path_1.join)(outDir, `${artifact.id}.json`);
+        (0, node_fs_1.writeFileSync)(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`);
+        let mirrored = false;
+        if (mirrorTarget !== undefined) {
+            await (0, engine_1.mirrorBakeoff)(artifact, {
+                supabaseUrl: mirrorTarget.supabaseUrl,
+                serviceRoleKey: mirrorTarget.serviceRoleKey,
+                sourceId: mirrorTarget.sourceId ?? (0, node_os_1.hostname)(),
+            });
+            mirrored = true;
+        }
+        if (asJson)
+            console.log(JSON.stringify(artifact, null, 2));
+        else
+            printMatrix(artifact);
+        console.log(`artifact → ${artifactPath}${mirrored ? " · mirrored to bakeoffs" : ""}`);
+        return 0;
+    }
+    finally {
+        await rig.close();
+    }
+}

package/dist/bakeoff/samples.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Bake-off review samples — Phase 4 addendum A3 (value-review-storage.md §11).
+ *
+ * A SEPARATE store from captures: rows exist only because a human ran
+ * `modelrig bakeoff --keep-outputs` intending a value review. They carry the
+ * rendered input + each arm's output so the human-in-the-loop diff (eval
+ * rung 1.5) happens in the console. Written by the CLI with the service-role
+ * key; read server-side only; deny-anon RLS. NEVER the anon staging posture.
+ *
+ * Transport is raw PostgREST over fetch (same pattern as the exporter).
+ */
+import type { BakeoffSamplesSink } from "./engine";
+/** 256KB per output, per the architect's caps; TOAST compresses at rest. */
+export declare const OUTPUT_MAX_BYTES: number;
+export type KeepOutputsMode = "winner" | "all";
+export interface BakeoffSample {
+    readonly bakeoffId: string;
+    readonly route: string;
+    readonly arm: string;
+    readonly armMeta: Record<string, unknown> | null;
+    readonly inputRef: string;
+    readonly inputText: string;
+    readonly outputText: string | null;
+    readonly conform: boolean;
+    readonly costUsd: number;
+    readonly latencyMs: number;
+    readonly expiresAt: string;
+}
+/** One arm's per-input samples, ready to persist. */
+export interface ArmSamples {
+    readonly arm: string;
+    readonly armMeta: Record<string, unknown> | null;
+    readonly samples: ReadonlyArray<{
+        readonly inputRef: string;
+        readonly inputText: string;
+        readonly outputText: string | null;
+        readonly conform: boolean;
+        readonly costUsd: number;
+        readonly latencyMs: number;
+    }>;
+}
+/** Default persists incumbent + winner only; "all" persists every arm. */
+export declare function selectArmsToKeep(mode: KeepOutputsMode, incumbentArm: string, winnerArm: string | null, allArms: readonly string[]): string[];
+export declare function truncateOutput(text: string | null): string | null;
+/** Default retention: 90 days from now (or pass a sooner stamp). Pure over
+ * an injected `now` so it stays testable and workflow-safe. */
+export declare function retentionExpiry(nowMs: number, days?: number): string;
+export declare function buildBakeoffSamples(bakeoffId: string, route: string, arms: readonly ArmSamples[], expiresAt: string): BakeoffSample[];
+export interface BakeoffSamplesClientConfig {
+    readonly supabaseUrl: string;
+    readonly serviceRoleKey: string;
+    readonly fetchFn?: typeof fetch;
+}
+export interface BakeoffSamplesClient {
+    insert(samples: readonly BakeoffSample[]): Promise<void>;
+    /** Purge by bakeoff id, or everything past its expiry when no id given. */
+    purge(opts?: {
+        bakeoffId?: string;
+        nowIso?: string;
+    }): Promise<number>;
+}
+export declare function createBakeoffSamplesClient(config: BakeoffSamplesClientConfig): BakeoffSamplesClient;
+/**
+ * Build the direct-mode bake-off review-sample sink (extracted from index.ts for
+ * the LOC cap, P1). Persists retained outputs to the SEPARATE `bakeoff_samples`
+ * store (service role; deny-anon RLS); captures never touch it.
+ *
+ * DIRECT MODE ONLY, and that is a real limitation, not an oversight:
+ * `bakeoff_samples` has no ingest route (telemetry-ingestion-boundary.md §2
+ * covers telemetry, not review samples), so an api-mode rig keeps its retained
+ * outputs local. Callers pass `undefined` for a non-direct rig.
+ */
+export declare function buildDirectSamplesSink(cfg: {
+    readonly supabaseUrl: string;
+    readonly serviceRoleKey: string;
+}): BakeoffSamplesSink;
+//# sourceMappingURL=samples.d.ts.map

package/dist/bakeoff/samples.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"samples.d.ts","sourceRoot":"","sources":["../../src/bakeoff/samples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEnD,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,QAAa,CAAC;AAE3C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,EAAE,CAKV;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOjE;AAED;+DAC+D;AAC/D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,MAAM,CAEhE;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,UAAU,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,aAAa,EAAE,CAoBjB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,0BAA0B,GAAG,oBAAoB,CAoEnG;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC,GAAG,kBAAkB,CAQrB"}

package/dist/bakeoff/samples.js

@@ -0,0 +1,148 @@
+"use strict";
+/**
+ * Bake-off review samples — Phase 4 addendum A3 (value-review-storage.md §11).
+ *
+ * A SEPARATE store from captures: rows exist only because a human ran
+ * `modelrig bakeoff --keep-outputs` intending a value review. They carry the
+ * rendered input + each arm's output so the human-in-the-loop diff (eval
+ * rung 1.5) happens in the console. Written by the CLI with the service-role
+ * key; read server-side only; deny-anon RLS. NEVER the anon staging posture.
+ *
+ * Transport is raw PostgREST over fetch (same pattern as the exporter).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OUTPUT_MAX_BYTES = void 0;
+exports.selectArmsToKeep = selectArmsToKeep;
+exports.truncateOutput = truncateOutput;
+exports.retentionExpiry = retentionExpiry;
+exports.buildBakeoffSamples = buildBakeoffSamples;
+exports.createBakeoffSamplesClient = createBakeoffSamplesClient;
+exports.buildDirectSamplesSink = buildDirectSamplesSink;
+/** 256KB per output, per the architect's caps; TOAST compresses at rest. */
+exports.OUTPUT_MAX_BYTES = 256 * 1024;
+/** Default persists incumbent + winner only; "all" persists every arm. */
+function selectArmsToKeep(mode, incumbentArm, winnerArm, allArms) {
+    if (mode === "all")
+        return [...allArms];
+    const keep = [incumbentArm];
+    if (winnerArm !== null && winnerArm !== incumbentArm)
+        keep.push(winnerArm);
+    return keep;
+}
+function truncateOutput(text) {
+    if (text === null)
+        return null;
+    if (Buffer.byteLength(text, "utf8") <= exports.OUTPUT_MAX_BYTES)
+        return text;
+    // Cut on a byte budget, then tag it so the reviewer knows it's clipped.
+    const marker = "\n\n[truncated at 256KB for review storage]";
+    const budget = exports.OUTPUT_MAX_BYTES - Buffer.byteLength(marker, "utf8");
+    return Buffer.from(text, "utf8").subarray(0, budget).toString("utf8") + marker;
+}
+/** Default retention: 90 days from now (or pass a sooner stamp). Pure over
+ * an injected `now` so it stays testable and workflow-safe. */
+function retentionExpiry(nowMs, days = 90) {
+    return new Date(nowMs + days * 24 * 3600 * 1000).toISOString();
+}
+function buildBakeoffSamples(bakeoffId, route, arms, expiresAt) {
+    const rows = [];
+    for (const arm of arms) {
+        for (const s of arm.samples) {
+            rows.push({
+                bakeoffId,
+                route,
+                arm: arm.arm,
+                armMeta: arm.armMeta,
+                inputRef: s.inputRef,
+                inputText: s.inputText,
+                outputText: truncateOutput(s.outputText),
+                conform: s.conform,
+                costUsd: s.costUsd,
+                latencyMs: s.latencyMs,
+                expiresAt,
+            });
+        }
+    }
+    return rows;
+}
+function createBakeoffSamplesClient(config) {
+    const fetchFn = config.fetchFn ?? fetch;
+    const base = config.supabaseUrl.replace(/\/$/, "");
+    async function request(path, init) {
+        const response = await fetchFn(`${base}/rest/v1/${path}`, {
+            ...init,
+            headers: {
+                "content-type": "application/json",
+                apikey: config.serviceRoleKey,
+                authorization: `Bearer ${config.serviceRoleKey}`,
+                ...(init.headers ?? {}),
+            },
+        });
+        if (!response.ok) {
+            throw new Error(`bakeoff_samples ${init.method ?? "GET"} ${path} → ${response.status} ` +
+                `${await response.text().catch(() => "")}`);
+        }
+        return response;
+    }
+    return {
+        async insert(samples) {
+            if (samples.length === 0)
+                return;
+            // Chunk the POST: at the ≤50-inputs cap, large-input routes (e.g. the
+            // cutover steps pass ~600KB assembled prompts) would make a single body
+            // tens of MB and blow past PostgREST's limit. Small batches keep every
+            // body well under it, at any cap.
+            const CHUNK = 8;
+            for (let i = 0; i < samples.length; i += CHUNK) {
+                const batch = samples.slice(i, i + CHUNK);
+                await request("bakeoff_samples", {
+                    method: "POST",
+                    headers: { prefer: "return=minimal" },
+                    body: JSON.stringify(batch.map((s) => ({
+                        bakeoff_id: s.bakeoffId,
+                        route: s.route,
+                        arm: s.arm,
+                        arm_meta: s.armMeta,
+                        input_ref: s.inputRef,
+                        input_text: s.inputText,
+                        output_text: s.outputText,
+                        conform: s.conform,
+                        cost_usd: s.costUsd,
+                        latency_ms: s.latencyMs,
+                        expires_at: s.expiresAt,
+                    }))),
+                });
+            }
+        },
+        async purge(opts) {
+            const filter = opts?.bakeoffId !== undefined
+                ? `bakeoff_id=eq.${encodeURIComponent(opts.bakeoffId)}`
+                : `expires_at=lt.${encodeURIComponent(opts?.nowIso ?? new Date().toISOString())}`;
+            const response = await request(`bakeoff_samples?${filter}`, {
+                method: "DELETE",
+                headers: { prefer: "return=representation" },
+            });
+            const deleted = (await response.json().catch(() => []));
+            return Array.isArray(deleted) ? deleted.length : 0;
+        },
+    };
+}
+/**
+ * Build the direct-mode bake-off review-sample sink (extracted from index.ts for
+ * the LOC cap, P1). Persists retained outputs to the SEPARATE `bakeoff_samples`
+ * store (service role; deny-anon RLS); captures never touch it.
+ *
+ * DIRECT MODE ONLY, and that is a real limitation, not an oversight:
+ * `bakeoff_samples` has no ingest route (telemetry-ingestion-boundary.md §2
+ * covers telemetry, not review samples), so an api-mode rig keeps its retained
+ * outputs local. Callers pass `undefined` for a non-direct rig.
+ */
+function buildDirectSamplesSink(cfg) {
+    return async ({ bakeoffId, route, arms }) => {
+        const rows = buildBakeoffSamples(bakeoffId, route, arms, retentionExpiry(Date.now()));
+        await createBakeoffSamplesClient({
+            supabaseUrl: cfg.supabaseUrl,
+            serviceRoleKey: cfg.serviceRoleKey,
+        }).insert(rows);
+    };
+}

package/dist/bakeoff/stats.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Bake-off statistics — Phase 3 spec WS5. Wilson 95% intervals ported from
+ * the probe suite's summarize math (same constants — results comparable);
+ * no cross-package import (the probes package is standalone since P2).
+ *
+ * The headline: effective $ per 1K conformant outputs — TOTAL sample cost
+ * (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
+ */
+import type { ReplaySampleResult } from "./replay";
+import type { VariantMetrics } from "./types";
+export declare function wilson95(successes: number, n: number): readonly [number, number];
+export declare function aggregateVariantMetrics(variant: string, samples: readonly ReplaySampleResult[]): VariantMetrics;
+//# sourceMappingURL=stats.d.ts.map

package/dist/bakeoff/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/bakeoff/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAShF;AAMD,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,kBAAkB,EAAE,GACrC,cAAc,CAuBhB"}

package/dist/bakeoff/stats.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * Bake-off statistics — Phase 3 spec WS5. Wilson 95% intervals ported from
+ * the probe suite's summarize math (same constants — results comparable);
+ * no cross-package import (the probes package is standalone since P2).
+ *
+ * The headline: effective $ per 1K conformant outputs — TOTAL sample cost
+ * (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wilson95 = wilson95;
+exports.aggregateVariantMetrics = aggregateVariantMetrics;
+function wilson95(successes, n) {
+    if (n === 0)
+        return [0, 1];
+    const z = 1.959963984540054;
+    const p = successes / n;
+    const z2 = z * z;
+    const denominator = 1 + z2 / n;
+    const center = (p + z2 / (2 * n)) / denominator;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denominator;
+    return [Math.max(0, center - half), Math.min(1, center + half)];
+}
+function mean(values) {
+    return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
+}
+function aggregateVariantMetrics(variant, samples) {
+    const n = samples.length;
+    const conformant = samples.filter((s) => s.conform);
+    const repaired = samples.filter((s) => s.repaired);
+    const accuracies = samples
+        .map((s) => s.valueAccuracy)
+        .filter((a) => a !== null);
+    const totalCostUsd = samples.reduce((a, s) => a + s.costUsd, 0);
+    return {
+        variant,
+        samples: n,
+        conformRate: n === 0 ? 0 : conformant.length / n,
+        conformCi95: wilson95(conformant.length, n),
+        repairRate: n === 0 ? 0 : repaired.length / n,
+        meanRepairCostUsd: mean(repaired.map((s) => s.repairCostUsd)),
+        valueAccuracyMean: accuracies.length === 0 ? null : mean(accuracies),
+        meanLatencyMs: mean(samples.map((s) => s.latencyMs)),
+        meanCostUsd: n === 0 ? 0 : totalCostUsd / n,
+        effectiveUsdPer1kConformant: conformant.length === 0 ? Number.POSITIVE_INFINITY : (totalCostUsd / conformant.length) * 1000,
+        pricingMissing: samples.some((s) => s.pricingMissing),
+    };
+}

package/dist/bakeoff/types.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Conformance bake-off contracts — Phase 3 spec WS5 (closed core).
+ * The headline metric: effective $ per 1K conformant outputs, with repair
+ * and search costs included (F1-corrected accounting).
+ */
+export interface VariantMetrics {
+    readonly variant: string;
+    readonly samples: number;
+    readonly conformRate: number;
+    readonly conformCi95: readonly [number, number];
+    readonly repairRate: number;
+    readonly meanRepairCostUsd: number;
+    /** Golden-set value accuracy where the route defines expectedValues; null otherwise. */
+    readonly valueAccuracyMean: number | null;
+    readonly meanLatencyMs: number;
+    readonly meanCostUsd: number;
+    /** (base + repair + search costs) / conformant outputs × 1000 — the number. */
+    readonly effectiveUsdPer1kConformant: number;
+    /** Addendum A1: any sample served by a model with no pricing entry —
+     * cost columns under-count; this variant is never ranked or proposed. */
+    readonly pricingMissing: boolean;
+}
+export interface BakeoffArtifact {
+    readonly id: string;
+    readonly route: string;
+    readonly routeVersion: number;
+    readonly ranAt: string;
+    readonly replayN: number;
+    readonly variants: readonly VariantMetrics[];
+    /** Winner by effectiveUsdPer1kConformant among variants meeting the gates;
+     * null when no variant beats the incumbent's gates. */
+    readonly proposal: {
+        readonly variant: string;
+        readonly vsIncumbentSavingsPct: number;
+    } | null;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/bakeoff/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;2DACuD;IACvD,QAAQ,CAAC,QAAQ,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAChG"}

package/dist/bakeoff/types.js

@@ -0,0 +1,10 @@
+"use strict";
+/**
+ * Conformance bake-off contracts — Phase 3 spec WS5 (closed core).
+ * The headline metric: effective $ per 1K conformant outputs, with repair
+ * and search costs included (F1-corrected accounting).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+// Implemented: engine.ts assembles this from replay results; stats.ts owns
+// the math (Wilson CI — ported from the probes' summarize approach, no
+// cross-package import).

package/dist/cli.d.ts

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+/**
+ * modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
+ *   modelrig bakeoff --route X --variants a,b,c --replay-last N
+ *     [--budget-usd usd] [--json] [--no-mirror]   (--envelope is a kept alias)
+ *   modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
+ *   modelrig watch run [--events fixtures.json] [--json]
+ *
+ * Bake-offs replay locally captured traffic, write the artifact JSON next
+ * to the telemetry db (.modelrig/bakeoffs/<id>.json), upsert the `bakeoffs`
+ * mirror when the Supabase sink is configured, and print the matrix.
+ * Captures never travel — only aggregate metrics. Watch cycles spend $0;
+ * --from-proposal runs draw from the monthly watch envelope and attach
+ * their evidence to the proposal (suggested → evidenced). Nothing here
+ * executes a swap — that is `modelrig swap execute --approve`, a separate,
+ * human command.
+ */
+export interface Flags {
+    readonly positional: readonly string[];
+    readonly named: ReadonlyMap<string, string>;
+    readonly bare: ReadonlySet<string>;
+}
+export declare function parseArgs(argv: readonly string[]): Flags;
+export declare function main(argv: readonly string[]): Promise<number>;
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAiCH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AA0gBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAqCnE"}