modelrig 0.0.1 → 0.1.0

package/dist/bakeoff/engine.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Bake-off engine — Phase 3 spec WS5 (closed core). Replays the last N
+ * captures through each named variant, aggregates per-variant metrics
+ * (stats.ts), and assembles the BakeoffArtifact with the proposal verdict.
+ *
+ * Proposal gate (builder decision, flagged): the incumbent is the "default"
+ * variant when compared, else the FIRST listed variant. A challenger
+ * proposes when (a) its conformance CI lower bound ≥ the incumbent's CI
+ * lower bound, and (b) its effective $/1K-conformant is strictly lower.
+ * Value gates beyond mechanical conformance arrive with Phase 4 judges —
+ * the artifact records value_accuracy where the hook supplies it.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runBakeoff = runBakeoff;
+exports.computeProposal = computeProposal;
+exports.mirrorBakeoff = mirrorBakeoff;
+const node_crypto_1 = require("node:crypto");
+const loader_1 = require("../routes/loader");
+const replay_1 = require("./replay");
+const stats_1 = require("./stats");
+const samples_1 = require("./samples");
+/** Arm metadata for the reviewer: which model/rung/scaffold served it. */
+function deriveArmMeta(route, arm) {
+    if (arm === "default") {
+        return { model: route.candidates[0]?.model ?? null, rung: route.policy.json };
+    }
+    const variant = route.variants.find((v) => v.name === arm);
+    const model = (variant?.candidates?.[0] ?? route.candidates[0])?.model ?? null;
+    return {
+        model,
+        rung: variant?.json ?? route.policy.json,
+        ...(variant?.scaffold !== undefined ? { scaffold: true } : {}),
+    };
+}
+async function runBakeoff(deps, captures, req, samplesSink) {
+    if (req.variants.length === 0) {
+        throw new loader_1.RouteConfigError("bakeoff needs at least one variant (use \"default\" for the base config)");
+    }
+    const route = deps.routes.get(req.route);
+    for (const name of req.variants) {
+        if (name !== "default" && !route.variants.some((v) => v.name === name)) {
+            throw new loader_1.RouteConfigError(`route "${req.route}" has no variant "${name}" — defined: [default, ${route.variants
+                .map((v) => v.name)
+                .join(", ")}]`);
+        }
+    }
+    const keepOutputs = req.keepOutputs !== undefined && samplesSink !== undefined;
+    const bakeoffId = (0, node_crypto_1.randomUUID)();
+    const metrics = [];
+    const rawByArm = new Map();
+    let replayN = 0;
+    for (const variant of req.variants) {
+        const samples = await (0, replay_1.replayVariant)(deps, captures, {
+            route: req.route,
+            variant,
+            lastN: req.replayLastN,
+            envelopeUsd: req.envelopeUsdPerVariant,
+            ...(keepOutputs ? { keepOutputs: true } : {}),
+        });
+        replayN = Math.max(replayN, samples.length);
+        metrics.push((0, stats_1.aggregateVariantMetrics)(variant, samples));
+        if (keepOutputs)
+            rawByArm.set(variant, samples);
+    }
+    const proposal = computeProposal(req.variants, metrics);
+    // Persist review samples (A3) — AFTER the winner is known, for the selected
+    // arms only. Never touches the artifact (payloads stay off the anon table).
+    if (keepOutputs && samplesSink !== undefined) {
+        const incumbentArm = req.variants.includes("default") ? "default" : req.variants[0];
+        const keptArms = (0, samples_1.selectArmsToKeep)(req.keepOutputs, incumbentArm, proposal?.variant ?? null, req.variants);
+        const arms = keptArms.map((arm) => ({
+            arm,
+            armMeta: deriveArmMeta(route, arm),
+            samples: (rawByArm.get(arm) ?? []).map((s) => ({
+                inputRef: s.captureId,
+                inputText: s.inputText ?? "",
+                outputText: s.outputText ?? null,
+                conform: s.conform,
+                costUsd: s.costUsd,
+                latencyMs: s.latencyMs,
+            })),
+        }));
+        // Best-effort: a hiccup persisting the SECONDARY review samples must not
+        // lose the PRIMARY metrics artifact of an expensive bake-off. Warn and
+        // continue; the artifact still returns, mirrors, and writes to disk.
+        try {
+            await samplesSink({ bakeoffId, route: route.route, arms });
+        }
+        catch (err) {
+            console.warn(`[modelrig] bake-off ${bakeoffId}: review samples not persisted ` +
+                `(${err instanceof Error ? err.message : String(err)}) — metrics unaffected; ` +
+                "re-run with --keep-outputs to retry the samples.");
+        }
+    }
+    return {
+        id: bakeoffId,
+        route: route.route,
+        routeVersion: route.version,
+        ranAt: new Date().toISOString(),
+        replayN,
+        variants: metrics,
+        proposal,
+    };
+}
+/** Winner among gate-passing challengers; null when nobody beats the
+ * incumbent. Exported for direct unit testing. */
+function computeProposal(requested, metrics) {
+    const incumbentName = requested.includes("default") ? "default" : requested[0];
+    const incumbent = metrics.find((m) => m.variant === incumbentName);
+    // A1: unpriced cost columns under-count — an unpriced incumbent can't be
+    // compared against, and an unpriced challenger must never look cheap.
+    if (incumbent === undefined || incumbent.samples === 0 || incumbent.pricingMissing)
+        return null;
+    const challengers = metrics.filter((m) => m.variant !== incumbentName &&
+        m.samples > 0 &&
+        !m.pricingMissing &&
+        m.conformCi95[0] >= incumbent.conformCi95[0] &&
+        m.effectiveUsdPer1kConformant < incumbent.effectiveUsdPer1kConformant);
+    if (challengers.length === 0)
+        return null;
+    const winner = [...challengers].sort((a, b) => a.effectiveUsdPer1kConformant - b.effectiveUsdPer1kConformant)[0];
+    const savings = incumbent.effectiveUsdPer1kConformant === 0 ||
+        !Number.isFinite(incumbent.effectiveUsdPer1kConformant)
+        ? 0
+        : ((incumbent.effectiveUsdPer1kConformant - winner.effectiveUsdPer1kConformant) /
+            incumbent.effectiveUsdPer1kConformant) *
+            100;
+    return { variant: winner.variant, vsIncumbentSavingsPct: savings };
+}
+/** Upsert the artifact into the `bakeoffs` mirror table (console read path).
+ * Same PostgREST transport pattern as the telemetry exporter; captures have
+ * no cloud surface — ONLY aggregate metrics travel. */
+async function mirrorBakeoff(artifact, config) {
+    const fetchFn = config.fetchFn ?? fetch;
+    const base = config.supabaseUrl.replace(/\/$/, "");
+    const response = await fetchFn(`${base}/rest/v1/bakeoffs`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/json",
+            apikey: config.serviceRoleKey,
+            authorization: `Bearer ${config.serviceRoleKey}`,
+            prefer: "resolution=merge-duplicates,return=minimal",
+        },
+        body: JSON.stringify([
+            {
+                id: artifact.id,
+                route: artifact.route,
+                route_version: artifact.routeVersion,
+                ran_at: artifact.ranAt,
+                variants: artifact.variants.map((m) => m.variant),
+                // JSON has no Infinity: a variant with zero conformant outputs
+                // serializes effectiveUsdPer1kConformant as null (console renders ∞).
+                matrix: JSON.parse(JSON.stringify(artifact.variants)),
+                replay_n: artifact.replayN,
+                source_id: config.sourceId,
+            },
+        ]),
+    });
+    if (!response.ok) {
+        throw new Error(`bakeoff mirror upsert → ${response.status} ${await response.text().catch(() => "")}`);
+    }
+}

package/dist/bakeoff/replay.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Offline shadow replay — Phase 3 spec WS4/WS5. Re-runs locally captured
+ * inputs against candidate variants with ZERO production exposure. Replay
+ * dispatches are real (billed) adapter calls, but: envelope-guarded, tagged
+ * as replay (distinct from production telemetry), and never written to
+ * captures (no replay-of-replay).
+ *
+ * Builder decisions (flagged for architect review):
+ *  - Replay rows go into the SAME local store under the `replay:<id>` TAG
+ *    NAMESPACE (the scaffold's sanctioned option A): every replay attempt row
+ *    carries tags {replay, replay_capture, variant} — production queries
+ *    filter them out by tag, and per-sample cost is reconstructed from the
+ *    same rows (base + retries + repair + search — F1-correct).
+ *  - ReplaySampleResult gained `repairCostUsd` (additive) so the engine can
+ *    report repair cost separately from total cost.
+ *  - Envelope breach stops the pass typed-ly: the breaching sample is
+ *    recorded with failureClass budget_exhausted and the PARTIAL result set
+ *    is returned (honest Ns beat discarded spend).
+ */
+import type { CaptureStore } from "../telemetry/capture";
+import type { RunDeps } from "../run/run";
+export interface ReplayRequest {
+    readonly route: string;
+    readonly variant: string;
+    readonly lastN: number;
+    readonly envelopeUsd: number;
+    /** Addendum A3: retain the rendered input + this arm's output on each
+     * sample for the human value review. Off by default (bounds memory + keeps
+     * the pure-metrics path clean). */
+    readonly keepOutputs?: boolean;
+}
+export interface ReplaySampleResult {
+    readonly captureId: string;
+    readonly conform: boolean;
+    readonly repaired: boolean;
+    readonly valueAccuracy: number | null;
+    readonly costUsd: number;
+    /** Portion of costUsd spent on repair calls (repaired_by rows). */
+    readonly repairCostUsd: number;
+    /** Addendum A1: any attempt row lacked pricing — costUsd under-counts and
+     * this sample must never rank a variant as cheap. */
+    readonly pricingMissing: boolean;
+    readonly latencyMs: number;
+    readonly failureClass: string | null;
+    /** Addendum A3 (keepOutputs only): the replayed input, verbatim. */
+    readonly inputText?: string;
+    /** Addendum A3 (keepOutputs only): this arm's output for the review, or
+     * null when the arm produced none (a failure). */
+    readonly outputText?: string | null;
+}
+export declare function replayVariant(deps: RunDeps, captures: CaptureStore, req: ReplayRequest): Promise<readonly ReplaySampleResult[]>;
+//# sourceMappingURL=replay.d.ts.map

package/dist/bakeoff/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAsExC"}

package/dist/bakeoff/replay.js

@@ -0,0 +1,94 @@
+"use strict";
+/**
+ * Offline shadow replay — Phase 3 spec WS4/WS5. Re-runs locally captured
+ * inputs against candidate variants with ZERO production exposure. Replay
+ * dispatches are real (billed) adapter calls, but: envelope-guarded, tagged
+ * as replay (distinct from production telemetry), and never written to
+ * captures (no replay-of-replay).
+ *
+ * Builder decisions (flagged for architect review):
+ *  - Replay rows go into the SAME local store under the `replay:<id>` TAG
+ *    NAMESPACE (the scaffold's sanctioned option A): every replay attempt row
+ *    carries tags {replay, replay_capture, variant} — production queries
+ *    filter them out by tag, and per-sample cost is reconstructed from the
+ *    same rows (base + retries + repair + search — F1-correct).
+ *  - ReplaySampleResult gained `repairCostUsd` (additive) so the engine can
+ *    report repair cost separately from total cost.
+ *  - Envelope breach stops the pass typed-ly: the breaching sample is
+ *    recorded with failureClass budget_exhausted and the PARTIAL result set
+ *    is returned (honest Ns beat discarded spend).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayVariant = replayVariant;
+const node_crypto_1 = require("node:crypto");
+const failures_1 = require("../types/failures");
+const run_1 = require("../run/run");
+async function replayVariant(deps, captures, req) {
+    const rows = captures.readLast(req.route, req.lastN);
+    const replayId = (0, node_crypto_1.randomUUID)();
+    const envelopeKey = `replay:${replayId}`;
+    // Zero production exposure knobs: no capture writes (no replay-of-replay),
+    // and the whole pass shares one hard envelope.
+    const replayDeps = {
+        ...deps,
+        captures: null,
+        envelopeBudgetUsd: req.envelopeUsd,
+    };
+    const results = [];
+    for (const capture of rows) {
+        const sampleTag = `${replayId}:${capture.inferenceId}`;
+        const started = Date.now();
+        let conform = false;
+        let repaired = false;
+        let failureClass = null;
+        let outputText = null;
+        try {
+            // Through the REAL run loop: the variant's template re-renders against
+            // the replay candidate's capability flags (never the captured prompt
+            // text — plan §10 replay-validity risk), repair/grounding rungs and
+            // retry budgets all behave exactly as production serving would.
+            const result = await (0, run_1.run)(replayDeps, req.route, {
+                input: JSON.parse(capture.variablesJson),
+                tags: { replay: replayId, replay_capture: sampleTag, variant: req.variant },
+                ...(req.variant !== "default" ? { variant: req.variant } : {}),
+                budget: { envelope: envelopeKey },
+            });
+            conform = result.meta.validated;
+            repaired = result.meta.repairedBy !== null;
+            if (req.keepOutputs)
+                outputText = JSON.stringify(result.output, null, 2);
+        }
+        catch (err) {
+            if (!(err instanceof failures_1.RigFailureError))
+                throw err;
+            failureClass = err.failure.class;
+        }
+        const latencyMs = Date.now() - started;
+        // Per-sample cost from the tagged attempt rows — base + retries + repair
+        // + search, exactly what telemetry billed (F1-corrected).
+        const attemptRows = deps.telemetry.queryByTag("replay_capture", sampleTag);
+        const costUsd = attemptRows.reduce((a, r) => a + r.costEstimateUsd, 0);
+        const repairCostUsd = attemptRows
+            .filter((r) => r.repairedBy !== null)
+            .reduce((a, r) => a + r.costEstimateUsd, 0);
+        const pricingMissing = attemptRows.some((r) => r.pricingMissing);
+        results.push({
+            captureId: capture.inferenceId,
+            conform,
+            repaired,
+            valueAccuracy: null, // golden-set hook — expected values are Phase 4's judge/goldens work
+            costUsd,
+            repairCostUsd,
+            pricingMissing,
+            latencyMs,
+            failureClass,
+            ...(req.keepOutputs
+                ? { inputText: capture.variablesJson, outputText }
+                : {}),
+        });
+        if (failureClass === "budget_exhausted") {
+            break; // typed stop at the envelope; partial results stand
+        }
+    }
+    return results;
+}

package/dist/bakeoff/samples.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Bake-off review samples — Phase 4 addendum A3 (value-review-storage.md §11).
+ *
+ * A SEPARATE store from captures: rows exist only because a human ran
+ * `modelrig bakeoff --keep-outputs` intending a value review. They carry the
+ * rendered input + each arm's output so the human-in-the-loop diff (eval
+ * rung 1.5) happens in the console. Written by the CLI with the service-role
+ * key; read server-side only; deny-anon RLS. NEVER the anon staging posture.
+ *
+ * Transport is raw PostgREST over fetch (same pattern as the exporter).
+ */
+/** 256KB per output, per the architect's caps; TOAST compresses at rest. */
+export declare const OUTPUT_MAX_BYTES: number;
+export type KeepOutputsMode = "winner" | "all";
+export interface BakeoffSample {
+    readonly bakeoffId: string;
+    readonly route: string;
+    readonly arm: string;
+    readonly armMeta: Record<string, unknown> | null;
+    readonly inputRef: string;
+    readonly inputText: string;
+    readonly outputText: string | null;
+    readonly conform: boolean;
+    readonly costUsd: number;
+    readonly latencyMs: number;
+    readonly expiresAt: string;
+}
+/** One arm's per-input samples, ready to persist. */
+export interface ArmSamples {
+    readonly arm: string;
+    readonly armMeta: Record<string, unknown> | null;
+    readonly samples: ReadonlyArray<{
+        readonly inputRef: string;
+        readonly inputText: string;
+        readonly outputText: string | null;
+        readonly conform: boolean;
+        readonly costUsd: number;
+        readonly latencyMs: number;
+    }>;
+}
+/** Default persists incumbent + winner only; "all" persists every arm. */
+export declare function selectArmsToKeep(mode: KeepOutputsMode, incumbentArm: string, winnerArm: string | null, allArms: readonly string[]): string[];
+export declare function truncateOutput(text: string | null): string | null;
+/** Default retention: 90 days from now (or pass a sooner stamp). Pure over
+ * an injected `now` so it stays testable and workflow-safe. */
+export declare function retentionExpiry(nowMs: number, days?: number): string;
+export declare function buildBakeoffSamples(bakeoffId: string, route: string, arms: readonly ArmSamples[], expiresAt: string): BakeoffSample[];
+export interface BakeoffSamplesClientConfig {
+    readonly supabaseUrl: string;
+    readonly serviceRoleKey: string;
+    readonly fetchFn?: typeof fetch;
+}
+export interface BakeoffSamplesClient {
+    insert(samples: readonly BakeoffSample[]): Promise<void>;
+    /** Purge by bakeoff id, or everything past its expiry when no id given. */
+    purge(opts?: {
+        bakeoffId?: string;
+        nowIso?: string;
+    }): Promise<number>;
+}
+export declare function createBakeoffSamplesClient(config: BakeoffSamplesClientConfig): BakeoffSamplesClient;
+//# sourceMappingURL=samples.d.ts.map

package/dist/bakeoff/samples.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"samples.d.ts","sourceRoot":"","sources":["../../src/bakeoff/samples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,QAAa,CAAC;AAE3C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,EAAE,CAKV;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOjE;AAED;+DAC+D;AAC/D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,MAAM,CAEhE;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,UAAU,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,aAAa,EAAE,CAoBjB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,0BAA0B,GAAG,oBAAoB,CAoEnG"}

package/dist/bakeoff/samples.js

@@ -0,0 +1,128 @@
+"use strict";
+/**
+ * Bake-off review samples — Phase 4 addendum A3 (value-review-storage.md §11).
+ *
+ * A SEPARATE store from captures: rows exist only because a human ran
+ * `modelrig bakeoff --keep-outputs` intending a value review. They carry the
+ * rendered input + each arm's output so the human-in-the-loop diff (eval
+ * rung 1.5) happens in the console. Written by the CLI with the service-role
+ * key; read server-side only; deny-anon RLS. NEVER the anon staging posture.
+ *
+ * Transport is raw PostgREST over fetch (same pattern as the exporter).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OUTPUT_MAX_BYTES = void 0;
+exports.selectArmsToKeep = selectArmsToKeep;
+exports.truncateOutput = truncateOutput;
+exports.retentionExpiry = retentionExpiry;
+exports.buildBakeoffSamples = buildBakeoffSamples;
+exports.createBakeoffSamplesClient = createBakeoffSamplesClient;
+/** 256KB per output, per the architect's caps; TOAST compresses at rest. */
+exports.OUTPUT_MAX_BYTES = 256 * 1024;
+/** Default persists incumbent + winner only; "all" persists every arm. */
+function selectArmsToKeep(mode, incumbentArm, winnerArm, allArms) {
+    if (mode === "all")
+        return [...allArms];
+    const keep = [incumbentArm];
+    if (winnerArm !== null && winnerArm !== incumbentArm)
+        keep.push(winnerArm);
+    return keep;
+}
+function truncateOutput(text) {
+    if (text === null)
+        return null;
+    if (Buffer.byteLength(text, "utf8") <= exports.OUTPUT_MAX_BYTES)
+        return text;
+    // Cut on a byte budget, then tag it so the reviewer knows it's clipped.
+    const marker = "\n\n[truncated at 256KB for review storage]";
+    const budget = exports.OUTPUT_MAX_BYTES - Buffer.byteLength(marker, "utf8");
+    return Buffer.from(text, "utf8").subarray(0, budget).toString("utf8") + marker;
+}
+/** Default retention: 90 days from now (or pass a sooner stamp). Pure over
+ * an injected `now` so it stays testable and workflow-safe. */
+function retentionExpiry(nowMs, days = 90) {
+    return new Date(nowMs + days * 24 * 3600 * 1000).toISOString();
+}
+function buildBakeoffSamples(bakeoffId, route, arms, expiresAt) {
+    const rows = [];
+    for (const arm of arms) {
+        for (const s of arm.samples) {
+            rows.push({
+                bakeoffId,
+                route,
+                arm: arm.arm,
+                armMeta: arm.armMeta,
+                inputRef: s.inputRef,
+                inputText: s.inputText,
+                outputText: truncateOutput(s.outputText),
+                conform: s.conform,
+                costUsd: s.costUsd,
+                latencyMs: s.latencyMs,
+                expiresAt,
+            });
+        }
+    }
+    return rows;
+}
+function createBakeoffSamplesClient(config) {
+    const fetchFn = config.fetchFn ?? fetch;
+    const base = config.supabaseUrl.replace(/\/$/, "");
+    async function request(path, init) {
+        const response = await fetchFn(`${base}/rest/v1/${path}`, {
+            ...init,
+            headers: {
+                "content-type": "application/json",
+                apikey: config.serviceRoleKey,
+                authorization: `Bearer ${config.serviceRoleKey}`,
+                ...(init.headers ?? {}),
+            },
+        });
+        if (!response.ok) {
+            throw new Error(`bakeoff_samples ${init.method ?? "GET"} ${path} → ${response.status} ` +
+                `${await response.text().catch(() => "")}`);
+        }
+        return response;
+    }
+    return {
+        async insert(samples) {
+            if (samples.length === 0)
+                return;
+            // Chunk the POST: at the ≤50-inputs cap, large-input routes (e.g. the
+            // cutover steps pass ~600KB assembled prompts) would make a single body
+            // tens of MB and blow past PostgREST's limit. Small batches keep every
+            // body well under it, at any cap.
+            const CHUNK = 8;
+            for (let i = 0; i < samples.length; i += CHUNK) {
+                const batch = samples.slice(i, i + CHUNK);
+                await request("bakeoff_samples", {
+                    method: "POST",
+                    headers: { prefer: "return=minimal" },
+                    body: JSON.stringify(batch.map((s) => ({
+                        bakeoff_id: s.bakeoffId,
+                        route: s.route,
+                        arm: s.arm,
+                        arm_meta: s.armMeta,
+                        input_ref: s.inputRef,
+                        input_text: s.inputText,
+                        output_text: s.outputText,
+                        conform: s.conform,
+                        cost_usd: s.costUsd,
+                        latency_ms: s.latencyMs,
+                        expires_at: s.expiresAt,
+                    }))),
+                });
+            }
+        },
+        async purge(opts) {
+            const filter = opts?.bakeoffId !== undefined
+                ? `bakeoff_id=eq.${encodeURIComponent(opts.bakeoffId)}`
+                : `expires_at=lt.${encodeURIComponent(opts?.nowIso ?? new Date().toISOString())}`;
+            const response = await request(`bakeoff_samples?${filter}`, {
+                method: "DELETE",
+                headers: { prefer: "return=representation" },
+            });
+            const deleted = (await response.json().catch(() => []));
+            return Array.isArray(deleted) ? deleted.length : 0;
+        },
+    };
+}

package/dist/bakeoff/stats.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Bake-off statistics — Phase 3 spec WS5. Wilson 95% intervals ported from
+ * the probe suite's summarize math (same constants — results comparable);
+ * no cross-package import (the probes package is standalone since P2).
+ *
+ * The headline: effective $ per 1K conformant outputs — TOTAL sample cost
+ * (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
+ */
+import type { ReplaySampleResult } from "./replay";
+import type { VariantMetrics } from "./types";
+export declare function wilson95(successes: number, n: number): readonly [number, number];
+export declare function aggregateVariantMetrics(variant: string, samples: readonly ReplaySampleResult[]): VariantMetrics;
+//# sourceMappingURL=stats.d.ts.map

package/dist/bakeoff/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/bakeoff/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAShF;AAMD,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,kBAAkB,EAAE,GACrC,cAAc,CAuBhB"}

package/dist/bakeoff/stats.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * Bake-off statistics — Phase 3 spec WS5. Wilson 95% intervals ported from
+ * the probe suite's summarize math (same constants — results comparable);
+ * no cross-package import (the probes package is standalone since P2).
+ *
+ * The headline: effective $ per 1K conformant outputs — TOTAL sample cost
+ * (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wilson95 = wilson95;
+exports.aggregateVariantMetrics = aggregateVariantMetrics;
+function wilson95(successes, n) {
+    if (n === 0)
+        return [0, 1];
+    const z = 1.959963984540054;
+    const p = successes / n;
+    const z2 = z * z;
+    const denominator = 1 + z2 / n;
+    const center = (p + z2 / (2 * n)) / denominator;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denominator;
+    return [Math.max(0, center - half), Math.min(1, center + half)];
+}
+function mean(values) {
+    return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
+}
+function aggregateVariantMetrics(variant, samples) {
+    const n = samples.length;
+    const conformant = samples.filter((s) => s.conform);
+    const repaired = samples.filter((s) => s.repaired);
+    const accuracies = samples
+        .map((s) => s.valueAccuracy)
+        .filter((a) => a !== null);
+    const totalCostUsd = samples.reduce((a, s) => a + s.costUsd, 0);
+    return {
+        variant,
+        samples: n,
+        conformRate: n === 0 ? 0 : conformant.length / n,
+        conformCi95: wilson95(conformant.length, n),
+        repairRate: n === 0 ? 0 : repaired.length / n,
+        meanRepairCostUsd: mean(repaired.map((s) => s.repairCostUsd)),
+        valueAccuracyMean: accuracies.length === 0 ? null : mean(accuracies),
+        meanLatencyMs: mean(samples.map((s) => s.latencyMs)),
+        meanCostUsd: n === 0 ? 0 : totalCostUsd / n,
+        effectiveUsdPer1kConformant: conformant.length === 0 ? Number.POSITIVE_INFINITY : (totalCostUsd / conformant.length) * 1000,
+        pricingMissing: samples.some((s) => s.pricingMissing),
+    };
+}

package/dist/bakeoff/types.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Conformance bake-off contracts — Phase 3 spec WS5 (closed core).
+ * The headline metric: effective $ per 1K conformant outputs, with repair
+ * and search costs included (F1-corrected accounting).
+ */
+export interface VariantMetrics {
+    readonly variant: string;
+    readonly samples: number;
+    readonly conformRate: number;
+    readonly conformCi95: readonly [number, number];
+    readonly repairRate: number;
+    readonly meanRepairCostUsd: number;
+    /** Golden-set value accuracy where the route defines expectedValues; null otherwise. */
+    readonly valueAccuracyMean: number | null;
+    readonly meanLatencyMs: number;
+    readonly meanCostUsd: number;
+    /** (base + repair + search costs) / conformant outputs × 1000 — the number. */
+    readonly effectiveUsdPer1kConformant: number;
+    /** Addendum A1: any sample served by a model with no pricing entry —
+     * cost columns under-count; this variant is never ranked or proposed. */
+    readonly pricingMissing: boolean;
+}
+export interface BakeoffArtifact {
+    readonly id: string;
+    readonly route: string;
+    readonly routeVersion: number;
+    readonly ranAt: string;
+    readonly replayN: number;
+    readonly variants: readonly VariantMetrics[];
+    /** Winner by effectiveUsdPer1kConformant among variants meeting the gates;
+     * null when no variant beats the incumbent's gates. */
+    readonly proposal: {
+        readonly variant: string;
+        readonly vsIncumbentSavingsPct: number;
+    } | null;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/bakeoff/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;2DACuD;IACvD,QAAQ,CAAC,QAAQ,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAChG"}

package/dist/bakeoff/types.js

@@ -0,0 +1,10 @@
+"use strict";
+/**
+ * Conformance bake-off contracts — Phase 3 spec WS5 (closed core).
+ * The headline metric: effective $ per 1K conformant outputs, with repair
+ * and search costs included (F1-corrected accounting).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+// Implemented: engine.ts assembles this from replay results; stats.ts owns
+// the math (Wilson CI — ported from the probes' summarize approach, no
+// cross-package import).

package/dist/cli.d.ts

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+/**
+ * modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
+ *   modelrig bakeoff --route X --variants a,b,c --replay-last N
+ *     [--envelope usd] [--json] [--no-mirror]
+ *   modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
+ *   modelrig watch run [--events fixtures.json] [--json]
+ *
+ * Bake-offs replay locally captured traffic, write the artifact JSON next
+ * to the telemetry db (.modelrig/bakeoffs/<id>.json), upsert the `bakeoffs`
+ * mirror when the Supabase sink is configured, and print the matrix.
+ * Captures never travel — only aggregate metrics. Watch cycles spend $0;
+ * --from-proposal runs draw from the monthly watch envelope and attach
+ * their evidence to the proposal (suggested → evidenced). Nothing here
+ * executes a swap — that is `modelrig swap execute --approve`, a separate,
+ * human command.
+ */
+import type { BakeoffArtifact } from "./bakeoff/types";
+export interface Flags {
+    readonly positional: readonly string[];
+    readonly named: ReadonlyMap<string, string>;
+    readonly bare: ReadonlySet<string>;
+}
+export declare function parseArgs(argv: readonly string[]): Flags;
+export declare function printMatrix(artifact: BakeoffArtifact): void;
+export declare function main(argv: readonly string[]): Promise<number>;
+//# sourceMappingURL=cli.d.ts.map