modelrig 0.7.0 → 0.8.1

package/dist/bakeoff/pairwise.d.ts

@@ -0,0 +1,102 @@
+/**
+ * N5 — pairwise preference vs the reference arm (near-optimal brief §3.2 N5).
+ *
+ * A second, complementary read on regret: for each replayed sample, is the SERVED
+ * arm's output BETTER than, tied with, or WORSE than the REFERENCE arm's on the
+ * same captured input? Aggregated to a `pairwise_win_rate` (win=1, tie=0.5,
+ * loss=0) with a seeded bootstrap CI — the RigIndex pairwise idea, scoped to one
+ * bake-off's arm-vs-reference comparison.
+ *
+ * DETERMINISTIC-FIRST (F5), exactly like the regret quality tiers:
+ *   1. When BOTH sides carry a per-sample value-accuracy score (eval case /
+ *      value_checks / Q1 gate), the verdict is the sign of their difference —
+ *      no model call. This is the free, always-preferred tier.
+ *   2. Only when a sample cannot be decided deterministically does an OPT-IN,
+ *      CAPPED, METERED pairwise judge (`bakeoff.pairwise.judge`) compare the two
+ *      outputs. Judge model ∈ declared candidates (#223 F3, enforced at load).
+ *
+ * The aggregation math is PURE and unit-tested directly; the judge reuses the
+ * real `run()` loop (no new dispatch path, no run.ts edit) exactly as judge.ts.
+ */
+import type { RunDeps } from "../run/run";
+import type { CandidateSpec, RouteBundle } from "../types/route";
+import type { ReplaySampleResult } from "./replay";
+/** The served arm's standing vs the reference on one sample. `win` = the arm's
+ * output is better than the reference's; `loss` = worse; `tie` = indistinguishable. */
+export type PairwiseOutcome = "win" | "tie" | "loss";
+/** How a single verdict was decided (F5): a free deterministic comparison of two
+ * value-accuracy scores, or a billed pairwise judge call. */
+export type PairwiseSource = "deterministic" | "judge";
+/** Aggregate pairwise standing of an arm vs the reference (N5). `winRate` counts
+ * a win as 1 and a tie as 0.5; the CI is a seeded bootstrap of the per-sample
+ * win values, so it is reproducible given the same sample set. */
+export interface PairwiseStats {
+    readonly winRate: number;
+    readonly winRateCi95: readonly [number, number];
+    /** Samples with a verdict (deterministic + judge); the CI's n. */
+    readonly nDecided: number;
+    readonly wins: number;
+    readonly ties: number;
+    readonly losses: number;
+    /** Paired samples that could NOT be decided (no deterministic score and no
+     * judge, or the judge was capped/failed) — excluded from the rate, counted. */
+    readonly nUndecided: number;
+    /** Where the decided verdicts came from — "deterministic", "judge", or "mixed". */
+    readonly source: "deterministic" | "judge" | "mixed";
+}
+/** Seeded so a bake-off's pairwise CI is reproducible (like the regret CI). A
+ * distinct seed from the regret bootstrap keeps the two resamplings independent. */
+export declare const PAIRWISE_BOOTSTRAP_REPS = 2000;
+export declare const PAIRWISE_BOOTSTRAP_SEED = 20260824;
+/** The win-value of an outcome (win=1, tie=0.5, loss=0) — the quantity the win
+ * rate averages. */
+export declare function winValue(o: PairwiseOutcome): number;
+/**
+ * The DETERMINISTIC pairwise verdict (tier 1): the sign of `armQuality −
+ * refQuality`. Returns null when EITHER side lacks a value-accuracy score — the
+ * caller then falls to the judge. A tie within EPS is a genuine tie.
+ */
+export declare function deterministicPairwise(armQuality: number | null, refQuality: number | null): PairwiseOutcome | null;
+/**
+ * Aggregate per-sample verdicts into `PairwiseStats` (N5). `decided` are the
+ * verdicts that resolved; `nUndecided` the paired samples that did not. The CI is
+ * a seeded bootstrap of the per-sample win values (win=1/tie=0.5/loss=0), so it
+ * is deterministic given the verdict list + seed. Pure.
+ */
+export declare function aggregatePairwise(decided: readonly {
+    readonly outcome: PairwiseOutcome;
+    readonly source: PairwiseSource;
+}[], nUndecided: number, seed?: number, reps?: number): PairwiseStats;
+/**
+ * Score ONE arm's pairwise standing vs the reference over the paired replay set
+ * (N5). Deterministic-first: each paired sample's verdict is the sign of the two
+ * value-accuracy scores; a sample neither side scored falls to the capped judge
+ * (when supplied), else it is undecided (counted, excluded from the rate). Pairs
+ * by captureId; unpaired samples are ignored. Returns null stats when there are
+ * NO paired samples at all. `outcomes` maps captureId → verdict (for N6 mining).
+ */
+export declare function scoreArmPairwise(armSamples: readonly ReplaySampleResult[], refSamples: readonly ReplaySampleResult[], judge: PairwiseJudge | null): Promise<{
+    stats: PairwiseStats | null;
+    outcomes: Map<string, PairwiseOutcome>;
+}>;
+/** Build the synthetic pairwise-judge route for a base route + judge model. The
+ * judge model is declared WITHOUT its trial flag so it serves live here. */
+export declare function buildPairwiseJudgeRoute(baseRoute: RouteBundle, model: CandidateSpec): RouteBundle;
+/** A per-arm pairwise scorer: `compare(armOutput, refOutput, taskInput)` returns
+ * the arm's outcome vs the reference, or null when the cap is hit or the judge
+ * could not produce a valid verdict. */
+export interface PairwiseJudge {
+    compare(armOutput: unknown, refOutput: unknown, taskInput: unknown): Promise<PairwiseOutcome | null>;
+    used(): number;
+}
+/**
+ * Build a pairwise judge over the run loop (mirrors makeJudgeScorer). Each arm
+ * gets a FRESH judge so `maxSamples` is per arm. The comparison is presented in a
+ * FIXED order (A = served arm, B = reference); position bias is a known limit,
+ * noted in verification — a fixed order keeps the judge DETERMINISTIC for tests.
+ */
+export declare function makePairwiseJudge(deps: RunDeps, baseRoute: RouteBundle, judge: {
+    readonly model: CandidateSpec;
+    readonly maxSamples: number;
+}): PairwiseJudge;
+//# sourceMappingURL=pairwise.d.ts.map

package/dist/bakeoff/pairwise.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.d.ts","sourceRoot":"","sources":["../../src/bakeoff/pairwise.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEnD;uFACuF;AACvF,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;AAErD;6DAC6D;AAC7D,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG,OAAO,CAAC;AAEvD;;kEAEkE;AAClE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;mFAC+E;IAC/E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,GAAG,OAAO,CAAC;CACtD;AAED;oFACoF;AACpF,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,uBAAuB,WAAW,CAAC;AAKhD;oBACoB;AACpB,wBAAgB,QAAQ,CAAC,CAAC,EAAE,eAAe,GAAG,MAAM,CAEnD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,UAAU,EAAE,MAAM,GAAG,IAAI,GACxB,eAAe,GAAG,IAAI,CAKxB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,SAAS;IAAE,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;CAAE,EAAE,EAC1F,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,MAAgC,EACtC,IAAI,GAAE,MAAgC,GACrC,aAAa,CA2Bf;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,KAAK,EAAE,aAAa,GAAG,IAAI,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;CAAE,CAAC,CA8BlF;AAmBD;4EAC4E;AAC5E,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,GAAG,WAAW,CAejG;AAED;;wCAEwC;AACxC,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IACrG,IAAI,IAAI,MAAM,CAAC;CAChB;AAMD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,WAAW,EACtB,KAAK,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACpE,aAAa,CAmCf"}

package/dist/bakeoff/pairwise.js

@@ -0,0 +1,214 @@
+"use strict";
+/**
+ * N5 — pairwise preference vs the reference arm (near-optimal brief §3.2 N5).
+ *
+ * A second, complementary read on regret: for each replayed sample, is the SERVED
+ * arm's output BETTER than, tied with, or WORSE than the REFERENCE arm's on the
+ * same captured input? Aggregated to a `pairwise_win_rate` (win=1, tie=0.5,
+ * loss=0) with a seeded bootstrap CI — the RigIndex pairwise idea, scoped to one
+ * bake-off's arm-vs-reference comparison.
+ *
+ * DETERMINISTIC-FIRST (F5), exactly like the regret quality tiers:
+ *   1. When BOTH sides carry a per-sample value-accuracy score (eval case /
+ *      value_checks / Q1 gate), the verdict is the sign of their difference —
+ *      no model call. This is the free, always-preferred tier.
+ *   2. Only when a sample cannot be decided deterministically does an OPT-IN,
+ *      CAPPED, METERED pairwise judge (`bakeoff.pairwise.judge`) compare the two
+ *      outputs. Judge model ∈ declared candidates (#223 F3, enforced at load).
+ *
+ * The aggregation math is PURE and unit-tested directly; the judge reuses the
+ * real `run()` loop (no new dispatch path, no run.ts edit) exactly as judge.ts.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PAIRWISE_BOOTSTRAP_SEED = exports.PAIRWISE_BOOTSTRAP_REPS = void 0;
+exports.winValue = winValue;
+exports.deterministicPairwise = deterministicPairwise;
+exports.aggregatePairwise = aggregatePairwise;
+exports.scoreArmPairwise = scoreArmPairwise;
+exports.buildPairwiseJudgeRoute = buildPairwiseJudgeRoute;
+exports.makePairwiseJudge = makePairwiseJudge;
+const node_crypto_1 = require("node:crypto");
+const bootstrap_1 = require("../rigindex/bootstrap");
+const run_1 = require("../run/run");
+const failures_1 = require("../types/failures");
+/** Seeded so a bake-off's pairwise CI is reproducible (like the regret CI). A
+ * distinct seed from the regret bootstrap keeps the two resamplings independent. */
+exports.PAIRWISE_BOOTSTRAP_REPS = 2000;
+exports.PAIRWISE_BOOTSTRAP_SEED = 20260824;
+/** Tie band: value-accuracy scores within EPS are a tie, not a hair-split win. */
+const PAIRWISE_EPS = 1e-9;
+/** The win-value of an outcome (win=1, tie=0.5, loss=0) — the quantity the win
+ * rate averages. */
+function winValue(o) {
+    return o === "win" ? 1 : o === "tie" ? 0.5 : 0;
+}
+/**
+ * The DETERMINISTIC pairwise verdict (tier 1): the sign of `armQuality −
+ * refQuality`. Returns null when EITHER side lacks a value-accuracy score — the
+ * caller then falls to the judge. A tie within EPS is a genuine tie.
+ */
+function deterministicPairwise(armQuality, refQuality) {
+    if (armQuality === null || refQuality === null)
+        return null;
+    const diff = armQuality - refQuality;
+    if (Math.abs(diff) <= PAIRWISE_EPS)
+        return "tie";
+    return diff > 0 ? "win" : "loss";
+}
+/**
+ * Aggregate per-sample verdicts into `PairwiseStats` (N5). `decided` are the
+ * verdicts that resolved; `nUndecided` the paired samples that did not. The CI is
+ * a seeded bootstrap of the per-sample win values (win=1/tie=0.5/loss=0), so it
+ * is deterministic given the verdict list + seed. Pure.
+ */
+function aggregatePairwise(decided, nUndecided, seed = exports.PAIRWISE_BOOTSTRAP_SEED, reps = exports.PAIRWISE_BOOTSTRAP_REPS) {
+    const wins = decided.filter((d) => d.outcome === "win").length;
+    const ties = decided.filter((d) => d.outcome === "tie").length;
+    const losses = decided.filter((d) => d.outcome === "loss").length;
+    const nDecided = decided.length;
+    const hasDet = decided.some((d) => d.source === "deterministic");
+    const hasJudge = decided.some((d) => d.source === "judge");
+    const source = hasDet && hasJudge ? "mixed" : hasJudge ? "judge" : "deterministic";
+    if (nDecided === 0) {
+        return { winRate: 0, winRateCi95: [0, 0], nDecided: 0, wins: 0, ties: 0, losses: 0, nUndecided, source };
+    }
+    const values = decided.map((d) => winValue(d.outcome));
+    const winRate = values.reduce((a, b) => a + b, 0) / nDecided;
+    // Bootstrap the per-sample win values (the pairing is already fixed), take each
+    // resample's mean, then the 2.5/97.5 percentiles.
+    const rng = (0, bootstrap_1.mulberry32)(seed);
+    const means = [];
+    for (let b = 0; b < reps; b += 1) {
+        let sum = 0;
+        for (let i = 0; i < nDecided; i += 1)
+            sum += values[Math.floor(rng() * nDecided)];
+        means.push(sum / nDecided);
+    }
+    means.sort((a, b) => a - b);
+    const lo = means[Math.floor(0.025 * reps)];
+    const hi = means[Math.min(reps - 1, Math.floor(0.975 * reps))];
+    return { winRate, winRateCi95: [lo, hi], nDecided, wins, ties, losses, nUndecided, source };
+}
+/**
+ * Score ONE arm's pairwise standing vs the reference over the paired replay set
+ * (N5). Deterministic-first: each paired sample's verdict is the sign of the two
+ * value-accuracy scores; a sample neither side scored falls to the capped judge
+ * (when supplied), else it is undecided (counted, excluded from the rate). Pairs
+ * by captureId; unpaired samples are ignored. Returns null stats when there are
+ * NO paired samples at all. `outcomes` maps captureId → verdict (for N6 mining).
+ */
+async function scoreArmPairwise(armSamples, refSamples, judge) {
+    const refByCapture = new Map(refSamples.map((s) => [s.captureId, s]));
+    const decided = [];
+    const outcomes = new Map();
+    let paired = 0;
+    let undecided = 0;
+    for (const arm of armSamples) {
+        const ref = refByCapture.get(arm.captureId);
+        if (ref === undefined)
+            continue;
+        paired += 1;
+        const det = deterministicPairwise(arm.valueAccuracy, ref.valueAccuracy);
+        if (det !== null) {
+            decided.push({ outcome: det, source: "deterministic" });
+            outcomes.set(arm.captureId, det);
+            continue;
+        }
+        // Deterministic can't decide (a side lacks value accuracy). Only a PRODUCED
+        // pair can be judged (both need an output); the judge is capped internally.
+        if (judge !== null && arm.failureClass === null && ref.failureClass === null) {
+            const verdict = await judge.compare(arm.outputText ?? null, ref.outputText ?? null, arm.inputText ?? null);
+            if (verdict !== null) {
+                decided.push({ outcome: verdict, source: "judge" });
+                outcomes.set(arm.captureId, verdict);
+                continue;
+            }
+        }
+        undecided += 1;
+    }
+    if (paired === 0)
+        return { stats: null, outcomes };
+    return { stats: aggregatePairwise(decided, undecided), outcomes };
+}
+// ── The opt-in, capped, metered pairwise judge (tier 2) ──────────────────────
+const PAIRWISE_SCHEMA = {
+    type: "object",
+    properties: { verdict: { type: "string", enum: ["A", "B", "tie"] }, reason: { type: "string" } },
+    required: ["verdict"],
+    additionalProperties: true,
+};
+const PAIRWISE_RUBRIC = "You are judging PAIRWISE PREFERENCE between two candidate outputs for ONE task — " +
+    "which is the better answer for VALUE (correctness and completeness), not merely which " +
+    "is better-formed. A is one candidate; B is the other.\n\nTASK INPUT:\n{{task_input}}\n\n" +
+    "OUTPUT A:\n{{output_a}}\n\nOUTPUT B:\n{{output_b}}\n\n" +
+    'Respond with ONLY compact JSON: {"verdict": "A" | "B" | "tie", "reason": "<one sentence>"}. ' +
+    'Answer "A" if A is better, "B" if B is better, "tie" if they are equally good.';
+/** Build the synthetic pairwise-judge route for a base route + judge model. The
+ * judge model is declared WITHOUT its trial flag so it serves live here. */
+function buildPairwiseJudgeRoute(baseRoute, model) {
+    return Object.freeze({
+        route: `__bakeoff_pairwise__:${baseRoute.route}`,
+        version: baseRoute.version,
+        outputSchema: Object.freeze({ ...PAIRWISE_SCHEMA }),
+        candidates: Object.freeze([{ provider: model.provider, model: model.model }]),
+        require: Object.freeze([]),
+        prefer: Object.freeze([]),
+        systemTemplate: PAIRWISE_RUBRIC,
+        variables: Object.freeze(["output_a", "output_b", "task_input"]),
+        policy: Object.freeze({ retries: Object.freeze({}), timeoutMs: baseRoute.policy.timeoutMs, tier: "standard", json: "json_mode" }),
+        variants: Object.freeze([]),
+        capture: false,
+        fingerprintCell: null,
+    });
+}
+function asText(v) {
+    return typeof v === "string" ? v : JSON.stringify(v ?? null);
+}
+/**
+ * Build a pairwise judge over the run loop (mirrors makeJudgeScorer). Each arm
+ * gets a FRESH judge so `maxSamples` is per arm. The comparison is presented in a
+ * FIXED order (A = served arm, B = reference); position bias is a known limit,
+ * noted in verification — a fixed order keeps the judge DETERMINISTIC for tests.
+ */
+function makePairwiseJudge(deps, baseRoute, judge) {
+    const judgeRoute = buildPairwiseJudgeRoute(baseRoute, judge.model);
+    const routes = {
+        get: (r) => (r === judgeRoute.route ? judgeRoute : deps.routes.get(r)),
+        list: () => Object.freeze([...deps.routes.list(), judgeRoute]),
+    };
+    const judgeDeps = { ...deps, routes, captures: null, mode: "live" };
+    const envelope = `bakeoff-pairwise:${(0, node_crypto_1.randomUUID)()}`;
+    let used = 0;
+    return {
+        used: () => used,
+        async compare(armOutput, refOutput, taskInput) {
+            if (used >= judge.maxSamples)
+                return null;
+            used += 1;
+            try {
+                const result = await (0, run_1.run)(judgeDeps, judgeRoute.route, {
+                    input: { output_a: asText(armOutput), output_b: asText(refOutput), task_input: asText(taskInput) },
+                    // `bakeoff_pairwise` marks the row a pairwise-judge inference (the typed
+                    // `rung` column is recorder-owned, so a distinct tag key is used); `lane:
+                    // trial` keeps judge spend off the route's live rollups/odometer.
+                    tags: { bakeoff_pairwise: "1", lane: "trial" },
+                    budget: { envelope },
+                });
+                const verdict = result.output?.verdict;
+                if (verdict === "A")
+                    return "win";
+                if (verdict === "B")
+                    return "loss";
+                if (verdict === "tie")
+                    return "tie";
+                return null;
+            }
+            catch (err) {
+                if (!(err instanceof failures_1.RigFailureError))
+                    throw err;
+                // The metered attempt row(s) still stand; this sample gets no verdict.
+                return null;
+            }
+        },
+    };
+}

package/dist/bakeoff/replay.d.ts

@@ -30,6 +30,12 @@ export interface ReplayRequest {
      * sample for the human value review. Off by default (bounds memory + keeps
      * the pure-metrics path clean). */
     readonly keepOutputs?: boolean;
+    /** N5: retain `inputText` + `outputText` on each sample for the pairwise judge
+     * (arm-vs-reference comparison in the engine), WITHOUT the keepOutputs sink
+     * persistence. Set by the engine only when `bakeoff.pairwise.judge` is declared.
+     * A superset trigger of `keepOutputs` for retention; the sink stays gated on
+     * `keepOutputs` alone so pairwise retention never persists review samples. */
+    readonly retainOutputForPairwise?: boolean;
     /** RB-1: corrected/golden expected outputs keyed by captureId (eval-case path).
      * Presence in the map ⇒ this sample has an expected value; `value` may be any
      * JSON (including null). Quality tier 1 (eval case) when present. */

package/dist/bakeoff/replay.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoIxC"}
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAyIxC"}

package/dist/bakeoff/replay.js

@@ -49,6 +49,10 @@ async function replayVariant(deps, captures, req) {
         envelopeBudgetUsd: req.envelopeUsd,
         mode,
     };
+    // Retain per-sample input/output text when the caller asked (keepOutputs sink,
+    // or N5 pairwise judge). Same fields either way; the sink persist stays gated
+    // on keepOutputs alone (engine.ts).
+    const retainText = req.keepOutputs === true || req.retainOutputForPairwise === true;
     const results = [];
     for (const capture of rows) {
         const sampleTag = `${replayId}:${capture.inferenceId}`;
@@ -81,7 +85,7 @@ async function replayVariant(deps, captures, req) {
             repaired = result.meta.repairedBy !== null;
             outputValue = result.output;
             produced = true;
-            if (req.keepOutputs)
+            if (retainText)
                 outputText = JSON.stringify(result.output, null, 2);
             if (req.qualityGate !== undefined) {
                 gated = { output: result.output, provider: result.meta.provider, model: result.meta.model, servedVariant: result.meta.servedVariant };
@@ -145,7 +149,7 @@ async function replayVariant(deps, captures, req) {
             pricingMissing,
             latencyMs,
             failureClass,
-            ...(req.keepOutputs
+            ...(retainText
                 ? { inputText: capture.variablesJson, outputText }
                 : {}),
         });

package/dist/bakeoff/report.d.ts

@@ -36,6 +36,13 @@ export interface CacheAdjustedRow {
  * Empty input ⇒ no lines (the section is omitted entirely — back-compat).
  */
 export declare function formatCacheAdjustedSection(rows: readonly CacheAdjustedRow[]): string[];
+/**
+ * N6 disagreement summary lines: the count of mined eval-case candidates (with
+ * the honest "of N found" when the list was capped), the per-reason breakdown,
+ * and the top candidates as captureId pointers an operator may promote. Pure
+ * (returns strings); empty when nothing disagreed. Exported for direct testing.
+ */
+export declare function formatDisagreementSummary(artifact: BakeoffArtifact): string[];
 export declare function printMatrix(artifact: BakeoffArtifact, cacheAdjusted?: readonly CacheAdjustedRow[]): void;
 /**
  * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's

package/dist/bakeoff/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAkCpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAkD5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAwD5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/bakeoff/report.js

@@ -8,6 +8,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.bakeoffHeadline = bakeoffHeadline;
 exports.formatCacheAdjustedSection = formatCacheAdjustedSection;
+exports.formatDisagreementSummary = formatDisagreementSummary;
 exports.printMatrix = printMatrix;
 exports.runEvalCaseBakeoffCli = runEvalCaseBakeoffCli;
 const node_fs_1 = require("node:fs");
@@ -35,6 +36,16 @@ function fmtRegret(m) {
 function fmtQuality(m) {
     return m.qualityMean === null ? "—" : `${(m.qualityMean * 100).toFixed(0)}% ${m.qualityMetric}`;
 }
+/** N5: an arm's pairwise-vs-reference cell — "63% [51…74] n=30", "ref", or "—". */
+function fmtPairwise(m) {
+    if (m.isReference === true)
+        return "ref";
+    const p = m.pairwise;
+    if (p == null)
+        return "—";
+    const pct = (v) => (v * 100).toFixed(0);
+    return `${pct(p.winRate)}% [${pct(p.winRateCi95[0])}…${pct(p.winRateCi95[1])}] n=${p.nDecided}`;
+}
 /** The reference $/1k or per-run ceiling from the objective, formatted, or null
  * when no ceiling is declared. */
 function fmtCeiling(artifact) {
@@ -107,14 +118,43 @@ function formatCacheAdjustedSection(rows) {
     }
     return lines;
 }
+/**
+ * N6 disagreement summary lines: the count of mined eval-case candidates (with
+ * the honest "of N found" when the list was capped), the per-reason breakdown,
+ * and the top candidates as captureId pointers an operator may promote. Pure
+ * (returns strings); empty when nothing disagreed. Exported for direct testing.
+ */
+function formatDisagreementSummary(artifact) {
+    const d = artifact.disagreement;
+    if (d === undefined || d.found === 0)
+        return [];
+    const reasons = Object.entries(d.counts)
+        .filter(([, n]) => n > 0)
+        .map(([r, n]) => `${r}=${n}`)
+        .join(", ");
+    const capNote = d.cappedAt !== null ? ` (showing top ${d.candidates.length} of ${d.found} — cap ${d.cappedAt})` : "";
+    const lines = [
+        `disagreement: ${d.found} eval-case candidate${d.found === 1 ? "" : "s"} mined from ${d.examined} samples${capNote} — ` +
+            "NOT auto-promoted; promote with artifact.promoteToEval.",
+    ];
+    if (reasons !== "")
+        lines.push(`  reasons: ${reasons}`);
+    for (const c of d.candidates.slice(0, 5)) {
+        lines.push(`  ${c.captureId} [${c.arm}] score ${c.score.toFixed(2)} — ${c.reasons.join(", ")}`);
+    }
+    return lines;
+}
 function printMatrix(artifact, cacheAdjusted = []) {
     console.log(`bakeoff ${artifact.id}\nroute ${artifact.route}@${artifact.routeVersion} · replayed ${artifact.replayN} captured inputs · ${artifact.ranAt}`);
     // RB-1: a reference arm is present ⇒ show quality + regret columns and badge
     // the reference row; otherwise the pre-RB-1 matrix stands byte-for-byte.
     const hasReference = artifact.variants.some((m) => m.isReference === true);
+    const hasPairwise = artifact.variants.some((m) => m.pairwise != null);
     const header = ["variant", "n", "conform", "ci95", "repair", "repair$", "lat(ms)", "eff-$/1K-conf"];
     if (hasReference)
         header.push("quality", "regret(pts)");
+    if (hasPairwise)
+        header.push("pairwise-win");
     const rows = artifact.variants.map((m) => {
         const base = [
             m.isReference === true ? `${m.variant} [ref]` : m.variant,
@@ -128,6 +168,8 @@ function printMatrix(artifact, cacheAdjusted = []) {
         ];
         if (hasReference)
             base.push(fmtQuality(m), fmtRegret(m));
+        if (hasPairwise)
+            base.push(fmtPairwise(m));
         return base;
     });
     const widths = header.map((h, c) => Math.max(h.length, ...rows.map((r) => r[c].length)));
@@ -153,6 +195,10 @@ function printMatrix(artifact, cacheAdjusted = []) {
     else {
         console.log("proposal: none (no variant beats the incumbent's gates)");
     }
+    // N6 disagreement summary — printed only when candidates were mined; absent ⇒
+    // byte-identical to the pre-N6 matrix output.
+    for (const l of formatDisagreementSummary(artifact))
+        console.log(l);
     // Cache-adjusted section (spec §5.4) — printed only when the caller supplied
     // simulator rows; absent ⇒ the matrix output is byte-identical to pre-WS-B.
     for (const l of formatCacheAdjustedSection(cacheAdjusted))

package/dist/bakeoff/types.d.ts

@@ -5,6 +5,9 @@
  * value accuracy, paired regret vs a reference arm, and an objective-aware gate.
  */
 import type { ObjectiveMode } from "../types/route";
+import type { PairwiseStats } from "./pairwise";
+import type { DisagreementReport } from "./disagreement";
+import type { MaskArmProof } from "../mask/proof";
 /** Which quality metric an arm's numbers are on (spec §2): value accuracy when a
  * golden / value check / judge scored it, else schema conformance. */
 export type QualityMetric = "value_accuracy" | "conformance";
@@ -57,6 +60,11 @@ export interface VariantMetrics {
     readonly regret?: RegretStats | null;
     /** RB-1: the reference arm hit its envelope — regret is on the paired subset. */
     readonly referencePartial?: boolean;
+    /** N5: this arm's pairwise standing vs the reference (win rate + seeded CI).
+     * Null on the reference arm and when no pairwise signal was decidable. A
+     * REPORTED signal beside regret; gates a proposal only when the route sets
+     * `objective.pairwiseAware`. */
+    readonly pairwise?: PairwiseStats | null;
 }
 /** RB-1: the resolved objective recorded on the artifact (evidence + report). */
 export interface BakeoffObjective {
@@ -67,6 +75,9 @@ export interface BakeoffObjective {
         readonly usdPerRun?: number;
     } | null;
     readonly lambda: number | null;
+    /** N5: the pairwise-aware gate knob (default false). Recorded on the artifact so
+     * the report/PR body/evidence say whether pairwise gated the proposal. */
+    readonly pairwiseAware?: boolean;
 }
 /** RB-1: the resolved reference arm recorded on the artifact. */
 export interface BakeoffReference {
@@ -104,5 +115,18 @@ export interface BakeoffArtifact {
     readonly reference?: BakeoffReference | null;
     /** RB-1: the metric the bake-off's quality numbers are on. */
     readonly qualityMetric?: QualityMetric;
+    /** N6: verdict-disagreement candidates mined from this bake-off's replay set —
+     * a ranked list of already-captured pointers an operator MAY promote to eval
+     * cases (never auto-promoted). Rides the LOCAL artifact only (captureIds are not
+     * mirrored). Absent on a pre-N5/N6 artifact or when no reference/arm signal
+     * disagreed. */
+    readonly disagreement?: DisagreementReport;
+    /** CS-1 Proven Masking (spec §7) — the masked-arm proof for a `--mask-policy`
+     * run: the Detector finds no span in the masked arm (`proven: true`), with the
+     * unmasked arm's detections as the Δ the mask removed. Counts by type only,
+     * never a value. Additive & optional (RB-1 fence — rides the matrix jsonb
+     * wholesale, does NOT restructure the replay engine); absent on an ordinary
+     * bake-off. The Chunk-B masked-vs-unmasked Δ-value-accuracy arm extends this. */
+    readonly maskArm?: MaskArmProof;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/bakeoff/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;sEACsE;AACtE,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAE7D;;8CAE8C;AAC9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;;sFAEkF;IAClF,QAAQ,CAAC,eAAe,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC3D;;sEAEkE;IAClE,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;iFAC6E;IAC7E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,iEAAiE;IACjE,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;6DAEyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;qCACiC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACrC,iFAAiF;IACjF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CACrC;AAED,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE;QAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;;8EAE0E;IAC1E,QAAQ,CAAC,QAAQ,EACb;QACE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;QACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;KACtC,GACD,IAAI,CAAC;IACT,sEAAsE;IACtE,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC7C,8DAA8D;IAC9D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;CACxC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD;sEACsE;AACtE,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAE7D;;8CAE8C;AAC9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;;sFAEkF;IAClF,QAAQ,CAAC,eAAe,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC3D;;sEAEkE;IAClE,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;iFAC6E;IAC7E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,iEAAiE;IACjE,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;6DAEyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;qCACiC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACrC,iFAAiF;IACjF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IACpC;;;oCAGgC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CAC1C;AAED,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE;QAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B;8EAC0E;IAC1E,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;;8EAE0E;IAC1E,QAAQ,CAAC,QAAQ,EACb;QACE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;QACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;KACtC,GACD,IAAI,CAAC;IACT,sEAAsE;IACtE,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC7C,8DAA8D;IAC9D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;;;oBAIgB;IAChB,QAAQ,CAAC,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAC3C;;;;;qFAKiF;IACjF,QAAQ,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC;CACjC"}

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAgDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AAqhBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCnE"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAiDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AAohBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCnE"}

package/dist/cli.js

@@ -30,7 +30,7 @@ const failures_1 = require("./types/failures");
 const failures_teaching_1 = require("./types/failures-teaching");
 const config_1 = require("./config");
 const index_1 = require("./index");
-const status_1 = require("./status");
+const status_cli_1 = require("./status-cli");
 const engine_1 = require("./bakeoff/engine");
 const samples_1 = require("./bakeoff/samples");
 const report_1 = require("./bakeoff/report");
@@ -48,6 +48,7 @@ function objectiveOf(flags) {
 const cli_run_1 = require("./verify/cli-run");
 const loader_1 = require("./routes/loader");
 const rawonly_1 = require("./run/rawonly");
+const mask_policy_cli_1 = require("./mask/mask-policy-cli");
 const validate_cli_1 = require("./validate-cli");
 const store_1 = require("./telemetry/store");
 const budget_1 = require("./watch/budget");
@@ -236,6 +237,11 @@ async function cmdBakeoff(flags) {
             throw new Error("--budget-usd must be positive");
         return (0, report_1.runEvalCaseBakeoffCli)(config, evalTask, vs, env, keepOutputsMode(flags), flags.bare.has("json"), objectiveOf(flags));
     }
+    // CS-1 (spec §7): --mask-policy proves no detected span leaves the masked arm
+    // (additive early-return; does not touch the RB-1 replay engine). Handler in mask/.
+    const maskPolicyRoute = flags.named.get("mask-policy");
+    if (maskPolicyRoute !== undefined)
+        return (0, mask_policy_cli_1.runMaskPolicyProofCli)(config, flags, maskPolicyRoute);
     const route = flags.named.get("route");
     const variantsRaw = flags.named.get("variants");
     const lastRaw = flags.named.get("replay-last");
@@ -486,13 +492,6 @@ async function cmdServe() {
     // The server owns the process from here (it holds the listener open).
     return 0;
 }
-/** `modelrig status` — print the runtime posture (§2): export mode, artifact
- * namespace on/off + WHY, content custody, OTLP. Reads config only (no rig, no
- * routes dir), so it works anywhere the env is set. */
-function cmdStatus() {
-    console.log((0, status_1.formatStatusReport)((0, status_1.buildRigStatus)((0, config_1.loadConfigFromEnv)())));
-    return 0;
-}
 /** `modelrig judge <run>` — E5 V.4: explicitly ask the bounded model-judge to
  * score a run. Goes through the API boundary (T12: the CLI never holds a DB
  * client) — POST /v1/runs/:id/judge on the configured server, authed with a
@@ -544,6 +543,7 @@ const USAGE = [
     "  modelrig serve                       (needs the modelrig-server package)",
     "  modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--objective quality-first|balanced|cost-first] [--keep-outputs[=all]] [--json] [--no-mirror]",
     "  modelrig bakeoff --from-proposal <id> [--replay-last N] [--keep-outputs[=all]] [--json]",
+    "  modelrig bakeoff --mask-policy <route> [--replay-last N]   (CS-1: prove the masked arm leaks no detected span)",
     "  modelrig watch run [--events fixtures.json] [--json]",
     "  modelrig verify-swap --route <route> --captures <fixture.json> [--budget-usd usd] [--out report.md] [--json]",
     "  modelrig swap execute <proposalId> --approve [--approved-by name]",
@@ -564,7 +564,7 @@ async function main(argv) {
     const flags = parseArgs(rest);
     switch (command) {
         case "status":
-            return cmdStatus();
+            return (0, status_cli_1.runStatusCli)();
         case "validate":
             return (0, validate_cli_1.runValidateCli)();
         case "bakeoff":