modelrig 0.6.0 → 0.8.0

package/dist/bakeoff/engine.js

@@ -22,6 +22,8 @@ const loader_1 = require("../routes/loader");
 const replay_1 = require("./replay");
 const stats_1 = require("./stats");
 const judge_1 = require("./judge");
+const pairwise_1 = require("./pairwise");
+const disagreement_1 = require("./disagreement");
 const reference_1 = require("./reference");
 const samples_1 = require("./samples");
 /** The arm name carrying the reference (ceiling) metrics. Reserved — a user
@@ -75,6 +77,10 @@ async function runBakeoff(deps, captures, req, samplesSink) {
     const keepOutputs = req.keepOutputs !== undefined && samplesSink !== undefined;
     const bakeoffId = (0, node_crypto_1.randomUUID)();
     const judgeCfg = route.bakeoff?.judge;
+    // N5: a declared pairwise judge needs each sample's output retained (arm-vs-
+    // reference comparison in the engine) — WITHOUT the keepOutputs sink persist.
+    const pairwiseJudgeCfg = route.bakeoff?.pairwise?.judge;
+    const retainForPairwise = pairwiseJudgeCfg !== undefined;
     // Replay every requested arm. Raw samples are kept ALWAYS (paired regret is
     // computed from per-sample quality); the keepOutputs sink is separate. A fresh
     // judge scorer per arm keeps `max_samples` per-arm (spec §4.3).
@@ -88,6 +94,7 @@ async function runBakeoff(deps, captures, req, samplesSink) {
             lastN: req.replayLastN,
             envelopeUsd: req.envelopeUsdPerVariant,
             ...(keepOutputs ? { keepOutputs: true } : {}),
+            ...(retainForPairwise ? { retainOutputForPairwise: true } : {}),
             ...(req.expected !== undefined ? { expected: req.expected } : {}),
             ...(judgeCfg !== undefined ? { judge: (0, judge_1.makeJudgeScorer)(deps, route, judgeCfg) } : {}),
             ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
@@ -101,6 +108,10 @@ async function runBakeoff(deps, captures, req, samplesSink) {
     // arm's paired regret vs it. Partial-result honesty is preserved.
     const referenceInfo = resolveReference(route, req);
     let reference = null;
+    // N6: pairwise verdicts keyed `${arm} ${captureId}` (a mining input) + the
+    // reference samples, hoisted so disagreement mining sees them after the block.
+    const pairwiseOutcomes = new Map();
+    let refSamplesForMining = [];
     if (referenceInfo !== null) {
         const { deps: refDeps } = withReferenceArm(deps, route, referenceInfo.candidate);
         const refEnvelope = route.bakeoff?.referenceEnvelopeUsd ?? req.envelopeUsdPerVariant;
@@ -109,16 +120,26 @@ async function runBakeoff(deps, captures, req, samplesSink) {
             variant: exports.REFERENCE_ARM,
             lastN: req.replayLastN,
             envelopeUsd: refEnvelope,
+            ...(retainForPairwise ? { retainOutputForPairwise: true } : {}),
             ...(req.expected !== undefined ? { expected: req.expected } : {}),
             ...(judgeCfg !== undefined ? { judge: (0, judge_1.makeJudgeScorer)(deps, route, judgeCfg) } : {}),
             ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
         });
         const partial = refSamples.length < replayN || refSamples.some((s) => s.failureClass === "budget_exhausted");
+        refSamplesForMining = refSamples;
         for (let i = 0; i < metrics.length; i += 1) {
             const raw = rawByArm.get(metrics[i].variant) ?? [];
+            // N5: pairwise standing vs the reference — deterministic where both sides
+            // carry value accuracy, else the opt-in capped judge (fresh per arm, so
+            // maxSamples is per-arm). A REPORTED signal beside regret.
+            const pwJudge = pairwiseJudgeCfg !== undefined ? (0, pairwise_1.makePairwiseJudge)(deps, route, pairwiseJudgeCfg) : null;
+            const { stats: pairwise, outcomes } = await (0, pairwise_1.scoreArmPairwise)(raw, refSamples, pwJudge);
+            for (const [captureId, outcome] of outcomes)
+                pairwiseOutcomes.set((0, disagreement_1.pairwiseKey)(metrics[i].variant, captureId), outcome);
             metrics[i] = {
                 ...metrics[i],
                 regret: (0, stats_1.pairedRegret)(raw, refSamples),
+                ...(pairwise !== null ? { pairwise } : {}),
                 ...(partial ? { referencePartial: true } : {}),
             };
         }
@@ -181,6 +202,15 @@ async function runBakeoff(deps, captures, req, samplesSink) {
                 "re-run with --keep-outputs to retry the samples.");
         }
     }
+    // N6: mine the replay/verdict set for disagreements → ranked eval-case
+    // candidates (never auto-promoted). Pure over the per-arm raw samples + the
+    // reference + pairwise verdicts. Rides the LOCAL artifact only — captureIds are
+    // pointers to already-captured inputs and are never mirrored.
+    const disagreement = (0, disagreement_1.mineDisagreements)({
+        arms: req.variants.map((variant) => ({ arm: variant, samples: rawByArm.get(variant) ?? [] })),
+        ...(refSamplesForMining.length > 0 ? { reference: refSamplesForMining } : {}),
+        ...(pairwiseOutcomes.size > 0 ? { pairwise: pairwiseOutcomes } : {}),
+    });
     return {
         id: bakeoffId,
         route: route.route,
@@ -192,6 +222,7 @@ async function runBakeoff(deps, captures, req, samplesSink) {
         objective,
         reference,
         qualityMetric,
+        disagreement,
     };
 }
 function isDeclared(route, candidate) {
@@ -224,6 +255,7 @@ function resolveObjective(route, req) {
         maxRegret: route.objective?.maxRegret ?? null,
         priceCeiling: route.objective?.priceCeiling ?? null,
         lambda: route.objective?.lambda ?? null,
+        ...(route.objective?.pairwiseAware === true ? { pairwiseAware: true } : {}),
     };
 }
 const EPS = 1e-9;
@@ -233,6 +265,16 @@ function savingsPct(incumbent, winner) {
         ? 0
         : ((inc - winner.effectiveUsdPer1kConformant) / inc) * 100;
 }
+/** N5 pairwise-aware gate clause (default off). When `objective.pairwiseAware` is
+ * set, an eligible arm must NOT be pairwise-worse than the reference within CI —
+ * its pairwise win-rate lower bound ≥ 0.5. Fail-closed: an arm with NO pairwise
+ * stats cannot be confirmed not-worse, so it is ineligible under the knob. When
+ * the knob is off this returns true for every arm ⇒ RB-1 behaviour byte-identical. */
+function pairwiseEligible(m, objective) {
+    if (objective.pairwiseAware !== true)
+        return true;
+    return m.pairwise != null && m.pairwise.winRateCi95[0] >= 0.5 - EPS;
+}
 /** A price ceiling test: usd_per_1k on the effective-$/1k axis, usd_per_run on
  * mean cost. Null ceiling ⇒ no cost constraint (regret alone gates). */
 function withinCeiling(m, ceiling) {
@@ -298,7 +340,8 @@ function computeProposal(requested, metrics, ctx) {
         const eligible = pool.filter((m) => m.regret != null &&
             m.regret.nPaired >= 10 &&
             (objective.maxRegret === null || m.regret.regretCi95[1] <= objective.maxRegret + EPS) &&
-            withinCeiling(m, objective.priceCeiling));
+            withinCeiling(m, objective.priceCeiling) &&
+            pairwiseEligible(m, objective));
         if (eligible.length === 0)
             return null;
         // Lowest regret, tie → lowest cost.
@@ -333,7 +376,8 @@ function computeProposal(requested, metrics, ctx) {
     const eligible = pool.filter((m) => m.regret != null &&
         m.regret.nPaired >= 10 &&
         (objective.maxRegret === null || m.regret.regretCi95[1] <= objective.maxRegret + EPS) &&
-        withinCeiling(m, objective.priceCeiling));
+        withinCeiling(m, objective.priceCeiling) &&
+        pairwiseEligible(m, objective));
     if (eligible.length === 0)
         return null;
     const winner = [...eligible].sort((a, b) => blended(a) - blended(b))[0];
@@ -395,6 +439,17 @@ function bakeoffEvidence(artifact, spentUsd) {
         reference: artifact.reference ?? null,
         regret: artifact.proposal?.regret ?? null,
         quality_metric: artifact.qualityMetric ?? null,
+        // N6: disagreement COUNTS only (numbers — meta-safe) ride the evidence; the
+        // captureId candidate list stays on the LOCAL artifact and is never sent to
+        // the control plane (content custody).
+        disagreement: artifact.disagreement === undefined
+            ? null
+            : {
+                examined: artifact.disagreement.examined,
+                found: artifact.disagreement.found,
+                capped_at: artifact.disagreement.cappedAt,
+                counts: artifact.disagreement.counts,
+            },
         matrix: artifact.variants.map((m) => ({
             variant: m.variant,
             samples: m.samples,
@@ -408,6 +463,7 @@ function bakeoffEvidence(artifact, spentUsd) {
             quality_mean: m.qualityMean,
             quality_metric: m.qualityMetric,
             regret: m.regret ?? null,
+            pairwise: m.pairwise ?? null,
         })),
     };
 }

package/dist/bakeoff/pairwise.d.ts

@@ -0,0 +1,102 @@
+/**
+ * N5 — pairwise preference vs the reference arm (near-optimal brief §3.2 N5).
+ *
+ * A second, complementary read on regret: for each replayed sample, is the SERVED
+ * arm's output BETTER than, tied with, or WORSE than the REFERENCE arm's on the
+ * same captured input? Aggregated to a `pairwise_win_rate` (win=1, tie=0.5,
+ * loss=0) with a seeded bootstrap CI — the RigIndex pairwise idea, scoped to one
+ * bake-off's arm-vs-reference comparison.
+ *
+ * DETERMINISTIC-FIRST (F5), exactly like the regret quality tiers:
+ *   1. When BOTH sides carry a per-sample value-accuracy score (eval case /
+ *      value_checks / Q1 gate), the verdict is the sign of their difference —
+ *      no model call. This is the free, always-preferred tier.
+ *   2. Only when a sample cannot be decided deterministically does an OPT-IN,
+ *      CAPPED, METERED pairwise judge (`bakeoff.pairwise.judge`) compare the two
+ *      outputs. Judge model ∈ declared candidates (#223 F3, enforced at load).
+ *
+ * The aggregation math is PURE and unit-tested directly; the judge reuses the
+ * real `run()` loop (no new dispatch path, no run.ts edit) exactly as judge.ts.
+ */
+import type { RunDeps } from "../run/run";
+import type { CandidateSpec, RouteBundle } from "../types/route";
+import type { ReplaySampleResult } from "./replay";
+/** The served arm's standing vs the reference on one sample. `win` = the arm's
+ * output is better than the reference's; `loss` = worse; `tie` = indistinguishable. */
+export type PairwiseOutcome = "win" | "tie" | "loss";
+/** How a single verdict was decided (F5): a free deterministic comparison of two
+ * value-accuracy scores, or a billed pairwise judge call. */
+export type PairwiseSource = "deterministic" | "judge";
+/** Aggregate pairwise standing of an arm vs the reference (N5). `winRate` counts
+ * a win as 1 and a tie as 0.5; the CI is a seeded bootstrap of the per-sample
+ * win values, so it is reproducible given the same sample set. */
+export interface PairwiseStats {
+    readonly winRate: number;
+    readonly winRateCi95: readonly [number, number];
+    /** Samples with a verdict (deterministic + judge); the CI's n. */
+    readonly nDecided: number;
+    readonly wins: number;
+    readonly ties: number;
+    readonly losses: number;
+    /** Paired samples that could NOT be decided (no deterministic score and no
+     * judge, or the judge was capped/failed) — excluded from the rate, counted. */
+    readonly nUndecided: number;
+    /** Where the decided verdicts came from — "deterministic", "judge", or "mixed". */
+    readonly source: "deterministic" | "judge" | "mixed";
+}
+/** Seeded so a bake-off's pairwise CI is reproducible (like the regret CI). A
+ * distinct seed from the regret bootstrap keeps the two resamplings independent. */
+export declare const PAIRWISE_BOOTSTRAP_REPS = 2000;
+export declare const PAIRWISE_BOOTSTRAP_SEED = 20260824;
+/** The win-value of an outcome (win=1, tie=0.5, loss=0) — the quantity the win
+ * rate averages. */
+export declare function winValue(o: PairwiseOutcome): number;
+/**
+ * The DETERMINISTIC pairwise verdict (tier 1): the sign of `armQuality −
+ * refQuality`. Returns null when EITHER side lacks a value-accuracy score — the
+ * caller then falls to the judge. A tie within EPS is a genuine tie.
+ */
+export declare function deterministicPairwise(armQuality: number | null, refQuality: number | null): PairwiseOutcome | null;
+/**
+ * Aggregate per-sample verdicts into `PairwiseStats` (N5). `decided` are the
+ * verdicts that resolved; `nUndecided` the paired samples that did not. The CI is
+ * a seeded bootstrap of the per-sample win values (win=1/tie=0.5/loss=0), so it
+ * is deterministic given the verdict list + seed. Pure.
+ */
+export declare function aggregatePairwise(decided: readonly {
+    readonly outcome: PairwiseOutcome;
+    readonly source: PairwiseSource;
+}[], nUndecided: number, seed?: number, reps?: number): PairwiseStats;
+/**
+ * Score ONE arm's pairwise standing vs the reference over the paired replay set
+ * (N5). Deterministic-first: each paired sample's verdict is the sign of the two
+ * value-accuracy scores; a sample neither side scored falls to the capped judge
+ * (when supplied), else it is undecided (counted, excluded from the rate). Pairs
+ * by captureId; unpaired samples are ignored. Returns null stats when there are
+ * NO paired samples at all. `outcomes` maps captureId → verdict (for N6 mining).
+ */
+export declare function scoreArmPairwise(armSamples: readonly ReplaySampleResult[], refSamples: readonly ReplaySampleResult[], judge: PairwiseJudge | null): Promise<{
+    stats: PairwiseStats | null;
+    outcomes: Map<string, PairwiseOutcome>;
+}>;
+/** Build the synthetic pairwise-judge route for a base route + judge model. The
+ * judge model is declared WITHOUT its trial flag so it serves live here. */
+export declare function buildPairwiseJudgeRoute(baseRoute: RouteBundle, model: CandidateSpec): RouteBundle;
+/** A per-arm pairwise scorer: `compare(armOutput, refOutput, taskInput)` returns
+ * the arm's outcome vs the reference, or null when the cap is hit or the judge
+ * could not produce a valid verdict. */
+export interface PairwiseJudge {
+    compare(armOutput: unknown, refOutput: unknown, taskInput: unknown): Promise<PairwiseOutcome | null>;
+    used(): number;
+}
+/**
+ * Build a pairwise judge over the run loop (mirrors makeJudgeScorer). Each arm
+ * gets a FRESH judge so `maxSamples` is per arm. The comparison is presented in a
+ * FIXED order (A = served arm, B = reference); position bias is a known limit,
+ * noted in verification — a fixed order keeps the judge DETERMINISTIC for tests.
+ */
+export declare function makePairwiseJudge(deps: RunDeps, baseRoute: RouteBundle, judge: {
+    readonly model: CandidateSpec;
+    readonly maxSamples: number;
+}): PairwiseJudge;
+//# sourceMappingURL=pairwise.d.ts.map

package/dist/bakeoff/pairwise.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.d.ts","sourceRoot":"","sources":["../../src/bakeoff/pairwise.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEnD;uFACuF;AACvF,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;AAErD;6DAC6D;AAC7D,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG,OAAO,CAAC;AAEvD;;kEAEkE;AAClE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;mFAC+E;IAC/E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,GAAG,OAAO,CAAC;CACtD;AAED;oFACoF;AACpF,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAC5C,eAAO,MAAM,uBAAuB,WAAW,CAAC;AAKhD;oBACoB;AACpB,wBAAgB,QAAQ,CAAC,CAAC,EAAE,eAAe,GAAG,MAAM,CAEnD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,UAAU,EAAE,MAAM,GAAG,IAAI,GACxB,eAAe,GAAG,IAAI,CAKxB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,SAAS;IAAE,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;CAAE,EAAE,EAC1F,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE,MAAgC,EACtC,IAAI,GAAE,MAAgC,GACrC,aAAa,CA2Bf;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,KAAK,EAAE,aAAa,GAAG,IAAI,GAC1B,OAAO,CAAC;IAAE,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;CAAE,CAAC,CA8BlF;AAmBD;4EAC4E;AAC5E,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,GAAG,WAAW,CAejG;AAED;;wCAEwC;AACxC,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IACrG,IAAI,IAAI,MAAM,CAAC;CAChB;AAMD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,WAAW,EACtB,KAAK,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACpE,aAAa,CAmCf"}

package/dist/bakeoff/pairwise.js

@@ -0,0 +1,214 @@
+"use strict";
+/**
+ * N5 — pairwise preference vs the reference arm (near-optimal brief §3.2 N5).
+ *
+ * A second, complementary read on regret: for each replayed sample, is the SERVED
+ * arm's output BETTER than, tied with, or WORSE than the REFERENCE arm's on the
+ * same captured input? Aggregated to a `pairwise_win_rate` (win=1, tie=0.5,
+ * loss=0) with a seeded bootstrap CI — the RigIndex pairwise idea, scoped to one
+ * bake-off's arm-vs-reference comparison.
+ *
+ * DETERMINISTIC-FIRST (F5), exactly like the regret quality tiers:
+ *   1. When BOTH sides carry a per-sample value-accuracy score (eval case /
+ *      value_checks / Q1 gate), the verdict is the sign of their difference —
+ *      no model call. This is the free, always-preferred tier.
+ *   2. Only when a sample cannot be decided deterministically does an OPT-IN,
+ *      CAPPED, METERED pairwise judge (`bakeoff.pairwise.judge`) compare the two
+ *      outputs. Judge model ∈ declared candidates (#223 F3, enforced at load).
+ *
+ * The aggregation math is PURE and unit-tested directly; the judge reuses the
+ * real `run()` loop (no new dispatch path, no run.ts edit) exactly as judge.ts.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PAIRWISE_BOOTSTRAP_SEED = exports.PAIRWISE_BOOTSTRAP_REPS = void 0;
+exports.winValue = winValue;
+exports.deterministicPairwise = deterministicPairwise;
+exports.aggregatePairwise = aggregatePairwise;
+exports.scoreArmPairwise = scoreArmPairwise;
+exports.buildPairwiseJudgeRoute = buildPairwiseJudgeRoute;
+exports.makePairwiseJudge = makePairwiseJudge;
+const node_crypto_1 = require("node:crypto");
+const bootstrap_1 = require("../rigindex/bootstrap");
+const run_1 = require("../run/run");
+const failures_1 = require("../types/failures");
+/** Seeded so a bake-off's pairwise CI is reproducible (like the regret CI). A
+ * distinct seed from the regret bootstrap keeps the two resamplings independent. */
+exports.PAIRWISE_BOOTSTRAP_REPS = 2000;
+exports.PAIRWISE_BOOTSTRAP_SEED = 20260824;
+/** Tie band: value-accuracy scores within EPS are a tie, not a hair-split win. */
+const PAIRWISE_EPS = 1e-9;
+/** The win-value of an outcome (win=1, tie=0.5, loss=0) — the quantity the win
+ * rate averages. */
+function winValue(o) {
+    return o === "win" ? 1 : o === "tie" ? 0.5 : 0;
+}
+/**
+ * The DETERMINISTIC pairwise verdict (tier 1): the sign of `armQuality −
+ * refQuality`. Returns null when EITHER side lacks a value-accuracy score — the
+ * caller then falls to the judge. A tie within EPS is a genuine tie.
+ */
+function deterministicPairwise(armQuality, refQuality) {
+    if (armQuality === null || refQuality === null)
+        return null;
+    const diff = armQuality - refQuality;
+    if (Math.abs(diff) <= PAIRWISE_EPS)
+        return "tie";
+    return diff > 0 ? "win" : "loss";
+}
+/**
+ * Aggregate per-sample verdicts into `PairwiseStats` (N5). `decided` are the
+ * verdicts that resolved; `nUndecided` the paired samples that did not. The CI is
+ * a seeded bootstrap of the per-sample win values (win=1/tie=0.5/loss=0), so it
+ * is deterministic given the verdict list + seed. Pure.
+ */
+function aggregatePairwise(decided, nUndecided, seed = exports.PAIRWISE_BOOTSTRAP_SEED, reps = exports.PAIRWISE_BOOTSTRAP_REPS) {
+    const wins = decided.filter((d) => d.outcome === "win").length;
+    const ties = decided.filter((d) => d.outcome === "tie").length;
+    const losses = decided.filter((d) => d.outcome === "loss").length;
+    const nDecided = decided.length;
+    const hasDet = decided.some((d) => d.source === "deterministic");
+    const hasJudge = decided.some((d) => d.source === "judge");
+    const source = hasDet && hasJudge ? "mixed" : hasJudge ? "judge" : "deterministic";
+    if (nDecided === 0) {
+        return { winRate: 0, winRateCi95: [0, 0], nDecided: 0, wins: 0, ties: 0, losses: 0, nUndecided, source };
+    }
+    const values = decided.map((d) => winValue(d.outcome));
+    const winRate = values.reduce((a, b) => a + b, 0) / nDecided;
+    // Bootstrap the per-sample win values (the pairing is already fixed), take each
+    // resample's mean, then the 2.5/97.5 percentiles.
+    const rng = (0, bootstrap_1.mulberry32)(seed);
+    const means = [];
+    for (let b = 0; b < reps; b += 1) {
+        let sum = 0;
+        for (let i = 0; i < nDecided; i += 1)
+            sum += values[Math.floor(rng() * nDecided)];
+        means.push(sum / nDecided);
+    }
+    means.sort((a, b) => a - b);
+    const lo = means[Math.floor(0.025 * reps)];
+    const hi = means[Math.min(reps - 1, Math.floor(0.975 * reps))];
+    return { winRate, winRateCi95: [lo, hi], nDecided, wins, ties, losses, nUndecided, source };
+}
+/**
+ * Score ONE arm's pairwise standing vs the reference over the paired replay set
+ * (N5). Deterministic-first: each paired sample's verdict is the sign of the two
+ * value-accuracy scores; a sample neither side scored falls to the capped judge
+ * (when supplied), else it is undecided (counted, excluded from the rate). Pairs
+ * by captureId; unpaired samples are ignored. Returns null stats when there are
+ * NO paired samples at all. `outcomes` maps captureId → verdict (for N6 mining).
+ */
+async function scoreArmPairwise(armSamples, refSamples, judge) {
+    const refByCapture = new Map(refSamples.map((s) => [s.captureId, s]));
+    const decided = [];
+    const outcomes = new Map();
+    let paired = 0;
+    let undecided = 0;
+    for (const arm of armSamples) {
+        const ref = refByCapture.get(arm.captureId);
+        if (ref === undefined)
+            continue;
+        paired += 1;
+        const det = deterministicPairwise(arm.valueAccuracy, ref.valueAccuracy);
+        if (det !== null) {
+            decided.push({ outcome: det, source: "deterministic" });
+            outcomes.set(arm.captureId, det);
+            continue;
+        }
+        // Deterministic can't decide (a side lacks value accuracy). Only a PRODUCED
+        // pair can be judged (both need an output); the judge is capped internally.
+        if (judge !== null && arm.failureClass === null && ref.failureClass === null) {
+            const verdict = await judge.compare(arm.outputText ?? null, ref.outputText ?? null, arm.inputText ?? null);
+            if (verdict !== null) {
+                decided.push({ outcome: verdict, source: "judge" });
+                outcomes.set(arm.captureId, verdict);
+                continue;
+            }
+        }
+        undecided += 1;
+    }
+    if (paired === 0)
+        return { stats: null, outcomes };
+    return { stats: aggregatePairwise(decided, undecided), outcomes };
+}
+// ── The opt-in, capped, metered pairwise judge (tier 2) ──────────────────────
+const PAIRWISE_SCHEMA = {
+    type: "object",
+    properties: { verdict: { type: "string", enum: ["A", "B", "tie"] }, reason: { type: "string" } },
+    required: ["verdict"],
+    additionalProperties: true,
+};
+const PAIRWISE_RUBRIC = "You are judging PAIRWISE PREFERENCE between two candidate outputs for ONE task — " +
+    "which is the better answer for VALUE (correctness and completeness), not merely which " +
+    "is better-formed. A is one candidate; B is the other.\n\nTASK INPUT:\n{{task_input}}\n\n" +
+    "OUTPUT A:\n{{output_a}}\n\nOUTPUT B:\n{{output_b}}\n\n" +
+    'Respond with ONLY compact JSON: {"verdict": "A" | "B" | "tie", "reason": "<one sentence>"}. ' +
+    'Answer "A" if A is better, "B" if B is better, "tie" if they are equally good.';
+/** Build the synthetic pairwise-judge route for a base route + judge model. The
+ * judge model is declared WITHOUT its trial flag so it serves live here. */
+function buildPairwiseJudgeRoute(baseRoute, model) {
+    return Object.freeze({
+        route: `__bakeoff_pairwise__:${baseRoute.route}`,
+        version: baseRoute.version,
+        outputSchema: Object.freeze({ ...PAIRWISE_SCHEMA }),
+        candidates: Object.freeze([{ provider: model.provider, model: model.model }]),
+        require: Object.freeze([]),
+        prefer: Object.freeze([]),
+        systemTemplate: PAIRWISE_RUBRIC,
+        variables: Object.freeze(["output_a", "output_b", "task_input"]),
+        policy: Object.freeze({ retries: Object.freeze({}), timeoutMs: baseRoute.policy.timeoutMs, tier: "standard", json: "json_mode" }),
+        variants: Object.freeze([]),
+        capture: false,
+        fingerprintCell: null,
+    });
+}
+function asText(v) {
+    return typeof v === "string" ? v : JSON.stringify(v ?? null);
+}
+/**
+ * Build a pairwise judge over the run loop (mirrors makeJudgeScorer). Each arm
+ * gets a FRESH judge so `maxSamples` is per arm. The comparison is presented in a
+ * FIXED order (A = served arm, B = reference); position bias is a known limit,
+ * noted in verification — a fixed order keeps the judge DETERMINISTIC for tests.
+ */
+function makePairwiseJudge(deps, baseRoute, judge) {
+    const judgeRoute = buildPairwiseJudgeRoute(baseRoute, judge.model);
+    const routes = {
+        get: (r) => (r === judgeRoute.route ? judgeRoute : deps.routes.get(r)),
+        list: () => Object.freeze([...deps.routes.list(), judgeRoute]),
+    };
+    const judgeDeps = { ...deps, routes, captures: null, mode: "live" };
+    const envelope = `bakeoff-pairwise:${(0, node_crypto_1.randomUUID)()}`;
+    let used = 0;
+    return {
+        used: () => used,
+        async compare(armOutput, refOutput, taskInput) {
+            if (used >= judge.maxSamples)
+                return null;
+            used += 1;
+            try {
+                const result = await (0, run_1.run)(judgeDeps, judgeRoute.route, {
+                    input: { output_a: asText(armOutput), output_b: asText(refOutput), task_input: asText(taskInput) },
+                    // `bakeoff_pairwise` marks the row a pairwise-judge inference (the typed
+                    // `rung` column is recorder-owned, so a distinct tag key is used); `lane:
+                    // trial` keeps judge spend off the route's live rollups/odometer.
+                    tags: { bakeoff_pairwise: "1", lane: "trial" },
+                    budget: { envelope },
+                });
+                const verdict = result.output?.verdict;
+                if (verdict === "A")
+                    return "win";
+                if (verdict === "B")
+                    return "loss";
+                if (verdict === "tie")
+                    return "tie";
+                return null;
+            }
+            catch (err) {
+                if (!(err instanceof failures_1.RigFailureError))
+                    throw err;
+                // The metered attempt row(s) still stand; this sample gets no verdict.
+                return null;
+            }
+        },
+    };
+}

package/dist/bakeoff/replay.d.ts

@@ -30,6 +30,12 @@ export interface ReplayRequest {
      * sample for the human value review. Off by default (bounds memory + keeps
      * the pure-metrics path clean). */
     readonly keepOutputs?: boolean;
+    /** N5: retain `inputText` + `outputText` on each sample for the pairwise judge
+     * (arm-vs-reference comparison in the engine), WITHOUT the keepOutputs sink
+     * persistence. Set by the engine only when `bakeoff.pairwise.judge` is declared.
+     * A superset trigger of `keepOutputs` for retention; the sink stays gated on
+     * `keepOutputs` alone so pairwise retention never persists review samples. */
+    readonly retainOutputForPairwise?: boolean;
     /** RB-1: corrected/golden expected outputs keyed by captureId (eval-case path).
      * Presence in the map ⇒ this sample has an expected value; `value` may be any
      * JSON (including null). Quality tier 1 (eval case) when present. */

package/dist/bakeoff/replay.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoIxC"}
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAyIxC"}

package/dist/bakeoff/replay.js

@@ -49,6 +49,10 @@ async function replayVariant(deps, captures, req) {
         envelopeBudgetUsd: req.envelopeUsd,
         mode,
     };
+    // Retain per-sample input/output text when the caller asked (keepOutputs sink,
+    // or N5 pairwise judge). Same fields either way; the sink persist stays gated
+    // on keepOutputs alone (engine.ts).
+    const retainText = req.keepOutputs === true || req.retainOutputForPairwise === true;
     const results = [];
     for (const capture of rows) {
         const sampleTag = `${replayId}:${capture.inferenceId}`;
@@ -81,7 +85,7 @@ async function replayVariant(deps, captures, req) {
             repaired = result.meta.repairedBy !== null;
             outputValue = result.output;
             produced = true;
-            if (req.keepOutputs)
+            if (retainText)
                 outputText = JSON.stringify(result.output, null, 2);
             if (req.qualityGate !== undefined) {
                 gated = { output: result.output, provider: result.meta.provider, model: result.meta.model, servedVariant: result.meta.servedVariant };
@@ -145,7 +149,7 @@ async function replayVariant(deps, captures, req) {
             pricingMissing,
             latencyMs,
             failureClass,
-            ...(req.keepOutputs
+            ...(retainText
                 ? { inputText: capture.variablesJson, outputText }
                 : {}),
         });

package/dist/bakeoff/report.d.ts

@@ -36,6 +36,13 @@ export interface CacheAdjustedRow {
  * Empty input ⇒ no lines (the section is omitted entirely — back-compat).
  */
 export declare function formatCacheAdjustedSection(rows: readonly CacheAdjustedRow[]): string[];
+/**
+ * N6 disagreement summary lines: the count of mined eval-case candidates (with
+ * the honest "of N found" when the list was capped), the per-reason breakdown,
+ * and the top candidates as captureId pointers an operator may promote. Pure
+ * (returns strings); empty when nothing disagreed. Exported for direct testing.
+ */
+export declare function formatDisagreementSummary(artifact: BakeoffArtifact): string[];
 export declare function printMatrix(artifact: BakeoffArtifact, cacheAdjusted?: readonly CacheAdjustedRow[]): void;
 /**
  * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's

package/dist/bakeoff/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAkCpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAkD5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAwD5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/bakeoff/report.js

@@ -8,6 +8,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.bakeoffHeadline = bakeoffHeadline;
 exports.formatCacheAdjustedSection = formatCacheAdjustedSection;
+exports.formatDisagreementSummary = formatDisagreementSummary;
 exports.printMatrix = printMatrix;
 exports.runEvalCaseBakeoffCli = runEvalCaseBakeoffCli;
 const node_fs_1 = require("node:fs");
@@ -35,6 +36,16 @@ function fmtRegret(m) {
 function fmtQuality(m) {
     return m.qualityMean === null ? "—" : `${(m.qualityMean * 100).toFixed(0)}% ${m.qualityMetric}`;
 }
+/** N5: an arm's pairwise-vs-reference cell — "63% [51…74] n=30", "ref", or "—". */
+function fmtPairwise(m) {
+    if (m.isReference === true)
+        return "ref";
+    const p = m.pairwise;
+    if (p == null)
+        return "—";
+    const pct = (v) => (v * 100).toFixed(0);
+    return `${pct(p.winRate)}% [${pct(p.winRateCi95[0])}…${pct(p.winRateCi95[1])}] n=${p.nDecided}`;
+}
 /** The reference $/1k or per-run ceiling from the objective, formatted, or null
  * when no ceiling is declared. */
 function fmtCeiling(artifact) {
@@ -107,14 +118,43 @@ function formatCacheAdjustedSection(rows) {
     }
     return lines;
 }
+/**
+ * N6 disagreement summary lines: the count of mined eval-case candidates (with
+ * the honest "of N found" when the list was capped), the per-reason breakdown,
+ * and the top candidates as captureId pointers an operator may promote. Pure
+ * (returns strings); empty when nothing disagreed. Exported for direct testing.
+ */
+function formatDisagreementSummary(artifact) {
+    const d = artifact.disagreement;
+    if (d === undefined || d.found === 0)
+        return [];
+    const reasons = Object.entries(d.counts)
+        .filter(([, n]) => n > 0)
+        .map(([r, n]) => `${r}=${n}`)
+        .join(", ");
+    const capNote = d.cappedAt !== null ? ` (showing top ${d.candidates.length} of ${d.found} — cap ${d.cappedAt})` : "";
+    const lines = [
+        `disagreement: ${d.found} eval-case candidate${d.found === 1 ? "" : "s"} mined from ${d.examined} samples${capNote} — ` +
+            "NOT auto-promoted; promote with artifact.promoteToEval.",
+    ];
+    if (reasons !== "")
+        lines.push(`  reasons: ${reasons}`);
+    for (const c of d.candidates.slice(0, 5)) {
+        lines.push(`  ${c.captureId} [${c.arm}] score ${c.score.toFixed(2)} — ${c.reasons.join(", ")}`);
+    }
+    return lines;
+}
 function printMatrix(artifact, cacheAdjusted = []) {
     console.log(`bakeoff ${artifact.id}\nroute ${artifact.route}@${artifact.routeVersion} · replayed ${artifact.replayN} captured inputs · ${artifact.ranAt}`);
     // RB-1: a reference arm is present ⇒ show quality + regret columns and badge
     // the reference row; otherwise the pre-RB-1 matrix stands byte-for-byte.
     const hasReference = artifact.variants.some((m) => m.isReference === true);
+    const hasPairwise = artifact.variants.some((m) => m.pairwise != null);
     const header = ["variant", "n", "conform", "ci95", "repair", "repair$", "lat(ms)", "eff-$/1K-conf"];
     if (hasReference)
         header.push("quality", "regret(pts)");
+    if (hasPairwise)
+        header.push("pairwise-win");
     const rows = artifact.variants.map((m) => {
         const base = [
             m.isReference === true ? `${m.variant} [ref]` : m.variant,
@@ -128,6 +168,8 @@ function printMatrix(artifact, cacheAdjusted = []) {
         ];
         if (hasReference)
             base.push(fmtQuality(m), fmtRegret(m));
+        if (hasPairwise)
+            base.push(fmtPairwise(m));
         return base;
     });
     const widths = header.map((h, c) => Math.max(h.length, ...rows.map((r) => r[c].length)));
@@ -153,6 +195,10 @@ function printMatrix(artifact, cacheAdjusted = []) {
     else {
         console.log("proposal: none (no variant beats the incumbent's gates)");
     }
+    // N6 disagreement summary — printed only when candidates were mined; absent ⇒
+    // byte-identical to the pre-N6 matrix output.
+    for (const l of formatDisagreementSummary(artifact))
+        console.log(l);
     // Cache-adjusted section (spec §5.4) — printed only when the caller supplied
     // simulator rows; absent ⇒ the matrix output is byte-identical to pre-WS-B.
     for (const l of formatCacheAdjustedSection(cacheAdjusted))