modelrig 0.8.0 → 0.9.0

package/dist/artifacts/run-context.js

@@ -14,7 +14,8 @@
  * write that fails is logged and dropped; the customer's pipeline is untouched.
  *
  * ── GATING ────────────────────────────────────────────────────────────────
- * The namespace is inert unless the rig enabled it (`MODELRIG_ARTIFACTS=1`).
+ * The namespace is inert unless the rig enabled it (a control plane is configured,
+ * or the dev-only local force-on; the org capture posture governs at runtime).
  * A disabled namespace's `run.start()` sets NO context, so `getActiveRunContext`
  * returns null everywhere and the run loop's auto-step is never reached —
  * v1 wire, zero new rows, zero behaviour change (spec §1.4).
@@ -129,30 +130,7 @@ function warnOnce(scope, err) {
 }
 function createRunNamespace(deps) {
     const { buffer, enabled } = deps;
-    const evalStore = deps.artifactStore;
-    /** Write ONE run-subject evaluation (artifactId null, runId set). Fail-open. */
-    function writeRunEvaluation(runId, opts) {
-        if (!enabled || evalStore === undefined || runId === "")
-            return;
-        try {
-            evalStore.recordArtifactEvaluation({
-                id: (0, node_crypto_1.randomUUID)(),
-                artifactId: null,
-                runId,
-                evaluator: opts.evaluator,
-                evaluatorKind: opts.kind,
-                score: opts.score ?? null,
-                passed: opts.passed ?? null,
-                label: opts.label ?? null,
-                explanation: opts.explanation ?? null,
-                createdBy: null,
-            });
-        }
-        catch (err) {
-            warnOnce("run.evaluate", err);
-        }
-    }
-    const inertHandle = { runId: "", end() { }, evaluate() { } };
+    const inertHandle = { runId: "", end() { } };
     function buildContext(startRow, traceId, zeroRetention) {
         const runId = startRow.id;
         const episodeKey = startRow.episodeKey;
@@ -250,7 +228,6 @@ function createRunNamespace(deps) {
             return {
                 runId: ctx.runId,
                 end: (status) => endContext(ctx, status),
-                evaluate: (opts2) => writeRunEvaluation(ctx.runId, opts2),
             };
         },
         end(status) {
@@ -258,19 +235,6 @@ function createRunNamespace(deps) {
             if (ctx !== undefined)
                 endContext(ctx, status);
         },
-        evaluate(opts) {
-            const ctx = storage.getStore();
-            // `ended` counts as gone — same rule as getActiveRunContext. `enterWith`
-            // has no lexical exit, so an ended run's context stays ambient on this
-            // async path; without this check ns.evaluate re-attributes verdicts to
-            // the ended run (post-end verdicts belong to the HANDLE, which closes
-            // over its runId — that path is unaffected).
-            if (ctx === undefined || ctx.ended) {
-                console.warn("[modelrig artifacts] run.evaluate() with no active run — dropped");
-                return;
-            }
-            writeRunEvaluation(ctx.runId, opts);
-        },
         async scope(opts, fn) {
             if (!enabled)
                 return fn();

package/dist/bakeoff/engine.d.ts

@@ -13,6 +13,7 @@
 import type { CaptureStore } from "../telemetry/capture";
 import type { RunDeps } from "../run/run";
 import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
+import type { GraderType } from "./quality";
 import type { QualityGate } from "../types/quality";
 import type { ArmSamples, KeepOutputsMode } from "./samples";
 import type { BakeoffArtifact, BakeoffObjective, VariantMetrics } from "./types";
@@ -52,6 +53,10 @@ export interface BakeoffRequest {
     readonly expected?: ReadonlyMap<string, {
         readonly value: unknown;
     }>;
+    /** LRE-1b: the golden set's grader for `expected` scoring (`exact` binary /
+     * `field-level` per-field / `llm-judge` deferred). Set by the from-eval-cases
+     * runner from the eval set's grader; absent ⇒ field-level (pre-LRE-1b). */
+    readonly graderType?: GraderType;
     /** Q1 quality gate (spec §4.2): scored on every replayed sample to populate
      * valueAccuracy + the quality pass rate, unlocking computeProposal's cost-first
      * third clause. A function, so it rides `rig.bakeoff` (programmatic) only — the
@@ -84,11 +89,35 @@ export interface BakeoffMirrorConfig {
  * Same PostgREST transport pattern as the telemetry exporter; captures have
  * no cloud surface — ONLY aggregate metrics travel. */
 export declare function mirrorBakeoff(artifact: BakeoffArtifact, config: BakeoffMirrorConfig): Promise<void>;
+/** N4 swap direction (near-optimal spec §4.2): the winner's effective $/1k vs the
+ * incumbent's. `up` = better-and-dearer (raises cost to cut regret); `down` = the
+ * shipped saving; `lateral` = within EPS (RB-1 never proposes a true lateral, but
+ * the label is honest at the boundary). Zero new measurement — both arms are
+ * already in the matrix. */
+export interface SwapDirection {
+    readonly direction: "up" | "down" | "lateral";
+    /** winnerEff − incumbentEff, signed (USD per 1k conformant); null if either ∞. */
+    readonly costDeltaUsdPer1k: number | null;
+}
+/**
+ * Derive a bake-off's proposed-swap direction from the winner's vs the
+ * incumbent's `effectiveUsdPer1kConformant` (spec §4.2). The incumbent is the
+ * `default` arm when present, else the first non-reference arm — matching
+ * `computeProposal`'s incumbent selection (requested variants keep their order in
+ * the matrix, with the reference pushed last). Null when there is no proposal.
+ * Pure, additive: `computeProposal` is untouched (the winner is only read back).
+ */
+export declare function deriveSwapDirection(artifact: BakeoffArtifact): SwapDirection | null;
 /**
  * The proposal-evidence jsonb for a bake-off artifact (near-optimal spec §4.6) —
  * the winning variant, the RB-1 objective/reference/regret/quality_metric, and a
  * per-arm matrix (with per-arm regret). Built here so the CLI stays thin; the PR
  * body reads these fields back off `proposal.evidence`. ZERO DDL — freeform jsonb.
+ *
+ * N4 (spec §4.2, ADDITIVE): when a proposal exists, stamp `swap_direction` +
+ * `cost_delta_usd_per_1k` so the PR body, the approval prompt, and the odometer
+ * treat a cost-raising promotion honestly. Absent on a no-proposal artifact (the
+ * existing evidence shape is byte-identical).
  */
 export declare function bakeoffEvidence(artifact: BakeoffArtifact, spentUsd: number): Record<string, unknown>;
 //# sourceMappingURL=engine.d.ts.map

package/dist/bakeoff/engine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAS9F,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CA8K1B;AAuCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA8G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAuCpG"}
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAG9F,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQ5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;+EAE2E;IAC3E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CAuL1B;AAuCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA8G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;4BAI4B;AAC5B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,SAAS,CAAC;IAC9C,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,IAAI,CAkBnF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA2CpG"}

package/dist/bakeoff/engine.js

@@ -16,12 +16,14 @@ exports.REFERENCE_ARM = void 0;
 exports.runBakeoff = runBakeoff;
 exports.computeProposal = computeProposal;
 exports.mirrorBakeoff = mirrorBakeoff;
+exports.deriveSwapDirection = deriveSwapDirection;
 exports.bakeoffEvidence = bakeoffEvidence;
 const node_crypto_1 = require("node:crypto");
 const loader_1 = require("../routes/loader");
 const replay_1 = require("./replay");
 const stats_1 = require("./stats");
 const judge_1 = require("./judge");
+const golden_judge_1 = require("./golden-judge");
 const pairwise_1 = require("./pairwise");
 const disagreement_1 = require("./disagreement");
 const reference_1 = require("./reference");
@@ -81,6 +83,11 @@ async function runBakeoff(deps, captures, req, samplesSink) {
     // reference comparison in the engine) — WITHOUT the keepOutputs sink persist.
     const pairwiseJudgeCfg = route.bakeoff?.pairwise?.judge;
     const retainForPairwise = pairwiseJudgeCfg !== undefined;
+    // LRE-2 llm-judge grader: when the eval set grades by `llm-judge`, each arm's
+    // output is scored against its GOLDEN via the pairwise judge. Reuse the route's
+    // declared (load-validated, #223 F3) pairwise/bakeoff judge model — a golden
+    // judge is built FRESH per arm (its cap is per arm) inside the replay calls.
+    const goldenJudgeCfg = req.graderType === "llm-judge" ? (route.bakeoff?.pairwise?.judge ?? route.bakeoff?.judge) : undefined;
     // Replay every requested arm. Raw samples are kept ALWAYS (paired regret is
     // computed from per-sample quality); the keepOutputs sink is separate. A fresh
     // judge scorer per arm keeps `max_samples` per-arm (spec §4.3).
@@ -96,7 +103,9 @@ async function runBakeoff(deps, captures, req, samplesSink) {
             ...(keepOutputs ? { keepOutputs: true } : {}),
             ...(retainForPairwise ? { retainOutputForPairwise: true } : {}),
             ...(req.expected !== undefined ? { expected: req.expected } : {}),
+            ...(req.graderType !== undefined ? { graderType: req.graderType } : {}),
             ...(judgeCfg !== undefined ? { judge: (0, judge_1.makeJudgeScorer)(deps, route, judgeCfg) } : {}),
+            ...(goldenJudgeCfg !== undefined ? { goldenJudge: (0, golden_judge_1.makeGoldenJudge)(deps, route, goldenJudgeCfg) } : {}),
             ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
         });
         replayN = Math.max(replayN, samples.length);
@@ -122,7 +131,9 @@ async function runBakeoff(deps, captures, req, samplesSink) {
             envelopeUsd: refEnvelope,
             ...(retainForPairwise ? { retainOutputForPairwise: true } : {}),
             ...(req.expected !== undefined ? { expected: req.expected } : {}),
+            ...(req.graderType !== undefined ? { graderType: req.graderType } : {}),
             ...(judgeCfg !== undefined ? { judge: (0, judge_1.makeJudgeScorer)(deps, route, judgeCfg) } : {}),
+            ...(goldenJudgeCfg !== undefined ? { goldenJudge: (0, golden_judge_1.makeGoldenJudge)(deps, route, goldenJudgeCfg) } : {}),
             ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
         });
         const partial = refSamples.length < replayN || refSamples.some((s) => s.failureClass === "budget_exhausted");
@@ -421,14 +432,50 @@ async function mirrorBakeoff(artifact, config) {
         throw new Error(`bakeoff mirror upsert → ${response.status} ${await response.text().catch(() => "")}`);
     }
 }
+/**
+ * Derive a bake-off's proposed-swap direction from the winner's vs the
+ * incumbent's `effectiveUsdPer1kConformant` (spec §4.2). The incumbent is the
+ * `default` arm when present, else the first non-reference arm — matching
+ * `computeProposal`'s incumbent selection (requested variants keep their order in
+ * the matrix, with the reference pushed last). Null when there is no proposal.
+ * Pure, additive: `computeProposal` is untouched (the winner is only read back).
+ */
+function deriveSwapDirection(artifact) {
+    const p = artifact.proposal;
+    if (p === null || p === undefined)
+        return null;
+    const winner = artifact.variants.find((m) => m.variant === p.variant);
+    const incumbent = artifact.variants.find((m) => m.variant === "default" && m.isReference !== true) ??
+        artifact.variants.find((m) => m.isReference !== true);
+    if (winner === undefined || incumbent === undefined)
+        return null;
+    const w = winner.effectiveUsdPer1kConformant;
+    const i = incumbent.effectiveUsdPer1kConformant;
+    const costDeltaUsdPer1k = Number.isFinite(w) && Number.isFinite(i) ? w - i : null;
+    const direction = costDeltaUsdPer1k === null || Math.abs(costDeltaUsdPer1k) <= EPS
+        ? "lateral"
+        : costDeltaUsdPer1k > 0
+            ? "up"
+            : "down";
+    return { direction, costDeltaUsdPer1k };
+}
 /**
  * The proposal-evidence jsonb for a bake-off artifact (near-optimal spec §4.6) —
  * the winning variant, the RB-1 objective/reference/regret/quality_metric, and a
  * per-arm matrix (with per-arm regret). Built here so the CLI stays thin; the PR
  * body reads these fields back off `proposal.evidence`. ZERO DDL — freeform jsonb.
+ *
+ * N4 (spec §4.2, ADDITIVE): when a proposal exists, stamp `swap_direction` +
+ * `cost_delta_usd_per_1k` so the PR body, the approval prompt, and the odometer
+ * treat a cost-raising promotion honestly. Absent on a no-proposal artifact (the
+ * existing evidence shape is byte-identical).
  */
 function bakeoffEvidence(artifact, spentUsd) {
+    const dir = deriveSwapDirection(artifact);
     return {
+        ...(dir !== null
+            ? { swap_direction: dir.direction, cost_delta_usd_per_1k: dir.costDeltaUsdPer1k }
+            : {}),
         bakeoff_id: artifact.id,
         replay_n: artifact.replayN,
         ran_at: artifact.ranAt,

package/dist/bakeoff/eval-cases-rig.d.ts

@@ -0,0 +1,36 @@
+/**
+ * The rig-facing from-eval-cases bake-off delegates — extracted from index.ts to
+ * keep it under the 700-LOC cap (P1). Both `rig.bakeoffFromEvalCases` (LOCAL
+ * artifact store + captures) and `rig.bakeoffFromEvalCasesWith` (LRE-1b: stores
+ * the CALLER injects — the server's cross-plane rehydration resolver) funnel
+ * through `runRigEvalCasesBakeoff`, so option defaulting lives in one place.
+ */
+import { type EvalCaseBakeoffResult, type EvalCaseStores } from "./eval-cases";
+import type { BakeoffSamplesSink } from "./engine";
+import type { GraderType } from "./quality";
+import type { KeepOutputsMode } from "./samples";
+import type { RunDeps } from "../run/run";
+import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
+import type { QualityGate } from "../types/quality";
+/** The shared option shape for both from-eval-cases rig entry points. */
+export interface EvalCasesBakeoffOpts {
+    readonly variants: readonly string[];
+    readonly envelopeUsdPerVariant?: number;
+    readonly keepOutputs?: KeepOutputsMode;
+    readonly objectiveMode?: ObjectiveMode;
+    readonly qualityGate?: QualityGate;
+    readonly graderType?: GraderType;
+}
+/** Run the from-eval-cases engine over the given deps + stores, defaulting the
+ * envelope and threading the reference-auto resolver + samples sink. Pure over
+ * its inputs — no local-store assumptions (the caller picks the stores). */
+export declare function runRigEvalCasesBakeoff(input: {
+    readonly deps: RunDeps;
+    readonly stores: EvalCaseStores;
+    readonly task: string;
+    readonly samplesSink?: BakeoffSamplesSink;
+    readonly resolveReferenceAuto: (route: RouteBundle) => CandidateSpec | null;
+    readonly defaultEnvelopeUsd: number;
+    readonly opts: EvalCasesBakeoffOpts;
+}): Promise<EvalCaseBakeoffResult>;
+//# sourceMappingURL=eval-cases-rig.d.ts.map

package/dist/bakeoff/eval-cases-rig.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-cases-rig.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases-rig.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAA2B,KAAK,qBAAqB,EAAE,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AACxG,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,yEAAyE;AACzE,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED;;4EAE4E;AAC5E,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;IAC1C,QAAQ,CAAC,oBAAoB,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC5E,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;CACrC,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAiBjC"}

package/dist/bakeoff/eval-cases-rig.js

@@ -0,0 +1,27 @@
+"use strict";
+/**
+ * The rig-facing from-eval-cases bake-off delegates — extracted from index.ts to
+ * keep it under the 700-LOC cap (P1). Both `rig.bakeoffFromEvalCases` (LOCAL
+ * artifact store + captures) and `rig.bakeoffFromEvalCasesWith` (LRE-1b: stores
+ * the CALLER injects — the server's cross-plane rehydration resolver) funnel
+ * through `runRigEvalCasesBakeoff`, so option defaulting lives in one place.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runRigEvalCasesBakeoff = runRigEvalCasesBakeoff;
+const eval_cases_1 = require("./eval-cases");
+/** Run the from-eval-cases engine over the given deps + stores, defaulting the
+ * envelope and threading the reference-auto resolver + samples sink. Pure over
+ * its inputs — no local-store assumptions (the caller picks the stores). */
+function runRigEvalCasesBakeoff(input) {
+    const { deps, stores, task, samplesSink, resolveReferenceAuto, defaultEnvelopeUsd, opts } = input;
+    return (0, eval_cases_1.runBakeoffFromEvalCases)(deps, stores, {
+        task,
+        variants: opts.variants,
+        envelopeUsdPerVariant: opts.envelopeUsdPerVariant ?? defaultEnvelopeUsd,
+        qualityGate: opts.qualityGate,
+        ...(opts.keepOutputs !== undefined ? { keepOutputs: opts.keepOutputs } : {}),
+        ...(opts.objectiveMode !== undefined ? { objectiveMode: opts.objectiveMode } : {}),
+        ...(opts.graderType !== undefined ? { graderType: opts.graderType } : {}),
+        resolveReferenceAuto,
+    }, samplesSink);
+}

package/dist/bakeoff/eval-cases.d.ts

@@ -21,6 +21,7 @@ import type { ExportableTelemetryStore } from "../telemetry/store";
 import type { EvalCaseRow } from "../types/telemetry";
 import type { RunDeps } from "../run/run";
 import { type BakeoffSamplesSink } from "./engine";
+import type { GraderType } from "./quality";
 import type { KeepOutputsMode } from "./samples";
 import type { BakeoffArtifact } from "./types";
 import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
@@ -39,6 +40,10 @@ export interface EvalCaseSelectionDeps {
 export interface RehydratedCase {
     readonly caseId: string;
     readonly capture: CaptureRow;
+    /** LRE-1b: the golden (corrected) artifact id, or null for a failure case (no
+     * golden to grade against — it scores by conformance only). Carried so the
+     * runner can build the `expected` map from the golden's content. */
+    readonly correctedArtifactId: string | null;
 }
 export interface EvalCaseRefusal {
     readonly caseId: string;
@@ -88,6 +93,15 @@ export interface EvalCaseStores {
     readonly artifacts: Pick<ArtifactStore, "readEvalCasesByTask" | "getArtifactById"> | null;
     readonly telemetry: Pick<ExportableTelemetryStore, "getStepInferenceId">;
     readonly captures: () => Pick<CaptureStore, "getByInferenceId">;
+    /** LRE-1b: resolve a golden (corrected) artifact's PARSED content value, or
+     * null when it is not readable (predates retention, evicted, or content custody
+     * is off). Injected so the runner can score a challenger's output against the
+     * golden (the `exact` grader). Absent ⇒ from-eval-cases scores by conformance
+     * only (pre-LRE-1b behavior) — a golden with no readable content is honest
+     * about it rather than pretending the case is graded. */
+    readonly golden?: (correctedArtifactId: string) => {
+        readonly value: unknown;
+    } | null;
 }
 export declare function runBakeoffFromEvalCases(deps: RunDeps, stores: EvalCaseStores, req: {
     readonly task: string;
@@ -97,5 +111,10 @@ export declare function runBakeoffFromEvalCases(deps: RunDeps, stores: EvalCaseS
     readonly objectiveMode?: ObjectiveMode;
     readonly resolveReferenceAuto?: (route: RouteBundle) => CandidateSpec | null;
     readonly qualityGate?: QualityGate;
+    /** The golden set's grader — all three wired: `exact` (binary), `field-level`
+     * (per-field partial credit), `llm-judge` (pairwise vs the golden, LRE-2).
+     * Applied to every case that HAS a readable golden; a failure case (no golden)
+     * still scores by conformance. */
+    readonly graderType?: GraderType;
 }, samplesSink?: BakeoffSamplesSink): Promise<EvalCaseBakeoffResult>;
 //# sourceMappingURL=eval-cases.d.ts.map

package/dist/bakeoff/eval-cases.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAgCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;CACjE;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CAgChC"}
+{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAgCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAChE;;;;;6DAKyD;IACzD,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,mBAAmB,EAAE,MAAM,KAAK;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACvF;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;sCAGkC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CA+ChC"}

package/dist/bakeoff/eval-cases.js

@@ -53,7 +53,7 @@ function selectEvalCaseCaptures(deps, task) {
             refuse("input not captured locally (route `capture` was off, or the capture was evicted)");
             continue;
         }
-        rehydratable.push({ caseId: c.id, capture });
+        rehydratable.push({ caseId: c.id, capture, correctedArtifactId: c.correctedArtifactId });
     }
     return { task, total: cases.length, rehydratable, refusals };
 }
@@ -94,11 +94,28 @@ async function runBakeoffFromEvalCases(deps, stores, req, samplesSink) {
     }, req.task);
     if (selection.rehydratable.length === 0)
         return { selection, artifact: null };
+    // LRE-1b: build the golden `expected` map — for each rehydratable case that has
+    // a corrected artifact whose content the injected `golden` resolver can read,
+    // key the golden value by the capture's inferenceId (the engine's `expected`
+    // key). A failure case (no golden) or an unreadable golden simply omits its
+    // key → that sample scores by conformance, never a silent zero.
+    const expected = new Map();
+    if (stores.golden !== undefined) {
+        for (const r of selection.rehydratable) {
+            if (r.correctedArtifactId === null)
+                continue;
+            const g = stores.golden(r.correctedArtifactId);
+            if (g !== null)
+                expected.set(r.capture.inferenceId, { value: g.value });
+        }
+    }
     const artifact = await (0, engine_1.runBakeoff)(deps, evalCaseCaptureStore(selection.rehydratable), {
         route: req.task,
         variants: req.variants,
         replayLastN: selection.rehydratable.length, // replay every rehydratable case
         envelopeUsdPerVariant: req.envelopeUsdPerVariant,
+        ...(expected.size > 0 ? { expected } : {}),
+        ...(req.graderType !== undefined ? { graderType: req.graderType } : {}),
         ...(req.keepOutputs !== undefined ? { keepOutputs: req.keepOutputs } : {}),
         ...(req.objectiveMode !== undefined ? { objectiveMode: req.objectiveMode } : {}),
         ...(req.resolveReferenceAuto !== undefined ? { resolveReferenceAuto: req.resolveReferenceAuto } : {}),

package/dist/bakeoff/golden-judge.d.ts

@@ -0,0 +1,47 @@
+/**
+ * The `llm-judge` grader (LRE-2, live-routing-evals §R-3/§4). When a golden eval
+ * set's `grader.type` is `llm-judge`, each rehydratable case is scored by an LLM
+ * COMPARISON of the challenger's output against the case's GOLDEN (`expected`)
+ * value — reusing the N5/N6 PAIRWISE judge machinery (pairwise.ts), not a new
+ * dispatch path.
+ *
+ * Why pairwise-vs-golden (R-3, "LLM-judge reuses the N5/N6 pairwise judge"):
+ *   - The pairwise judge already asks "which of A / B is the better answer for
+ *     VALUE (correctness + completeness)". Presenting A = the challenger output
+ *     and B = the golden turns that into "is the challenger at least as good as
+ *     the known-correct golden?" — exactly a grade against a reference.
+ *   - The verdict maps to a per-sample value accuracy via the SAME `winValue`
+ *     the pairwise aggregation uses (win=1, tie=0.5, loss=0). This is the
+ *     CONSERVATIVE mapping (L5, never round in our favor): a tie against the
+ *     golden is 0.5, never rounded up to full credit; a loss is 0. Builder
+ *     decision — flagged in the PR body for QA.
+ *   - The judge model MUST be a declared candidate (#223 F3) — enforced at load
+ *     for `bakeoff.pairwise.judge` / `bakeoff.judge`, whose model this reuses, so
+ *     it is a real, reachable model here.
+ *
+ * CAPPED + METERED + fail-closed exactly like the pairwise judge (it IS the
+ * pairwise judge): past `maxSamples` per arm `score()` returns null and the
+ * sample falls back to conformance (counted by omission), an unparseable verdict
+ * returns null, and the metered attempt row still stands.
+ */
+import type { RunDeps } from "../run/run";
+import type { CandidateSpec, RouteBundle } from "../types/route";
+/** A per-arm golden grader: `score(output, golden, taskInput)` returns the
+ * challenger output's value accuracy in [0,1] judged against the golden, or null
+ * when the cap is hit or the judge could not produce a valid verdict. */
+export interface GoldenJudgeScorer {
+    score(output: unknown, golden: unknown, taskInput: unknown): Promise<number | null>;
+}
+/**
+ * Build a golden judge over the pairwise-judge loop. Each arm gets a FRESH judge
+ * (the underlying pairwise judge caps `maxSamples` per arm). A = the challenger
+ * output, B = the golden `expected` value; the pairwise verdict (win/tie/loss)
+ * maps to [0,1] via `winValue` — an output judged at least as good as the golden
+ * scores high, a worse output scores 0. Position bias is the pairwise judge's
+ * known limit (fixed A/B order); noted in verification.
+ */
+export declare function makeGoldenJudge(deps: RunDeps, baseRoute: RouteBundle, judge: {
+    readonly model: CandidateSpec;
+    readonly maxSamples: number;
+}): GoldenJudgeScorer;
+//# sourceMappingURL=golden-judge.d.ts.map

package/dist/bakeoff/golden-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden-judge.d.ts","sourceRoot":"","sources":["../../src/bakeoff/golden-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAGjE;;yEAEyE;AACzE,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CACrF;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,WAAW,EACtB,KAAK,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACpE,iBAAiB,CAUnB"}

package/dist/bakeoff/golden-judge.js

@@ -0,0 +1,49 @@
+"use strict";
+/**
+ * The `llm-judge` grader (LRE-2, live-routing-evals §R-3/§4). When a golden eval
+ * set's `grader.type` is `llm-judge`, each rehydratable case is scored by an LLM
+ * COMPARISON of the challenger's output against the case's GOLDEN (`expected`)
+ * value — reusing the N5/N6 PAIRWISE judge machinery (pairwise.ts), not a new
+ * dispatch path.
+ *
+ * Why pairwise-vs-golden (R-3, "LLM-judge reuses the N5/N6 pairwise judge"):
+ *   - The pairwise judge already asks "which of A / B is the better answer for
+ *     VALUE (correctness + completeness)". Presenting A = the challenger output
+ *     and B = the golden turns that into "is the challenger at least as good as
+ *     the known-correct golden?" — exactly a grade against a reference.
+ *   - The verdict maps to a per-sample value accuracy via the SAME `winValue`
+ *     the pairwise aggregation uses (win=1, tie=0.5, loss=0). This is the
+ *     CONSERVATIVE mapping (L5, never round in our favor): a tie against the
+ *     golden is 0.5, never rounded up to full credit; a loss is 0. Builder
+ *     decision — flagged in the PR body for QA.
+ *   - The judge model MUST be a declared candidate (#223 F3) — enforced at load
+ *     for `bakeoff.pairwise.judge` / `bakeoff.judge`, whose model this reuses, so
+ *     it is a real, reachable model here.
+ *
+ * CAPPED + METERED + fail-closed exactly like the pairwise judge (it IS the
+ * pairwise judge): past `maxSamples` per arm `score()` returns null and the
+ * sample falls back to conformance (counted by omission), an unparseable verdict
+ * returns null, and the metered attempt row still stands.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeGoldenJudge = makeGoldenJudge;
+const pairwise_1 = require("./pairwise");
+/**
+ * Build a golden judge over the pairwise-judge loop. Each arm gets a FRESH judge
+ * (the underlying pairwise judge caps `maxSamples` per arm). A = the challenger
+ * output, B = the golden `expected` value; the pairwise verdict (win/tie/loss)
+ * maps to [0,1] via `winValue` — an output judged at least as good as the golden
+ * scores high, a worse output scores 0. Position bias is the pairwise judge's
+ * known limit (fixed A/B order); noted in verification.
+ */
+function makeGoldenJudge(deps, baseRoute, judge) {
+    const pairwise = (0, pairwise_1.makePairwiseJudge)(deps, baseRoute, judge);
+    return {
+        async score(output, golden, taskInput) {
+            // compare(A=output, B=golden): "win" ⇒ output beats the golden, "tie" ⇒ as
+            // good as, "loss" ⇒ worse. winValue gives 1 / 0.5 / 0.
+            const outcome = await pairwise.compare(output, golden, taskInput);
+            return outcome === null ? null : (0, pairwise_1.winValue)(outcome);
+        },
+    };
+}

package/dist/bakeoff/quality.d.ts

@@ -13,6 +13,15 @@
  * ordering.
  */
 import type { ValueCheck } from "../types/route";
+/**
+ * The grader a golden eval set scores with (live-routing-evals §2.2/§4). All
+ * three are wired: `exact` (binary normalized equality, LRE-1b), `field-level`
+ * (schema-aware per-field partial credit via {@link fieldAccuracy}, first-class
+ * as of LRE-2), and `llm-judge` (a PAIRWISE comparison of the output against the
+ * golden — NOT a deterministic tier, so this resolver returns null for it and the
+ * caller's golden judge (golden-judge.ts, LRE-2) scores it against `expected`).
+ */
+export type GraderType = "exact" | "field-level" | "llm-judge";
 /**
  * Schema-aware field accuracy in [0,1]: the fraction of the EXPECTED value's leaf
  * fields that the actual output matches. Recurses into objects/arrays keyed by
@@ -21,6 +30,18 @@ import type { ValueCheck } from "../types/route";
  * expected ⇒ 1 (nothing to be wrong about).
  */
 export declare function fieldAccuracy(expected: unknown, actual: unknown): number;
+/**
+ * The `exact` grader (LRE-1b, spec §4): normalized EQUALITY of the parsed output
+ * against the golden → 1 (match) or 0 (mismatch). Binary, never partial credit —
+ * that is the whole point of `exact` vs `field-level` (`fieldAccuracy`).
+ *
+ * Normalization is `normalizeDeep`: object keys are order-insensitive (sorted),
+ * string leaves are whitespace-trimmed, arrays stay ORDER-sensitive (a reordered
+ * list is a different value), and types are strict (`1` ≠ `"1"`, `null` ≠
+ * `undefined`). So `{a:1, b:" x "}` matches `{b:"x", a:1}` but `[1,2]` does not
+ * match `[2,1]`. Pure + unit-tested (exact-grader.test.ts).
+ */
+export declare function exactMatch(expected: unknown, actual: unknown): number;
 /** Resolve an RFC 6901 JSON pointer against a value. Returns `undefined` when any
  * segment is missing (a missing field fails its value check). */
 export declare function resolveJsonPointer(value: unknown, pointer: string): unknown;
@@ -35,6 +56,15 @@ export interface DeterministicQualityInput {
     /** Set when `expected` is meaningfully present (distinguishes a golden `null`
      * expected value from "no eval case"). */
     readonly hasExpected?: boolean;
+    /** The golden set's grader. Only consulted alongside `hasExpected`:
+     *   • `exact`      → binary normalized equality (exactMatch).
+     *   • `field-level`→ schema-aware per-field partial credit (fieldAccuracy) —
+     *     the fraction of the golden's leaf fields the output matches, in [0,1].
+     *   • `llm-judge`  → NOT a deterministic tier; returns null so the caller's
+     *     golden judge (golden-judge.ts) scores the output against the golden via
+     *     the N5/N6 pairwise judge (LRE-2).
+     * Absent ⇒ `field-level` behavior (byte-identical to pre-LRE-1b default). */
+    readonly graderType?: GraderType;
 }
 /**
  * Deterministic value accuracy for one sample, or null when neither an eval-case

package/dist/bakeoff/quality.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAiBjD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAgBxE;AAED;iEACiE;AACjE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmB3E;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAWvE;AAYD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;oDACgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;8CAC0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,GAAG,IAAI,CAU3F"}
+{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,aAAa,GAAG,WAAW,CAAC;AAiB/D;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAgBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAErE;AAED;iEACiE;AACjE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmB3E;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAWvE;AAYD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;oDACgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;8CAC0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;;;;iFAO6E;IAC7E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,GAAG,IAAI,CAc3F"}

package/dist/bakeoff/quality.js

@@ -15,6 +15,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fieldAccuracy = fieldAccuracy;
+exports.exactMatch = exactMatch;
 exports.resolveJsonPointer = resolveJsonPointer;
 exports.checkPasses = checkPasses;
 exports.resolveDeterministicQuality = resolveDeterministicQuality;
@@ -58,6 +59,20 @@ function fieldAccuracy(expected, actual) {
     }
     return scalarEqual(expected, actual) ? 1 : 0;
 }
+/**
+ * The `exact` grader (LRE-1b, spec §4): normalized EQUALITY of the parsed output
+ * against the golden → 1 (match) or 0 (mismatch). Binary, never partial credit —
+ * that is the whole point of `exact` vs `field-level` (`fieldAccuracy`).
+ *
+ * Normalization is `normalizeDeep`: object keys are order-insensitive (sorted),
+ * string leaves are whitespace-trimmed, arrays stay ORDER-sensitive (a reordered
+ * list is a different value), and types are strict (`1` ≠ `"1"`, `null` ≠
+ * `undefined`). So `{a:1, b:" x "}` matches `{b:"x", a:1}` but `[1,2]` does not
+ * match `[2,1]`. Pure + unit-tested (exact-grader.test.ts).
+ */
+function exactMatch(expected, actual) {
+    return JSON.stringify(normalizeDeep(expected)) === JSON.stringify(normalizeDeep(actual)) ? 1 : 0;
+}
 /** Resolve an RFC 6901 JSON pointer against a value. Returns `undefined` when any
  * segment is missing (a missing field fails its value check). */
 function resolveJsonPointer(value, pointer) {
@@ -115,6 +130,12 @@ function normalizeDeep(v) {
  */
 function resolveDeterministicQuality(input) {
     if (input.hasExpected === true) {
+        // llm-judge is not deterministic — defer to the judge tier (LRE-2 seam).
+        if (input.graderType === "llm-judge")
+            return null;
+        if (input.graderType === "exact")
+            return exactMatch(input.expected, input.output);
+        // field-level (explicit) or unspecified default → schema-aware field accuracy.
         return fieldAccuracy(input.expected, input.output);
     }
     const checks = input.valueChecks;

package/dist/bakeoff/replay.d.ts

@@ -19,7 +19,9 @@
  */
 import type { CaptureStore } from "../telemetry/capture";
 import type { RunDeps } from "../run/run";
+import type { GraderType } from "./quality";
 import type { JudgeScorer } from "./judge";
+import type { GoldenJudgeScorer } from "./golden-judge";
 import type { QualityGate } from "../types/quality";
 export interface ReplayRequest {
     readonly route: string;
@@ -42,9 +44,18 @@ export interface ReplayRequest {
     readonly expected?: ReadonlyMap<string, {
         readonly value: unknown;
     }>;
+    /** LRE-1b: the golden set's grader — how an `expected` value is scored against
+     * the output (`exact` binary, `field-level` per-field, `llm-judge` deferred to
+     * the judge tier). Absent ⇒ field-level (pre-LRE-1b behavior). */
+    readonly graderType?: GraderType;
     /** RB-1: an opt-in judge scorer (tier 3), used only when neither an eval-case
      * expected value nor a declared value check yields a score. Capped internally. */
     readonly judge?: JudgeScorer;
+    /** LRE-2: the `llm-judge` grader — scores a produced output against its GOLDEN
+     * (`expected`) value via the N5/N6 pairwise judge. Used ONLY when
+     * `graderType==='llm-judge'` AND this sample has an eval-case expected value (a
+     * failure case with no golden still scores by conformance). Capped internally. */
+    readonly goldenJudge?: GoldenJudgeScorer;
     /** Q1 quality gate (spec §4.2): scored on each replayed sample's output to
      * populate `valueAccuracy` + `qualityPassed`. NOT passed into the inner
      * `run()` — a bake-off MEASURES quality, it does not gate serving; injecting