modelrig 0.4.0 → 0.6.0

package/dist/bakeoff/engine.js

@@ -12,14 +12,22 @@
  * the artifact records value_accuracy where the hook supplies it.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.REFERENCE_ARM = void 0;
 exports.runBakeoff = runBakeoff;
 exports.computeProposal = computeProposal;
 exports.mirrorBakeoff = mirrorBakeoff;
+exports.bakeoffEvidence = bakeoffEvidence;
 const node_crypto_1 = require("node:crypto");
 const loader_1 = require("../routes/loader");
 const replay_1 = require("./replay");
 const stats_1 = require("./stats");
+const judge_1 = require("./judge");
+const reference_1 = require("./reference");
 const samples_1 = require("./samples");
+/** The arm name carrying the reference (ceiling) metrics. Reserved — a user
+ * variant may not be named this (they cannot start a name with "__"; and it is
+ * never in `req.variants`). */
+exports.REFERENCE_ARM = "__reference__";
 /** Arm metadata for the reviewer: which model/rung/scaffold served it. */
 function deriveArmMeta(route, arm) {
     if (arm === "default") {
@@ -33,6 +41,25 @@ function deriveArmMeta(route, arm) {
         ...(variant?.scaffold !== undefined ? { scaffold: true } : {}),
     };
 }
+/** Overlay one synthetic route (reference/judge arm) onto a routes store. */
+function overlayRoute(routes, extra) {
+    return {
+        get: (r) => (r === extra.route ? extra : routes.get(r)),
+        list: () => Object.freeze([...routes.list(), extra]),
+    };
+}
+/** Resolve the reference candidate from `route.bakeoff.reference` (auto/explicit),
+ * or null when none is declared/resolvable. */
+function resolveReference(route, req) {
+    const ref = route.bakeoff?.reference;
+    if (ref === undefined)
+        return null;
+    if (ref === "auto") {
+        const candidate = req.resolveReferenceAuto?.(route) ?? null;
+        return candidate === null ? null : { candidate, auto: true };
+    }
+    return { candidate: { provider: ref.provider, model: ref.model }, auto: false };
+}
 async function runBakeoff(deps, captures, req, samplesSink) {
     if (req.variants.length === 0) {
         throw new loader_1.RouteConfigError("bakeoff needs at least one variant (use \"default\" for the base config)");
@@ -47,6 +74,10 @@ async function runBakeoff(deps, captures, req, samplesSink) {
     }
     const keepOutputs = req.keepOutputs !== undefined && samplesSink !== undefined;
     const bakeoffId = (0, node_crypto_1.randomUUID)();
+    const judgeCfg = route.bakeoff?.judge;
+    // Replay every requested arm. Raw samples are kept ALWAYS (paired regret is
+    // computed from per-sample quality); the keepOutputs sink is separate. A fresh
+    // judge scorer per arm keeps `max_samples` per-arm (spec §4.3).
     const metrics = [];
     const rawByArm = new Map();
     let replayN = 0;
@@ -57,13 +88,70 @@ async function runBakeoff(deps, captures, req, samplesSink) {
             lastN: req.replayLastN,
             envelopeUsd: req.envelopeUsdPerVariant,
             ...(keepOutputs ? { keepOutputs: true } : {}),
+            ...(req.expected !== undefined ? { expected: req.expected } : {}),
+            ...(judgeCfg !== undefined ? { judge: (0, judge_1.makeJudgeScorer)(deps, route, judgeCfg) } : {}),
+            ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
         });
         replayN = Math.max(replayN, samples.length);
+        rawByArm.set(variant, samples);
         metrics.push((0, stats_1.aggregateVariantMetrics)(variant, samples));
-        if (keepOutputs)
-            rawByArm.set(variant, samples);
     }
-    const proposal = computeProposal(req.variants, metrics);
+    // RB-1 reference (ceiling) arm — replay the reference model on the SAME set
+    // under its own envelope (it is dear by definition), then attach each requested
+    // arm's paired regret vs it. Partial-result honesty is preserved.
+    const referenceInfo = resolveReference(route, req);
+    let reference = null;
+    if (referenceInfo !== null) {
+        const { deps: refDeps } = withReferenceArm(deps, route, referenceInfo.candidate);
+        const refEnvelope = route.bakeoff?.referenceEnvelopeUsd ?? req.envelopeUsdPerVariant;
+        const refSamples = await (0, replay_1.replayVariant)(refDeps, captures, {
+            route: req.route,
+            variant: exports.REFERENCE_ARM,
+            lastN: req.replayLastN,
+            envelopeUsd: refEnvelope,
+            ...(req.expected !== undefined ? { expected: req.expected } : {}),
+            ...(judgeCfg !== undefined ? { judge: (0, judge_1.makeJudgeScorer)(deps, route, judgeCfg) } : {}),
+            ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
+        });
+        const partial = refSamples.length < replayN || refSamples.some((s) => s.failureClass === "budget_exhausted");
+        for (let i = 0; i < metrics.length; i += 1) {
+            const raw = rawByArm.get(metrics[i].variant) ?? [];
+            metrics[i] = {
+                ...metrics[i],
+                regret: (0, stats_1.pairedRegret)(raw, refSamples),
+                ...(partial ? { referencePartial: true } : {}),
+            };
+        }
+        metrics.push({
+            ...(0, stats_1.aggregateVariantMetrics)(exports.REFERENCE_ARM, refSamples),
+            isReference: true,
+            regret: null,
+            ...(partial ? { referencePartial: true } : {}),
+        });
+        reference = {
+            variant: exports.REFERENCE_ARM,
+            model: `${referenceInfo.candidate.provider}/${referenceInfo.candidate.model}`,
+            auto: referenceInfo.auto,
+            rule: referenceInfo.auto ? reference_1.REFERENCE_AUTO_RULE : null,
+            external: !isDeclared(route, referenceInfo.candidate),
+            partial,
+        };
+    }
+    const objective = resolveObjective(route, req);
+    const qualityMetric = metrics.some((m) => m.isReference !== true && m.qualityMetric === "value_accuracy")
+        ? "value_accuracy"
+        : "conformance";
+    const proposal = computeProposal(req.variants, metrics, {
+        objective,
+        ...(reference !== null ? { referenceVariant: reference.variant } : {}),
+    });
+    // Architect B-3: stamp the proposed arm onto the matrix so the mirror/console
+    // read the artifact's decision instead of recomputing a mode-blind winner.
+    // Every arm carries the flag (false included) so a no-proposal artifact is
+    // distinguishable from a pre-RB-1 row — the console never recomputes for it.
+    for (let i = 0; i < metrics.length; i += 1) {
+        metrics[i] = { ...metrics[i], proposed: proposal !== null && metrics[i].variant === proposal.variant };
+    }
     // Persist review samples (A3) — AFTER the winner is known, for the selected
     // arms only. Never touches the artifact (payloads stay off the anon table).
     if (keepOutputs && samplesSink !== undefined) {
@@ -101,32 +189,160 @@ async function runBakeoff(deps, captures, req, samplesSink) {
         replayN,
         variants: metrics,
         proposal,
+        objective,
+        reference,
+        qualityMetric,
     };
 }
-/** Winner among gate-passing challengers; null when nobody beats the
- * incumbent. Exported for direct unit testing. */
-function computeProposal(requested, metrics) {
+function isDeclared(route, candidate) {
+    return route.candidates.some((c) => c.provider === candidate.provider && c.model === candidate.model);
+}
+/** Build a deps whose route store carries a synthetic `__reference__` variant
+ * narrowing to the reference candidate (added to the declared set when external,
+ * so the candidate-set guard is satisfied). */
+function withReferenceArm(deps, route, candidate) {
+    const refVariant = { name: exports.REFERENCE_ARM, candidates: [{ provider: candidate.provider, model: candidate.model }] };
+    const augmented = Object.freeze({
+        ...route,
+        candidates: isDeclared(route, candidate)
+            ? route.candidates
+            : Object.freeze([...route.candidates, { provider: candidate.provider, model: candidate.model }]),
+        variants: Object.freeze([...route.variants, refVariant]),
+    });
+    return { deps: { ...deps, routes: overlayRoute(deps.routes, augmented) } };
+}
+function resolveObjective(route, req) {
+    // Architect B-2: a route that declares NEITHER an `objective:` NOR a `bakeoff:`
+    // block has not opted into RB-1 — it defaults to `cost-first` (the pre-RB-1
+    // gate, byte-identical), so legacy routes and the hosted R3 swap loop keep
+    // proposing. A route that DID opt in (either block) defaults to quality-first,
+    // where "no bound ⇒ no proposal" (loader-warned) prevents silent actuation.
+    const optedIn = route.objective !== undefined || route.bakeoff !== undefined;
+    const mode = req.objectiveMode ?? route.objective?.mode ?? (optedIn ? "quality-first" : "cost-first");
+    return {
+        mode,
+        maxRegret: route.objective?.maxRegret ?? null,
+        priceCeiling: route.objective?.priceCeiling ?? null,
+        lambda: route.objective?.lambda ?? null,
+    };
+}
+const EPS = 1e-9;
+function savingsPct(incumbent, winner) {
+    const inc = incumbent.effectiveUsdPer1kConformant;
+    return inc === 0 || !Number.isFinite(inc)
+        ? 0
+        : ((inc - winner.effectiveUsdPer1kConformant) / inc) * 100;
+}
+/** A price ceiling test: usd_per_1k on the effective-$/1k axis, usd_per_run on
+ * mean cost. Null ceiling ⇒ no cost constraint (regret alone gates). */
+function withinCeiling(m, ceiling) {
+    if (ceiling === null)
+        return true;
+    if (ceiling.usdPer1k !== undefined)
+        return m.effectiveUsdPer1kConformant <= ceiling.usdPer1k + EPS;
+    if (ceiling.usdPerRun !== undefined)
+        return m.meanCostUsd <= ceiling.usdPerRun + EPS;
+    return true;
+}
+/**
+ * Winner among gate-passing challengers; null when nobody beats the incumbent.
+ * Exported for direct unit testing. `ctx` absent OR `ctx.objective` absent ⇒ the
+ * legacy cost-first gate (goldens). Otherwise per the objective's mode.
+ */
+function computeProposal(requested, metrics, ctx) {
+    const mode = ctx?.objective?.mode ?? "cost-first";
     const incumbentName = requested.includes("default") ? "default" : requested[0];
     const incumbent = metrics.find((m) => m.variant === incumbentName);
     // A1: unpriced cost columns under-count — an unpriced incumbent can't be
     // compared against, and an unpriced challenger must never look cheap.
     if (incumbent === undefined || incumbent.samples === 0 || incumbent.pricingMissing)
         return null;
-    const challengers = metrics.filter((m) => m.variant !== incumbentName &&
+    // Q1 (spec §4.2): when a quality gate ran on the incumbent, the cost-first
+    // comparison gains a THIRD clause (challenger's quality-pass CI-lower ≥
+    // incumbent's) AND the headline denominator switches from conformant to
+    // conformant-AND-passing (effectiveUsdPer1kPassing, reported beside — never
+    // redefining — the conformant metric; flag F6). Absent a gate, byte-identical.
+    const gated = incumbent.qualityPassCi95 !== null;
+    const metricOf = (m) => gated ? m.effectiveUsdPer1kPassing ?? Number.POSITIVE_INFINITY : m.effectiveUsdPer1kConformant;
+    // RB-1: the challenger pool is membership + priced; the per-mode gate below
+    // decides eligibility (cost-first uses Q1's conformance/quality/cost clauses;
+    // quality-first/balanced use regret vs the reference under the ceiling).
+    const pool = metrics.filter((m) => m.variant !== incumbentName &&
+        m.variant !== ctx?.referenceVariant &&
+        requested.includes(m.variant) &&
         m.samples > 0 &&
-        !m.pricingMissing &&
-        m.conformCi95[0] >= incumbent.conformCi95[0] &&
-        m.effectiveUsdPer1kConformant < incumbent.effectiveUsdPer1kConformant);
-    if (challengers.length === 0)
+        !m.pricingMissing);
+    if (mode === "cost-first") {
+        // Q1 gate: not-worse on conformance CI, (when gated) not-worse on quality-pass
+        // CI, AND cheaper on the effective metric (passing when gated, else conformant).
+        const challengers = pool.filter((m) => m.conformCi95[0] >= incumbent.conformCi95[0] &&
+            (!gated ||
+                (m.qualityPassCi95 !== null &&
+                    incumbent.qualityPassCi95 !== null &&
+                    m.qualityPassCi95[0] >= incumbent.qualityPassCi95[0])) &&
+            metricOf(m) < metricOf(incumbent));
+        if (challengers.length === 0)
+            return null;
+        const winner = [...challengers].sort((a, b) => metricOf(a) - metricOf(b))[0];
+        const incumbentMetric = metricOf(incumbent);
+        const savings = incumbentMetric === 0 || !Number.isFinite(incumbentMetric)
+            ? 0
+            : ((incumbentMetric - metricOf(winner)) / incumbentMetric) * 100;
+        return { variant: winner.variant, vsIncumbentSavingsPct: savings };
+    }
+    const objective = ctx?.objective;
+    if (mode === "quality-first") {
+        // No bound declared ⇒ regret is REPORTED but nothing is proposed (spec §3).
+        if (objective.maxRegret === null && objective.priceCeiling === null)
+            return null;
+        const eligible = pool.filter((m) => m.regret != null &&
+            m.regret.nPaired >= 10 &&
+            (objective.maxRegret === null || m.regret.regretCi95[1] <= objective.maxRegret + EPS) &&
+            withinCeiling(m, objective.priceCeiling));
+        if (eligible.length === 0)
+            return null;
+        // Lowest regret, tie → lowest cost.
+        const winner = [...eligible].sort((a, b) => a.regret.regretMean - b.regret.regretMean ||
+            a.effectiveUsdPer1kConformant - b.effectiveUsdPer1kConformant)[0];
+        // Propose iff the arm improves on the incumbent — never a lateral, never a
+        // quality regression to buy cost (architect B-1). The conformance clause is
+        // kept (a swap must not regress shape), and a COST-motivated proposal must be
+        // not-worse on regret: a cheaper-but-worse-quality arm is never proposed under
+        // quality-first. A regret win is a quality improvement and proposes on its own.
+        const incRegret = incumbent.regret?.regretMean ?? Number.POSITIVE_INFINITY;
+        const winRegret = winner.regret.regretMean;
+        const notWorseConform = winner.conformCi95[0] >= incumbent.conformCi95[0] - EPS;
+        const notWorseRegret = winRegret <= incRegret + EPS;
+        const betterRegret = winRegret < incRegret - EPS;
+        const betterCost = winner.effectiveUsdPer1kConformant < incumbent.effectiveUsdPer1kConformant - EPS;
+        if (!notWorseConform)
+            return null;
+        if (!(betterRegret || (betterCost && notWorseRegret)))
+            return null;
+        return { variant: winner.variant, vsIncumbentSavingsPct: savingsPct(incumbent, winner), regret: winner.regret ?? null };
+    }
+    // balanced: minimize regret_mean + λ·cost_norm, where cost_norm = the arm's
+    // effective $/1k divided by the incumbent's (dimensionless; 1.0 = the
+    // incumbent's cost). Must beat the incumbent's own blended score, under ceiling.
+    const lambda = objective.lambda ?? 0;
+    const incCost = incumbent.effectiveUsdPer1kConformant;
+    const norm = (m) => Number.isFinite(incCost) && incCost > 0 ? m.effectiveUsdPer1kConformant / incCost : m.effectiveUsdPer1kConformant;
+    const blended = (m) => (m.regret?.regretMean ?? Number.POSITIVE_INFINITY) + lambda * norm(m);
+    // `max_regret` is a HARD bound in every mode (architect B-1, round 2): λ may
+    // trade regret for cost only inside the declared bound, never across it.
+    const eligible = pool.filter((m) => m.regret != null &&
+        m.regret.nPaired >= 10 &&
+        (objective.maxRegret === null || m.regret.regretCi95[1] <= objective.maxRegret + EPS) &&
+        withinCeiling(m, objective.priceCeiling));
+    if (eligible.length === 0)
         return null;
-    const winner = [...challengers].sort((a, b) => a.effectiveUsdPer1kConformant - b.effectiveUsdPer1kConformant)[0];
-    const savings = incumbent.effectiveUsdPer1kConformant === 0 ||
-        !Number.isFinite(incumbent.effectiveUsdPer1kConformant)
-        ? 0
-        : ((incumbent.effectiveUsdPer1kConformant - winner.effectiveUsdPer1kConformant) /
-            incumbent.effectiveUsdPer1kConformant) *
-            100;
-    return { variant: winner.variant, vsIncumbentSavingsPct: savings };
+    const winner = [...eligible].sort((a, b) => blended(a) - blended(b))[0];
+    if (blended(winner) >= blended(incumbent) - EPS)
+        return null;
+    // Keep the conformance clause (B-1): a swap must not regress shape.
+    if (winner.conformCi95[0] < incumbent.conformCi95[0] - EPS)
+        return null;
+    return { variant: winner.variant, vsIncumbentSavingsPct: savingsPct(incumbent, winner), regret: winner.regret ?? null };
 }
 /** Upsert the artifact into the `bakeoffs` mirror table (console read path).
  * Same PostgREST transport pattern as the telemetry exporter; captures have
@@ -161,3 +377,37 @@ async function mirrorBakeoff(artifact, config) {
         throw new Error(`bakeoff mirror upsert → ${response.status} ${await response.text().catch(() => "")}`);
     }
 }
+/**
+ * The proposal-evidence jsonb for a bake-off artifact (near-optimal spec §4.6) —
+ * the winning variant, the RB-1 objective/reference/regret/quality_metric, and a
+ * per-arm matrix (with per-arm regret). Built here so the CLI stays thin; the PR
+ * body reads these fields back off `proposal.evidence`. ZERO DDL — freeform jsonb.
+ */
+function bakeoffEvidence(artifact, spentUsd) {
+    return {
+        bakeoff_id: artifact.id,
+        replay_n: artifact.replayN,
+        ran_at: artifact.ranAt,
+        spent_usd: spentUsd,
+        winning_variant: artifact.proposal?.variant ?? null,
+        proposal: artifact.proposal,
+        objective: artifact.objective ?? null,
+        reference: artifact.reference ?? null,
+        regret: artifact.proposal?.regret ?? null,
+        quality_metric: artifact.qualityMetric ?? null,
+        matrix: artifact.variants.map((m) => ({
+            variant: m.variant,
+            samples: m.samples,
+            conform_rate: m.conformRate,
+            conform_ci95: m.conformCi95,
+            effective_usd_per_1k_conformant: Number.isFinite(m.effectiveUsdPer1kConformant)
+                ? m.effectiveUsdPer1kConformant
+                : null,
+            pricing_missing: m.pricingMissing,
+            is_reference: m.isReference ?? false,
+            quality_mean: m.qualityMean,
+            quality_metric: m.qualityMetric,
+            regret: m.regret ?? null,
+        })),
+    };
+}

package/dist/bakeoff/eval-cases.d.ts

@@ -23,6 +23,8 @@ import type { RunDeps } from "../run/run";
 import { type BakeoffSamplesSink } from "./engine";
 import type { KeepOutputsMode } from "./samples";
 import type { BakeoffArtifact } from "./types";
+import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
+import type { QualityGate } from "../types/quality";
 /** The narrow store surface the selection reads — kept minimal so this module
  * depends on four reads, not three whole stores. */
 export interface EvalCaseSelectionDeps {
@@ -92,5 +94,8 @@ export declare function runBakeoffFromEvalCases(deps: RunDeps, stores: EvalCaseS
     readonly variants: readonly string[];
     readonly envelopeUsdPerVariant: number;
     readonly keepOutputs?: KeepOutputsMode;
+    readonly objectiveMode?: ObjectiveMode;
+    readonly resolveReferenceAuto?: (route: RouteBundle) => CandidateSpec | null;
+    readonly qualityGate?: QualityGate;
 }, samplesSink?: BakeoffSamplesSink): Promise<EvalCaseBakeoffResult>;
 //# sourceMappingURL=eval-cases.d.ts.map

package/dist/bakeoff/eval-cases.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAgCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;CACjE;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;CACxC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CA6BhC"}
+{"version":3,"file":"eval-cases.d.ts","sourceRoot":"","sources":["../../src/bakeoff/eval-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD;oDACoD;AACpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IAC9D,QAAQ,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;IAClG,QAAQ,CAAC,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IAC/E,QAAQ,CAAC,uBAAuB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,UAAU,GAAG,IAAI,CAAC;CAC9E;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,SAAS,cAAc,EAAE,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,CAAC;CAC/C;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAgCnG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAclF;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;uFACmF;IACnF,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAC3C;AAED;;;;;;GAMG;AACH;;;0DAG0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,qBAAqB,GAAG,iBAAiB,CAAC,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,oBAAoB,CAAC,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;CACjE;AAED,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,cAAc,EACtB,GAAG,EAAE;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC,EACD,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,qBAAqB,CAAC,CAgChC"}

package/dist/bakeoff/eval-cases.js

@@ -100,6 +100,9 @@ async function runBakeoffFromEvalCases(deps, stores, req, samplesSink) {
         replayLastN: selection.rehydratable.length, // replay every rehydratable case
         envelopeUsdPerVariant: req.envelopeUsdPerVariant,
         ...(req.keepOutputs !== undefined ? { keepOutputs: req.keepOutputs } : {}),
+        ...(req.objectiveMode !== undefined ? { objectiveMode: req.objectiveMode } : {}),
+        ...(req.resolveReferenceAuto !== undefined ? { resolveReferenceAuto: req.resolveReferenceAuto } : {}),
+        ...(req.qualityGate !== undefined ? { qualityGate: req.qualityGate } : {}),
     }, samplesSink);
     return { selection, artifact };
 }

package/dist/bakeoff/judge.d.ts

@@ -0,0 +1,39 @@
+/**
+ * SDK-side bake-off judge (RB-1, near-optimal spec §4.3, tier 3). When a sample
+ * has NO eval-case expected value and NO declared value check, an opt-in judge
+ * (`bakeoff.judge: { model, max_samples }`) scores the arm's output on value.
+ *
+ * How it stays honest and bounded, reusing the real run loop (no new dispatch
+ * path, no run.ts edit):
+ *   - The judge is a normal `run()` against a synthetic single-candidate route
+ *     whose only candidate is the judge model — so the call is METERED exactly
+ *     like any inference (a telemetry attempt row with real cost), tagged
+ *     `rung: "judge"`, and no capture is written (captures severed).
+ *   - CAPPED at `max_samples` PER ARM: past the cap `score()` returns null and
+ *     the sample falls back to conformance (counted by omission).
+ *   - The judge model MUST be a declared candidate (#223 F3) — enforced at load
+ *     (objective-loader.ts), so it is a real, reachable model here.
+ *   - Fail-closed: an unparseable/short-of-schema judge answer throws in the run
+ *     loop; the metered row still stands and the sample's score is null.
+ */
+import type { RunDeps } from "../run/run";
+import type { CandidateSpec, RouteBundle } from "../types/route";
+/** Build the synthetic judge route for a base route + judge model. The judge
+ * model is declared WITHOUT its trial flag so it serves live on this route. */
+export declare function buildJudgeRoute(baseRoute: RouteBundle, model: CandidateSpec): RouteBundle;
+/** A per-arm judge scorer: `score(output, taskInput)` returns a value in [0,1],
+ * or null when the cap is hit or the judge could not produce a valid answer. */
+export interface JudgeScorer {
+    score(output: unknown, taskInput: unknown): Promise<number | null>;
+    used(): number;
+}
+/**
+ * Build a judge scorer over the run loop. `deps` are the bake-off's deps (the
+ * judge model's adapter must be present, as for any candidate). Each arm gets a
+ * FRESH scorer so the `max_samples` cap is per arm (spec: max_samples per arm).
+ */
+export declare function makeJudgeScorer(deps: RunDeps, baseRoute: RouteBundle, judge: {
+    readonly model: CandidateSpec;
+    readonly maxSamples: number;
+}): JudgeScorer;
+//# sourceMappingURL=judge.d.ts.map

package/dist/bakeoff/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/bakeoff/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAejE;+EAC+E;AAC/E,wBAAgB,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,GAAG,WAAW,CAezF;AAED;gFACgF;AAChF,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACnE,IAAI,IAAI,MAAM,CAAC;CAChB;AAMD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,WAAW,EACtB,KAAK,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACpE,WAAW,CAqCb"}

package/dist/bakeoff/judge.js

@@ -0,0 +1,102 @@
+"use strict";
+/**
+ * SDK-side bake-off judge (RB-1, near-optimal spec §4.3, tier 3). When a sample
+ * has NO eval-case expected value and NO declared value check, an opt-in judge
+ * (`bakeoff.judge: { model, max_samples }`) scores the arm's output on value.
+ *
+ * How it stays honest and bounded, reusing the real run loop (no new dispatch
+ * path, no run.ts edit):
+ *   - The judge is a normal `run()` against a synthetic single-candidate route
+ *     whose only candidate is the judge model — so the call is METERED exactly
+ *     like any inference (a telemetry attempt row with real cost), tagged
+ *     `rung: "judge"`, and no capture is written (captures severed).
+ *   - CAPPED at `max_samples` PER ARM: past the cap `score()` returns null and
+ *     the sample falls back to conformance (counted by omission).
+ *   - The judge model MUST be a declared candidate (#223 F3) — enforced at load
+ *     (objective-loader.ts), so it is a real, reachable model here.
+ *   - Fail-closed: an unparseable/short-of-schema judge answer throws in the run
+ *     loop; the metered row still stands and the sample's score is null.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildJudgeRoute = buildJudgeRoute;
+exports.makeJudgeScorer = makeJudgeScorer;
+const node_crypto_1 = require("node:crypto");
+const run_1 = require("../run/run");
+const failures_1 = require("../types/failures");
+const JUDGE_SCHEMA = {
+    type: "object",
+    properties: { score: { type: "number" }, reason: { type: "string" } },
+    required: ["score"],
+    additionalProperties: true,
+};
+const JUDGE_RUBRIC = "You are grading ONE candidate output for a task, for VALUE ACCURACY — how correct and " +
+    "complete the answer is, not merely whether it is well-formed. Score from 0 (wrong/empty) " +
+    "to 1 (fully correct).\n\nTASK INPUT:\n{{task_input}}\n\nCANDIDATE OUTPUT:\n{{candidate_output}}\n\n" +
+    'Respond with ONLY compact JSON: {"score": <number 0..1>, "reason": "<one sentence>"}.';
+/** Build the synthetic judge route for a base route + judge model. The judge
+ * model is declared WITHOUT its trial flag so it serves live on this route. */
+function buildJudgeRoute(baseRoute, model) {
+    return Object.freeze({
+        route: `__bakeoff_judge__:${baseRoute.route}`,
+        version: baseRoute.version,
+        outputSchema: Object.freeze({ ...JUDGE_SCHEMA }),
+        candidates: Object.freeze([{ provider: model.provider, model: model.model }]),
+        require: Object.freeze([]),
+        prefer: Object.freeze([]),
+        systemTemplate: JUDGE_RUBRIC,
+        variables: Object.freeze(["candidate_output", "task_input"]),
+        policy: Object.freeze({ retries: Object.freeze({}), timeoutMs: baseRoute.policy.timeoutMs, tier: "standard", json: "json_mode" }),
+        variants: Object.freeze([]),
+        capture: false,
+        fingerprintCell: null,
+    });
+}
+function clamp01(n) {
+    return typeof n === "number" && Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : null;
+}
+/**
+ * Build a judge scorer over the run loop. `deps` are the bake-off's deps (the
+ * judge model's adapter must be present, as for any candidate). Each arm gets a
+ * FRESH scorer so the `max_samples` cap is per arm (spec: max_samples per arm).
+ */
+function makeJudgeScorer(deps, baseRoute, judge) {
+    const judgeRoute = buildJudgeRoute(baseRoute, judge.model);
+    const routes = {
+        get: (r) => (r === judgeRoute.route ? judgeRoute : deps.routes.get(r)),
+        list: () => Object.freeze([...deps.routes.list(), judgeRoute]),
+    };
+    // Judge dispatches are real, billed calls — bounded by their own envelope and
+    // never written to captures (no replay-of-replay). live mode: the judge model
+    // is a plain declared candidate on the synthetic route.
+    const judgeDeps = { ...deps, routes, captures: null, mode: "live" };
+    const envelope = `bakeoff-judge:${(0, node_crypto_1.randomUUID)()}`;
+    let used = 0;
+    return {
+        used: () => used,
+        async score(output, taskInput) {
+            if (used >= judge.maxSamples)
+                return null;
+            used += 1;
+            try {
+                const result = await (0, run_1.run)(judgeDeps, judgeRoute.route, {
+                    input: {
+                        candidate_output: typeof output === "string" ? output : JSON.stringify(output ?? null),
+                        task_input: typeof taskInput === "string" ? taskInput : JSON.stringify(taskInput ?? null),
+                    },
+                    // `bakeoff_judge` marks the row a judge inference (the typed `rung`
+                    // column is recorder-owned, so a distinct tag key is used); `lane:
+                    // trial` keeps judge spend off the route's live rollups/odometer.
+                    tags: { bakeoff_judge: "1", lane: "trial" },
+                    budget: { envelope },
+                });
+                return clamp01(result.output?.score);
+            }
+            catch (err) {
+                if (!(err instanceof failures_1.RigFailureError))
+                    throw err;
+                // The metered attempt row(s) still stand; this sample gets no judge score.
+                return null;
+            }
+        },
+    };
+}

package/dist/bakeoff/quality.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Per-sample value-accuracy resolution for a bake-off (RB-1, near-optimal spec
+ * §4.3). DETERMINISTIC tiers only — the judge tier lives in judge.ts because it
+ * dispatches a metered inference. Precedence:
+ *
+ *   1. eval-case expected value (a corrected/golden artifact) ⇒ exact/normalized
+ *      match for a primitive, or schema-aware field accuracy in [0,1] for an
+ *      object/array.
+ *   2. declared `value_checks` (JSON-pointer assertions) ⇒ fraction passing.
+ *   3. null (the caller falls back to the judge, then to conformance).
+ *
+ * Pure; unit-tested directly. F5 (deterministic before judge) is exactly this
+ * ordering.
+ */
+import type { ValueCheck } from "../types/route";
+/**
+ * Schema-aware field accuracy in [0,1]: the fraction of the EXPECTED value's leaf
+ * fields that the actual output matches. Recurses into objects/arrays keyed by
+ * the expected shape (extra keys in `actual` are ignored — the golden defines
+ * what "correct" covers). A scalar expected ⇒ 1 or 0. An empty object/array
+ * expected ⇒ 1 (nothing to be wrong about).
+ */
+export declare function fieldAccuracy(expected: unknown, actual: unknown): number;
+/** Resolve an RFC 6901 JSON pointer against a value. Returns `undefined` when any
+ * segment is missing (a missing field fails its value check). */
+export declare function resolveJsonPointer(value: unknown, pointer: string): unknown;
+/** True when `output` satisfies one value check (RFC 6901 pointer + literal|regex). */
+export declare function checkPasses(output: unknown, check: ValueCheck): boolean;
+export interface DeterministicQualityInput {
+    readonly output: unknown;
+    readonly valueChecks?: readonly ValueCheck[];
+    /** The corrected/golden expected output for this capture, when a bake-off runs
+     * from eval cases with a corrected artifact. */
+    readonly expected?: unknown;
+    /** Set when `expected` is meaningfully present (distinguishes a golden `null`
+     * expected value from "no eval case"). */
+    readonly hasExpected?: boolean;
+}
+/**
+ * Deterministic value accuracy for one sample, or null when neither an eval-case
+ * expected value nor a value check applies (the caller then tries the judge).
+ */
+export declare function resolveDeterministicQuality(input: DeterministicQualityInput): number | null;
+//# sourceMappingURL=quality.d.ts.map

package/dist/bakeoff/quality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAiBjD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAgBxE;AAED;iEACiE;AACjE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmB3E;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAWvE;AAYD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;oDACgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;8CAC0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,GAAG,IAAI,CAU3F"}