modelrig 0.4.0 → 0.6.0

package/dist/bakeoff/samples.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"samples.d.ts","sourceRoot":"","sources":["../../src/bakeoff/samples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEnD,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,QAAa,CAAC;AAE3C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,EAAE,CAKV;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOjE;AAED;+DAC+D;AAC/D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,MAAM,CAEhE;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,UAAU,EAAE,EAC3B,SAAS,EAAE,MAAM,GAChB,aAAa,EAAE,CAoBjB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,0BAA0B,GAAG,oBAAoB,CAoEnG;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC,GAAG,kBAAkB,CAQrB"}
+{"version":3,"file":"samples.d.ts","sourceRoot":"","sources":["../../src/bakeoff/samples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAA2D,KAAK,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC7G,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,QAAa,CAAC;AAE3C,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,KAAK,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;qFACiF;IACjF,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,MAAM,EAAE,CAKV;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAOjE;AAED;+DAC+D;AAC/D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,MAAM,CAEhE;AA+BD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,UAAU,EAAE,EAC3B,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,SAAiB,GACtB,aAAa,EAAE,CAyBjB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,2EAA2E;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,0BAA0B,GAAG,oBAAoB,CAwEnG;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAChD;4EACwE;IACxE,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS,CAAC;CACnC,GAAG,kBAAkB,CASrB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,EACrC,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,SAAS,GACvB,SAAS,CAEX"}

package/dist/bakeoff/samples.js

@@ -9,6 +9,15 @@
  * key; read server-side only; deny-anon RLS. NEVER the anon staging posture.
  *
  * Transport is raw PostgREST over fetch (same pattern as the exporter).
+ *
+ * ── SCRUB BEFORE PERSIST (S7, screening B0-a) ───────────────────────────────
+ * `bakeoff_samples` is the largest server-side customer-content path, so the
+ * task's scrub setting — the SAME `ScrubMode` the content-custody + capture-write
+ * paths honor — runs on `inputText` AND `outputText` BEFORE any row is built or
+ * POSTed (mirrors src/run/capture-write.ts). off/detect ⇒ verbatim (detect still
+ * COUNTS findings onto the row's `arm_meta.scrub`, never values); redact ⇒ masked
+ * copy; block ⇒ a sample with any finding becomes metadata-only (input/output
+ * dropped) and counted. Findings are `{type,count}` only — never a matched value.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OUTPUT_MAX_BYTES = void 0;
@@ -18,6 +27,8 @@ exports.retentionExpiry = retentionExpiry;
 exports.buildBakeoffSamples = buildBakeoffSamples;
 exports.createBakeoffSamplesClient = createBakeoffSamplesClient;
 exports.buildDirectSamplesSink = buildDirectSamplesSink;
+exports.resolveSampleScrubMode = resolveSampleScrubMode;
+const scrub_1 = require("../artifacts/scrub");
 /** 256KB per output, per the architect's caps; TOAST compresses at rest. */
 exports.OUTPUT_MAX_BYTES = 256 * 1024;
 /** Default persists incumbent + winner only; "all" persists every arm. */
@@ -44,18 +55,53 @@ function truncateOutput(text) {
 function retentionExpiry(nowMs, days = 90) {
     return new Date(nowMs + days * 24 * 3600 * 1000).toISOString();
 }
-function buildBakeoffSamples(bakeoffId, route, arms, expiresAt) {
+/** Sum two finding lists by type (counts only), sorted by type — the scrub
+ * invariant: a finding is `{type, count}`, never a value. */
+function mergeFindings(a, b) {
+    const counts = new Map();
+    for (const f of [...a, ...b])
+        counts.set(f.type, (counts.get(f.type) ?? 0) + f.count);
+    return [...counts.entries()]
+        .map(([type, count]) => ({ type, count }))
+        .sort((x, y) => (x.type < y.type ? -1 : x.type > y.type ? 1 : 0));
+}
+/** One sample's input + output run through scrub `mode` (S7). off ⇒ verbatim,
+ * no findings; detect ⇒ verbatim, findings counted; redact ⇒ masked; block ⇒ a
+ * sample with any finding is dropped to metadata-only (both texts null). Pure. */
+function scrubSample(inputText, outputText, mode) {
+    if (mode === "off")
+        return { inputText, outputText, findings: [] };
+    const inScrub = (0, scrub_1.scrubContent)(inputText, mode);
+    const outScrub = outputText === null ? null : (0, scrub_1.scrubContent)(outputText, mode);
+    const findings = mergeFindings(inScrub.findings, outScrub?.findings ?? []);
+    // block: ANY finding on either side ⇒ the whole sample is metadata-only.
+    if (mode === "block" && (inScrub.blocked || (outScrub?.blocked ?? false))) {
+        return { inputText: null, outputText: null, findings };
+    }
+    return { inputText: inScrub.text, outputText: outScrub === null ? null : outScrub.text, findings };
+}
+/**
+ * Flatten arms×samples into rows, scrubbing input + output with the task's
+ * `mode` (S7, default `off` ⇒ verbatim, byte-identical). Scrub findings ride the
+ * row's `arm_meta.scrub` as counts-by-type (never values); a `block`-mode sample
+ * with any finding is stored metadata-only (input/output dropped).
+ */
+function buildBakeoffSamples(bakeoffId, route, arms, expiresAt, mode = "off") {
     const rows = [];
     for (const arm of arms) {
         for (const s of arm.samples) {
+            const scrubbed = scrubSample(s.inputText, s.outputText, mode);
+            const armMeta = scrubbed.findings.length > 0
+                ? { ...(arm.armMeta ?? {}), scrub: { mode, findings: scrubbed.findings } }
+                : arm.armMeta;
             rows.push({
                 bakeoffId,
                 route,
                 arm: arm.arm,
-                armMeta: arm.armMeta,
+                armMeta,
                 inputRef: s.inputRef,
-                inputText: s.inputText,
-                outputText: truncateOutput(s.outputText),
+                inputText: scrubbed.inputText,
+                outputText: truncateOutput(scrubbed.outputText),
                 conform: s.conform,
                 costUsd: s.costUsd,
                 latencyMs: s.latencyMs,
@@ -104,7 +150,11 @@ function createBakeoffSamplesClient(config) {
                         arm: s.arm,
                         arm_meta: s.armMeta,
                         input_ref: s.inputRef,
-                        input_text: s.inputText,
+                        // `input_text` is NOT NULL (00008); a block-mode drop persists ""
+                        // (metadata-only — zero content), never SQL null. `output_text` is
+                        // nullable and rides as-is. (Spec §6/B0-a asks for null on both; the
+                        // live column disallows it and this mission is ZERO-DDL — see PR body.)
+                        input_text: s.inputText ?? "",
                         output_text: s.outputText,
                         conform: s.conform,
                         cost_usd: s.costUsd,
@@ -139,10 +189,21 @@ function createBakeoffSamplesClient(config) {
  */
 function buildDirectSamplesSink(cfg) {
     return async ({ bakeoffId, route, arms }) => {
-        const rows = buildBakeoffSamples(bakeoffId, route, arms, retentionExpiry(Date.now()));
+        const mode = resolveSampleScrubMode(cfg.resolver ?? null, route, cfg.defaultScrub ?? "off");
+        const rows = buildBakeoffSamples(bakeoffId, route, arms, retentionExpiry(Date.now()), mode);
         await createBakeoffSamplesClient({
             supabaseUrl: cfg.supabaseUrl,
             serviceRoleKey: cfg.serviceRoleKey,
         }).insert(rows);
     };
 }
+/**
+ * The effective scrub mode for a bake-off sample write — resolved EXACTLY as
+ * capture-write.ts does: a configured control plane WINS (`resolver.get(route)`),
+ * else the rig's config default. A local-only / direct-mode rig has no resolver,
+ * so it falls to `configDefault`. Pure and synchronous (the resolver read is
+ * cache-only, never a network hop).
+ */
+function resolveSampleScrubMode(resolver, route, configDefault) {
+    return resolver !== null ? resolver.get(route).scrub : configDefault;
+}

package/dist/bakeoff/stats.d.ts

@@ -7,7 +7,23 @@
  * (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
  */
 import type { ReplaySampleResult } from "./replay";
-import type { VariantMetrics } from "./types";
+import type { RegretStats, VariantMetrics } from "./types";
+/** Seeded so a bake-off's regret CI is reproducible given the same sample set
+ * (spec §4.2 — "seeded for determinism"). */
+export declare const REGRET_BOOTSTRAP_REPS = 2000;
+export declare const REGRET_BOOTSTRAP_SEED = 20260823;
+/** Per-sample quality (spec §2): value accuracy when a golden/deterministic
+ * check/judge scored it, else 1/0 conformance. Always defined — so paired regret
+ * is always computable. */
+export declare function sampleQuality(s: ReplaySampleResult): number;
 export declare function wilson95(successes: number, n: number): readonly [number, number];
 export declare function aggregateVariantMetrics(variant: string, samples: readonly ReplaySampleResult[]): VariantMetrics;
+/**
+ * Paired regret of an arm vs the reference (spec §4.2). Pairs samples by
+ * captureId, takes per-sample quality differences `q(reference) − q(arm)`, and
+ * reports the mean, a seeded paired-bootstrap 95% CI, and n_paired. Samples
+ * present on only one side are excluded (and, by omission, counted: n_paired vs
+ * each arm's own sample count). Deterministic given the sample set + seed.
+ */
+export declare function pairedRegret(armSamples: readonly ReplaySampleResult[], refSamples: readonly ReplaySampleResult[], seed?: number, reps?: number): RegretStats;
 //# sourceMappingURL=stats.d.ts.map

package/dist/bakeoff/stats.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/bakeoff/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAShF;AAMD,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,kBAAkB,EAAE,GACrC,cAAc,CAuBhB"}
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/bakeoff/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAiB,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE1E;6CAC6C;AAC7C,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAC1C,eAAO,MAAM,qBAAqB,WAAW,CAAC;AAE9C;;2BAE2B;AAC3B,wBAAgB,aAAa,CAAC,CAAC,EAAE,kBAAkB,GAAG,MAAM,CAE3D;AAED,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAShF;AAMD,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,SAAS,kBAAkB,EAAE,GACrC,cAAc,CA0ChB;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,UAAU,EAAE,SAAS,kBAAkB,EAAE,EACzC,IAAI,GAAE,MAA8B,EACpC,IAAI,GAAE,MAA8B,GACnC,WAAW,CAwBb"}

package/dist/bakeoff/stats.js

@@ -8,8 +8,22 @@
  * (base + retries + repair + search, F1-corrected) ÷ conformant count × 1000.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.REGRET_BOOTSTRAP_SEED = exports.REGRET_BOOTSTRAP_REPS = void 0;
+exports.sampleQuality = sampleQuality;
 exports.wilson95 = wilson95;
 exports.aggregateVariantMetrics = aggregateVariantMetrics;
+exports.pairedRegret = pairedRegret;
+const bootstrap_1 = require("../rigindex/bootstrap");
+/** Seeded so a bake-off's regret CI is reproducible given the same sample set
+ * (spec §4.2 — "seeded for determinism"). */
+exports.REGRET_BOOTSTRAP_REPS = 2000;
+exports.REGRET_BOOTSTRAP_SEED = 20260823;
+/** Per-sample quality (spec §2): value accuracy when a golden/deterministic
+ * check/judge scored it, else 1/0 conformance. Always defined — so paired regret
+ * is always computable. */
+function sampleQuality(s) {
+    return s.valueAccuracy ?? (s.conform ? 1 : 0);
+}
 function wilson95(successes, n) {
     if (n === 0)
         return [0, 1];
@@ -32,6 +46,16 @@ function aggregateVariantMetrics(variant, samples) {
         .map((s) => s.valueAccuracy)
         .filter((a) => a !== null);
     const totalCostUsd = samples.reduce((a, s) => a + s.costUsd, 0);
+    // RB-1: the metric is value_accuracy iff at least one sample was scored on
+    // value (eval case / value check / judge / Q1 gate); otherwise the arm's
+    // quality is 1/0 conformance. qualityMean averages per-sample quality either way.
+    const qualityMetric = accuracies.length > 0 ? "value_accuracy" : "conformance";
+    // Q1 quality pass rate (spec §4.2): a variant is "gated" once ANY sample
+    // carried a boolean verdict; a failed (unscored) sample counts against the
+    // pass rate as a non-pass, exactly as a conformance miss counts against
+    // conformance. null when no gate ran, so computeProposal can tell.
+    const gated = samples.some((s) => s.qualityPassed !== null && s.qualityPassed !== undefined);
+    const passing = samples.filter((s) => s.qualityPassed === true).length;
     return {
         variant,
         samples: n,
@@ -43,6 +67,49 @@ function aggregateVariantMetrics(variant, samples) {
         meanLatencyMs: mean(samples.map((s) => s.latencyMs)),
         meanCostUsd: n === 0 ? 0 : totalCostUsd / n,
         effectiveUsdPer1kConformant: conformant.length === 0 ? Number.POSITIVE_INFINITY : (totalCostUsd / conformant.length) * 1000,
+        qualityPassCi95: gated ? wilson95(passing, n) : null,
+        effectiveUsdPer1kPassing: !gated
+            ? null
+            : passing === 0
+                ? Number.POSITIVE_INFINITY
+                : (totalCostUsd / passing) * 1000,
         pricingMissing: samples.some((s) => s.pricingMissing),
+        qualityMean: n === 0 ? null : mean(samples.map(sampleQuality)),
+        qualityMetric,
     };
 }
+/**
+ * Paired regret of an arm vs the reference (spec §4.2). Pairs samples by
+ * captureId, takes per-sample quality differences `q(reference) − q(arm)`, and
+ * reports the mean, a seeded paired-bootstrap 95% CI, and n_paired. Samples
+ * present on only one side are excluded (and, by omission, counted: n_paired vs
+ * each arm's own sample count). Deterministic given the sample set + seed.
+ */
+function pairedRegret(armSamples, refSamples, seed = exports.REGRET_BOOTSTRAP_SEED, reps = exports.REGRET_BOOTSTRAP_REPS) {
+    const refByCapture = new Map(refSamples.map((s) => [s.captureId, s]));
+    const diffs = [];
+    for (const arm of armSamples) {
+        const ref = refByCapture.get(arm.captureId);
+        if (ref === undefined)
+            continue;
+        diffs.push(sampleQuality(ref) - sampleQuality(arm));
+    }
+    const nPaired = diffs.length;
+    if (nPaired === 0)
+        return { regretMean: 0, regretCi95: [0, 0], nPaired: 0 };
+    const regretMean = mean(diffs);
+    // Paired bootstrap: resample the DIFFERENCES with replacement (the pairing is
+    // already fixed), take each resample's mean, then the 2.5/97.5 percentiles.
+    const rng = (0, bootstrap_1.mulberry32)(seed);
+    const means = [];
+    for (let b = 0; b < reps; b += 1) {
+        let sum = 0;
+        for (let i = 0; i < nPaired; i += 1)
+            sum += diffs[Math.floor(rng() * nPaired)];
+        means.push(sum / nPaired);
+    }
+    means.sort((a, b) => a - b);
+    const lo = means[Math.floor(0.025 * reps)];
+    const hi = means[Math.min(reps - 1, Math.floor(0.975 * reps))];
+    return { regretMean, regretCi95: [lo, hi], nPaired };
+}

package/dist/bakeoff/types.d.ts

@@ -1,8 +1,22 @@
 /**
- * Conformance bake-off contracts — Phase 3 spec WS5 (closed core).
- * The headline metric: effective $ per 1K conformant outputs, with repair
- * and search costs included (F1-corrected accounting).
+ * Conformance bake-off contracts — Phase 3 spec WS5 (closed core), extended for
+ * RB-1 regret-bounded objectives (near-optimal spec §4). The headline metric is
+ * effective $ per 1K conformant outputs (F1-corrected); RB-1 adds per-sample
+ * value accuracy, paired regret vs a reference arm, and an objective-aware gate.
  */
+import type { ObjectiveMode } from "../types/route";
+/** Which quality metric an arm's numbers are on (spec §2): value accuracy when a
+ * golden / value check / judge scored it, else schema conformance. */
+export type QualityMetric = "value_accuracy" | "conformance";
+/** An arm's quality gap to the reference on the SAME paired samples (spec §2).
+ * 0 = matches the reference; negative = beats it. CI is a seeded paired
+ * bootstrap of the per-sample differences. */
+export interface RegretStats {
+    readonly regretMean: number;
+    readonly regretCi95: readonly [number, number];
+    /** Samples present on BOTH the arm and the reference; the CI's n. */
+    readonly nPaired: number;
+}
 export interface VariantMetrics {
     readonly variant: string;
     readonly samples: number;
@@ -16,9 +30,58 @@ export interface VariantMetrics {
     readonly meanCostUsd: number;
     /** (base + repair + search costs) / conformant outputs × 1000 — the number. */
     readonly effectiveUsdPer1kConformant: number;
+    /** Q1 quality gate (spec §4.2): Wilson 95% CI on the quality PASS rate
+     * (samples the gate accepted / all samples). null when no gate ran on this
+     * variant — computeProposal's third clause only fires when both arms have one. */
+    readonly qualityPassCi95: readonly [number, number] | null;
+    /** Q1: (base + repair + search costs) / quality-PASSING outputs × 1000 —
+     * reported BESIDE effectiveUsdPer1kConformant (never redefining it; flag F6).
+     * ∞ when a gate ran but nothing passed; null when no gate ran. */
+    readonly effectiveUsdPer1kPassing: number | null;
     /** Addendum A1: any sample served by a model with no pricing entry —
      * cost columns under-count; this variant is never ranked or proposed. */
     readonly pricingMissing: boolean;
+    /** RB-1: mean per-sample quality (value accuracy where present, else 1/0
+     * conformance). The number regret is computed from. Null when no samples. */
+    readonly qualityMean: number | null;
+    /** RB-1: which metric qualityMean is on, named on the report. */
+    readonly qualityMetric: QualityMetric;
+    /** RB-1: this arm is the reference (ceiling) arm. */
+    readonly isReference?: boolean;
+    /** RB-1 (architect B-3): the objective gate proposed THIS arm — stamped on the
+     * winning variant so the mirror/console read the artifact's decision rather
+     * than recomputing a (mode-blind) winner client-side. */
+    readonly proposed?: boolean;
+    /** RB-1: this arm's regret vs the reference; null on the reference arm and
+     * when there is no reference. */
+    readonly regret?: RegretStats | null;
+    /** RB-1: the reference arm hit its envelope — regret is on the paired subset. */
+    readonly referencePartial?: boolean;
+}
+/** RB-1: the resolved objective recorded on the artifact (evidence + report). */
+export interface BakeoffObjective {
+    readonly mode: ObjectiveMode;
+    readonly maxRegret: number | null;
+    readonly priceCeiling: {
+        readonly usdPer1k?: number;
+        readonly usdPerRun?: number;
+    } | null;
+    readonly lambda: number | null;
+}
+/** RB-1: the resolved reference arm recorded on the artifact. */
+export interface BakeoffReference {
+    /** The arm name carrying the reference metrics. */
+    readonly variant: string;
+    /** "provider/model". */
+    readonly model: string;
+    /** Resolved via `reference: auto` (vs an explicit declaration). */
+    readonly auto: boolean;
+    /** The `auto` rule + source, when auto. */
+    readonly rule: string | null;
+    /** Out-of-candidate reference (reference_allow_external). */
+    readonly external: boolean;
+    /** The reference replay hit its envelope (partial paired set). */
+    readonly partial: boolean;
 }
 export interface BakeoffArtifact {
     readonly id: string;
@@ -27,11 +90,19 @@ export interface BakeoffArtifact {
     readonly ranAt: string;
     readonly replayN: number;
     readonly variants: readonly VariantMetrics[];
-    /** Winner by effectiveUsdPer1kConformant among variants meeting the gates;
-     * null when no variant beats the incumbent's gates. */
+    /** Winner among variants meeting the objective's gates; null when nobody beats
+     * the incumbent. `vsIncumbentSavingsPct` is the effective-cost saving (kept for
+     * every mode); `regret` is the proposed arm's regret vs the reference. */
     readonly proposal: {
         readonly variant: string;
         readonly vsIncumbentSavingsPct: number;
+        readonly regret?: RegretStats | null;
     } | null;
+    /** RB-1: the objective the gate ran under (default quality-first). */
+    readonly objective?: BakeoffObjective;
+    /** RB-1: the reference arm, or null when no reference was declared/resolved. */
+    readonly reference?: BakeoffReference | null;
+    /** RB-1: the metric the bake-off's quality numbers are on. */
+    readonly qualityMetric?: QualityMetric;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/bakeoff/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;2DACuD;IACvD,QAAQ,CAAC,QAAQ,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAChG"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/bakeoff/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;sEACsE;AACtE,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAE7D;;8CAE8C;AAC9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,wFAAwF;IACxF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C;;sFAEkF;IAClF,QAAQ,CAAC,eAAe,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC3D;;sEAEkE;IAClE,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD;6EACyE;IACzE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;iFAC6E;IAC7E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,iEAAiE;IACjE,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;6DAEyD;IACzD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;qCACiC;IACjC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACrC,iFAAiF;IACjF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CACrC;AAED,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE;QAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC1F,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C;;8EAE0E;IAC1E,QAAQ,CAAC,QAAQ,EACb;QACE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;QACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;KACtC,GACD,IAAI,CAAC;IACT,sEAAsE;IACtE,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC7C,8DAA8D;IAC9D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;CACxC"}

package/dist/bakeoff/types.js

@@ -1,8 +1,9 @@
 "use strict";
 /**
- * Conformance bake-off contracts — Phase 3 spec WS5 (closed core).
- * The headline metric: effective $ per 1K conformant outputs, with repair
- * and search costs included (F1-corrected accounting).
+ * Conformance bake-off contracts — Phase 3 spec WS5 (closed core), extended for
+ * RB-1 regret-bounded objectives (near-optimal spec §4). The headline metric is
+ * effective $ per 1K conformant outputs (F1-corrected); RB-1 adds per-sample
+ * value accuracy, paired regret vs a reference arm, and an objective-aware gate.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 // Implemented: engine.ts assembles this from replay results; stats.ts owns

package/dist/cli-query.d.ts

@@ -0,0 +1,14 @@
+/**
+ * `modelrig query` — WI-1b. Runs a telemetry-grammar query against the analytics
+ * surface of the configured server (`POST /v1/analytics/query`), authed with a
+ * `rig_sk_` key. Goes through the API boundary (T12: the CLI never holds a DB
+ * client); the server scopes to the key's org and folds the grammar.
+ *
+ *   modelrig query "SELECT subject, sum(cost) FROM inferences GROUP BY subject"
+ *   modelrig query --saved cost-per-subject
+ *
+ * Kept in its own file so cli.ts stays under the LOC cap (like validate-cli.ts).
+ */
+import type { Flags } from "./cli";
+export declare function cmdQuery(flags: Flags): Promise<number>;
+//# sourceMappingURL=cli-query.d.ts.map

package/dist/cli-query.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-query.d.ts","sourceRoot":"","sources":["../src/cli-query.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAMnC,wBAAsB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAyC5D"}

package/dist/cli-query.js

@@ -0,0 +1,59 @@
+"use strict";
+/**
+ * `modelrig query` — WI-1b. Runs a telemetry-grammar query against the analytics
+ * surface of the configured server (`POST /v1/analytics/query`), authed with a
+ * `rig_sk_` key. Goes through the API boundary (T12: the CLI never holds a DB
+ * client); the server scopes to the key's org and folds the grammar.
+ *
+ *   modelrig query "SELECT subject, sum(cost) FROM inferences GROUP BY subject"
+ *   modelrig query --saved cost-per-subject
+ *
+ * Kept in its own file so cli.ts stays under the LOC cap (like validate-cli.ts).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cmdQuery = cmdQuery;
+const config_1 = require("./config");
+const grammar_1 = require("./query/grammar");
+function savedCatalogue() {
+    return grammar_1.SAVED_QUERIES.map((q) => `  ${q.id.padEnd(20)} ${q.title}`).join("\n");
+}
+async function cmdQuery(flags) {
+    const saved = flags.named.get("saved");
+    const grammar = flags.positional[0];
+    // Config is the only env reader (package-invariants). The analytics endpoint
+    // shares the api-mode ingest server, so base = that URL's origin, key = the api key.
+    const exp = (0, config_1.loadConfigFromEnv)().export;
+    const api = exp !== undefined && exp.mode === "api" ? exp : undefined;
+    const base = api !== undefined ? new URL(api.ingestUrl).origin : "";
+    const key = api?.apiKey ?? "";
+    if ((saved === undefined && (grammar === undefined || grammar === "")) || base === "" || key === "") {
+        console.error([
+            'usage: modelrig query "<grammar>"   |   modelrig query --saved <id>',
+            "",
+            "Runs a telemetry query against your org's analytics surface (metadata only —",
+            "enums, numbers, hashes, identifiers; never content). Needs api-mode telemetry:",
+            "  MODELRIG_API_KEY    a rig_sk_ key with the `query` scope",
+            "  MODELRIG_INGEST_URL optional — the server base; defaults to the hosted API",
+            "",
+            "Saved questions:",
+            savedCatalogue(),
+        ].join("\n"));
+        return 1;
+    }
+    const body = saved !== undefined ? { saved } : { query: grammar };
+    const res = await fetch(`${base}/v1/analytics/query`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+        body: JSON.stringify(body),
+    });
+    const payload = (await res.json().catch(() => ({})));
+    if (!res.ok) {
+        console.error(`query ${res.status}: ${typeof payload.error === "string" ? payload.error : "failed"}`);
+        return 1;
+    }
+    console.log(JSON.stringify(payload, null, 2));
+    if (payload.truncated === true) {
+        console.error("note: result truncated (LIMIT or the 10k-row scan cap) — narrow the window or raise LIMIT");
+    }
+    return 0;
+}

package/dist/cli.d.ts

@@ -2,7 +2,8 @@
 /**
  * modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
  *   modelrig bakeoff --route X --variants a,b,c --replay-last N
- *     [--budget-usd usd] [--json] [--no-mirror]   (--envelope is a kept alias)
+ *     [--budget-usd usd] [--objective quality-first|balanced|cost-first]
+ *     [--json] [--no-mirror]   (--envelope is a kept alias)
  *   modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
  *   modelrig watch run [--events fixtures.json] [--json]
  *

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;GAeG;AAkCH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AA0gBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAqCnE"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAgDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AAqhBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCnE"}

package/dist/cli.js

@@ -3,7 +3,8 @@
 /**
  * modelrig CLI — Phase 3 plan §3 + Phase 4 plan §3:
  *   modelrig bakeoff --route X --variants a,b,c --replay-last N
- *     [--budget-usd usd] [--json] [--no-mirror]   (--envelope is a kept alias)
+ *     [--budget-usd usd] [--objective quality-first|balanced|cost-first]
+ *     [--json] [--no-mirror]   (--envelope is a kept alias)
  *   modelrig bakeoff --from-proposal <id> [--replay-last N] [--json]
  *   modelrig watch run [--events fixtures.json] [--json]
  *
@@ -33,6 +34,17 @@ const status_1 = require("./status");
 const engine_1 = require("./bakeoff/engine");
 const samples_1 = require("./bakeoff/samples");
 const report_1 = require("./bakeoff/report");
+const OBJECTIVE_MODES = new Set(["quality-first", "balanced", "cost-first"]);
+/** Parse & validate `--objective <mode>`; undefined when absent. */
+function objectiveOf(flags) {
+    const raw = flags.named.get("objective");
+    if (raw === undefined)
+        return undefined;
+    if (!OBJECTIVE_MODES.has(raw)) {
+        throw new Error(`--objective must be one of [${[...OBJECTIVE_MODES].join(", ")}]`);
+    }
+    return raw;
+}
 const cli_run_1 = require("./verify/cli-run");
 const loader_1 = require("./routes/loader");
 const rawonly_1 = require("./run/rawonly");
@@ -45,6 +57,7 @@ const state_1 = require("./watch/state");
 const state_store_1 = require("./watch/state-store");
 const swaps_client_1 = require("./savings/swaps-client");
 const swap_execute_1 = require("./savings/swap-execute");
+const cli_query_1 = require("./cli-query");
 const BARE_FLAGS = new Set([
     "json",
     "no-mirror",
@@ -99,7 +112,7 @@ function keepOutputsMode(flags) {
         return "winner";
     return undefined;
 }
-async function runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope) {
+async function runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope, objectiveMode) {
     const keep = keepOutputsMode(flags);
     if (keep !== undefined)
         (0, config_1.directExportOf)(config, "--keep-outputs (bakeoff_samples lives there)");
@@ -111,6 +124,7 @@ async function runAndReportBakeoff(config, flags, route, variants, replayLastN,
             replayLastN,
             ...(envelope !== undefined ? { envelopeUsdPerVariant: envelope } : {}),
             ...(keep !== undefined ? { keepOutputs: keep } : {}),
+            ...(objectiveMode !== undefined ? { objectiveMode } : {}),
         });
         const outDir = (0, node_path_1.join)((0, node_path_1.dirname)(config.telemetryDbPath), "bakeoffs");
         (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
@@ -159,7 +173,27 @@ async function cmdBakeoff(flags) {
         }
         const routes = (0, loader_1.loadRoutes)((0, rawonly_1.requireRoutesDir)(config, "modelrig bakeoff"));
         const bundle = routes.get(proposal.route);
-        const variants = ["default", ...bundle.variants.map((v) => v.name)];
+        // Declare-first (routing-console spec §4.4, R3): a manual_challenger proposal
+        // names ONE trial variant — bake off default vs that variant only. The trial
+        // variant must be DECLARED in the local routes (the human merged the
+        // declare-YAML); until then it isn't loadable, so fail with a teaching error
+        // rather than a bake-off that silently omits the challenger.
+        let variants;
+        const trigger = proposal.trigger;
+        if (trigger.type === "manual_challenger") {
+            const trialVariant = trigger.variant;
+            if (typeof trialVariant !== "string" || trialVariant === "") {
+                throw new Error(`proposal ${proposalId} is a manual_challenger with no trial variant name`);
+            }
+            if (!bundle.variants.some((v) => v.name === trialVariant)) {
+                throw new Error(`route "${proposal.route}" does not declare the trial variant "${trialVariant}" yet — ` +
+                    "merge the declare-YAML from the proposal first (modelrig ... declare-body), then re-run");
+            }
+            variants = ["default", trialVariant];
+        }
+        else {
+            variants = ["default", ...bundle.variants.map((v) => v.name)];
+        }
         const lastRaw = flags.named.get("replay-last");
         const replayLastN = lastRaw !== undefined ? Number(lastRaw) : 25;
         if (!Number.isInteger(replayLastN) || replayLastN < 1) {
@@ -174,26 +208,12 @@ async function cmdBakeoff(flags) {
             const perVariant = reservation.remainingUsd / variants.length;
             console.log(`watch budget ${reservation.key}: $${reservation.remainingUsd.toFixed(2)} remaining → ` +
                 `$${perVariant.toFixed(2)}/variant cap`);
-            const artifact = await runAndReportBakeoff(config, flags, proposal.route, variants, replayLastN, perVariant);
+            const artifact = await runAndReportBakeoff(config, flags, proposal.route, variants, replayLastN, perVariant, objectiveOf(flags));
             const spentUsd = artifact.variants.reduce((a, m) => a + m.samples * m.meanCostUsd, 0);
             budgetStore.accrue(reservation.key, spentUsd);
-            await client.attachEvidence(proposal.id, {
-                bakeoff_id: artifact.id,
-                replay_n: artifact.replayN,
-                ran_at: artifact.ranAt,
-                spent_usd: spentUsd,
-                proposal: artifact.proposal,
-                matrix: artifact.variants.map((m) => ({
-                    variant: m.variant,
-                    samples: m.samples,
-                    conform_rate: m.conformRate,
-                    conform_ci95: m.conformCi95,
-                    effective_usd_per_1k_conformant: Number.isFinite(m.effectiveUsdPer1kConformant)
-                        ? m.effectiveUsdPer1kConformant
-                        : null,
-                    pricing_missing: m.pricingMissing,
-                })),
-            });
+            // Evidence jsonb (winner + RB-1 objective/reference/regret/quality) is
+            // built by the engine so the CLI stays thin; the PR body reads it back.
+            await client.attachEvidence(proposal.id, (0, engine_1.bakeoffEvidence)(artifact, spentUsd));
             console.log(`evidence attached → proposal ${proposal.id} is now evidenced ` +
                 `($${spentUsd.toFixed(2)} accrued to ${reservation.key}). ` +
                 "Approval stays human: modelrig swap execute <proposalId> --approve");
@@ -214,7 +234,7 @@ async function cmdBakeoff(flags) {
         const env = envRaw !== undefined ? Number(envRaw) : undefined;
         if (env !== undefined && (!Number.isFinite(env) || env <= 0))
             throw new Error("--budget-usd must be positive");
-        return (0, report_1.runEvalCaseBakeoffCli)(config, evalTask, vs, env, keepOutputsMode(flags), flags.bare.has("json"));
+        return (0, report_1.runEvalCaseBakeoffCli)(config, evalTask, vs, env, keepOutputsMode(flags), flags.bare.has("json"), objectiveOf(flags));
     }
     const route = flags.named.get("route");
     const variantsRaw = flags.named.get("variants");
@@ -233,7 +253,7 @@ async function cmdBakeoff(flags) {
     if (envelope !== undefined && (!Number.isFinite(envelope) || envelope <= 0)) {
         throw new Error("--budget-usd must be a positive number of dollars (per replay variant)");
     }
-    await runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope);
+    await runAndReportBakeoff(config, flags, route, variants, replayLastN, envelope, objectiveOf(flags));
     return 0;
 }
 const DEFAULT_WATCH_THRESHOLD_PCT = 10;
@@ -522,13 +542,14 @@ const USAGE = [
     "  modelrig validate                    (structurally check rig.yaml + every route bundle — no provider key needed)",
     "  modelrig status                      (print export/artifact/otlp posture)",
     "  modelrig serve                       (needs the modelrig-server package)",
-    "  modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--keep-outputs[=all]] [--json] [--no-mirror]",
+    "  modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--objective quality-first|balanced|cost-first] [--keep-outputs[=all]] [--json] [--no-mirror]",
     "  modelrig bakeoff --from-proposal <id> [--replay-last N] [--keep-outputs[=all]] [--json]",
     "  modelrig watch run [--events fixtures.json] [--json]",
     "  modelrig verify-swap --route <route> --captures <fixture.json> [--budget-usd usd] [--out report.md] [--json]",
     "  modelrig swap execute <proposalId> --approve [--approved-by name]",
     "  modelrig swap revert <swapId>",
     "  modelrig judge <runId>               (E5: score a run with the bounded model-judge; explicit + metered)",
+    '  modelrig query "<grammar>" | query --saved <id>   (WI-1b: telemetry analytics — metadata only, never content)',
     "  modelrig samples purge [--bakeoff <id>]",
 ].join("\n");
 async function main(argv) {
@@ -562,6 +583,8 @@ async function main(argv) {
             return cmdVerifySwap(flags);
         case "judge":
             return cmdJudge(flags);
+        case "query":
+            return (0, cli_query_1.cmdQuery)(flags);
         case "serve":
             return cmdServe();
         default: