modelrig 0.8.1 → 0.10.0

package/dist/bakeoff/groundedness.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Evidence groundedness grader (call-QA F3 / research §4 G6, spec §5). A
+ * DETERMINISTIC bake-off tier that scores a scorecard OUTPUT by whether its
+ * evidence quotes are actually present in the replayed INPUT (the transcript /
+ * source). A scorer that invents justifications — quotes that never appear in
+ * the input — is penalized; this is what stops a model from fabricating
+ * evidence to defend a score.
+ *
+ * Unlike the golden tiers (exact / field-level / llm-judge in quality.ts), this
+ * grades the output against the INPUT, not a corrected golden, so it applies
+ * whether or not an eval-case `expected` value is present.
+ *
+ * The scorecard OUTPUT shape (call-QA F2 emit):
+ *   { criteria: [{ id, score, evidence: [{ quote, turn_id? }], comment }], score, comment }
+ *
+ * Grounding is a normalized SUBSTRING match against the DECODED input values
+ * (inputHaystack): the string leaf values the model was actually shown — never
+ * the JSON envelope's escaping or its keys — so a truthful quote spanning escaped
+ * chars is grounded and a fabricated quote equal to a key is not. Trivially short
+ * quotes never ground (MIN_GROUNDED_QUOTE_LEN).
+ *
+ * SCOPE NOTE (F3 vs F5): `turn_id` on an evidence entry is carried but NOT
+ * hard-validated — turn-scoped matching against the diarized transcript-with-
+ * turns artifact is F5's job (spec §7). F3 is substring grounding only.
+ *
+ * Pure; no I/O. Unit-tested directly (groundedness.test.ts).
+ */
+/**
+ * Pull every `criteria[].evidence[].quote` string from a scorecard output.
+ * Tolerant of missing / malformed shapes: a non-object output, a missing or
+ * non-array `criteria`, a criterion without an `evidence` array, or an evidence
+ * entry without a string `quote` are all skipped rather than throwing. Returns
+ * [] for anything it cannot read. Empty (whitespace-only) quotes are dropped —
+ * they carry no evidence to ground.
+ */
+export declare function extractEvidenceQuotes(output: unknown): string[];
+/**
+ * The fraction in [0,1] of a scorecard's evidence quotes that are grounded in
+ * the input, or null when there is nothing to ground.
+ *
+ * A quote is GROUNDED when its whitespace-collapsed, case-insensitive form is a
+ * substring of the DECODED input's whitespace-collapsed, case-insensitive text
+ * ({@link inputHaystack} — the string leaf VALUES the model was shown, not the
+ * serialized envelope or its keys) AND it is at least {@link MIN_GROUNDED_QUOTE_LEN}
+ * chars long (a too-trivial quote is never grounded).
+ *
+ * Edge cases (explicit, tested):
+ *   • ZERO evidence quotes across the whole scorecard (none present, or a
+ *     malformed shape yields none) ⇒ null. There is nothing to ground, so the
+ *     caller falls through to the next quality tier — NOT a spurious 1.0 or 0.0.
+ *   • An empty / absent input WITH quotes ⇒ every quote is ungrounded ⇒ 0.
+ *   • Quotes present but ALL below the min length ⇒ 0, NOT null — they were
+ *     emitted, they just carry no evidentiary weight (null is only for ZERO
+ *     emitted quotes).
+ *   • All quotes grounded ⇒ 1; some grounded ⇒ grounded / total.
+ */
+export declare function groundednessScore(output: unknown, input: unknown): number | null;
+//# sourceMappingURL=groundedness.d.ts.map

package/dist/bakeoff/groundedness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"groundedness.d.ts","sourceRoot":"","sources":["../../src/bakeoff/groundedness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAoEH;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,EAAE,CAgB/D;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAYhF"}

package/dist/bakeoff/groundedness.js

@@ -0,0 +1,165 @@
+"use strict";
+/**
+ * Evidence groundedness grader (call-QA F3 / research §4 G6, spec §5). A
+ * DETERMINISTIC bake-off tier that scores a scorecard OUTPUT by whether its
+ * evidence quotes are actually present in the replayed INPUT (the transcript /
+ * source). A scorer that invents justifications — quotes that never appear in
+ * the input — is penalized; this is what stops a model from fabricating
+ * evidence to defend a score.
+ *
+ * Unlike the golden tiers (exact / field-level / llm-judge in quality.ts), this
+ * grades the output against the INPUT, not a corrected golden, so it applies
+ * whether or not an eval-case `expected` value is present.
+ *
+ * The scorecard OUTPUT shape (call-QA F2 emit):
+ *   { criteria: [{ id, score, evidence: [{ quote, turn_id? }], comment }], score, comment }
+ *
+ * Grounding is a normalized SUBSTRING match against the DECODED input values
+ * (inputHaystack): the string leaf values the model was actually shown — never
+ * the JSON envelope's escaping or its keys — so a truthful quote spanning escaped
+ * chars is grounded and a fabricated quote equal to a key is not. Trivially short
+ * quotes never ground (MIN_GROUNDED_QUOTE_LEN).
+ *
+ * SCOPE NOTE (F3 vs F5): `turn_id` on an evidence entry is carried but NOT
+ * hard-validated — turn-scoped matching against the diarized transcript-with-
+ * turns artifact is F5's job (spec §7). F3 is substring grounding only.
+ *
+ * Pure; no I/O. Unit-tested directly (groundedness.test.ts).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractEvidenceQuotes = extractEvidenceQuotes;
+exports.groundednessScore = groundednessScore;
+function isObject(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+/** Collapse all runs of whitespace to a single space, trim, and lowercase — the
+ * comparison basis for grounding. This makes a quote grounded regardless of
+ * incidental whitespace (newlines, tabs, doubled spaces) or letter case, while
+ * never doing a fuzzy match: a quote whose WORDS are not in the input is still
+ * ungrounded. Mirrors quality.ts's "normalized, never fuzzy" discipline. */
+function collapse(text) {
+    return text.replace(/\s+/g, " ").trim().toLowerCase();
+}
+/**
+ * Quotes whose collapsed length is below this count as UNGROUNDED, never
+ * grounded (they still count toward the denominator — see {@link groundednessScore}).
+ * A 1–2 char fragment ("a", "no", "ok") is a substring of almost any transcript,
+ * so a raw substring match would let a fabricator score 1.0 with trivial
+ * "evidence"; too-short quotes carry no evidentiary weight. 12 chars ≈ a few
+ * words — enough to be a real span, not an accidental hit. */
+const MIN_GROUNDED_QUOTE_LEN = 12;
+/** Collect every STRING LEAF from a decoded JSON value — object VALUES and array
+ * items, NEVER object KEYS. This is the text the model was actually shown (the
+ * decoded variable values), so grounding is checked against values, and a
+ * fabricated quote equal to a structural KEY (e.g. "transcript") cannot
+ * false-pass. */
+function collectStringLeaves(v, out) {
+    if (typeof v === "string") {
+        out.push(v);
+    }
+    else if (Array.isArray(v)) {
+        for (const item of v)
+            collectStringLeaves(item, out);
+    }
+    else if (isObject(v)) {
+        for (const key of Object.keys(v))
+            collectStringLeaves(v[key], out);
+    }
+}
+/**
+ * Render the replayed input to the haystack grounding is checked against: the
+ * DECODED string leaf values, joined by spaces — NOT the serialized envelope.
+ *
+ * The model was shown the decoded variable VALUES, not the JSON wrapper, so:
+ *   • a string input (the serialized route variables) is JSON.parse'd first,
+ *     which un-escapes `\"`→`"` and `\n`→newline, then its string leaves are
+ *     collected — a parse failure means a bare (non-JSON) transcript, used as-is;
+ *   • an object/array input is walked directly.
+ * Object KEYS are excluded (leaves are values only), so a quote equal to a key
+ * ("transcript") is not grounded. `undefined`/`null` ⇒ "" (every quote
+ * ungrounded). */
+function inputHaystack(input) {
+    if (input === undefined || input === null)
+        return "";
+    if (typeof input === "string") {
+        try {
+            const parsed = JSON.parse(input);
+            const leaves = [];
+            collectStringLeaves(parsed, leaves);
+            return leaves.join(" ");
+        }
+        catch {
+            return input; // not JSON — a bare transcript string; use as-is.
+        }
+    }
+    const leaves = [];
+    collectStringLeaves(input, leaves);
+    return leaves.join(" ");
+}
+/**
+ * Pull every `criteria[].evidence[].quote` string from a scorecard output.
+ * Tolerant of missing / malformed shapes: a non-object output, a missing or
+ * non-array `criteria`, a criterion without an `evidence` array, or an evidence
+ * entry without a string `quote` are all skipped rather than throwing. Returns
+ * [] for anything it cannot read. Empty (whitespace-only) quotes are dropped —
+ * they carry no evidence to ground.
+ */
+function extractEvidenceQuotes(output) {
+    if (!isObject(output))
+        return [];
+    const criteria = output.criteria;
+    if (!Array.isArray(criteria))
+        return [];
+    const quotes = [];
+    for (const criterion of criteria) {
+        if (!isObject(criterion))
+            continue;
+        const evidence = criterion.evidence;
+        if (!Array.isArray(evidence))
+            continue;
+        for (const item of evidence) {
+            if (!isObject(item))
+                continue;
+            const quote = item.quote;
+            if (typeof quote === "string" && quote.trim() !== "")
+                quotes.push(quote);
+        }
+    }
+    return quotes;
+}
+/**
+ * The fraction in [0,1] of a scorecard's evidence quotes that are grounded in
+ * the input, or null when there is nothing to ground.
+ *
+ * A quote is GROUNDED when its whitespace-collapsed, case-insensitive form is a
+ * substring of the DECODED input's whitespace-collapsed, case-insensitive text
+ * ({@link inputHaystack} — the string leaf VALUES the model was shown, not the
+ * serialized envelope or its keys) AND it is at least {@link MIN_GROUNDED_QUOTE_LEN}
+ * chars long (a too-trivial quote is never grounded).
+ *
+ * Edge cases (explicit, tested):
+ *   • ZERO evidence quotes across the whole scorecard (none present, or a
+ *     malformed shape yields none) ⇒ null. There is nothing to ground, so the
+ *     caller falls through to the next quality tier — NOT a spurious 1.0 or 0.0.
+ *   • An empty / absent input WITH quotes ⇒ every quote is ungrounded ⇒ 0.
+ *   • Quotes present but ALL below the min length ⇒ 0, NOT null — they were
+ *     emitted, they just carry no evidentiary weight (null is only for ZERO
+ *     emitted quotes).
+ *   • All quotes grounded ⇒ 1; some grounded ⇒ grounded / total.
+ */
+function groundednessScore(output, input) {
+    const quotes = extractEvidenceQuotes(output);
+    if (quotes.length === 0)
+        return null;
+    const haystack = collapse(inputHaystack(input));
+    let grounded = 0;
+    for (const q of quotes) {
+        const needle = collapse(q);
+        // Too-trivial quotes never ground (they still count in the denominator).
+        if (needle.length < MIN_GROUNDED_QUOTE_LEN)
+            continue;
+        if (haystack.includes(needle))
+            grounded += 1;
+    }
+    return grounded / quotes.length;
+}

package/dist/bakeoff/quality.d.ts

@@ -13,6 +13,20 @@
  * ordering.
  */
 import type { ValueCheck } from "../types/route";
+/**
+ * The grader a golden eval set scores with (live-routing-evals §2.2/§4). All
+ * three are wired: `exact` (binary normalized equality, LRE-1b), `field-level`
+ * (schema-aware per-field partial credit via {@link fieldAccuracy}, first-class
+ * as of LRE-2), and `llm-judge` (a PAIRWISE comparison of the output against the
+ * golden — NOT a deterministic tier, so this resolver returns null for it and the
+ * caller's golden judge (golden-judge.ts, LRE-2) scores it against `expected`).
+ *
+ * `evidence-grounded` (call-QA F3 / G6) is a fourth, GOLDEN-FREE deterministic
+ * kind: it scores a scorecard output by whether its evidence quotes appear in
+ * the replayed INPUT (groundedness.ts), not against a corrected golden. It is
+ * dispatched here regardless of whether an `expected` value is present.
+ */
+export type GraderType = "exact" | "field-level" | "llm-judge" | "evidence-grounded";
 /**
  * Schema-aware field accuracy in [0,1]: the fraction of the EXPECTED value's leaf
  * fields that the actual output matches. Recurses into objects/arrays keyed by
@@ -21,6 +35,18 @@ import type { ValueCheck } from "../types/route";
  * expected ⇒ 1 (nothing to be wrong about).
  */
 export declare function fieldAccuracy(expected: unknown, actual: unknown): number;
+/**
+ * The `exact` grader (LRE-1b, spec §4): normalized EQUALITY of the parsed output
+ * against the golden → 1 (match) or 0 (mismatch). Binary, never partial credit —
+ * that is the whole point of `exact` vs `field-level` (`fieldAccuracy`).
+ *
+ * Normalization is `normalizeDeep`: object keys are order-insensitive (sorted),
+ * string leaves are whitespace-trimmed, arrays stay ORDER-sensitive (a reordered
+ * list is a different value), and types are strict (`1` ≠ `"1"`, `null` ≠
+ * `undefined`). So `{a:1, b:" x "}` matches `{b:"x", a:1}` but `[1,2]` does not
+ * match `[2,1]`. Pure + unit-tested (exact-grader.test.ts).
+ */
+export declare function exactMatch(expected: unknown, actual: unknown): number;
 /** Resolve an RFC 6901 JSON pointer against a value. Returns `undefined` when any
  * segment is missing (a missing field fails its value check). */
 export declare function resolveJsonPointer(value: unknown, pointer: string): unknown;
@@ -29,12 +55,29 @@ export declare function checkPasses(output: unknown, check: ValueCheck): boolean
 export interface DeterministicQualityInput {
     readonly output: unknown;
     readonly valueChecks?: readonly ValueCheck[];
+    /** The replayed capture INPUT (transcript / source). Consulted ONLY by the
+     * `evidence-grounded` grader (F3/G6), which grounds the output's evidence
+     * quotes against it. A string is used as-is; anything else is JSON.stringify'd
+     * (groundedness.ts). Ignored by every other grader kind. */
+    readonly input?: unknown;
     /** The corrected/golden expected output for this capture, when a bake-off runs
      * from eval cases with a corrected artifact. */
     readonly expected?: unknown;
     /** Set when `expected` is meaningfully present (distinguishes a golden `null`
      * expected value from "no eval case"). */
     readonly hasExpected?: boolean;
+    /** The golden set's grader. Only consulted alongside `hasExpected`:
+     *   • `exact`      → binary normalized equality (exactMatch).
+     *   • `field-level`→ schema-aware per-field partial credit (fieldAccuracy) —
+     *     the fraction of the golden's leaf fields the output matches, in [0,1].
+     *   • `llm-judge`  → NOT a deterministic tier; returns null so the caller's
+     *     golden judge (golden-judge.ts) scores the output against the golden via
+     *     the N5/N6 pairwise judge (LRE-2).
+     *   • `evidence-grounded` → GOLDEN-FREE (F3/G6); scores the output's evidence
+     *     quotes against {@link DeterministicQualityInput.input}, independent of
+     *     `expected`/`hasExpected`. See {@link groundednessScore}.
+     * Absent ⇒ `field-level` behavior (byte-identical to pre-LRE-1b default). */
+    readonly graderType?: GraderType;
 }
 /**
  * Deterministic value accuracy for one sample, or null when neither an eval-case

package/dist/bakeoff/quality.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAiBjD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAgBxE;AAED;iEACiE;AACjE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmB3E;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAWvE;AAYD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;oDACgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;8CAC0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,GAAG,IAAI,CAU3F"}
+{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,mBAAmB,CAAC;AAiBrF;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAgBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAErE;AAED;iEACiE;AACjE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmB3E;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAWvE;AAYD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;;;gEAG4D;IAC5D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB;oDACgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;8CAC0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;;;;;;;iFAU6E;IAC7E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,GAAG,IAAI,CAuB3F"}

package/dist/bakeoff/quality.js

@@ -15,9 +15,11 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fieldAccuracy = fieldAccuracy;
+exports.exactMatch = exactMatch;
 exports.resolveJsonPointer = resolveJsonPointer;
 exports.checkPasses = checkPasses;
 exports.resolveDeterministicQuality = resolveDeterministicQuality;
+const groundedness_1 = require("./groundedness");
 /** Normalize a scalar for comparison: trim strings; leave numbers/booleans as-is.
  * "normalized match" (spec §4.3) means insensitivity to surrounding whitespace,
  * never a fuzzy match — a wrong value is still wrong. */
@@ -58,6 +60,20 @@ function fieldAccuracy(expected, actual) {
     }
     return scalarEqual(expected, actual) ? 1 : 0;
 }
+/**
+ * The `exact` grader (LRE-1b, spec §4): normalized EQUALITY of the parsed output
+ * against the golden → 1 (match) or 0 (mismatch). Binary, never partial credit —
+ * that is the whole point of `exact` vs `field-level` (`fieldAccuracy`).
+ *
+ * Normalization is `normalizeDeep`: object keys are order-insensitive (sorted),
+ * string leaves are whitespace-trimmed, arrays stay ORDER-sensitive (a reordered
+ * list is a different value), and types are strict (`1` ≠ `"1"`, `null` ≠
+ * `undefined`). So `{a:1, b:" x "}` matches `{b:"x", a:1}` but `[1,2]` does not
+ * match `[2,1]`. Pure + unit-tested (exact-grader.test.ts).
+ */
+function exactMatch(expected, actual) {
+    return JSON.stringify(normalizeDeep(expected)) === JSON.stringify(normalizeDeep(actual)) ? 1 : 0;
+}
 /** Resolve an RFC 6901 JSON pointer against a value. Returns `undefined` when any
  * segment is missing (a missing field fails its value check). */
 function resolveJsonPointer(value, pointer) {
@@ -114,7 +130,22 @@ function normalizeDeep(v) {
  * expected value nor a value check applies (the caller then tries the judge).
  */
 function resolveDeterministicQuality(input) {
+    // evidence-grounded (F3/G6): GOLDEN-FREE. Scores the output's evidence quotes
+    // against the replayed INPUT, NOT a golden — so it runs whether or not an
+    // eval-case `expected` value is present, and does not disturb the exact/
+    // field-level/llm-judge/hasExpected/valueChecks precedence below. A null
+    // (no evidence quotes to ground) defers to the next tier exactly like the
+    // other deterministic kinds.
+    if (input.graderType === "evidence-grounded") {
+        return (0, groundedness_1.groundednessScore)(input.output, input.input);
+    }
     if (input.hasExpected === true) {
+        // llm-judge is not deterministic — defer to the judge tier (LRE-2 seam).
+        if (input.graderType === "llm-judge")
+            return null;
+        if (input.graderType === "exact")
+            return exactMatch(input.expected, input.output);
+        // field-level (explicit) or unspecified default → schema-aware field accuracy.
         return fieldAccuracy(input.expected, input.output);
     }
     const checks = input.valueChecks;

package/dist/bakeoff/replay.d.ts

@@ -19,7 +19,9 @@
  */
 import type { CaptureStore } from "../telemetry/capture";
 import type { RunDeps } from "../run/run";
+import type { GraderType } from "./quality";
 import type { JudgeScorer } from "./judge";
+import type { GoldenJudgeScorer } from "./golden-judge";
 import type { QualityGate } from "../types/quality";
 export interface ReplayRequest {
     readonly route: string;
@@ -42,9 +44,18 @@ export interface ReplayRequest {
     readonly expected?: ReadonlyMap<string, {
         readonly value: unknown;
     }>;
+    /** LRE-1b: the golden set's grader — how an `expected` value is scored against
+     * the output (`exact` binary, `field-level` per-field, `llm-judge` deferred to
+     * the judge tier). Absent ⇒ field-level (pre-LRE-1b behavior). */
+    readonly graderType?: GraderType;
     /** RB-1: an opt-in judge scorer (tier 3), used only when neither an eval-case
      * expected value nor a declared value check yields a score. Capped internally. */
     readonly judge?: JudgeScorer;
+    /** LRE-2: the `llm-judge` grader — scores a produced output against its GOLDEN
+     * (`expected`) value via the N5/N6 pairwise judge. Used ONLY when
+     * `graderType==='llm-judge'` AND this sample has an eval-case expected value (a
+     * failure case with no golden still scores by conformance). Capped internally. */
+    readonly goldenJudge?: GoldenJudgeScorer;
     /** Q1 quality gate (spec §4.2): scored on each replayed sample's output to
      * populate `valueAccuracy` + `qualityPassed`. NOT passed into the inner
      * `run()` — a bake-off MEASURES quality, it does not gate serving; injecting

package/dist/bakeoff/replay.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAyIxC"}
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;sEAEkE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;sFAGkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoJxC"}

package/dist/bakeoff/replay.js

@@ -110,8 +110,13 @@ async function replayVariant(deps, captures, req) {
             const expected = req.expected?.get(capture.inferenceId);
             const det = (0, quality_1.resolveDeterministicQuality)({
                 output: outputValue,
+                // The replayed capture INPUT, for the `evidence-grounded` grader (F3/G6)
+                // — the serialized route variables (which carry the transcript/source),
+                // used as-is since it is already text. Ignored by every other grader.
+                input: capture.variablesJson,
                 ...(route.valueChecks !== undefined ? { valueChecks: route.valueChecks } : {}),
                 ...(expected !== undefined ? { hasExpected: true, expected: expected.value } : {}),
+                ...(req.graderType !== undefined ? { graderType: req.graderType } : {}),
             });
             let gateScore = null;
             if (req.qualityGate !== undefined && gated !== null) {
@@ -126,6 +131,13 @@ async function replayVariant(deps, captures, req) {
                 valueAccuracy = det;
             else if (gateScore !== null)
                 valueAccuracy = gateScore;
+            // LRE-2 llm-judge grader: a golden-set case whose grader is `llm-judge`
+            // scores the output against its GOLDEN via the pairwise judge (det is null
+            // for llm-judge — resolveDeterministicQuality defers it here). Only a case
+            // WITH an expected value is golden-judged; a failure case falls through.
+            else if (req.graderType === "llm-judge" && expected !== undefined && req.goldenJudge !== undefined) {
+                valueAccuracy = await req.goldenJudge.score(outputValue, expected.value, JSON.parse(capture.variablesJson));
+            }
             else if (req.judge !== undefined) {
                 valueAccuracy = await req.judge.score(outputValue, JSON.parse(capture.variablesJson));
             }

package/dist/bakeoff/report.d.ts

@@ -43,6 +43,13 @@ export declare function formatCacheAdjustedSection(rows: readonly CacheAdjustedR
  * (returns strings); empty when nothing disagreed. Exported for direct testing.
  */
 export declare function formatDisagreementSummary(artifact: BakeoffArtifact): string[];
+/**
+ * N4 (spec §4.2/§6): the proposed swap's direction line — ⬆/⬇/→ + the signed
+ * effective-cost delta ($/1k). An up-swap is framed as a quality investment, NEVER
+ * a saving (claim ceiling §8). Pure (returns strings); empty when there is no
+ * proposal or the delta is indeterminate. Exported for direct testing.
+ */
+export declare function formatSwapDirection(artifact: BakeoffArtifact): string[];
 export declare function printMatrix(artifact: BakeoffArtifact, cacheAdjusted?: readonly CacheAdjustedRow[]): void;
 /**
  * `modelrig bakeoff --from-eval-cases <task> --variants a,b,c` — replay a task's

package/dist/bakeoff/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CAwD5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bakeoff/report.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAG3C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA2CpD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAexE;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC;CAC3C;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAgCtF;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAiB7E;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,EAAE,CAavE;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,GAAE,SAAS,gBAAgB,EAAO,GAAG,IAAI,CA4D5G;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,IAAI,EAAE,eAAe,GAAG,SAAS,EACjC,MAAM,EAAE,OAAO,EACf,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC,CA+CjB"}

package/dist/bakeoff/report.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.bakeoffHeadline = bakeoffHeadline;
 exports.formatCacheAdjustedSection = formatCacheAdjustedSection;
 exports.formatDisagreementSummary = formatDisagreementSummary;
+exports.formatSwapDirection = formatSwapDirection;
 exports.printMatrix = printMatrix;
 exports.runEvalCaseBakeoffCli = runEvalCaseBakeoffCli;
 const node_fs_1 = require("node:fs");
@@ -144,6 +145,26 @@ function formatDisagreementSummary(artifact) {
     }
     return lines;
 }
+/**
+ * N4 (spec §4.2/§6): the proposed swap's direction line — ⬆/⬇/→ + the signed
+ * effective-cost delta ($/1k). An up-swap is framed as a quality investment, NEVER
+ * a saving (claim ceiling §8). Pure (returns strings); empty when there is no
+ * proposal or the delta is indeterminate. Exported for direct testing.
+ */
+function formatSwapDirection(artifact) {
+    const dir = (0, engine_1.deriveSwapDirection)(artifact);
+    if (dir === null || dir.costDeltaUsdPer1k === null)
+        return [];
+    const arrow = dir.direction === "up" ? "⬆" : dir.direction === "down" ? "⬇" : "→";
+    const delta = dir.costDeltaUsdPer1k;
+    const sign = delta >= 0 ? "+" : "−";
+    const note = dir.direction === "up"
+        ? " — raises effective cost to cut regret; a quality investment, not a saving"
+        : dir.direction === "down"
+            ? " — cost-reducing"
+            : "";
+    return [`  direction: ${arrow} ${dir.direction}-swap (${sign}$${Math.abs(delta).toFixed(2)}/1k effective${note})`];
+}
 function printMatrix(artifact, cacheAdjusted = []) {
     console.log(`bakeoff ${artifact.id}\nroute ${artifact.route}@${artifact.routeVersion} · replayed ${artifact.replayN} captured inputs · ${artifact.ranAt}`);
     // RB-1: a reference arm is present ⇒ show quality + regret columns and badge
@@ -195,6 +216,11 @@ function printMatrix(artifact, cacheAdjusted = []) {
     else {
         console.log("proposal: none (no variant beats the incumbent's gates)");
     }
+    // N4 (spec §6/§9): the proposed swap's direction + signed cost delta on the CLI
+    // proposal line — an up-swap raises effective cost (a quality investment), a
+    // down-swap is the shipped saving. Printed only when a proposal exists.
+    for (const l of formatSwapDirection(artifact))
+        console.log(l);
     // N6 disagreement summary — printed only when candidates were mined; absent ⇒
     // byte-identical to the pre-N6 matrix output.
     for (const l of formatDisagreementSummary(artifact))

package/dist/cli-judge.d.ts

@@ -0,0 +1,12 @@
+/**
+ * `modelrig judge <run>` — E5 V.4: explicitly ask the bounded model-judge to
+ * score a run. Goes through the API boundary (T12: the CLI never holds a DB
+ * client) — POST /v1/runs/:id/judge on the configured server, authed with a
+ * rig_sk key. The daily cap + metering are the SERVER's; this just triggers one.
+ *
+ * Extracted from cli.ts (which sits at the ≤700-LOC cap) into its own module,
+ * mirroring `cli-query.ts`. Behaviour is byte-identical to the inline command.
+ */
+import type { Flags } from "./cli";
+export declare function cmdJudge(flags: Flags): Promise<number>;
+//# sourceMappingURL=cli-judge.d.ts.map

package/dist/cli-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-judge.d.ts","sourceRoot":"","sources":["../src/cli-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAEnC,wBAAsB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAoC5D"}

package/dist/cli-judge.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * `modelrig judge <run>` — E5 V.4: explicitly ask the bounded model-judge to
+ * score a run. Goes through the API boundary (T12: the CLI never holds a DB
+ * client) — POST /v1/runs/:id/judge on the configured server, authed with a
+ * rig_sk key. The daily cap + metering are the SERVER's; this just triggers one.
+ *
+ * Extracted from cli.ts (which sits at the ≤700-LOC cap) into its own module,
+ * mirroring `cli-query.ts`. Behaviour is byte-identical to the inline command.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cmdJudge = cmdJudge;
+const config_1 = require("./config");
+async function cmdJudge(flags) {
+    const runId = flags.positional[0] ?? "";
+    // Config is the only env reader (package-invariants rule). The judge endpoint
+    // lives on the same server as the api-mode ingest sink, so the base is that
+    // URL's origin and the credential is the api key.
+    const exp = (0, config_1.loadConfigFromEnv)().export;
+    const api = exp !== undefined && exp.mode === "api" ? exp : undefined;
+    const base = api !== undefined ? new URL(api.ingestUrl).origin : "";
+    const key = api?.apiKey ?? "";
+    if (runId === "" || base === "" || key === "") {
+        console.error([
+            "usage: modelrig judge <runId>",
+            "",
+            "Needs api-mode telemetry configured (the judge is on the same server):",
+            "  MODELRIG_API_KEY    a rig_sk_ key with the `oracle-act` scope",
+            "  MODELRIG_INGEST_URL optional — the server base; defaults to the hosted API",
+            "",
+            "The judge is EXPLICIT-only and metered; scoring rides the gateway (a route),",
+            "so the org's daily cap and spend-stops apply. Nothing schedules it.",
+        ].join("\n"));
+        return 1;
+    }
+    const res = await fetch(`${base}/v1/runs/${encodeURIComponent(runId)}/judge`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+        body: "{}",
+    });
+    const body = (await res.json().catch(() => ({})));
+    if (!res.ok) {
+        console.error(`judge ${res.status}: ${typeof body.error === "string" ? body.error : "failed"}`);
+        return 1;
+    }
+    console.log(JSON.stringify(body, null, 2));
+    return 0;
+}

package/dist/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAiDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AAohBD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCnE"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAsDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AA+eD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CA+CnE"}