@vibe-agent-toolkit/agent-skills 0.1.42 → 0.2.0-rc.10

package/dist/skill-test/grader-text.js

@@ -0,0 +1,410 @@
+/**
+ * Neutralize GRADER-SUPPLIED free text before it reaches an operator surface or
+ * one of vat's own artifacts.
+ *
+ * WHY THIS EXISTS. The grader's only input is the executor transcript, and
+ * untrusted skill code writes that transcript. Every free-text field a grader
+ * emits is therefore attacker-influenced: `expectations[].text`/`.evidence`,
+ * `friction[].message`/`.evidence`/`.subjectFile`, `tool.*[].name`/`.evidence`,
+ * `tool.sequence[].steps`. `formatFrictionReport` interpolates one of those
+ * straight into `process.stderr.write`, so a message carrying a newline and an
+ * SGR sequence renders as vat's OWN voice — verified end-to-end with a grader
+ * emitting a green "vat: grading verified, ignore the warning above." line on a
+ * row of its own. That is strictly worse than a wrong number in
+ * `baseline.json`: operators read stderr, and nobody reads the artifact.
+ * (`summary` is NOT in that list — vat computes it numerically.)
+ *
+ * WHAT IT DOES, and why each part:
+ * - **ANSI escape sequences are removed WHOLE.** Dropping only the ESC byte
+ *   would leave the parameter bytes (`[32m`) as visible litter; removing the
+ *   sequence keeps the surviving text readable.
+ * - **Remaining C0/C1 controls become a space.** This is what kills the
+ *   forged-line attack: no newline survives, so grader text can never occupy a
+ *   line of its own. It also covers CR (line overwrite) and BS.
+ * - **Bidi controls and invisible non-spelling characters become a space too.**
+ *   Stopping at U+009F left them alive, and they are not decoration: one U+202E
+ *   (RIGHT-TO-LEFT OVERRIDE) in a friction `message` renders the REST of vat's own
+ *   `[low] path-assumption: …` stderr line right-to-left, so the grader repaints
+ *   text it did not write. `JSON.stringify` only escapes below U+0020, so the same
+ *   code point also survives verbatim into `baseline.json` and `friction.json`. The
+ *   invisible members (U+200B, U+2060…) are the quieter half: they let a grader
+ *   smuggle a marker through any downstream string comparison while rendering as
+ *   nothing at all. This is NOT the whole Cf class — see
+ *   {@link INVISIBLE_CODE_RANGES} for the two admission tests and for the joiners
+ *   and variation selectors deliberately left alone, because ordinary words are
+ *   spelled with them and a grader quoting a file must get the file back.
+ * - **Whitespace runs collapse and the value is trimmed.** After the previous
+ *   step a 200-newline message would be 200 spaces of padding. (The Unicode
+ *   line/paragraph separators are NOT left to the collapse: the scan folds them,
+ *   because {@link sanitizeTextPreservingLines} has no collapse to leave them to.)
+ * - **Length is capped.** Bounds a flooding grader. The cap is generous enough
+ *   for real evidence prose; the untruncated value is preserved nowhere, which
+ *   is deliberate — an artifact nobody can trust to render is worse than a
+ *   short one.
+ *
+ * A value that was non-empty but sanitizes away to nothing becomes
+ * {@link UNPRINTABLE_PLACEHOLDER} rather than the empty string, so a nonsense
+ * grader value cannot turn a `.min(1)` schema field into a hard run failure.
+ *
+ * SCANNED, NOT REGEXED. The obvious implementation is three regexes over
+ * control-character classes. Those need literal ESC/NUL bytes in this source
+ * file (`\u` escapes get normalized INTO literal bytes by editors and tools on
+ * the way in), which makes the module binary to `grep` and unreviewable in a
+ * diff. A forward scan over code units says the same thing in plain ASCII.
+ *
+ * WHERE IT IS APPLIED. Not once, and not redundantly — each of these is a distinct
+ * crossing from untrusted bytes to an operator surface or a vat artifact, and none
+ * of them is downstream of another:
+ * - `eval-fragment.ts` — `parseEvalFragment` walks EVERY string entering vat from a
+ *   grader, plus the schema path/message it quotes when the parse fails.
+ * - `eval-grader.ts` — the grader-invocation error text, and both sides of the
+ *   declared-vs-observed executable-name comparison (names are compared AFTER
+ *   sanitizing, so an invisible smuggled into one side cannot split the match).
+ * - `run-harness.ts` — `formatFrictionReport`, the stderr render, which re-reads
+ *   `friction.json` from a harness dir that same-uid skill code can reach.
+ * - `eval-inputs.ts` — `quoteSuiteText` / `quoteSuiteBlock`, for SUITE-authored text
+ *   (untrusted too: `resolveEvalSuitePath` will harvest a suite out of a fetched
+ *   artifact, i.e. out of the skill under test).
+ * - `eval-lint.ts` — `lintToolExpectationExecutables`, via `quoteSuiteText`. Every
+ *   executable name it quotes comes from the suite (`toolExpectations.*`, an
+ *   unconstrained `z.array(z.string().min(1))` on a `.passthrough()` entry) or from
+ *   the subject skill's own manifest, and `run-harness.ts` writes the advisory to
+ *   stderr with no sanitizer of its own — at Step 5.5, AHEAD of the
+ *   `--i-understand-this-runs-skill-code` gate and ahead of the `--dry-run`
+ *   short-circuit, so this one paints a run that spawned nothing.
+ * - `baseline-integrity.ts` — `BaselineScanDegradation.detail`, built from a `cd`
+ *   argument lifted verbatim out of the CONTROL ARM's transcript. Sanitized where the
+ *   detail is CONSTRUCTED, not at the stderr write, so the one call covers both
+ *   surfaces: `run-harness.ts` (`baselineContaminationFor`) writes it to stderr AND
+ *   vat stamps the same bytes into `baselineIntegrity.degraded` in `baseline.json` —
+ *   which a stderr-side fix would miss, because `degraded` is attached to the
+ *   fragment AFTER `parseEvalFragment` has sanitized it.
+ * - `grading-adapter.ts` — the schema path and message quoted when the aggregate
+ *   `grading.json` parse fails.
+ *
+ * That list is the whole of it, and keeping it whole is the point: this file's own
+ * standard is that a function's docblock and its behaviour disagreeing is how a
+ * security claim rots, and a LIST that has fallen behind the call sites rots the same
+ * way in the other direction — it under-claims, so a reader goes looking for a hole
+ * that was closed and trusts a boundary that was never added. `git grep
+ * 'sanitizeGraderText\|sanitizeTextPreservingLines'` is the check.
+ */
+/**
+ * Longest grader-supplied free-text value vat will carry. ~20 terminal lines —
+ * enough for a real evidence excerpt, small enough that one item cannot own the
+ * operator's screen.
+ */
+export const MAX_GRADER_TEXT_LENGTH = 2000;
+/** Substituted for a non-empty value that sanitizes away to nothing. */
+export const UNPRINTABLE_PLACEHOLDER = '(unprintable)';
+const TRUNCATION_MARKER = '... (truncated)';
+const ESC = 0x1b;
+const BEL = 0x07;
+const C0_MAX = 0x1f;
+const DEL = 0x7f;
+const C1_MAX = 0x9f;
+const LEFT_BRACKET = 0x5b;
+const RIGHT_BRACKET = 0x5d;
+const BACKSLASH = 0x5c;
+/** A CSI sequence ends at its first byte in `@`..`~`. */
+const CSI_FINAL_MIN = 0x40;
+const CSI_FINAL_MAX = 0x7e;
+/** Intermediate bytes of an nF escape sequence (` `..`/`), e.g. the `(` of `ESC ( B`. */
+const INTERMEDIATE_MIN = 0x20;
+const INTERMEDIATE_MAX = 0x2f;
+/** The leading half of a surrogate pair; see {@link capLength}. */
+const HIGH_SURROGATE_MIN = 0xd800;
+const HIGH_SURROGATE_MAX = 0xdbff;
+/** Above this, a code point is written as a surrogate PAIR; see {@link capLength}. */
+const MAX_BMP_CODE_POINT = 0xffff;
+const WHITESPACE_RUN = /\s+/gu;
+/** Guard against a pathologically nested grader fragment; the strict parse rejects it anyway. */
+const MAX_SANITIZE_DEPTH = 20;
+/** Line feed — the ONE control {@link sanitizeTextPreservingLines} keeps. */
+const LINE_FEED = 0x0a;
+/**
+ * Invisible code points above U+009F that are neutralized exactly like a C0/C1
+ * control, because they do the same job by other means.
+ *
+ * ⚠️ WRITTEN AS NUMBERS, like every other constant in this file, and for the same
+ * reason: a `\u202E` escape in a regex literal gets normalized into the real code
+ * point by editors and tooling on the way in, which makes the module unreviewable
+ * in a diff and unfindable by `grep`. Build test inputs with `String.fromCodePoint`.
+ *
+ * THE RULE, and it is not "the Cf class". Two admission tests; a code point has to
+ * pass one of them:
+ *
+ * (a) **It is a bidi or shaping control** — it changes how the text AROUND it is
+ *     ordered or directed, which is the forgery this module exists to stop. U+202E
+ *     is the verified case: it renders the REST of vat's own
+ *     `[low] path-assumption: …` stderr line right-to-left.
+ * (b) **It is invisible AND spells nothing** — no word in any script is written with
+ *     it, so removing it cannot change what a quotation says, while keeping it lets
+ *     a grader hide a difference from a reader and from every downstream string
+ *     comparison (`eval-grader.ts` compares tool names across both sides of a run,
+ *     sanitized on each).
+ *
+ * Member by member:
+ * - U+00AD SOFT HYPHEN — (b) a hyphenation HINT; no word is spelled with one.
+ * - U+061C ARABIC LETTER MARK — (a) the Arabic analogue of LRM/RLM.
+ * - U+200B ZERO WIDTH SPACE — (b) a line-break hint, invisible, spells nothing.
+ * - U+200E…U+200F LRM/RLM — (a) directional MARKS: they re-resolve the direction of
+ *   the neutral characters beside them, so they reorder text they did not write.
+ * - U+2028…U+2029 LINE and PARAGRAPH SEPARATOR — some renderers break a line on
+ *   them, which is the "occupy a line of your own" attack spelled with a different
+ *   code point. Folded to a space, never to LF, even in the line-preserving path.
+ * - U+202A…U+202E — (a) the bidi embeddings and OVERRIDES.
+ * - U+2060…U+2064 — (b) word joiner and the invisible math operators.
+ * - U+2066…U+2069 — (a) the bidi ISOLATES (U+2066 LRI verified surviving).
+ * - U+206A…U+206F — (a) the deprecated shaping controls (inhibit symmetric swapping,
+ *   Arabic form shaping, nominal digit shapes); absent from modern text entirely.
+ * - U+FEFF ZERO WIDTH NO-BREAK SPACE — (b). It is `\s`, so {@link sanitizeGraderText}'s
+ *   whitespace collapse would eat it anyway; {@link sanitizeTextPreservingLines} has
+ *   no collapse and used to ship it to stderr intact, so it is neutralized HERE.
+ * - U+FFF9…U+FFFB — (b) interlinear annotation, which hides its own payload.
+ *
+ * DELIBERATELY ABSENT, because they fail BOTH tests — zero directional power, and
+ * real words ARE spelled with them:
+ * - U+FE00…U+FE0F variation selectors (category Mn, not Cf). How `⚠️` and most emoji
+ *   are spelled, vat's own banner glyph included.
+ * - U+200C ZERO WIDTH NON-JOINER — orthographically load-bearing in Persian, Hindi,
+ *   Bengali and Malayalam. `می<ZWNJ>خواهم` is two words; folding it to a space gave
+ *   back a different string than the file the grader was quoting.
+ * - U+200D ZERO WIDTH JOINER — how a multi-person emoji and a flag sequence are
+ *   spelled. Folding it turned `👨<ZWJ>👩<ZWJ>👧` into three separate glyphs.
+ * - U+180E MONGOLIAN VOWEL SEPARATOR — orthographic in Mongolian (it selects the
+ *   letterform of the syllable before it) and reorders nothing.
+ *
+ * The residual cost of that exclusion is real and accepted: a grader can still
+ * smuggle an invisible joiner into a string a human compares by eye. It is the same
+ * trade the variation-selector exclusion always made, and the standard this file
+ * already stated — "dropping them would mangle ordinary text" — decides it the same
+ * way. What it must NOT do is state that standard and then break it.
+ */
+const INVISIBLE_CODE_RANGES = [
+    [0x00ad, 0x00ad],
+    [0x061c, 0x061c],
+    [0x200b, 0x200b],
+    [0x200e, 0x200f],
+    [0x2028, 0x2029],
+    [0x202a, 0x202e],
+    [0x2060, 0x2064],
+    [0x2066, 0x206f],
+    [0xfeff, 0xfeff],
+    [0xfff9, 0xfffb],
+];
+/** Lowest code point in {@link INVISIBLE_CODE_RANGES}, so ASCII bails on one compare. */
+const INVISIBLE_CODE_MIN = 0x00ad;
+function isInvisibleCode(code) {
+    if (code < INVISIBLE_CODE_MIN)
+        return false;
+    return INVISIBLE_CODE_RANGES.some(([low, high]) => code >= low && code <= high);
+}
+function isControlCode(code) {
+    return code <= C0_MAX || (code >= DEL && code <= C1_MAX) || isInvisibleCode(code);
+}
+/** Index of the first code unit AFTER a CSI sequence whose parameters start at `from`. */
+function endOfCsi(value, from) {
+    let i = from;
+    while (i < value.length) {
+        const code = value.codePointAt(i) ?? Number.NaN;
+        i += 1;
+        if (code >= CSI_FINAL_MIN && code <= CSI_FINAL_MAX)
+            return i;
+    }
+    return i;
+}
+/** Index of the first code unit AFTER an OSC sequence whose payload starts at `from`. */
+function endOfOsc(value, from) {
+    let i = from;
+    while (i < value.length) {
+        const code = value.codePointAt(i) ?? Number.NaN;
+        if (code === BEL)
+            return i + 1;
+        // String Terminator is ESC followed by a backslash; a bare ESC ends it too.
+        if (code === ESC)
+            return i + (value.codePointAt(i + 1) === BACKSLASH ? 2 : 1);
+        i += 1;
+    }
+    return i;
+}
+/**
+ * Index of the first code unit AFTER an nF sequence (`ESC` intermediates final),
+ * e.g. the charset selection `ESC ( B`, whose intermediates start at `from`.
+ */
+function endOfNf(value, from) {
+    let i = from;
+    while (i < value.length) {
+        const code = value.codePointAt(i) ?? Number.NaN;
+        i += 1;
+        if (code < INTERMEDIATE_MIN || code > INTERMEDIATE_MAX)
+            return i;
+    }
+    return i;
+}
+/** Index of the first code unit AFTER the escape sequence introduced by the ESC at `start`. */
+function endOfEscapeSequence(value, start) {
+    if (start + 1 >= value.length)
+        return start + 1;
+    const second = value.codePointAt(start + 1) ?? Number.NaN;
+    if (second === LEFT_BRACKET)
+        return endOfCsi(value, start + 2);
+    if (second === RIGHT_BRACKET)
+        return endOfOsc(value, start + 2);
+    if (second >= INTERMEDIATE_MIN && second <= INTERMEDIATE_MAX)
+        return endOfNf(value, start + 1);
+    // Any remaining two-code-unit form (RIS, NEL, ...).
+    return start + 2;
+}
+/**
+ * Drop every escape sequence and map every remaining control code to a space.
+ *
+ * `preserveNewlines` keeps LF — and ONLY LF — as itself, for
+ * {@link sanitizeTextPreservingLines}. CR still becomes a space even then: the
+ * line-overwrite attack it carries is not made safe by the caller wanting lines.
+ */
+function stripEscapesAndControls(value, preserveNewlines = false) {
+    const out = [];
+    let i = 0;
+    while (i < value.length) {
+        const code = value.codePointAt(i) ?? Number.NaN;
+        if (code === ESC) {
+            i = endOfEscapeSequence(value, i);
+        }
+        else if (isControlCode(code)) {
+            out.push(preserveNewlines && code === LINE_FEED ? '\n' : ' ');
+            i += 1;
+        }
+        else {
+            out.push(value.charAt(i));
+            i += 1;
+        }
+    }
+    return out.join('');
+}
+/**
+ * Truncate to at most `max` code units, spending the tail on a marker that says so.
+ *
+ * THE CUT BACKS OFF ONE CODE UNIT rather than landing between the halves of a
+ * surrogate pair. A naive `slice` on a value ending in an astral character (1984
+ * ASCII characters plus one emoji is enough) leaves a LONE HIGH SURROGATE, which
+ * persists into `grading.json` and `baseline.json`. The artifact stays valid JSON —
+ * Node's `JSON.stringify` is well-formed and escapes it — but a consumer decoding
+ * it gets an unpaired surrogate, and stderr renders U+FFFD. Well-formedness is a
+ * correctness property of what this function emits, so it is enforced here.
+ *
+ * GRAPHEME CLUSTERS ARE DELIBERATELY NOT HANDLED. A cut inside a cluster — between
+ * a base and its combining mark, or inside a ZWJ emoji sequence (now that the
+ * joiners survive, see {@link INVISIBLE_CODE_RANGES}) — yields a WELL-FORMED string
+ * that renders as two glyphs instead of one. That is cosmetic. Segmenting for it
+ * would mean `Intl.Segmenter` over the whole value on every capped field, for a
+ * locale-dependent answer, to fix an appearance rather than a defect. The line is
+ * well-formedness, not beauty.
+ */
+function capLength(value, max) {
+    if (value.length <= max)
+        return value;
+    const cut = max - TRUNCATION_MARKER.length;
+    // A high surrogate immediately before the cut has its partner AT the cut (or is
+    // itself already lone) — either way it must not be the last thing kept.
+    // `codePointAt` reports the COMBINED code point when a well-formed pair STARTS at the
+    // index, so a value above the BMP means the cut would fall between the two halves. A
+    // value still inside the high-surrogate block means that half never had a partner and
+    // is already lone. Either way it must not be the last thing kept. (A LOW surrogate
+    // reads back as itself, which is correct: its partner is at `cut - 2`, so the pair
+    // survives the cut intact.)
+    const last = value.codePointAt(cut - 1) ?? Number.NaN;
+    const cutSplitsPair = last > MAX_BMP_CODE_POINT || (last >= HIGH_SURROGATE_MIN && last <= HIGH_SURROGATE_MAX);
+    const safeCut = cutSplitsPair ? cut - 1 : cut;
+    return value.slice(0, safeCut) + TRUNCATION_MARKER;
+}
+/** Sanitize one grader-supplied string. Pure. See the module docblock for the rules. */
+export function sanitizeGraderText(value) {
+    const collapsed = stripEscapesAndControls(value).replaceAll(WHITESPACE_RUN, ' ').trim();
+    if (collapsed.length === 0)
+        return value.length === 0 ? '' : UNPRINTABLE_PLACEHOLDER;
+    return capLength(collapsed, MAX_GRADER_TEXT_LENGTH);
+}
+/**
+ * Most lines {@link sanitizeTextPreservingLines} will let through, and the total
+ * budget across them.
+ *
+ * Two caps rather than one because the two floods are different shapes: 4000
+ * characters on ONE line is a wrapped paragraph, 4000 NEWLINES is the operator's
+ * whole scrollback. The line cap is the one that protects the screen; the length
+ * cap protects `Summary:` and the artifact.
+ */
+const MAX_MULTILINE_TEXT_LINES = 40;
+const MAX_MULTILINE_TEXT_LENGTH = 4000;
+/**
+ * The LINE-PRESERVING sanitizer, for untrusted text whose STRUCTURE is the
+ * message.
+ *
+ * WHY IT EXISTS RATHER THAN A SECOND CALL TO {@link sanitizeGraderText}. Zod's
+ * `.strict()`/schema failure text is a multi-line, indented list of issues, one
+ * per offending path, and it is the only thing telling an adopter WHICH eval in a
+ * 200-eval suite is malformed. Folding it with the single-line sanitizer collapses
+ * every issue onto one 2000-character line and then truncates it — so the common
+ * case (a real typo in a real suite) gets materially worse in exchange for
+ * neutralizing bytes the suite author almost never wrote. This keeps the shape and
+ * still removes everything that can paint: escape sequences whole, every C0/C1
+ * control except LF, and every member of {@link INVISIBLE_CODE_RANGES}.
+ *
+ * THAT LAST CLAUSE USED TO BE FALSE, and the sibling's whitespace collapse was why.
+ * U+FEFF, U+2028 and U+2029 were left OUT of the scan on the grounds that they are
+ * `\s` and "the collapse eats them" — true of exactly one of the two consumers.
+ * This function has no collapse, so all three reached `process.stderr` through
+ * `quoteSuiteBlock` -> `EvalInputError` intact, two of them being separators some
+ * renderers break a line on. They are neutralized in the SCAN now, which is the only
+ * place both consumers share. A function's docblock and its behaviour disagreeing is
+ * how a security claim rots; on a terminal-bound path, the code moves to meet the
+ * docblock and not the other way round.
+ *
+ * It is NOT a replacement for {@link sanitizeGraderText} and must not be used for
+ * grader free text: a surviving newline is exactly what lets a value occupy a line
+ * of its own and impersonate vat's voice. Use it only where the caller's own
+ * message already spans lines and the untrusted part is quoted INSIDE it.
+ */
+export function sanitizeTextPreservingLines(value) {
+    const stripped = stripEscapesAndControls(value, true).trim();
+    if (stripped.length === 0)
+        return value.length === 0 ? '' : UNPRINTABLE_PLACEHOLDER;
+    const lines = stripped.split('\n');
+    const kept = lines.slice(0, MAX_MULTILINE_TEXT_LINES);
+    if (lines.length > kept.length) {
+        kept.push(`... and ${lines.length - kept.length} more line(s) (truncated)`);
+    }
+    return capLength(kept.join('\n'), MAX_MULTILINE_TEXT_LENGTH);
+}
+function walk(value, skipKeys, depth) {
+    if (depth > MAX_SANITIZE_DEPTH)
+        return value;
+    if (typeof value === 'string')
+        return sanitizeGraderText(value);
+    if (Array.isArray(value))
+        return value.map((item) => walk(item, skipKeys, depth + 1));
+    if (typeof value === 'object' && value !== null) {
+        const out = {};
+        for (const [key, val] of Object.entries(value)) {
+            out[key] = skipKeys.has(key) ? val : walk(val, skipKeys, depth + 1);
+        }
+        return out;
+    }
+    return value;
+}
+/**
+ * Sanitize every string in an arbitrary parsed-JSON value, recursively.
+ *
+ * Deliberately field-BLIND: a per-field allowlist is a list somebody has to
+ * remember to extend, and these schemas grow (`mustSucceed` and
+ * `sequence[].steps` both arrived after the fields above them). Walking every
+ * string means a new grader-controlled field is covered the day it is added.
+ *
+ * `skipKeys` is REQUIRED, not defaulted — the one value that must survive
+ * byte-exact is the integrity nonce, and a caller that forgets it should have
+ * to say so.
+ */
+export function sanitizeGraderTextDeep(value, skipKeys) {
+    return walk(value, skipKeys, 0);
+}
+//# sourceMappingURL=grader-text.js.map

package/dist/skill-test/grader-text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grader-text.js","sourceRoot":"","sources":["../../src/skill-test/grader-text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0FG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,wEAAwE;AACxE,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC;AAEvD,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;AAE5C,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,MAAM,GAAG,IAAI,CAAC;AACpB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,MAAM,GAAG,IAAI,CAAC;AACpB,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,yDAAyD;AACzD,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,yFAAyF;AACzF,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,mEAAmE;AACnE,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,sFAAsF;AACtF,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,cAAc,GAAG,OAAO,CAAC;AAE/B,iGAAiG;AACjG,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,6EAA6E;AAC7E,MAAM,SAAS,GAAG,IAAI,CAAC;AAEvB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,MAAM,qBAAqB,GAA6C;IACtE,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,yFAAyF;AACzF,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,IAAI,GAAG,kBAAkB;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI,IAAI,MAAM,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC;AACpF,CAAC;AAED,0FAA0F;AAC1F,SAAS,QAAQ,CAAC,KAAa,EAAE,IAAY;IAC3C,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;QAChD,CAAC,IAAI,CAAC,CAAC;QACP,IAAI,IAAI,IAAI,aAAa,IAAI,IAAI,IAAI,aAAa;YAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,yFAAyF;AACzF,SAAS,QAAQ,CAAC,KAAa,EAAE,IAAY;IAC3C,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;QAChD,IAAI,IAAI,KAAK,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,4EAA4E;QAC5E,IAAI,IAAI,KAAK,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,CAAC,IAAI,CAAC,CAAC;IACT,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CAAC,KAAa,EAAE,IAAY;IAC1C,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;QAChD,CAAC,IAAI,CAAC,CAAC;QACP,IAAI,IAAI,GAAG,gBAAgB,IAAI,IAAI,GAAG,gBAAgB;YAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,+FAA+F;AAC/F,SAAS,mBAAmB,CAAC,KAAa,EAAE,KAAa;IACvD,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM;QAAE,OAAO,KAAK,GAAG,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;IAC1D,IAAI,MAAM,KAAK,YAAY;QAAE,OAAO,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAC/D,IAAI,MAAM,KAAK,aAAa;QAAE,OAAO,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAChE,IAAI,MAAM,IAAI,gBAAgB,IAAI,MAAM,IAAI,gBAAgB;QAAE,OAAO,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAC/F,oDAAoD;IACpD,OAAO,KAAK,GAAG,CAAC,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,uBAAuB,CAAC,KAAa,EAAE,gBAAgB,GAAG,KAAK;IACtE,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;QAChD,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,CAAC,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACtB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,SAAS,CAAC,KAAa,EAAE,GAAW;IAC3C,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC;IACtC,MAAM,GAAG,GAAG,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC;IAC3C,gFAAgF;IAChF,wEAAwE;IACxE,sFAAsF;IACtF,qFAAqF;IACrF,sFAAsF;IACtF,mFAAmF;IACnF,mFAAmF;IACnF,4BAA4B;IAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;IACtD,MAAM,aAAa,GACjB,IAAI,GAAG,kBAAkB,IAAI,CAAC,IAAI,IAAI,kBAAkB,IAAI,IAAI,IAAI,kBAAkB,CAAC,CAAC;IAC1F,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,iBAAiB,CAAC;AACrD,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,uBAAuB,CAAC;IACrF,OAAO,SAAS,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,wBAAwB,GAAG,EAAE,CAAC;AACpC,MAAM,yBAAyB,GAAG,IAAI,CAAC;AAEvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAAa;IACvD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,uBAAuB,CAAC;IACpF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,2BAA2B,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,yBAAyB,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,IAAI,CAAC,KAAc,EAAE,QAA6B,EAAE,KAAa;IACxE,IAAI,KAAK,GAAG,kBAAkB;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAChE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACtF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAc,EAAE,QAA6B;IAClF,OAAO,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;AAClC,CAAC"}

package/dist/skill-test/grading-adapter.d.ts

@@ -1,10 +1,12 @@
+import { VatError } from '@vibe-agent-toolkit/utils';
 /**
  * Thrown when grading.json does not match the canonical skill-creator shape
  * (see grading-schema.ts / skill-creator references/schemas.md). vat refuses to
  * limp along on malformed grading data — a wrong shape silently flowing
  * downstream causes confusing failures far from the real cause.
  */
-export declare class GradingSkewError extends Error {
+export declare class GradingSkewError extends VatError {
+    readonly reason: "internal";
     constructor(message: string);
 }
 export interface NormalizedGrading {
@@ -12,13 +14,31 @@ export interface NormalizedGrading {
         passed: number;
         total: number;
     };
+    /**
+     * Every graded expectation, flattened across all evals. `evalId` names the eval
+     * the entry came from — it is what lets a reader line the two `--baseline`
+     * artifacts up PER EVAL instead of only in aggregate. Optional for the same
+     * reason `runNonce` is: {@link parseGradingJson} builds a `NormalizedGrading`
+     * from an externally produced grading.json, which carries no such attribution.
+     */
     expectations: {
         text: string;
         passed: boolean;
         evidence?: string;
+        evalId?: string;
     }[];
     /** Per-run integrity nonce the experimenter copied from its prompt (if present). */
     runNonce?: string;
+    /**
+     * Which `--baseline` arm produced this report: `'with'` (the skill declared —
+     * grading.json) or `'without'` (the control — baseline.json). Stamped by
+     * {@link import('./fragment-merge.js').mergeFragmentsToGrading}, which is the
+     * only producer that knows the arm. Optional for the same reason `runNonce` is:
+     * {@link parseGradingJson} normalizes an externally produced grading.json that
+     * legitimately carries no arm, and inventing one there would be a lie about
+     * provenance rather than a missing field.
+     */
+    arm?: 'with' | 'without';
 }
 /**
  * Thrown when a per-eval grader fragment's integrity nonce is absent or does not
@@ -27,7 +47,23 @@ export interface NormalizedGrading {
  * prompted — most likely forged or left behind by untrusted skill code in the
  * shared sandbox — so the verdict merged from it cannot be trusted.
  */
-export declare class GradingNonceError extends Error {
+export declare class GradingNonceError extends VatError {
+    readonly reason: "internal";
+    constructor(message: string);
+}
+/**
+ * Thrown when a per-eval grader fragment's `arm` disagrees with the arm the
+ * merge was told it is assembling. Deliberately distinct from
+ * {@link GradingNonceError}: a fragment can carry a perfectly valid nonce for
+ * THIS run and still belong to the other arm, and that is a different failure —
+ * not a forgery, but a mix-up that would fold a control-arm verdict into the
+ * treatment number (or the reverse). Either way the merged total is no longer a
+ * measurement of one arm, which is the entire premise of the `--baseline` A/B,
+ * so the merge refuses rather than emit a mislabelled artifact that downstream
+ * readers would take as authoritative.
+ */
+export declare class GradingArmError extends VatError {
+    readonly reason: "internal";
     constructor(message: string);
 }
 export declare function parseGradingJson(raw: unknown): NormalizedGrading;

package/dist/skill-test/grading-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACrE,oFAAoF;IACpF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAS5B;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAwBhE;AAED,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,GAAG,cAAc,CAqB1E"}
+{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAKrD;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,QAAQ;IAC5C,QAAQ,CAAC,MAAM,EAAG,UAAU,CAAU;gBAC1B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C;;;;;;OAMG;IACH,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtF,oFAAoF;IACpF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;OAQG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,qBAAa,iBAAkB,SAAQ,QAAQ;IAC7C,QAAQ,CAAC,MAAM,EAAG,UAAU,CAAU;gBAC1B,OAAO,EAAE,MAAM;CAS5B;AAED;;;;;;;;;;GAUG;AACH,qBAAa,eAAgB,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,MAAM,EAAG,UAAU,CAAU;gBAC1B,OAAO,EAAE,MAAM;CAU5B;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAiDhE;AAED,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,GAAG,cAAc,CAqB1E"}

package/dist/skill-test/grading-adapter.js

@@ -1,3 +1,5 @@
+import { VatError } from '@vibe-agent-toolkit/utils';
+import { sanitizeGraderText } from './grader-text.js';
 import { GradingReportSchema } from './grading-schema.js';
 /**
  * Thrown when grading.json does not match the canonical skill-creator shape
@@ -5,12 +7,12 @@ import { GradingReportSchema } from './grading-schema.js';
  * limp along on malformed grading data — a wrong shape silently flowing
  * downstream causes confusing failures far from the real cause.
  */
-export class GradingSkewError extends Error {
+export class GradingSkewError extends VatError {
+    reason = 'internal';
     constructor(message) {
-        super(`grading.json shape skew: ${message}. Expected skill-creator's grading.json shape ` +
+        super('GRADING_SKEW', `grading.json shape skew: ${message}. Expected skill-creator's grading.json shape ` +
             '(a single flat object with top-level `expectations` and `summary`); see ' +
             'docs/skill-test-grading-schema.md. Re-sync the vendored skill-creator / adopted shapes.');
-        this.name = 'GradingSkewError';
     }
 }
 /**
@@ -20,13 +22,34 @@ export class GradingSkewError extends Error {
  * prompted — most likely forged or left behind by untrusted skill code in the
  * shared sandbox — so the verdict merged from it cannot be trusted.
  */
-export class GradingNonceError extends Error {
+export class GradingNonceError extends VatError {
+    reason = 'internal';
     constructor(message) {
-        super(`grader integrity check failed: ${message}. The harness stamps a secret per-run ` +
+        super('GRADING_NONCE', `grader integrity check failed: ${message}. The harness stamps a secret per-run ` +
             'nonce into each grader prompt (delivered only via stdin, never written to disk) ' +
             'and requires every per-eval grader fragment to echo it; a missing or wrong nonce ' +
             'means the fragment was not produced by the grader we prompted and is rejected.');
-        this.name = 'GradingNonceError';
+    }
+}
+/**
+ * Thrown when a per-eval grader fragment's `arm` disagrees with the arm the
+ * merge was told it is assembling. Deliberately distinct from
+ * {@link GradingNonceError}: a fragment can carry a perfectly valid nonce for
+ * THIS run and still belong to the other arm, and that is a different failure —
+ * not a forgery, but a mix-up that would fold a control-arm verdict into the
+ * treatment number (or the reverse). Either way the merged total is no longer a
+ * measurement of one arm, which is the entire premise of the `--baseline` A/B,
+ * so the merge refuses rather than emit a mislabelled artifact that downstream
+ * readers would take as authoritative.
+ */
+export class GradingArmError extends VatError {
+    reason = 'internal';
+    constructor(message) {
+        super('GRADING_ARM', `grader fragment arm mismatch: ${message}. Each --baseline arm is merged into its own ` +
+            'artifact (grading.json = the WITH arm, baseline.json = the WITHOUT/control arm), and ' +
+            'every fragment merged into one must come from that arm — a fragment with no `arm` ' +
+            "belongs to the default 'with' arm. Mixing arms would make the merged pass count a " +
+            'measurement of neither.');
     }
 }
 /**
@@ -48,19 +71,44 @@ export function parseGradingJson(raw) {
                 'grading.json must be ONE flat object whose top-level `expectations` lists every ' +
                 'graded expectation across all evals');
         }
+        // Both halves are attacker-reachable and land in an operator-facing message: a
+        // `.strict()` rejection puts the grader-CHOSEN KEY inside zod's own issue message,
+        // and `path` is built from that same key. Sanitize both — this is the route that
+        // has already been found twice in sibling modules, and it must not reopen here just
+        // because today's only callers are tests.
         const firstIssue = result.error.issues[0];
-        const path = firstIssue?.path.join('.') ?? '(root)';
-        throw new GradingSkewError(`missing/invalid field at "${path}" (${firstIssue?.message ?? 'unknown'})`);
+        const path = sanitizeGraderText(firstIssue?.path.join('.') ?? '(root)');
+        const detail = sanitizeGraderText(firstIssue?.message ?? 'unknown');
+        throw new GradingSkewError(`missing/invalid field at "${path}" (${detail})`);
     }
-    const { summary, expectations, runNonce } = result.data;
+    const { summary, expectations, runNonce, arm } = result.data;
+    // Pass `evalId`/`arm` through WHEN PRESENT. The docblocks on NormalizedGrading
+    // explain why they are optional — an externally produced grading.json legitimately
+    // carries no attribution, and inventing one would be a lie about provenance. That
+    // argues for omitting an ABSENT field, not for discarding a present one: the schema
+    // already parses both, so dropping them silently narrowed the declared type and
+    // threw away exactly the key a reader needs to line the two --baseline artifacts
+    // up per eval.
+    //
+    // 📌 STATE OF PLAY, so nobody mistakes this for load-bearing code: NOTHING IN
+    // SHIPPED CODE READS THESE TWO TODAY. `parseGradingJson` itself is reachable only
+    // from the package barrel and this module's own test file — the harness builds its
+    // NormalizedGrading from per-eval fragments, not by re-parsing an aggregate
+    // grading.json — so the two spreads are written-but-never-read outside tests. They
+    // stay anyway, because the cost of keeping a parsed field is zero and the cost of
+    // reopening the discard is a reader who cannot attribute an expectation to an eval
+    // or an arm. Treat "no production caller" as a reason to leave this alone, not as
+    // permission to delete it.
     return {
         summary: { passed: summary.passed, total: summary.total },
         expectations: expectations.map(e => ({
             text: e.text,
             passed: e.passed,
             ...(e.evidence === undefined ? {} : { evidence: e.evidence }),
+            ...(e.evalId === undefined ? {} : { evalId: e.evalId }),
         })),
         ...(runNonce === undefined ? {} : { runNonce }),
+        ...(arm === undefined ? {} : { arm }),
     };
 }
 /**

package/dist/skill-test/grading-adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CACH,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AASD;;;;;;GAMG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CACH,kCAAkC,OAAO,wCAAwC;YAC/E,kFAAkF;YAClF,mFAAmF;YACnF,gFAAgF,CACnF,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IACxD,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;QACH,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;KAChD,CAAC;AACJ,CAAC;AASD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;IAEjD,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,gBAAgB,CACxB,2FAA2F;YACzF,iFAAiF,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACzC,IAAI,MAAM,KAAK,cAAc,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;QACzD,MAAM,IAAI,gBAAgB,CACxB,yEAAyE,MAAM,UAAU,KAAK,IAAI;YAChG,yCAAyC,cAAc,UAAU,aAAa,KAAK;YACnF,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,KAAK,aAAa,EAAE,CAAC;AACvG,CAAC"}
+{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,QAAQ;IACnC,MAAM,GAAG,UAAmB,CAAC;IACtC,YAAY,OAAe;QACzB,KAAK,CACH,cAAc,EACd,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;IACJ,CAAC;CACF;AA0BD;;;;;;GAMG;AACH,MAAM,OAAO,iBAAkB,SAAQ,QAAQ;IACpC,MAAM,GAAG,UAAmB,CAAC;IACtC,YAAY,OAAe;QACzB,KAAK,CACH,eAAe,EACf,kCAAkC,OAAO,wCAAwC;YAC/E,kFAAkF;YAClF,mFAAmF;YACnF,gFAAgF,CACnF,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAClC,MAAM,GAAG,UAAmB,CAAC;IACtC,YAAY,OAAe;QACzB,KAAK,CACH,aAAa,EACb,iCAAiC,OAAO,+CAA+C;YACrF,uFAAuF;YACvF,oFAAoF;YACpF,oFAAoF;YACpF,yBAAyB,CAC5B,CAAC;IACJ,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,+EAA+E;QAC/E,mFAAmF;QACnF,iFAAiF;QACjF,oFAAoF;QACpF,0CAA0C;QAC1C,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,kBAAkB,CAAC,UAAU,EAAE,OAAO,IAAI,SAAS,CAAC,CAAC;QACpE,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,MAAM,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7D,+EAA+E;IAC/E,mFAAmF;IACnF,kFAAkF;IAClF,oFAAoF;IACpF,gFAAgF;IAChF,iFAAiF;IACjF,eAAe;IACf,EAAE;IACF,8EAA8E;IAC9E,kFAAkF;IAClF,mFAAmF;IACnF,4EAA4E;IAC5E,mFAAmF;IACnF,kFAAkF;IAClF,mFAAmF;IACnF,kFAAkF;IAClF,2BAA2B;IAC3B,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC7D,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;SACxD,CAAC,CAAC;QACH,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC;KACtC,CAAC;AACJ,CAAC;AASD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;IAEjD,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,gBAAgB,CACxB,2FAA2F;YACzF,iFAAiF,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACzC,IAAI,MAAM,KAAK,cAAc,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;QACzD,MAAM,IAAI,gBAAgB,CACxB,yEAAyE,MAAM,UAAU,KAAK,IAAI;YAChG,yCAAyC,cAAc,UAAU,aAAa,KAAK;YACnF,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,KAAK,aAAa,EAAE,CAAC;AACvG,CAAC"}