@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.3

package/dist/skill-test/eval-grader.js

@@ -3,6 +3,7 @@ import { mkdirSyncReal, parseStreamJsonTranscript, safePath, spawnHeadlessClaude
 import { EvalFragmentError, parseEvalFragment } from './eval-fragment.js';
 import { InternalHarnessError } from './exit-codes.js';
 import { assertGraderPromptInvariants, buildGraderPrompt } from './grader-prompt.js';
+import { sanitizeGraderText } from './grader-text.js';
 import { GradingNonceError } from './grading-adapter.js';
 import { computeToolPassed } from './tool-eval-schema.js';
 /**
@@ -40,7 +41,16 @@ function fragmentWarnRouter(onProgress) {
     return (message) => onProgress(`[skill-test] ${message}\n`);
 }
 export async function runGraderForEval(input) {
-    mkdirSyncReal(input.graderOutDir, { recursive: true });
+    // 0700, like every other vat-only directory this codebase creates (the grader
+    // root in `resolveGraderOutDir`, the workspaces root and every eval workspace in
+    // `stageEvalWorkspaces`). This is the per-ARM subdirectory of an already-0700
+    // parent, so nothing is exposed today — traversal into it is blocked one level
+    // up. It is made explicit because the rule is only worth anything while it is
+    // uniform: an unmoded `mkdir` here is the one place a reader would have to prove
+    // the parent's mode to know the fragment (which echoes the run's integrity nonce)
+    // is unreadable, and the day this dir is created somewhere else that proof is
+    // gone. Fragments are consumed-on-read anyway; this is the resting-state half.
+    mkdirSyncReal(input.graderOutDir, { recursive: true, mode: 0o700 });
     // evalId is validated `[A-Za-z0-9_-]+` upstream (eval-inputs.ts) so a plain
     // join is safe, but `joinUnderRoot` is used defensively — it costs nothing
     // and closes off any future path where an unvalidated id reaches here.
@@ -58,10 +68,14 @@ export async function runGraderForEval(input) {
     });
     // Defense-in-depth (parity with eval-executor.ts's build→assert→use pattern):
     // re-verify the built grader prompt still carries its required invariants
-    // (STOP, fragment path, browser/iteration forbids, nonce directive) before
-    // spawning, so a regressed buildGraderPrompt is caught here, not by a grader
-    // that silently misbehaves.
-    assertGraderPromptInvariants(prompt, input.transcript);
+    // (STOP, fragment path, browser/iteration forbids, nonce directive, and the
+    // untrusted-data fences) before spawning, so a regressed buildGraderPrompt is
+    // caught here, not by a grader that silently misbehaves. The nonce is handed
+    // over so the assertion can cut out every fenced region — transcript, subject
+    // manifest, and the suite's own expectations/expected_output — and none of
+    // that attacker-controlled text can satisfy an invariant on the builder's
+    // behalf.
+    assertGraderPromptInvariants(prompt, input.nonce);
     const spawn = input.spawn ?? spawnHeadlessClaude;
     // Accumulate the grader's stream-json stdout so we can read its terminal
     // `total_cost_usd` for run-wide spend aggregation (adopter follow-up). Held in
@@ -117,8 +131,16 @@ export async function runGraderForEval(input) {
     if (fragment.runNonce !== input.nonce) {
         throw new GradingNonceError(`eval "${input.evalId}" fragment \`runNonce\` does not match this run`);
     }
+    assertExpectationCountDeclared(input.evalId, input.expectations, fragment.expectations);
     assertToolVerdictConsistent(input.evalId, input.toolExpectations, fragment.tool);
-    return fragment;
+    // The grader's only input is the executor transcript, which untrusted skill code
+    // controls, and `evalId` is schema-typed as any non-empty string. It is echoed
+    // verbatim into the run summary and into `baseline.json` — including the
+    // CONTAMINATED banner — so a grader talked into emitting an id containing
+    // newlines and ANSI escapes can paint a reassuring "known false positive, the
+    // delta is valid" line directly beneath vat's own warning. VAT knows which eval
+    // it asked about; take the id from the request, never from the answer.
+    return { ...fragment, evalId: input.evalId };
 }
 /**
  * Read the grader's fragment JSON into memory and unlink it immediately —
@@ -146,7 +168,16 @@ function readAndConsumeFragmentFile(fragmentOut, evalId, status) {
         raw = JSON.parse(readFileSync(fragmentOut, 'utf-8'));
     }
     catch (err) {
-        throw new EvalFragmentError(`grader fragment for eval "${evalId}" at ${fragmentOut} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
+        // V8 embeds a VERBATIM slice of the offending bytes in its SyntaxError
+        // message, unescaped — and this is one of the few grader-controlled strings
+        // that reaches an operator WITHOUT passing `parseEvalFragment`, the
+        // documented text boundary, because the parse it would have gone through is
+        // the one that just failed. The CLI writes it as `Error: ${err.message}`, so
+        // an unsanitized fragment of `ESC[2K CR ESC[32m…` wipes the line vat just
+        // printed and continues in vat's own colour (and a <=16-byte file gets its
+        // whole `ESC[2J ESC[H` echoed, clearing the screen).
+        throw new EvalFragmentError(`grader fragment for eval "${evalId}" at ${fragmentOut} is not valid JSON: ` +
+            sanitizeGraderText(err instanceof Error ? err.message : String(err)));
     }
     try {
         // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
@@ -157,6 +188,57 @@ function readAndConsumeFragmentFile(fragmentOut, evalId, status) {
     }
     return raw;
 }
+/**
+ * The OUTPUT channel's declared-vs-reported check — the exact thing
+ * {@link assertToolCheckNamesDeclared} does for the TOOL channel, applied to the
+ * channel every reported number is computed from.
+ *
+ * Each arm's `summary`, the pass/fail verdict `reconcileGrading` produces, and
+ * the `--baseline` delta between the arms are all computed over
+ * `fragment.expectations` — i.e. over however many entries the grader chose to
+ * write. The prompt ASKS for "one per expectation above" and, until this check,
+ * nothing verified it:
+ *
+ * - **Omission laundered a pass, on EVERY run.** A grader that skips a FAILING
+ *   expectation leaves a fragment where the survivors all passed, so
+ *   `reconcileGrading` reports `allPassed: true` and the run exits 0.
+ * - **Invention inflated the delta.** A suite declaring 1 expectation whose two
+ *   graders each emit 3 of their own yields `Baseline delta: +3` where the
+ *   maximum legal lift is +1.
+ *
+ * The existing `armExpectationSkew` cannot see either: it compares the two ARMS
+ * to each other, and both graders receive the same declared list from the same
+ * template, so they drift together and parity holds while a confident wrong
+ * number prints.
+ *
+ * COUNT, not text. The grader is asked to echo each expectation's `text`, but it
+ * routinely reworks the wording, and a reworded text is not a wrong verdict —
+ * whereas a missing or extra ENTRY always is, because the entry is the unit
+ * every score counts.
+ *
+ * Fail-closed like its siblings below: this is vat's own grading infrastructure
+ * failing its one job, not adopter data being audited.
+ *
+ * FAIL-CLOSED IS NOT FAIL-THE-RUN, and the distinction is the caller's to make, not
+ * this function's. Throwing here is unconditional and stays that way: a fragment
+ * whose entry count does not match the declaration is not a verdict, and returning
+ * it "leniently" for the control arm would put an unchecked number straight into the
+ * delta — which is the hole this check was added to close. What changed is where the
+ * throw lands. `runEvalWorker` (run-harness.ts) catches a CONTROL-arm throw, records
+ * it as a control-arm failure, and lets the run finish; a TREATMENT-arm throw still
+ * propagates and fails the run. So a control-arm grader miscount now withholds the
+ * delta as `null` with the reason attached, instead of destroying fully-billed
+ * treatment work that was never in doubt. Do not soften this assert to get that
+ * behaviour — it is already there, one level up, where the arm is known.
+ */
+function assertExpectationCountDeclared(evalId, declared, reported) {
+    if (reported.length === declared.length)
+        return;
+    throw new InternalHarnessError(`Grader for eval "${evalId}" returned ${reported.length} expectation entr${reported.length === 1 ? 'y' : 'ies'} ` +
+        `for an eval that declares ${declared.length}. Every score vat reports — each arm's summary and the ` +
+        `--baseline delta between them — is computed over the entries the grader emitted, so an invented or ` +
+        `omitted entry silently changes it.`);
+}
 /**
  * Fail-open guards on the grader's tool verdict, split out of
  * {@link runGraderForEval} to keep its cognitive complexity within budget:
@@ -171,6 +253,8 @@ function readAndConsumeFragmentFile(fragmentOut, evalId, status) {
  *   that disagrees with its own sub-checks is grader malfunction (a false green
  *   or a false red), surfaced loudly rather than merged. Mirrors
  *   `reconcileGrading`'s summary/expectations reconciliation (grading-adapter.ts).
+ * - Fix #4: the check NAMES must be exactly the eval's declared ones. See
+ *   {@link assertToolCheckNamesDeclared}.
  */
 function assertToolVerdictConsistent(evalId, toolExpectations, tool) {
     if (toolExpectations !== undefined && tool === undefined) {
@@ -180,5 +264,57 @@ function assertToolVerdictConsistent(evalId, toolExpectations, tool) {
         throw new InternalHarnessError(`Grader for eval "${evalId}" emitted a tool verdict whose \`passed\` (${tool.passed}) ` +
             `disagrees with its own sub-checks (recomputed ${computeToolPassed(tool)}).`);
     }
+    assertToolCheckNamesDeclared(evalId, toolExpectations, tool);
+}
+/** Render grader-supplied names for an error message, already sanitized by the fragment parse. */
+function quoteNames(names) {
+    return names.map((name) => `"${name}"`).join(', ');
+}
+/**
+ * One channel's check names must be EXACTLY the declared ones — no invention,
+ * no omission.
+ *
+ * `computeToolPassed` iterates the checks the GRADER emitted, not the ones the
+ * eval declared, so both directions are load-bearing and neither is cosmetic:
+ *
+ * - **Omission laundered a pass.** Drop the `mustRun` entry for an executable
+ *   that never ran and the channel is vacuously true — the verdict reads
+ *   `passed: true` for an expectation nobody checked.
+ * - **Invention gates on a name vat never asked about.** A grader talked into
+ *   emitting `{name: "totally-fine", ran: true}` adds a check the eval does not
+ *   declare, and it counts toward the verdict exactly like a real one.
+ *
+ * Names are compared AFTER {@link sanitizeGraderText} on both sides. The
+ * reported side is already sanitized (parseEvalFragment); normalizing the
+ * declared side too means the comparison happens in one space, so an adopter
+ * who declared a name containing a tab does not get an unexplainable mismatch.
+ *
+ * Fail-closed like its siblings above: this is vat's own grading infrastructure
+ * failing to do its one job, not adopter data being audited.
+ */
+function assertToolCheckNamesDeclared(evalId, toolExpectations, tool) {
+    if (tool === undefined)
+        return;
+    const declared = toolExpectations ?? {};
+    assertChannelNames(evalId, 'mustRun', declared.mustRun, (tool.mustRun ?? []).map((c) => c.name));
+    assertChannelNames(evalId, 'mustNotRun', declared.mustNotRun, (tool.mustNotRun ?? []).map((c) => c.name));
+    assertChannelNames(evalId, 'mustSucceed', declared.mustSucceed, (tool.mustSucceed ?? []).map((c) => c.name));
+    assertChannelNames(evalId, 'sequence', declared.sequence, (tool.sequence ?? []).flatMap((c) => c.steps));
+}
+function assertChannelNames(evalId, channel, declared, reported) {
+    const declaredSet = new Set((declared ?? []).map(sanitizeGraderText));
+    const reportedSet = new Set(reported);
+    const invented = [...reportedSet].filter((name) => !declaredSet.has(name));
+    const missing = [...declaredSet].filter((name) => !reportedSet.has(name));
+    if (invented.length === 0 && missing.length === 0)
+        return;
+    const problems = [];
+    if (invented.length > 0)
+        problems.push(`invented ${quoteNames(invented)}`);
+    if (missing.length > 0)
+        problems.push(`omitted ${quoteNames(missing)}`);
+    throw new InternalHarnessError(`Grader for eval "${evalId}" emitted \`tool.${channel}\` checks that do not match the eval's ` +
+        `declared toolExpectations: ${problems.join('; ')}. The verdict is computed from the checks the ` +
+        `grader emitted, so an invented or omitted name silently changes it.`);
 }
 //# sourceMappingURL=eval-grader.js.map

package/dist/skill-test/eval-grader.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEpH,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAgD1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH;;;;;GAKG;AACH,SAAS,kBAAkB,CACzB,UAAwC;IAExC,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC/C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,8EAA8E;IAC9E,0EAA0E;IAC1E,2EAA2E;IAC3E,6EAA6E;IAC7E,4BAA4B;IAC5B,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAE9E,6EAA6E;IAC7E,6EAA6E;IAC7E,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,0BAA0B,CAAC,WAAmB,EAAE,MAAc,EAAE,MAAc;IACrF,kIAAkI;IAClI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,kIAAkI;QAClI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,iBAAiB,CACzB,6BAA6B,MAAM,QAAQ,WAAW,uBAAuB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAChI,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,kIAAkI;QAClI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,sDAAsD;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEpH,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAgD1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH;;;;;GAKG;AACH,SAAS,kBAAkB,CACzB,UAAwC;IAExC,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC/C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,8EAA8E;IAC9E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,kFAAkF;IAClF,8EAA8E;IAC9E,+EAA+E;IAC/E,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,8EAA8E;IAC9E,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,UAAU;IACV,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAE9E,6EAA6E;IAC7E,6EAA6E;IAC7E,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IACxF,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,iFAAiF;IACjF,+EAA+E;IAC/E,yEAAyE;IACzE,0EAA0E;IAC1E,8EAA8E;IAC9E,gFAAgF;IAChF,uEAAuE;IACvE,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;AAC/C,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,0BAA0B,CAAC,WAAmB,EAAE,MAAc,EAAE,MAAc;IACrF,kIAAkI;IAClI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,kIAAkI;QAClI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,4EAA4E;QAC5E,oEAAoE;QACpE,4EAA4E;QAC5E,6EAA6E;QAC7E,0EAA0E;QAC1E,2EAA2E;QAC3E,qDAAqD;QACrD,MAAM,IAAI,iBAAiB,CACzB,6BAA6B,MAAM,QAAQ,WAAW,sBAAsB;YAC1E,kBAAkB,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,kIAAkI;QAClI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,sDAAsD;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA2B,EAC3B,QAAsC;IAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO;IAChD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,cAAc,QAAQ,CAAC,MAAM,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;QAC/G,6BAA6B,QAAQ,CAAC,MAAM,yDAAyD;QACrG,qGAAqG;QACrG,oCAAoC,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAwB;IAC1C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,4BAA4B,CACnC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,MAAM,QAAQ,GAAG,gBAAgB,IAAI,EAAE,CAAC;IACxC,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7G,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAc,EACd,OAAe,EACf,QAAuC,EACvC,QAA2B;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC1D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,oBAAoB,OAAO,yCAAyC;QAC5F,8BAA8B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD;QACjG,qEAAqE,CACxE,CAAC;AACJ,CAAC"}

package/dist/skill-test/eval-inputs.d.ts

@@ -1,8 +1,48 @@
 import { z } from 'zod';
-/** Raised for any eval-input problem (bad JSON, schema failure, missing input file). Maps to exit 2. */
+/**
+ * Raised for any eval-input problem (bad JSON, schema failure, missing input file).
+ * Maps to exit 2.
+ *
+ * The FIELD is what makes that sentence true, and until it was added only one of this
+ * class's two routes delivered it. `attemptStageWorkspaces` catches the staging
+ * instance and maps it explicitly; the instance thrown from {@link armDirSegment}
+ * fires deep in the eval loop, outside that handler, and reached
+ * `mapErrorToExitCode`'s Internal default — reporting 1 for a preflight-class
+ * problem while the docblock above claimed 2. Exit 2 for the internal-invariant case
+ * too is consistent with every sibling in this feature.
+ */
 export declare class EvalInputError extends Error {
+    readonly exitCode: 2;
     constructor(message: string);
 }
+/**
+ * Neutralize a SUITE-AUTHORED string before it is quoted into an
+ * {@link EvalInputError}, whose message reaches `process.stdout` on the
+ * `Summary:` line — the one channel deliberately kept machine-readable — and
+ * renders twice on the way there.
+ *
+ * The suite is nominally adopter-authored, but `resolveEvalSuitePath` will
+ * harvest one out of a FETCHED npm/url artifact, i.e. out of the skill under
+ * test, so treat it as untrusted. `files[]` is the sharp edge: it is
+ * `z.array(z.string().min(1))` with NO charset constraint, in deliberate
+ * contrast to `id`, which is regex-pinned precisely because it names a
+ * directory. A derived path or an OS error text built from such an entry
+ * carries the same bytes, so those are sanitized here too, not just the entry.
+ *
+ * Reuses {@link sanitizeGraderText} rather than growing a second neutralizer:
+ * the threat and the remedy (drop escape sequences whole, fold every remaining
+ * control code to a space, collapse, cap) are identical — only the source of
+ * the untrusted text differs.
+ *
+ * EXPORTED because the suite reaches an operator surface from more than this
+ * module: `eval-lint.ts` quotes `toolExpectations.*` entries — the same
+ * unconstrained `z.array(z.string().min(1))` shape as `files[]` — into an advisory
+ * that `run-harness.ts` writes to stderr verbatim, and does it at Step 5.5, ahead
+ * of both the security-ack gate and the `--dry-run` short-circuit. Every new
+ * suite-authored string bound for a terminal goes through HERE, so the trust model
+ * above stays stated in one place.
+ */
+export declare function quoteSuiteText(value: string): string;
 /**
  * True when `key` is exactly one edit (insert/delete/substitute one char) from
  * `target`. Used to catch typos of recognized fields (here) and of declared
@@ -496,19 +536,80 @@ export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
 export type ToolExpectations = NonNullable<EvalEntry['toolExpectations']>;
 /** Parse + validate a skill-test eval suite. Throws {@link EvalInputError} on any problem. */
 export declare function parseEvalSuite(jsonText: string): EvalSuite;
+/**
+ * One side of a `--baseline` A/B: `'with'` runs the skill staged and declared,
+ * `'without'` is the control. Declared here rather than in the run orchestrator
+ * because the staging layer needs it to give each arm its own workspace tree.
+ */
+export type EvalArm = 'with' | 'without';
+/**
+ * The opaque directory segment each arm's workspaces live under, minted fresh per
+ * run. `with` is required and `without` present only under `--baseline`, so the
+ * set of arms is the keys — a shape that cannot express "stage nothing", unlike
+ * the `readonly EvalArm[]` this replaced (an empty array was a legal input that
+ * silently left every executor with a non-existent cwd).
+ *
+ * ⚠️ The VALUES must not be the literal arm names, and must not be derivable from
+ * each other. See {@link stageEvalWorkspaces}.
+ */
+export interface ArmWorkspaceDirs {
+    with: string;
+    without?: string;
+}
+/** The arms an {@link ArmWorkspaceDirs} covers, in dispatch order. */
+export declare function armsOf(dirs: ArmWorkspaceDirs): readonly EvalArm[];
+/**
+ * The directory segment for one arm. Throws rather than falling back to the arm
+ * NAME: a silent fallback would reinstate the unblinding this indirection exists
+ * to remove, and would do it invisibly.
+ */
+export declare function armDirSegment(dirs: ArmWorkspaceDirs, arm: EvalArm): string;
 export interface StageEvalWorkspacesInput {
     /** Parsed suite (Task 1). */
     suite: EvalSuite;
     /** Directory containing evals.json — the base for each eval's relative `files`. */
     evalsDir: string;
-    /** `<harnessRoot>/workspaces` — per-eval dirs are created beneath it. */
+    /** Per-run workspaces root — per-arm, per-eval dirs are created beneath it. */
     workspacesRoot: string;
+    /**
+     * Opaque per-arm directory segments. Each arm gets its OWN copy of every
+     * workspace; see the note on the layout below for why that is not an
+     * optimization to undo.
+     */
+    armDirs: ArmWorkspaceDirs;
 }
 /**
- * Materialize each eval's declared input `files` into `<workspacesRoot>/<id>/<relpath>`,
- * preserving relative structure. Evals without `files` are skipped. Throws
- * {@link EvalInputError} if a listed file does not exist (the eval cannot run without it).
- * Returns `workspacesRoot`.
+ * Materialize each eval's declared input `files` into
+ * `<workspacesRoot>/<armDir>/<id>/<relpath>`, preserving relative structure. Throws
+ * {@link EvalInputError} if a listed file does not exist (the eval cannot run
+ * without it). Returns `workspacesRoot`.
+ *
+ * EVERY eval gets a directory, including one that declares no `files` — it is
+ * simply left empty. An eval without a workspace used to leave the executor with
+ * no cwd of its own, so it ran inside the staged subject dir instead; for the
+ * skill-absent arm of a `--baseline` run that placed the control's cwd inside the
+ * very skill the arm exists to withhold. An empty dir costs nothing and keeps cwd
+ * shaped identically across both arms, which is what makes the A/B a comparison.
+ *
+ * WHY THE `<arm>` SEGMENT. The two arms of one eval are dispatched ADJACENTLY into
+ * a bounded-parallel pool, so they run at the same time. When they shared one
+ * directory, the control arm could simply `ls` and read whatever the treatment arm
+ * had just written — recovering the treatment's answer without ever touching the
+ * skill, and without producing a single detectable harness path. That is a channel
+ * BETWEEN the arms rather than into one of them, which is why auditing the four
+ * channels into a process (prompt, argv, cwd, env) did not surface it: each arm's
+ * own cwd was correct in isolation, and wrong as a pair. The arms must be
+ * byte-identical at start and unable to observe each other after it.
+ *
+ * WHY THE SEGMENT IS AN OPAQUE TOKEN and not the arm's name. It used to be the
+ * literal `with`/`without`, which made the executor's own working directory the
+ * loudest possible unblinding: the prompt tells each arm `Your working directory
+ * is …/vat-skill-test-ws-<token>/without/e1`, so the skill-absent arm was told, in
+ * the one string it cannot avoid reading, which side of the experiment it was on —
+ * while `buildExecutorPrompt`'s docblock promises it "must NEVER be told (directly
+ * or by implication) that it is being tested, evaluated, or graded". It also named
+ * the sibling: `../with/<id>/` was a guessable path to the treatment arm's live
+ * output. Independent random tokens fix both, and cost nothing.
  */
 export declare function stageEvalWorkspaces(input: StageEvalWorkspacesInput): string;
 //# sourceMappingURL=eval-inputs.d.ts.map

package/dist/skill-test/eval-inputs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAyBD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAWD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAsDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAmB1D;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAoC3E"}
+{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AA2CD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAiDD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAyDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAsB1D;AAED;;;;GAIG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAEzC;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sEAAsE;AACtE,wBAAgB,MAAM,CAAC,IAAI,EAAE,gBAAgB,GAAG,SAAS,OAAO,EAAE,CAEjE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAU1E;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAK3E"}

package/dist/skill-test/eval-inputs.js

@@ -1,13 +1,73 @@
 import { cpSync, existsSync } from 'node:fs';
 import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
 import { z } from 'zod';
-/** Raised for any eval-input problem (bad JSON, schema failure, missing input file). Maps to exit 2. */
+import { sanitizeGraderText, sanitizeTextPreservingLines } from './grader-text.js';
+/**
+ * Raised for any eval-input problem (bad JSON, schema failure, missing input file).
+ * Maps to exit 2.
+ *
+ * The FIELD is what makes that sentence true, and until it was added only one of this
+ * class's two routes delivered it. `attemptStageWorkspaces` catches the staging
+ * instance and maps it explicitly; the instance thrown from {@link armDirSegment}
+ * fires deep in the eval loop, outside that handler, and reached
+ * `mapErrorToExitCode`'s Internal default — reporting 1 for a preflight-class
+ * problem while the docblock above claimed 2. Exit 2 for the internal-invariant case
+ * too is consistent with every sibling in this feature.
+ */
 export class EvalInputError extends Error {
+    exitCode = 2;
     constructor(message) {
         super(message);
         this.name = 'EvalInputError';
     }
 }
+/**
+ * Neutralize a SUITE-AUTHORED string before it is quoted into an
+ * {@link EvalInputError}, whose message reaches `process.stdout` on the
+ * `Summary:` line — the one channel deliberately kept machine-readable — and
+ * renders twice on the way there.
+ *
+ * The suite is nominally adopter-authored, but `resolveEvalSuitePath` will
+ * harvest one out of a FETCHED npm/url artifact, i.e. out of the skill under
+ * test, so treat it as untrusted. `files[]` is the sharp edge: it is
+ * `z.array(z.string().min(1))` with NO charset constraint, in deliberate
+ * contrast to `id`, which is regex-pinned precisely because it names a
+ * directory. A derived path or an OS error text built from such an entry
+ * carries the same bytes, so those are sanitized here too, not just the entry.
+ *
+ * Reuses {@link sanitizeGraderText} rather than growing a second neutralizer:
+ * the threat and the remedy (drop escape sequences whole, fold every remaining
+ * control code to a space, collapse, cap) are identical — only the source of
+ * the untrusted text differs.
+ *
+ * EXPORTED because the suite reaches an operator surface from more than this
+ * module: `eval-lint.ts` quotes `toolExpectations.*` entries — the same
+ * unconstrained `z.array(z.string().min(1))` shape as `files[]` — into an advisory
+ * that `run-harness.ts` writes to stderr verbatim, and does it at Step 5.5, ahead
+ * of both the security-ack gate and the `--dry-run` short-circuit. Every new
+ * suite-authored string bound for a terminal goes through HERE, so the trust model
+ * above stays stated in one place.
+ */
+export function quoteSuiteText(value) {
+    return sanitizeGraderText(value);
+}
+/**
+ * The same neutralization for a suite-derived string that is ALREADY multi-line
+ * and whose lines carry the meaning — today, exactly one: zod's schema-failure
+ * text.
+ *
+ * That message is a list of issues, one per offending path, and every one of the
+ * paths in it is a SUITE-AUTHORED KEY (`evals[17].toolExpecations`) reaching
+ * stderr unsanitized. It cannot go through {@link quoteSuiteText}: that collapses
+ * the list onto one capped line, which is the difference between "your suite has
+ * a typo, here is where" and "your suite has a typo". Degrading the common case to
+ * neutralize the rare one is a bad trade, so the sanitizer that keeps lines is
+ * used instead. See {@link sanitizeTextPreservingLines} for why it is not the
+ * default anywhere else.
+ */
+function quoteSuiteBlock(value) {
+    return sanitizeTextPreservingLines(value);
+}
 /**
  * String eval ids name a per-eval working directory ({@link stageEvalWorkspaces}),
  * so they must be safe path segments on every platform. Letters, digits, hyphen,
@@ -52,6 +112,42 @@ export function isSingleEditAway(key, target) {
         return key.slice(i + 1) === target.slice(i); // deletion from key
     return key.slice(i) === target.slice(i + 1); // insertion into key
 }
+/**
+ * Reject an expectation string an eval declares TWICE.
+ *
+ * The entry, not the text, is the unit every reported score counts:
+ * `runGraderForEval`'s declared-vs-graded check compares `fragment.expectations`
+ * against `expectations.length`, and each arm's summary and the `--baseline` delta
+ * are computed over those entries. A suite that declares the same string twice
+ * hands the grader "1. x / 2. x" and gets ONE entry back for it — a perfectly
+ * reasonable grader response, and one that fails the count check mid-run with an
+ * `InternalHarnessError`, destroying a fully-billed treatment run over an input
+ * defect nothing looked at.
+ *
+ * So it is caught at PARSE, exactly like the duplicate-id rule in
+ * {@link parseEvalSuite}: exit 2, user-correctable, before anything is spawned.
+ * Scoped to one eval — two different evals may legitimately share expectation
+ * text, since each is graded from its own transcript.
+ *
+ * `expectations` is `z.array(z.string().min(1))` with no charset constraint and
+ * the message reaches stdout, so the offending string is sanitized on the way in.
+ */
+function addDuplicateExpectationIssues(expectations, ctx) {
+    const seen = new Set();
+    for (const [index, expectation] of expectations.entries()) {
+        if (!seen.has(expectation)) {
+            seen.add(expectation);
+            continue;
+        }
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `duplicate expectation "${quoteSuiteText(expectation)}" — an eval must declare each expectation ` +
+                'once. Every reported score counts fragment ENTRIES, and a grader handed the same text twice ' +
+                'returns one entry for it, which fails the declared-count check mid-run.',
+            path: ['expectations', index],
+        });
+    }
+}
 // evals.json is adopter-authored input that VAT *reads* — so per the project's
 // Postel's Law (read the outside world liberally), we validate only the fields
 // VAT actually consumes and pass everything else through untouched. `id` accepts
@@ -101,6 +197,7 @@ export const EvalEntrySchema = z
 })
     .passthrough()
     .superRefine((entry, ctx) => {
+    addDuplicateExpectationIssues(entry.expectations, ctx);
     for (const key of Object.keys(entry)) {
         if (RECOGNIZED_EVAL_FIELDS.includes(key))
             continue;
@@ -108,7 +205,9 @@ export const EvalEntrySchema = z
         if (near !== undefined) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: `unknown eval field "${key}" — did you mean "${near}"? (other custom fields are allowed and ignored)`,
+                // `key` is a suite-authored object key with no charset constraint, and
+                // this message is surfaced through `result.error.message` below.
+                message: `unknown eval field "${quoteSuiteText(key)}" — did you mean "${near}"? (other custom fields are allowed and ignored)`,
                 path: [key],
             });
         }
@@ -128,11 +227,14 @@ export function parseEvalSuite(jsonText) {
         raw = JSON.parse(jsonText);
     }
     catch (e) {
-        throw new EvalInputError(`evals.json is not valid JSON: ${e.message}`);
+        // V8 quotes a verbatim, unescaped slice of the offending bytes into its
+        // SyntaxError message — a suite file that is nothing but ANSI escapes would
+        // otherwise repaint the operator's terminal from vat's own error line.
+        throw new EvalInputError(`evals.json is not valid JSON: ${quoteSuiteText(e.message)}`);
     }
     const result = EvalSuiteSchema.safeParse(raw);
     if (!result.success) {
-        throw new EvalInputError(`evals.json failed schema validation: ${result.error.message}`);
+        throw new EvalInputError(`evals.json failed schema validation: ${quoteSuiteBlock(result.error.message)}`);
     }
     // Compare ids as strings: each id names a working directory via String(id),
     // so a numeric `1` and a string `"1"` would collide on disk even though they
@@ -143,18 +245,72 @@ export function parseEvalSuite(jsonText) {
     }
     return result.data;
 }
+/** The arms an {@link ArmWorkspaceDirs} covers, in dispatch order. */
+export function armsOf(dirs) {
+    return dirs.without === undefined ? ['with'] : ['with', 'without'];
+}
+/**
+ * The directory segment for one arm. Throws rather than falling back to the arm
+ * NAME: a silent fallback would reinstate the unblinding this indirection exists
+ * to remove, and would do it invisibly.
+ */
+export function armDirSegment(dirs, arm) {
+    const segment = arm === 'with' ? dirs.with : dirs.without;
+    if (segment === undefined || segment === '') {
+        throw new EvalInputError(`Internal: no workspace directory was minted for the "${arm}" arm. ` +
+            'Every arm this run dispatches must have one; falling back to the arm name would ' +
+            'tell the executor which side of the A/B it is on.');
+    }
+    return segment;
+}
 /**
- * Materialize each eval's declared input `files` into `<workspacesRoot>/<id>/<relpath>`,
- * preserving relative structure. Evals without `files` are skipped. Throws
- * {@link EvalInputError} if a listed file does not exist (the eval cannot run without it).
- * Returns `workspacesRoot`.
+ * Materialize each eval's declared input `files` into
+ * `<workspacesRoot>/<armDir>/<id>/<relpath>`, preserving relative structure. Throws
+ * {@link EvalInputError} if a listed file does not exist (the eval cannot run
+ * without it). Returns `workspacesRoot`.
+ *
+ * EVERY eval gets a directory, including one that declares no `files` — it is
+ * simply left empty. An eval without a workspace used to leave the executor with
+ * no cwd of its own, so it ran inside the staged subject dir instead; for the
+ * skill-absent arm of a `--baseline` run that placed the control's cwd inside the
+ * very skill the arm exists to withhold. An empty dir costs nothing and keeps cwd
+ * shaped identically across both arms, which is what makes the A/B a comparison.
+ *
+ * WHY THE `<arm>` SEGMENT. The two arms of one eval are dispatched ADJACENTLY into
+ * a bounded-parallel pool, so they run at the same time. When they shared one
+ * directory, the control arm could simply `ls` and read whatever the treatment arm
+ * had just written — recovering the treatment's answer without ever touching the
+ * skill, and without producing a single detectable harness path. That is a channel
+ * BETWEEN the arms rather than into one of them, which is why auditing the four
+ * channels into a process (prompt, argv, cwd, env) did not surface it: each arm's
+ * own cwd was correct in isolation, and wrong as a pair. The arms must be
+ * byte-identical at start and unable to observe each other after it.
+ *
+ * WHY THE SEGMENT IS AN OPAQUE TOKEN and not the arm's name. It used to be the
+ * literal `with`/`without`, which made the executor's own working directory the
+ * loudest possible unblinding: the prompt tells each arm `Your working directory
+ * is …/vat-skill-test-ws-<token>/without/e1`, so the skill-absent arm was told, in
+ * the one string it cannot avoid reading, which side of the experiment it was on —
+ * while `buildExecutorPrompt`'s docblock promises it "must NEVER be told (directly
+ * or by implication) that it is being tested, evaluated, or graded". It also named
+ * the sibling: `../with/<id>/` was a guessable path to the treatment arm's live
+ * output. Independent random tokens fix both, and cost nothing.
  */
 export function stageEvalWorkspaces(input) {
+    for (const arm of armsOf(input.armDirs)) {
+        stageEvalWorkspacesForArm(input, arm);
+    }
+    return input.workspacesRoot;
+}
+function stageEvalWorkspacesForArm(input, arm) {
+    const armRoot = safePath.joinUnderRoot(input.workspacesRoot, armDirSegment(input.armDirs, arm));
+    mkdirSyncReal(armRoot, { recursive: true, mode: 0o700 });
     for (const entry of input.suite.evals) {
-        if (entry.files === undefined || entry.files.length === 0)
-            continue;
-        const evalWorkspace = safePath.joinUnderRoot(input.workspacesRoot, String(entry.id));
-        for (const rel of entry.files) {
+        const evalWorkspace = safePath.joinUnderRoot(armRoot, String(entry.id));
+        // 0700 like the root above it — created for every eval, populated only by
+        // those declaring `files`.
+        mkdirSyncReal(evalWorkspace, { recursive: true, mode: 0o700 });
+        for (const rel of entry.files ?? []) {
             // Containment first: a `rel` that escapes evalsDir or the workspace is a
             // genuine "escapes the eval directory" problem and is reported as such.
             let src;
@@ -164,11 +320,12 @@ export function stageEvalWorkspaces(input) {
                 dest = safePath.joinUnderRoot(evalWorkspace, rel);
             }
             catch (err) {
-                throw new EvalInputError(`eval ${entry.id} declares input file "${rel}" that escapes the eval directory: ${err.message}`);
+                throw new EvalInputError(`eval ${entry.id} declares input file "${quoteSuiteText(rel)}" that escapes the eval directory: ` +
+                    quoteSuiteText(err.message));
             }
             // eslint-disable-next-line security/detect-non-literal-fs-filename -- src is contained under evalsDir via joinUnderRoot; suite is developer-authored
             if (!existsSync(src)) {
-                throw new EvalInputError(`eval ${entry.id} declares input file "${rel}" but it is absent at ${src}`);
+                throw new EvalInputError(`eval ${entry.id} declares input file "${quoteSuiteText(rel)}" but it is absent at ${quoteSuiteText(src)}`);
             }
             // Copy failures (permissions, illegal filename on the host, disk) are
             // reported accurately rather than mislabeled as a containment escape.
@@ -179,10 +336,10 @@ export function stageEvalWorkspaces(input) {
                 cpSync(src, dest, { recursive: true });
             }
             catch (err) {
-                throw new EvalInputError(`eval ${entry.id} failed to stage input file "${rel}" into the workspace: ${err.message}`);
+                throw new EvalInputError(`eval ${entry.id} failed to stage input file "${quoteSuiteText(rel)}" into the workspace: ` +
+                    quoteSuiteText(err.message));
             }
         }
     }
-    return input.workspacesRoot;
 }
 //# sourceMappingURL=eval-inputs.js.map