@vibe-agent-toolkit/agent-skills 0.1.42 → 0.2.0-rc.10

package/dist/skill-test/eval-grader.js

@@ -1,16 +1,53 @@
-import { existsSync, readFileSync, unlinkSync } from 'node:fs';
-import { mkdirSyncReal, parseStreamJsonTranscript, safePath, spawnHeadlessClaude } from '@vibe-agent-toolkit/utils';
-import { EvalFragmentError, parseEvalFragment } from './eval-fragment.js';
-import { InternalHarnessError } from './exit-codes.js';
+import { randomBytes } from 'node:crypto';
+import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { mkdirSyncReal, safePath, VatError } from '@vibe-agent-toolkit/utils';
+import { parseStreamJsonTranscript, spawnHeadlessClaude, } from '@vibe-agent-toolkit/utils/skill-test';
+import { parseEvalFragment } from './eval-fragment.js';
+import { InternalHarnessError } from './failure-reason.js';
 import { assertGraderPromptInvariants, buildGraderPrompt } from './grader-prompt.js';
+import { sanitizeGraderText } from './grader-text.js';
 import { GradingNonceError } from './grading-adapter.js';
 import { computeToolPassed } from './tool-eval-schema.js';
+/** The reason named in {@link GraderFragmentUnparseableError}'s message and in every `evidence` of the failing fragment vat synthesizes from it. */
+export const GRADER_FRAGMENT_UNPARSEABLE = 'grader-fragment-unparseable';
+/**
+ * Thrown when the grader's fragment was not valid JSON on BOTH attempts (see
+ * {@link runGraderForEval}). A per-eval GRADING failure, deliberately neither
+ * {@link InternalHarnessError} nor `EvalFragmentError`: the harness did its
+ * job twice and the model could not write strict JSON, which is a verdict about
+ * this eval's grading, not about the suite — so it ends on the verdict path
+ * (`FINDINGS`), never on `ERROR` with `Reason: internal`. `runEvalWorker` (run-harness.ts)
+ * turns it into that eval's failing fragment (reason
+ * {@link GRADER_FRAGMENT_UNPARSEABLE}) on the treatment arm, and into a recorded
+ * control-arm failure on the other — where a fabricated `fail` would read as
+ * skill lift. `detail` is already sanitized. Carries no `reason` on purpose:
+ * `skillTestFailureReason` would read it only if this escaped, and escaping IS
+ * the bug — the worker catches it on both arms.
+ */
+export class GraderFragmentUnparseableError extends VatError {
+    evalId;
+    detail;
+    constructor(evalId, detail) {
+        super('SKILL_TEST_GRADER_FRAGMENT_UNPARSEABLE', `${GRADER_FRAGMENT_UNPARSEABLE}: the grader for eval "${evalId}" wrote a fragment that was not valid JSON ` +
+            `on either attempt (${detail}).`);
+        this.evalId = evalId;
+        this.detail = detail;
+    }
+}
+/**
+ * Adapt the grader's stdout `onProgress` sink into the `onWarn` callback
+ * {@link parseEvalFragment} requires (called only when it drops malformed
+ * friction items).
+ */
+function fragmentWarnRouter(onProgress) {
+    return (message) => onProgress(`[skill-test] ${message}\n`);
+}
 /**
  * Run ONE eval's grader: a blind `claude -p` spawn (grader model, NO
  * skill/plugin loaded) that reads the executor's captured transcript via its
- * prompt and writes a nonce'd fragment JSON to `graderOutDir` (issue #145
- * Task 8). `graderOutDir` is vat-only and separate from the skill sandbox the
- * executor ran in (Task 7) — the grader's `sandboxDir` --add-dir is
+ * prompt and writes a nonce'd fragment JSON to `graderOutDir`.
+ * `graderOutDir` is vat-only and separate from the skill sandbox the
+ * executor ran in — the grader's `sandboxDir` --add-dir is
  * `graderOutDir` itself, and `pluginDirs: []` means no skill code ever runs
  * inside the grader's process, so skill code cannot forge or tamper with the
  * fragment this run trusts.
@@ -19,49 +56,100 @@ import { computeToolPassed } from './tool-eval-schema.js';
  * executor (see eval-executor.ts), there is NO clean-failure path here. A
  * spawn rejection, watchdog kill (`timedOut`/`stalled`), non-zero exit, or a
  * missing fragment file all mean the grader failed to do its ONE job and are
- * thrown as {@link InternalHarnessError} (exit 1) — never laundered into a
+ * thrown as {@link InternalHarnessError} (`ERROR`, `Reason: internal`) — never laundered into a
  * passing (or even a valid failing) verdict.
  *
- * Once the fragment file exists, shape/JSON problems throw
- * {@link EvalFragmentError} (via {@link parseEvalFragment}) and a nonce that
- * doesn't match this run's expected nonce throws {@link GradingNonceError} —
- * both distinct from harness breakage because the grader DID run and produce
- * *something*, just not something we can trust.
- */
-/**
- * Adapt the grader's stdout `onProgress` sink into the `onWarn` callback
- * {@link parseEvalFragment} expects (used only when it drops malformed friction
- * items). Extracted so the ternary does not add a branch to `runGraderForEval`'s
- * cognitive complexity budget.
+ * A fragment that exists but is not valid JSON is the one exception, because
+ * the grader MODEL writes it: one bad escape in a free-text `evidence` took a
+ * six-eval suite down as harness-broke after five evals had graded fine. That
+ * attempt is re-run ONCE — same transcript, a FRESH nonce (so the stale file, or
+ * anything left behind, cannot satisfy the retry), the bad file consumed first,
+ * and the prompt told what went wrong. Only a second unparseable fragment
+ * surfaces, as {@link GraderFragmentUnparseableError}. Once a fragment parses,
+ * shape problems throw `EvalFragmentError` (via {@link parseEvalFragment})
+ * and a nonce that doesn't match the nonce THAT attempt was prompted with throws
+ * {@link GradingNonceError} — both fail-closed with no retry, because the grader
+ * DID produce JSON, just not JSON we can trust.
  */
-function fragmentWarnRouter(onProgress) {
-    if (onProgress === undefined)
-        return undefined;
-    return (message) => onProgress(`[skill-test] ${message}\n`);
-}
 export async function runGraderForEval(input) {
-    mkdirSyncReal(input.graderOutDir, { recursive: true });
+    // 0700, like every other vat-only directory this codebase creates (the grader
+    // root in `resolveGraderOutDir`, the workspaces root and every eval workspace in
+    // `stageEvalWorkspaces`). This is the per-ARM subdirectory of an already-0700
+    // parent, so nothing is exposed today — traversal into it is blocked one level
+    // up. It is made explicit because the rule is only worth anything while it is
+    // uniform: an unmoded `mkdir` here is the one place a reader would have to prove
+    // the parent's mode to know the fragment (which echoes the run's integrity nonce)
+    // is unreadable, and the day this dir is created somewhere else that proof is
+    // gone. Fragments are consumed-on-read anyway; this is the resting-state half.
+    mkdirSyncReal(input.graderOutDir, { recursive: true, mode: 0o700 });
     // evalId is validated `[A-Za-z0-9_-]+` upstream (eval-inputs.ts) so a plain
     // join is safe, but `joinUnderRoot` is used defensively — it costs nothing
     // and closes off any future path where an unvalidated id reaches here.
     const fragmentOut = safePath.joinUnderRoot(input.graderOutDir, `${input.evalId}.json`);
-    const prompt = buildGraderPrompt({
+    let attempt = await runGraderAttempt(input, fragmentOut, input.nonce, undefined);
+    if (attempt.unparseable !== undefined) {
+        input.onProgress(`[skill-test] grader fragment for eval "${input.evalId}" was not valid JSON (${attempt.unparseable}); re-grading once.\n`);
+        // Minted here, not derived from the run nonce: the retry must accept nothing
+        // written under the first prompt.
+        attempt = await runGraderAttempt(input, fragmentOut, randomBytes(16).toString('hex'), attempt.unparseable);
+        if (attempt.unparseable !== undefined) {
+            throw new GraderFragmentUnparseableError(input.evalId, attempt.unparseable);
+        }
+    }
+    const fragment = parseEvalFragment(attempt.parsed, fragmentWarnRouter(input.onProgress));
+    // Integrity gate: a missing/wrong nonce means this fragment was not produced
+    // by the grader we prompted for THIS attempt — most likely forged or left
+    // behind by untrusted skill code — so the verdict cannot be trusted, hard error.
+    if (fragment.runNonce !== attempt.nonce) {
+        throw new GradingNonceError(`eval "${input.evalId}" fragment \`runNonce\` does not match this run`);
+    }
+    assertExpectationCountDeclared(input.evalId, input.expectations, fragment.expectations);
+    assertToolVerdictConsistent(input.evalId, input.toolExpectations, fragment.tool);
+    // The grader's only input is the executor transcript, which untrusted skill code
+    // controls, and `evalId` is schema-typed as any non-empty string. It is echoed
+    // verbatim into the run summary and into `baseline.json` — including the
+    // CONTAMINATED banner — so a grader talked into emitting an id containing
+    // newlines and ANSI escapes can paint a reassuring "known false positive, the
+    // delta is valid" line directly beneath vat's own warning. VAT knows which eval
+    // it asked about; take the id from the request, never from the answer.
+    //
+    // `runNonce` is re-stamped for the same reason: the merge re-verifies every
+    // fragment against the ONE nonce the run minted, and a re-graded fragment has
+    // already passed the gate above against the nonce its own prompt carried.
+    return { ...fragment, evalId: input.evalId, runNonce: input.nonce };
+}
+/**
+ * ONE grader spawn under `nonce`: build and assert the prompt, spawn, apply the
+ * harness-breakage gates, report cost, consume the fragment file. `previousFailure`
+ * is the re-grade's only difference — a trailing line telling the grader what its
+ * first fragment got wrong (sanitized where it was caught, at the bottom of this
+ * function). Required, not defaulted, so both call sites say which attempt they are.
+ */
+async function runGraderAttempt(input, fragmentOut, nonce, previousFailure) {
+    const built = buildGraderPrompt({
         evalId: input.evalId,
         transcript: input.transcript,
         expectations: input.expectations,
         ...(input.expectedOutput === undefined ? {} : { expectedOutput: input.expectedOutput }),
         rubricPath: input.rubricPath,
         fragmentOut,
-        nonce: input.nonce,
+        nonce,
         ...(input.toolExpectations === undefined ? {} : { toolExpectations: input.toolExpectations }),
         ...(input.declaredExecutables === undefined ? {} : { declaredExecutables: input.declaredExecutables }),
     });
+    const prompt = previousFailure === undefined
+        ? built
+        : `${built}\n\nYour previous fragment was not valid JSON: ${previousFailure}. Write strict JSON.`;
     // Defense-in-depth (parity with eval-executor.ts's build→assert→use pattern):
     // re-verify the built grader prompt still carries its required invariants
-    // (STOP, fragment path, browser/iteration forbids, nonce directive) before
-    // spawning, so a regressed buildGraderPrompt is caught here, not by a grader
-    // that silently misbehaves.
-    assertGraderPromptInvariants(prompt, input.transcript);
+    // (STOP, fragment path, browser/iteration forbids, nonce directive, and the
+    // untrusted-data fences) before spawning, so a regressed buildGraderPrompt is
+    // caught here, not by a grader that silently misbehaves. The nonce is handed
+    // over so the assertion can cut out every fenced region — transcript, subject
+    // manifest, and the suite's own expectations/expected_output — and none of
+    // that attacker-controlled text can satisfy an invariant on the builder's
+    // behalf.
+    assertGraderPromptInvariants(prompt, nonce);
     const spawn = input.spawn ?? spawnHeadlessClaude;
     // Accumulate the grader's stream-json stdout so we can read its terminal
     // `total_cost_usd` for run-wide spend aggregation (adopter follow-up). Held in
@@ -69,7 +157,7 @@ export async function runGraderForEval(input) {
     let graderTranscript = '';
     const onStdout = (chunk) => {
         graderTranscript += chunk;
-        input.onProgress?.(chunk);
+        input.onProgress(chunk);
     };
     let spawnResult;
     try {
@@ -109,53 +197,110 @@ export async function runGraderForEval(input) {
     // the in-memory transcript (not the fragment) so it reflects real API spend even
     // under subscription auth; `undefined` when the transcript carried no result.
     input.costSink?.(parseStreamJsonTranscript(graderTranscript).result?.totalCostUsd);
-    const raw = readAndConsumeFragmentFile(fragmentOut, input.evalId, spawnResult.status);
-    const fragment = parseEvalFragment(raw, fragmentWarnRouter(input.onProgress));
-    // Integrity gate: a missing/wrong nonce means this fragment was not produced
-    // by the grader we prompted for THIS run — most likely forged or left behind
-    // by untrusted skill code — so the verdict cannot be trusted, hard error.
-    if (fragment.runNonce !== input.nonce) {
-        throw new GradingNonceError(`eval "${input.evalId}" fragment \`runNonce\` does not match this run`);
+    const warn = fragmentWarnRouter(input.onProgress);
+    const text = readAndConsumeFragmentFile(fragmentOut, input.evalId, spawnResult.status, warn);
+    try {
+        return { nonce, parsed: JSON.parse(text) };
+    }
+    catch (err) {
+        if (!(err instanceof SyntaxError))
+            throw err;
+        // V8 embeds a VERBATIM slice of the offending bytes in its SyntaxError
+        // message, unescaped — and this is one of the few grader-controlled strings
+        // that reaches an operator WITHOUT passing `parseEvalFragment`, the
+        // documented text boundary, because the parse it would have gone through is
+        // the one that just failed. It lands in a thrown message (which the CLI writes
+        // as `Error: ${err.message}`, so an unsanitized `ESC[2K CR ESC[32m…` wipes the
+        // line vat just printed and continues in vat's own colour) AND in the re-grade
+        // prompt.
+        return { nonce, unparseable: sanitizeGraderText(err.message) };
     }
-    assertToolVerdictConsistent(input.evalId, input.toolExpectations, fragment.tool);
-    return fragment;
 }
 /**
- * Read the grader's fragment JSON into memory and unlink it immediately —
- * consume-on-read. Split out of {@link runGraderForEval} to keep its cognitive
- * complexity within budget. Throws {@link InternalHarnessError} if the grader
- * wrote no fragment, {@link EvalFragmentError} if it is not valid JSON.
+ * Read the grader's fragment file into memory and remove it immediately —
+ * consume-on-read, whether or not it will parse. Throws {@link InternalHarnessError}
+ * if the grader wrote no fragment.
  *
- * The unlink is the security-relevant half: the grader dir is same-uid (see
+ * The removal is the security-relevant half: the grader dir is same-uid (see
  * `resolveGraderOutDir`), so a fragment left on disk lets skill code that
  * survived the process-group kill read the echoed nonce at leisure and forge
  * LATER fragments. Consuming it on read leaves no persisted copy — exposure
- * shrinks to each fragment's own read window (no cross-eval harvest). The read
- * here is the ONLY read of the file; all downstream logic runs off the returned
- * value. Best-effort: a failed unlink is not a run failure (end-of-run cleanup
- * removes the whole dir), and true isolation from same-uid code is issue #149.
+ * shrinks to each fragment's own read window (no cross-eval harvest). It is also
+ * what makes the re-grade honest: a retry that writes nothing finds no file, not
+ * the unparseable one. The read here is the ONLY read of the file; all downstream
+ * logic runs off the returned text. A failed removal is not a run failure
+ * (end-of-run cleanup removes the whole dir) but it IS the one degradation this
+ * design exists to prevent, so it is reported through `onWarn` rather than
+ * swallowed: the operator learns the fragment lingered, with the errno. True
+ * isolation from same-uid code is a separate, unbuilt mechanism (a different uid
+ * or a sandbox), not this removal.
  */
-function readAndConsumeFragmentFile(fragmentOut, evalId, status) {
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
+function readAndConsumeFragmentFile(fragmentOut, evalId, status, onWarn) {
     if (!existsSync(fragmentOut)) {
         throw new InternalHarnessError(`Grader exited (status ${status}) without writing a fragment at ${fragmentOut} for eval "${evalId}".`);
     }
-    let raw;
+    const text = readFileSync(fragmentOut, 'utf-8');
     try {
-        // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
-        raw = JSON.parse(readFileSync(fragmentOut, 'utf-8'));
+        // `force` covers the one benign case — the file is already gone — so what the
+        // catch sees is a refusal (EACCES/EPERM/EBUSY), never an absence.
+        rmSync(fragmentOut, { force: true });
     }
     catch (err) {
-        throw new EvalFragmentError(`grader fragment for eval "${evalId}" at ${fragmentOut} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
-    }
-    try {
-        // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
-        unlinkSync(fragmentOut);
-    }
-    catch {
-        // Swallow: cleanup removes the grader dir regardless.
+        onWarn(`grader fragment for eval "${evalId}" at ${fragmentOut} could not be removed after it was read ` +
+            `(${err instanceof Error ? err.message : String(err)}); it stays on disk until end-of-run cleanup.`);
     }
-    return raw;
+    return text;
+}
+/**
+ * The OUTPUT channel's declared-vs-reported check — the exact thing
+ * {@link assertToolCheckNamesDeclared} does for the TOOL channel, applied to the
+ * channel every reported number is computed from.
+ *
+ * Each arm's `summary`, the pass/fail verdict `reconcileGrading` produces, and
+ * the `--baseline` delta between the arms are all computed over
+ * `fragment.expectations` — i.e. over however many entries the grader chose to
+ * write. The prompt ASKS for "one per expectation above" and, until this check,
+ * nothing verified it:
+ *
+ * - **Omission laundered a pass, on EVERY run.** A grader that skips a FAILING
+ *   expectation leaves a fragment where the survivors all passed, so
+ *   `reconcileGrading` reports `allPassed: true` and the run exits 0.
+ * - **Invention inflated the delta.** A suite declaring 1 expectation whose two
+ *   graders each emit 3 of their own yields `Baseline delta: +3` where the
+ *   maximum legal lift is +1.
+ *
+ * The existing `armExpectationSkew` cannot see either: it compares the two ARMS
+ * to each other, and both graders receive the same declared list from the same
+ * template, so they drift together and parity holds while a confident wrong
+ * number prints.
+ *
+ * COUNT, not text. The grader is asked to echo each expectation's `text`, but it
+ * routinely reworks the wording, and a reworded text is not a wrong verdict —
+ * whereas a missing or extra ENTRY always is, because the entry is the unit
+ * every score counts.
+ *
+ * Fail-closed like its siblings below: this is vat's own grading infrastructure
+ * failing its one job, not adopter data being audited.
+ *
+ * FAIL-CLOSED IS NOT FAIL-THE-RUN, and the distinction is the caller's to make, not
+ * this function's. Throwing here is unconditional and stays that way: a fragment
+ * whose entry count does not match the declaration is not a verdict, and returning
+ * it "leniently" for the control arm would put an unchecked number straight into the
+ * delta — which is the hole this check was added to close. What changed is where the
+ * throw lands. `runEvalWorker` (run-harness.ts) catches a CONTROL-arm throw, records
+ * it as a control-arm failure, and lets the run finish; a TREATMENT-arm throw still
+ * propagates and fails the run. So a control-arm grader miscount now withholds the
+ * delta as `null` with the reason attached, instead of destroying fully-billed
+ * treatment work that was never in doubt. Do not soften this assert to get that
+ * behaviour — it is already there, one level up, where the arm is known.
+ */
+function assertExpectationCountDeclared(evalId, declared, reported) {
+    if (reported.length === declared.length)
+        return;
+    throw new InternalHarnessError(`Grader for eval "${evalId}" returned ${reported.length} expectation entr${reported.length === 1 ? 'y' : 'ies'} ` +
+        `for an eval that declares ${declared.length}. Every score vat reports — each arm's summary and the ` +
+        `--baseline delta between them — is computed over the entries the grader emitted, so an invented or ` +
+        `omitted entry silently changes it.`);
 }
 /**
  * Fail-open guards on the grader's tool verdict, split out of
@@ -171,6 +316,8 @@ function readAndConsumeFragmentFile(fragmentOut, evalId, status) {
  *   that disagrees with its own sub-checks is grader malfunction (a false green
  *   or a false red), surfaced loudly rather than merged. Mirrors
  *   `reconcileGrading`'s summary/expectations reconciliation (grading-adapter.ts).
+ * - Fix #4: the check NAMES must be exactly the eval's declared ones. See
+ *   {@link assertToolCheckNamesDeclared}.
  */
 function assertToolVerdictConsistent(evalId, toolExpectations, tool) {
     if (toolExpectations !== undefined && tool === undefined) {
@@ -180,5 +327,57 @@ function assertToolVerdictConsistent(evalId, toolExpectations, tool) {
         throw new InternalHarnessError(`Grader for eval "${evalId}" emitted a tool verdict whose \`passed\` (${tool.passed}) ` +
             `disagrees with its own sub-checks (recomputed ${computeToolPassed(tool)}).`);
     }
+    assertToolCheckNamesDeclared(evalId, toolExpectations, tool);
+}
+/** Render grader-supplied names for an error message, already sanitized by the fragment parse. */
+function quoteNames(names) {
+    return names.map((name) => `"${name}"`).join(', ');
+}
+/**
+ * One channel's check names must be EXACTLY the declared ones — no invention,
+ * no omission.
+ *
+ * `computeToolPassed` iterates the checks the GRADER emitted, not the ones the
+ * eval declared, so both directions are load-bearing and neither is cosmetic:
+ *
+ * - **Omission laundered a pass.** Drop the `mustRun` entry for an executable
+ *   that never ran and the channel is vacuously true — the verdict reads
+ *   `passed: true` for an expectation nobody checked.
+ * - **Invention gates on a name vat never asked about.** A grader talked into
+ *   emitting `{name: "totally-fine", ran: true}` adds a check the eval does not
+ *   declare, and it counts toward the verdict exactly like a real one.
+ *
+ * Names are compared AFTER {@link sanitizeGraderText} on both sides. The
+ * reported side is already sanitized (parseEvalFragment); normalizing the
+ * declared side too means the comparison happens in one space, so an adopter
+ * who declared a name containing a tab does not get an unexplainable mismatch.
+ *
+ * Fail-closed like its siblings above: this is vat's own grading infrastructure
+ * failing to do its one job, not adopter data being audited.
+ */
+function assertToolCheckNamesDeclared(evalId, toolExpectations, tool) {
+    if (tool === undefined)
+        return;
+    const declared = toolExpectations ?? {};
+    assertChannelNames(evalId, 'mustRun', declared.mustRun, (tool.mustRun ?? []).map((c) => c.name));
+    assertChannelNames(evalId, 'mustNotRun', declared.mustNotRun, (tool.mustNotRun ?? []).map((c) => c.name));
+    assertChannelNames(evalId, 'mustSucceed', declared.mustSucceed, (tool.mustSucceed ?? []).map((c) => c.name));
+    assertChannelNames(evalId, 'sequence', declared.sequence, (tool.sequence ?? []).flatMap((c) => c.steps));
+}
+function assertChannelNames(evalId, channel, declared, reported) {
+    const declaredSet = new Set((declared ?? []).map(sanitizeGraderText));
+    const reportedSet = new Set(reported);
+    const invented = [...reportedSet].filter((name) => !declaredSet.has(name));
+    const missing = [...declaredSet].filter((name) => !reportedSet.has(name));
+    if (invented.length === 0 && missing.length === 0)
+        return;
+    const problems = [];
+    if (invented.length > 0)
+        problems.push(`invented ${quoteNames(invented)}`);
+    if (missing.length > 0)
+        problems.push(`omitted ${quoteNames(missing)}`);
+    throw new InternalHarnessError(`Grader for eval "${evalId}" emitted \`tool.${channel}\` checks that do not match the eval's ` +
+        `declared toolExpectations: ${problems.join('; ')}. The verdict is computed from the checks the ` +
+        `grader emitted, so an invented or omitted name silently changes it.`);
 }
 //# sourceMappingURL=eval-grader.js.map

package/dist/skill-test/eval-grader.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEpH,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAgD1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH;;;;;GAKG;AACH,SAAS,kBAAkB,CACzB,UAAwC;IAExC,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC/C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,8EAA8E;IAC9E,0EAA0E;IAC1E,2EAA2E;IAC3E,6EAA6E;IAC7E,4BAA4B;IAC5B,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAE9E,6EAA6E;IAC7E,6EAA6E;IAC7E,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,0BAA0B,CAAC,WAAmB,EAAE,MAAc,EAAE,MAAc;IACrF,kIAAkI;IAClI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,kIAAkI;QAClI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,iBAAiB,CACzB,6BAA6B,MAAM,QAAQ,WAAW,uBAAuB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAChI,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,kIAAkI;QAClI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,sDAAsD;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EACL,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE1E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAuD1D,oJAAoJ;AACpJ,MAAM,CAAC,MAAM,2BAA2B,GAAG,6BAA6B,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,8BAA+B,SAAQ,QAAQ;IAExC;IACA;IAFlB,YACkB,MAAc,EACd,MAAc;QAE9B,KAAK,CACH,wCAAwC,EACxC,GAAG,2BAA2B,0BAA0B,MAAM,6CAA6C;YACzG,sBAAsB,MAAM,IAAI,CACnC,CAAC;QAPc,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAQ;IAOhC,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,UAAwC;IAClE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,8EAA8E;IAC9E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,kFAAkF;IAClF,8EAA8E;IAC9E,+EAA+E;IAC/E,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,IAAI,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,KAAK,CAAC,UAAU,CACd,0CAA0C,KAAK,CAAC,MAAM,yBAAyB,OAAO,CAAC,WAAW,uBAAuB,CAC1H,CAAC;QACF,6EAA6E;QAC7E,kCAAkC;QAClC,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3G,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAEzF,6EAA6E;IAC7E,0EAA0E;IAC1E,iFAAiF;IACjF,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IACxF,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,iFAAiF;IACjF,+EAA+E;IAC/E,yEAAyE;IACzE,0EAA0E;IAC1E,8EAA8E;IAC9E,gFAAgF;IAChF,uEAAuE;IACvE,EAAE;IACF,4EAA4E;IAC5E,8EAA8E;IAC9E,0EAA0E;IAC1E,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;AACtE,CAAC;AAUD;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB,CAC7B,KAAqB,EACrB,WAAmB,EACnB,KAAa,EACb,eAAmC;IAEnC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK;QACL,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,MAAM,MAAM,GACV,eAAe,KAAK,SAAS;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,GAAG,KAAK,kDAAkD,eAAe,sBAAsB,CAAC;IACtG,8EAA8E;IAC9E,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,UAAU;IACV,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7F,IAAI,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;YAAE,MAAM,GAAG,CAAC;QAC7C,uEAAuE;QACvE,4EAA4E;QAC5E,oEAAoE;QACpE,4EAA4E;QAC5E,+EAA+E;QAC/E,+EAA+E;QAC/E,+EAA+E;QAC/E,UAAU;QACV,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;IACjE,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,0BAA0B,CACjC,WAAmB,EACnB,MAAc,EACd,MAAc,EACd,MAAiC;IAEjC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC;QACH,8EAA8E;QAC9E,kEAAkE;QAClE,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CACJ,6BAA6B,MAAM,QAAQ,WAAW,0CAA0C;YAC9F,IAAI,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,+CAA+C,CACtG,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA2B,EAC3B,QAAsC;IAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO;IAChD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,cAAc,QAAQ,CAAC,MAAM,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;QAC/G,6BAA6B,QAAQ,CAAC,MAAM,yDAAyD;QACrG,qGAAqG;QACrG,oCAAoC,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAwB;IAC1C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,4BAA4B,CACnC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,MAAM,QAAQ,GAAG,gBAAgB,IAAI,EAAE,CAAC;IACxC,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7G,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAc,EACd,OAAe,EACf,QAAuC,EACvC,QAA2B;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC1D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,oBAAoB,OAAO,yCAAyC;QAC5F,8BAA8B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD;QACjG,qEAAqE,CACxE,CAAC;AACJ,CAAC"}

package/dist/skill-test/eval-inputs.d.ts

@@ -1,8 +1,49 @@
+import { VatError } from '@vibe-agent-toolkit/utils';
 import { z } from 'zod';
-/** Raised for any eval-input problem (bad JSON, schema failure, missing input file). Maps to exit 2. */
-export declare class EvalInputError extends Error {
+/**
+ * Raised for any eval-input problem (bad JSON, schema failure, missing input file).
+ * Reason `preflight`.
+ *
+ * The FIELD is what makes that sentence true, and until it was added only one of this
+ * class's two routes delivered it. `attemptStageWorkspaces` catches the staging
+ * instance and maps it explicitly; the instance thrown from {@link armDirSegment}
+ * fires deep in the eval loop, outside that handler, and reached
+ * `skillTestFailureReason`'s `internal` default — reporting a harness bug for a
+ * preflight-class problem while the docblock above claimed otherwise. `preflight`
+ * for the internal-invariant case too is consistent with every sibling in this feature.
+ */
+export declare class EvalInputError extends VatError {
+    readonly reason: "preflight";
     constructor(message: string);
 }
+/**
+ * Neutralize a SUITE-AUTHORED string before it is quoted into an
+ * {@link EvalInputError}, whose message reaches `process.stdout` on the
+ * `Summary:` line — the one channel deliberately kept machine-readable — and
+ * renders twice on the way there.
+ *
+ * The suite is nominally adopter-authored, but `resolveEvalSuitePath` will
+ * harvest one out of a FETCHED npm/url artifact, i.e. out of the skill under
+ * test, so treat it as untrusted. `files[]` is the sharp edge: it is
+ * `z.array(z.string().min(1))` with NO charset constraint, in deliberate
+ * contrast to `id`, which is regex-pinned precisely because it names a
+ * directory. A derived path or an OS error text built from such an entry
+ * carries the same bytes, so those are sanitized here too, not just the entry.
+ *
+ * Reuses {@link sanitizeGraderText} rather than growing a second neutralizer:
+ * the threat and the remedy (drop escape sequences whole, fold every remaining
+ * control code to a space, collapse, cap) are identical — only the source of
+ * the untrusted text differs.
+ *
+ * EXPORTED because the suite reaches an operator surface from more than this
+ * module: `eval-lint.ts` quotes `toolExpectations.*` entries — the same
+ * unconstrained `z.array(z.string().min(1))` shape as `files[]` — into an advisory
+ * that `run-harness.ts` writes to stderr verbatim, and does it at Step 5.5, ahead
+ * of both the security-ack gate and the `--dry-run` short-circuit. Every new
+ * suite-authored string bound for a terminal goes through HERE, so the trust model
+ * above stays stated in one place.
+ */
+export declare function quoteSuiteText(value: string): string;
 /**
  * True when `key` is exactly one edit (insert/delete/substitute one char) from
  * `target`. Used to catch typos of recognized fields (here) and of declared
@@ -488,7 +529,7 @@ export declare const EvalSuiteSchema: z.ZodObject<{
 export type EvalEntry = z.infer<typeof EvalEntrySchema>;
 export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
 /**
- * The declared tool expectations of ONE eval (issue #145 Phase T). Derived from
+ * The declared tool expectations of ONE eval. Derived from
  * {@link EvalEntry} so the grader input + prompt builder share the EXACT shape the
  * parser produces — no drift, and no duplicated inline `{ mustRun?; mustNotRun?;
  * sequence? }` literal across those consumers.
@@ -496,19 +537,80 @@ export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
 export type ToolExpectations = NonNullable<EvalEntry['toolExpectations']>;
 /** Parse + validate a skill-test eval suite. Throws {@link EvalInputError} on any problem. */
 export declare function parseEvalSuite(jsonText: string): EvalSuite;
+/**
+ * One side of a `--baseline` A/B: `'with'` runs the skill staged and declared,
+ * `'without'` is the control. Declared here rather than in the run orchestrator
+ * because the staging layer needs it to give each arm its own workspace tree.
+ */
+export type EvalArm = 'with' | 'without';
+/**
+ * The opaque directory segment each arm's workspaces live under, minted fresh per
+ * run. `with` is required and `without` present only under `--baseline`, so the
+ * set of arms is the keys — a shape that cannot express "stage nothing", unlike
+ * the `readonly EvalArm[]` this replaced (an empty array was a legal input that
+ * silently left every executor with a non-existent cwd).
+ *
+ * ⚠️ The VALUES must not be the literal arm names, and must not be derivable from
+ * each other. See {@link stageEvalWorkspaces}.
+ */
+export interface ArmWorkspaceDirs {
+    with: string;
+    without?: string;
+}
+/** The arms an {@link ArmWorkspaceDirs} covers, in dispatch order. */
+export declare function armsOf(dirs: ArmWorkspaceDirs): readonly EvalArm[];
+/**
+ * The directory segment for one arm. Throws rather than falling back to the arm
+ * NAME: a silent fallback would reinstate the unblinding this indirection exists
+ * to remove, and would do it invisibly.
+ */
+export declare function armDirSegment(dirs: ArmWorkspaceDirs, arm: EvalArm): string;
 export interface StageEvalWorkspacesInput {
     /** Parsed suite (Task 1). */
     suite: EvalSuite;
     /** Directory containing evals.json — the base for each eval's relative `files`. */
     evalsDir: string;
-    /** `<harnessRoot>/workspaces` — per-eval dirs are created beneath it. */
+    /** Per-run workspaces root — per-arm, per-eval dirs are created beneath it. */
     workspacesRoot: string;
+    /**
+     * Opaque per-arm directory segments. Each arm gets its OWN copy of every
+     * workspace; see the note on the layout below for why that is not an
+     * optimization to undo.
+     */
+    armDirs: ArmWorkspaceDirs;
 }
 /**
- * Materialize each eval's declared input `files` into `<workspacesRoot>/<id>/<relpath>`,
- * preserving relative structure. Evals without `files` are skipped. Throws
- * {@link EvalInputError} if a listed file does not exist (the eval cannot run without it).
- * Returns `workspacesRoot`.
+ * Materialize each eval's declared input `files` into
+ * `<workspacesRoot>/<armDir>/<id>/<relpath>`, preserving relative structure. Throws
+ * {@link EvalInputError} if a listed file does not exist (the eval cannot run
+ * without it). Returns `workspacesRoot`.
+ *
+ * EVERY eval gets a directory, including one that declares no `files` — it is
+ * simply left empty. An eval without a workspace used to leave the executor with
+ * no cwd of its own, so it ran inside the staged subject dir instead; for the
+ * skill-absent arm of a `--baseline` run that placed the control's cwd inside the
+ * very skill the arm exists to withhold. An empty dir costs nothing and keeps cwd
+ * shaped identically across both arms, which is what makes the A/B a comparison.
+ *
+ * WHY THE `<arm>` SEGMENT. The two arms of one eval are dispatched ADJACENTLY into
+ * a bounded-parallel pool, so they run at the same time. When they shared one
+ * directory, the control arm could simply `ls` and read whatever the treatment arm
+ * had just written — recovering the treatment's answer without ever touching the
+ * skill, and without producing a single detectable harness path. That is a channel
+ * BETWEEN the arms rather than into one of them, which is why auditing the four
+ * channels into a process (prompt, argv, cwd, env) did not surface it: each arm's
+ * own cwd was correct in isolation, and wrong as a pair. The arms must be
+ * byte-identical at start and unable to observe each other after it.
+ *
+ * WHY THE SEGMENT IS AN OPAQUE TOKEN and not the arm's name. It used to be the
+ * literal `with`/`without`, which made the executor's own working directory the
+ * loudest possible unblinding: the prompt tells each arm `Your working directory
+ * is …/vat-skill-test-ws-<token>/without/e1`, so the skill-absent arm was told, in
+ * the one string it cannot avoid reading, which side of the experiment it was on —
+ * while `buildExecutorPrompt`'s docblock promises it "must NEVER be told (directly
+ * or by implication) that it is being tested, evaluated, or graded". It also named
+ * the sibling: `../with/<id>/` was a guessable path to the treatment arm's live
+ * output. Independent random tokens fix both, and cost nothing.
  */
 export declare function stageEvalWorkspaces(input: StageEvalWorkspacesInput): string;
 //# sourceMappingURL=eval-inputs.d.ts.map

package/dist/skill-test/eval-inputs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAyBD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAWD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAsDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAmB1D;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAoC3E"}
+{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBAC3B,OAAO,EAAE,MAAM;CAG5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AA2CD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAiDD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAyDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAsB1D;AAED;;;;GAIG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAEzC;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sEAAsE;AACtE,wBAAgB,MAAM,CAAC,IAAI,EAAE,gBAAgB,GAAG,SAAS,OAAO,EAAE,CAEjE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAU1E;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAK3E"}