@vibe-agent-toolkit/agent-skills 0.1.40 → 0.1.41-rc.2

package/dist/skill-test/eval-grader.js

@@ -0,0 +1,184 @@
+import { existsSync, readFileSync, unlinkSync } from 'node:fs';
+import { mkdirSyncReal, parseStreamJsonTranscript, safePath, spawnHeadlessClaude } from '@vibe-agent-toolkit/utils';
+import { EvalFragmentError, parseEvalFragment } from './eval-fragment.js';
+import { InternalHarnessError } from './exit-codes.js';
+import { assertGraderPromptInvariants, buildGraderPrompt } from './grader-prompt.js';
+import { GradingNonceError } from './grading-adapter.js';
+import { computeToolPassed } from './tool-eval-schema.js';
+/**
+ * Run ONE eval's grader: a blind `claude -p` spawn (grader model, NO
+ * skill/plugin loaded) that reads the executor's captured transcript via its
+ * prompt and writes a nonce'd fragment JSON to `graderOutDir` (issue #145
+ * Task 8). `graderOutDir` is vat-only and separate from the skill sandbox the
+ * executor ran in (Task 7) — the grader's `sandboxDir` --add-dir is
+ * `graderOutDir` itself, and `pluginDirs: []` means no skill code ever runs
+ * inside the grader's process, so skill code cannot forge or tamper with the
+ * fragment this run trusts.
+ *
+ * The grader is vat infrastructure, not the thing under test: unlike the
+ * executor (see eval-executor.ts), there is NO clean-failure path here. A
+ * spawn rejection, watchdog kill (`timedOut`/`stalled`), non-zero exit, or a
+ * missing fragment file all mean the grader failed to do its ONE job and are
+ * thrown as {@link InternalHarnessError} (exit 1) — never laundered into a
+ * passing (or even a valid failing) verdict.
+ *
+ * Once the fragment file exists, shape/JSON problems throw
+ * {@link EvalFragmentError} (via {@link parseEvalFragment}) and a nonce that
+ * doesn't match this run's expected nonce throws {@link GradingNonceError} —
+ * both distinct from harness breakage because the grader DID run and produce
+ * *something*, just not something we can trust.
+ */
+/**
+ * Adapt the grader's stdout `onProgress` sink into the `onWarn` callback
+ * {@link parseEvalFragment} expects (used only when it drops malformed friction
+ * items). Extracted so the ternary does not add a branch to `runGraderForEval`'s
+ * cognitive complexity budget.
+ */
+function fragmentWarnRouter(onProgress) {
+    if (onProgress === undefined)
+        return undefined;
+    return (message) => onProgress(`[skill-test] ${message}\n`);
+}
+export async function runGraderForEval(input) {
+    mkdirSyncReal(input.graderOutDir, { recursive: true });
+    // evalId is validated `[A-Za-z0-9_-]+` upstream (eval-inputs.ts) so a plain
+    // join is safe, but `joinUnderRoot` is used defensively — it costs nothing
+    // and closes off any future path where an unvalidated id reaches here.
+    const fragmentOut = safePath.joinUnderRoot(input.graderOutDir, `${input.evalId}.json`);
+    const prompt = buildGraderPrompt({
+        evalId: input.evalId,
+        transcript: input.transcript,
+        expectations: input.expectations,
+        ...(input.expectedOutput === undefined ? {} : { expectedOutput: input.expectedOutput }),
+        rubricPath: input.rubricPath,
+        fragmentOut,
+        nonce: input.nonce,
+        ...(input.toolExpectations === undefined ? {} : { toolExpectations: input.toolExpectations }),
+        ...(input.declaredExecutables === undefined ? {} : { declaredExecutables: input.declaredExecutables }),
+    });
+    // Defense-in-depth (parity with eval-executor.ts's build→assert→use pattern):
+    // re-verify the built grader prompt still carries its required invariants
+    // (STOP, fragment path, browser/iteration forbids, nonce directive) before
+    // spawning, so a regressed buildGraderPrompt is caught here, not by a grader
+    // that silently misbehaves.
+    assertGraderPromptInvariants(prompt, input.transcript);
+    const spawn = input.spawn ?? spawnHeadlessClaude;
+    // Accumulate the grader's stream-json stdout so we can read its terminal
+    // `total_cost_usd` for run-wide spend aggregation (adopter follow-up). Held in
+    // memory only, like the executor's transcript — never written to disk.
+    let graderTranscript = '';
+    const onStdout = (chunk) => {
+        graderTranscript += chunk;
+        input.onProgress?.(chunk);
+    };
+    let spawnResult;
+    try {
+        spawnResult = await spawn({
+            prompt,
+            // No skill/plugin ever loads inside the grader's process — this is the
+            // forgery-proofing half of the contract (the other half is that
+            // graderOutDir is not the skill sandbox).
+            pluginDirs: [],
+            sandboxDir: input.graderOutDir,
+            cwd: input.graderOutDir,
+            env: input.env,
+            timeoutMs: input.timeoutMs,
+            onStdout,
+            maxTurns: input.maxTurns,
+            maxBudgetUsd: input.maxBudgetUsd,
+            model: input.graderModel,
+            ...(input.stallMs === undefined ? {} : { stallMs: input.stallMs }),
+        });
+    }
+    catch (err) {
+        throw new InternalHarnessError(`Grader spawn failed for eval "${input.evalId}": ${err instanceof Error ? err.message : String(err)}`);
+    }
+    // The grader is vat infrastructure: any of these mean it failed to do its
+    // ONE job, so all of them are harness breakage — there is no clean-failure
+    // path here (contrast eval-executor.ts's cleanFailure).
+    if (spawnResult.stalled) {
+        throw new InternalHarnessError(`Grader stalled for eval "${input.evalId}" (no output for ${input.stallMs ?? 0}ms).`);
+    }
+    if (spawnResult.timedOut) {
+        throw new InternalHarnessError(`Grader timed out for eval "${input.evalId}" (limit ${input.timeoutMs}ms).`);
+    }
+    if (spawnResult.status !== 0) {
+        throw new InternalHarnessError(`Grader exited non-zero (status ${spawnResult.status}) for eval "${input.evalId}".`);
+    }
+    // Report the grader session's cost for run-wide spend aggregation. Parsed from
+    // the in-memory transcript (not the fragment) so it reflects real API spend even
+    // under subscription auth; `undefined` when the transcript carried no result.
+    input.costSink?.(parseStreamJsonTranscript(graderTranscript).result?.totalCostUsd);
+    const raw = readAndConsumeFragmentFile(fragmentOut, input.evalId, spawnResult.status);
+    const fragment = parseEvalFragment(raw, fragmentWarnRouter(input.onProgress));
+    // Integrity gate: a missing/wrong nonce means this fragment was not produced
+    // by the grader we prompted for THIS run — most likely forged or left behind
+    // by untrusted skill code — so the verdict cannot be trusted, hard error.
+    if (fragment.runNonce !== input.nonce) {
+        throw new GradingNonceError(`eval "${input.evalId}" fragment \`runNonce\` does not match this run`);
+    }
+    assertToolVerdictConsistent(input.evalId, input.toolExpectations, fragment.tool);
+    return fragment;
+}
+/**
+ * Read the grader's fragment JSON into memory and unlink it immediately —
+ * consume-on-read. Split out of {@link runGraderForEval} to keep its cognitive
+ * complexity within budget. Throws {@link InternalHarnessError} if the grader
+ * wrote no fragment, {@link EvalFragmentError} if it is not valid JSON.
+ *
+ * The unlink is the security-relevant half: the grader dir is same-uid (see
+ * `resolveGraderOutDir`), so a fragment left on disk lets skill code that
+ * survived the process-group kill read the echoed nonce at leisure and forge
+ * LATER fragments. Consuming it on read leaves no persisted copy — exposure
+ * shrinks to each fragment's own read window (no cross-eval harvest). The read
+ * here is the ONLY read of the file; all downstream logic runs off the returned
+ * value. Best-effort: a failed unlink is not a run failure (end-of-run cleanup
+ * removes the whole dir), and true isolation from same-uid code is issue #149.
+ */
+function readAndConsumeFragmentFile(fragmentOut, evalId, status) {
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
+    if (!existsSync(fragmentOut)) {
+        throw new InternalHarnessError(`Grader exited (status ${status}) without writing a fragment at ${fragmentOut} for eval "${evalId}".`);
+    }
+    let raw;
+    try {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
+        raw = JSON.parse(readFileSync(fragmentOut, 'utf-8'));
+    }
+    catch (err) {
+        throw new EvalFragmentError(`grader fragment for eval "${evalId}" at ${fragmentOut} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    try {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- fragmentOut is our own derived path (joinUnderRoot-guarded)
+        unlinkSync(fragmentOut);
+    }
+    catch {
+        // Swallow: cleanup removes the grader dir regardless.
+    }
+    return raw;
+}
+/**
+ * Fail-open guards on the grader's tool verdict, split out of
+ * {@link runGraderForEval} to keep its cognitive complexity within budget:
+ *
+ * - Fix #1: the eval DECLARED tool expectations, so the grader was asked to
+ *   judge them and emit a `tool` verdict — a fragment WITHOUT one means the
+ *   grader silently skipped half its job. That is grader breakage (exit 1),
+ *   never a laundered pass: an absent tool block must not merge as "there were
+ *   no tool expectations to check".
+ * - Fix #3: the grader self-reports `tool.passed`, but we never trust it —
+ *   recompute from the sub-checks via {@link computeToolPassed}. A `passed`
+ *   that disagrees with its own sub-checks is grader malfunction (a false green
+ *   or a false red), surfaced loudly rather than merged. Mirrors
+ *   `reconcileGrading`'s summary/expectations reconciliation (grading-adapter.ts).
+ */
+function assertToolVerdictConsistent(evalId, toolExpectations, tool) {
+    if (toolExpectations !== undefined && tool === undefined) {
+        throw new InternalHarnessError(`Grader for eval "${evalId}" was asked to judge declared tool expectations but returned no \`tool\` verdict in its fragment.`);
+    }
+    if (tool !== undefined && computeToolPassed(tool) !== tool.passed) {
+        throw new InternalHarnessError(`Grader for eval "${evalId}" emitted a tool verdict whose \`passed\` (${tool.passed}) ` +
+            `disagrees with its own sub-checks (recomputed ${computeToolPassed(tool)}).`);
+    }
+}
+//# sourceMappingURL=eval-grader.js.map

package/dist/skill-test/eval-grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEpH,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAgD1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH;;;;;GAKG;AACH,SAAS,kBAAkB,CACzB,UAAwC;IAExC,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC/C,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,8EAA8E;IAC9E,0EAA0E;IAC1E,2EAA2E;IAC3E,6EAA6E;IAC7E,4BAA4B;IAC5B,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAE9E,6EAA6E;IAC7E,6EAA6E;IAC7E,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,0BAA0B,CAAC,WAAmB,EAAE,MAAc,EAAE,MAAc;IACrF,kIAAkI;IAClI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,kIAAkI;QAClI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,iBAAiB,CACzB,6BAA6B,MAAM,QAAQ,WAAW,uBAAuB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAChI,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,kIAAkI;QAClI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,sDAAsD;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;AACH,CAAC"}