@vibe-agent-toolkit/agent-skills 0.1.40 → 0.1.41-rc.2

package/dist/skill-test/eval-executor.d.ts

@@ -0,0 +1,50 @@
+import { spawnHeadlessClaude, type ParsedTranscript, type SpawnResult } from '@vibe-agent-toolkit/utils';
+export interface RunExecutorInput {
+    evalId: string;
+    task: string;
+    subjectStagedDir: string;
+    /** `<workspacesRoot>/<id>` when the eval declares input `files`. */
+    workspaceDir?: string;
+    pluginDirs: string[];
+    env: NodeJS.ProcessEnv;
+    model?: string;
+    maxTurns: number;
+    maxBudgetUsd: number;
+    timeoutMs: number;
+    stallMs?: number;
+    /** Injectable seam for tests; defaults to the real {@link spawnHeadlessClaude}. */
+    spawn?: typeof spawnHeadlessClaude;
+    /** Called with each stdout chunk as it streams (for progress echo). */
+    onProgress?: (chunk: string) => void;
+}
+export interface ExecutorOutcome {
+    /** Full captured stdout (stream-json), held IN MEMORY only — never written to disk. */
+    transcript: string;
+    parsed: ParsedTranscript;
+    spawnResult: SpawnResult;
+    /** true = the executor exited non-zero WITHIN budget — a valid eval failure, not a harness break. */
+    cleanFailure: boolean;
+}
+/**
+ * Run ONE eval's executor: a blind `claude -p` spawn that performs `task`
+ * against the staged subject (issue #145). The executor's entire stream-json
+ * stdout is accumulated in memory (never written to a skill-writable disk
+ * path — the "stop discarding stdout" fix) and echoed to `onProgress` chunk
+ * by chunk for visibility.
+ *
+ * Encodes the R1 exit bifurcation at the executor boundary:
+ * - A spawn error (the promise rejects), or our own watchdog firing
+ *   (`timedOut`/`stalled`), is HARNESS breakage → thrown as
+ *   {@link InternalHarnessError} (exit 1).
+ * - A mid-stream rate-limit that cuts the run off before it completes (status
+ *   !== 0) is NOT a harness failure but also not gradeable → thrown as
+ *   {@link RateLimitSignal} so the caller's pool can back off and retry this
+ *   eval. A rate-limit event that the run still completed past (status 0) is
+ *   kept and returned normally.
+ * - Anything else — including a clean non-zero exit within budget — is a
+ *   valid eval OUTCOME, not a harness break: returned with `cleanFailure`
+ *   reflecting whether the executor exited non-zero. Never laundered into a
+ *   thrown error; the grader decides whether the transcript passes.
+ */
+export declare function runExecutorForEval(input: RunExecutorInput): Promise<ExecutorOutcome>;
+//# sourceMappingURL=eval-executor.d.ts.map

package/dist/skill-test/eval-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-executor.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,mBAAmB,EACnB,KAAK,gBAAgB,EACrB,KAAK,WAAW,EACjB,MAAM,2BAA2B,CAAC;AAMnC,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mFAAmF;IACnF,KAAK,CAAC,EAAE,OAAO,mBAAmB,CAAC;IACnC,uEAAuE;IACvE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,eAAe;IAC9B,uFAAuF;IACvF,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,gBAAgB,CAAC;IACzB,WAAW,EAAE,WAAW,CAAC;IACzB,qGAAqG;IACrG,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsE1F"}

package/dist/skill-test/eval-executor.js

@@ -0,0 +1,85 @@
+import { parseStreamJsonTranscript, spawnHeadlessClaude, } from '@vibe-agent-toolkit/utils';
+import { assertExecutorPromptInvariants, buildExecutorPrompt } from './executor-prompt.js';
+import { InternalHarnessError } from './exit-codes.js';
+import { RateLimitSignal } from './pipeline.js';
+/**
+ * Run ONE eval's executor: a blind `claude -p` spawn that performs `task`
+ * against the staged subject (issue #145). The executor's entire stream-json
+ * stdout is accumulated in memory (never written to a skill-writable disk
+ * path — the "stop discarding stdout" fix) and echoed to `onProgress` chunk
+ * by chunk for visibility.
+ *
+ * Encodes the R1 exit bifurcation at the executor boundary:
+ * - A spawn error (the promise rejects), or our own watchdog firing
+ *   (`timedOut`/`stalled`), is HARNESS breakage → thrown as
+ *   {@link InternalHarnessError} (exit 1).
+ * - A mid-stream rate-limit that cuts the run off before it completes (status
+ *   !== 0) is NOT a harness failure but also not gradeable → thrown as
+ *   {@link RateLimitSignal} so the caller's pool can back off and retry this
+ *   eval. A rate-limit event that the run still completed past (status 0) is
+ *   kept and returned normally.
+ * - Anything else — including a clean non-zero exit within budget — is a
+ *   valid eval OUTCOME, not a harness break: returned with `cleanFailure`
+ *   reflecting whether the executor exited non-zero. Never laundered into a
+ *   thrown error; the grader decides whether the transcript passes.
+ */
+export async function runExecutorForEval(input) {
+    const workDir = input.workspaceDir ?? input.subjectStagedDir;
+    const prompt = buildExecutorPrompt({
+        task: input.task,
+        subjectPath: input.subjectStagedDir,
+        ...(input.workspaceDir === undefined ? {} : { workspaceDir: input.workspaceDir }),
+    });
+    // Defense-in-depth (parity with run-harness's build→assert→use pattern):
+    // re-verify the built prompt didn't accidentally blind-break the executor
+    // before spawning. Only OUR scaffolding is policed — the adopter task is
+    // excluded, so a legit task containing denylist words does not trip this.
+    assertExecutorPromptInvariants(prompt, input.task);
+    const spawn = input.spawn ?? spawnHeadlessClaude;
+    let transcript = '';
+    const onStdout = (chunk) => {
+        transcript += chunk;
+        input.onProgress?.(chunk);
+    };
+    let spawnResult;
+    try {
+        spawnResult = await spawn({
+            prompt,
+            pluginDirs: input.pluginDirs,
+            sandboxDir: workDir,
+            cwd: workDir,
+            env: input.env,
+            timeoutMs: input.timeoutMs,
+            onStdout,
+            maxTurns: input.maxTurns,
+            maxBudgetUsd: input.maxBudgetUsd,
+            ...(input.model === undefined ? {} : { model: input.model }),
+            ...(input.stallMs === undefined ? {} : { stallMs: input.stallMs }),
+        });
+    }
+    catch (err) {
+        throw new InternalHarnessError(`Executor spawn failed for eval "${input.evalId}": ${err instanceof Error ? err.message : String(err)}`);
+    }
+    // Watchdog kills (our own harness code, not the executor) are harness
+    // breakage regardless of what the executor produced before being killed.
+    if (spawnResult.stalled) {
+        throw new InternalHarnessError(`Executor stalled for eval "${input.evalId}" (no output for ${input.stallMs ?? 0}ms).`);
+    }
+    if (spawnResult.timedOut) {
+        throw new InternalHarnessError(`Executor timed out for eval "${input.evalId}" (limit ${input.timeoutMs}ms).`);
+    }
+    const parsed = parseStreamJsonTranscript(transcript);
+    // A rate-limit that cut the run off before completion (non-zero status) is
+    // not gradeable — signal the pool to back off and retry. A rate-limit event
+    // the run otherwise completed past (status 0) is kept as-is.
+    if (parsed.rateLimited && spawnResult.status !== 0) {
+        throw new RateLimitSignal(`Executor rate-limited for eval "${input.evalId}".`);
+    }
+    return {
+        transcript,
+        parsed,
+        spawnResult,
+        cleanFailure: spawnResult.status !== 0,
+    };
+}
+//# sourceMappingURL=eval-executor.js.map

package/dist/skill-test/eval-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-executor.js","sourceRoot":"","sources":["../../src/skill-test/eval-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yBAAyB,EACzB,mBAAmB,GAGpB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AA8BhD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAuB;IAC9D,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,gBAAgB,CAAC;IAC7D,MAAM,MAAM,GAAG,mBAAmB,CAAC;QACjC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,gBAAgB;QACnC,GAAG,CAAC,KAAK,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC;KAClF,CAAC,CAAC;IACH,yEAAyE;IACzE,0EAA0E;IAC1E,yEAAyE;IACzE,0EAA0E;IAC1E,8BAA8B,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAEnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,UAAU,IAAI,KAAK,CAAC;QACpB,KAAK,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC;IAEF,IAAI,WAAwB,CAAC;IAC7B,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,OAAO;YACZ,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5D,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,mCAAmC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxG,CAAC;IACJ,CAAC;IAED,sEAAsE;IACtE,yEAAyE;IACzE,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACvF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,gCAAgC,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC9E,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,yBAAyB,CAAC,UAAU,CAAC,CAAC;IAErD,2EAA2E;IAC3E,4EAA4E;IAC5E,6DAA6D;IAC7D,IAAI,MAAM,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,eAAe,CAAC,mCAAmC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACjF,CAAC;IAED,OAAO;QACL,UAAU;QACV,MAAM;QACN,WAAW;QACX,YAAY,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC;KACvC,CAAC;AACJ,CAAC"}

package/dist/skill-test/eval-fragment.d.ts

@@ -0,0 +1,278 @@
+import { z } from 'zod';
+/** One graded expectation inside a per-eval grader fragment. */
+export declare const EvalFragmentExpectationSchema: z.ZodObject<{
+    text: z.ZodString;
+    passed: z.ZodBoolean;
+    evidence: z.ZodOptional<z.ZodString>;
+}, "strict", z.ZodTypeAny, {
+    passed: boolean;
+    text: string;
+    evidence?: string | undefined;
+}, {
+    passed: boolean;
+    text: string;
+    evidence?: string | undefined;
+}>;
+export type EvalFragmentExpectation = z.infer<typeof EvalFragmentExpectationSchema>;
+/**
+ * Strict shape of a single grader subagent's output (issue #145 Task 4). One
+ * GRADER spawn grades ONE eval's expectations from that eval's captured
+ * transcript and writes exactly one fragment matching this schema; vat reads
+ * every eval's fragment back and merges them into the run's aggregate grading
+ * result (skill-creator's flat `grading.json` shape — see grading-adapter.ts).
+ *
+ * `.strict()`: this is vat's OWN output artifact contract, not external data
+ * we're auditing (see CLAUDE.md Postel's Law) — an unrecognized field is a
+ * grader bug we want surfaced immediately, not silently dropped.
+ *
+ * `arm` distinguishes a WITH/WITHOUT baseline run's fragment; absent means the
+ * default 'with' (skill present) arm.
+ *
+ * `tool` (issue #145 Phase T) carries this eval's tool-expectation verdict —
+ * a SEPARATE channel from `expectations[]` and from `friction` (see
+ * tool-eval-schema.ts). It is {@link ToolVerdictBodySchema}, i.e. a
+ * {@link import('./tool-eval-schema.js').ToolVerdict} WITHOUT `evalId`: the
+ * fragment already carries `evalId` at its top level, so the merge step
+ * (`mergeFragmentsToToolEval` in fragment-merge.ts) re-attaches it from the
+ * fragment when assembling vat's `tool-eval.json`.
+ */
+export declare const EvalFragmentSchema: z.ZodObject<{
+    runNonce: z.ZodString;
+    evalId: z.ZodString;
+    arm: z.ZodOptional<z.ZodEnum<["with", "without"]>>;
+    expectations: z.ZodArray<z.ZodObject<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, "strict", z.ZodTypeAny, {
+        passed: boolean;
+        text: string;
+        evidence?: string | undefined;
+    }, {
+        passed: boolean;
+        text: string;
+        evidence?: string | undefined;
+    }>, "many">;
+    friction: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        severity: z.ZodEnum<["high", "medium", "low"]>;
+        category: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file", "tool-expectation"]>;
+        message: z.ZodString;
+        subjectFile: z.ZodOptional<z.ZodString>;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, "strict", z.ZodTypeAny, {
+        message: string;
+        severity: "high" | "medium" | "low";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
+        evidence?: string | undefined;
+        subjectFile?: string | undefined;
+    }, {
+        message: string;
+        severity: "high" | "medium" | "low";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
+        evidence?: string | undefined;
+        subjectFile?: string | undefined;
+    }>, "many">>;
+    tool: z.ZodOptional<z.ZodObject<{
+        mustRun: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            ran: z.ZodBoolean;
+            evidence: z.ZodOptional<z.ZodString>;
+        }, "strict", z.ZodTypeAny, {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }, {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }>, "many">>;
+        mustNotRun: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            ran: z.ZodBoolean;
+            evidence: z.ZodOptional<z.ZodString>;
+        }, "strict", z.ZodTypeAny, {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }, {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }>, "many">>;
+        mustSucceed: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            succeeded: z.ZodBoolean;
+            evidence: z.ZodOptional<z.ZodString>;
+        }, "strict", z.ZodTypeAny, {
+            name: string;
+            succeeded: boolean;
+            evidence?: string | undefined;
+        }, {
+            name: string;
+            succeeded: boolean;
+            evidence?: string | undefined;
+        }>, "many">>;
+        sequence: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            steps: z.ZodArray<z.ZodString, "many">;
+            satisfied: z.ZodBoolean;
+            evidence: z.ZodOptional<z.ZodString>;
+        }, "strict", z.ZodTypeAny, {
+            steps: string[];
+            satisfied: boolean;
+            evidence?: string | undefined;
+        }, {
+            steps: string[];
+            satisfied: boolean;
+            evidence?: string | undefined;
+        }>, "many">>;
+        passed: z.ZodBoolean;
+    }, "strict", z.ZodTypeAny, {
+        passed: boolean;
+        mustRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustNotRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustSucceed?: {
+            name: string;
+            succeeded: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        sequence?: {
+            steps: string[];
+            satisfied: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+    }, {
+        passed: boolean;
+        mustRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustNotRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustSucceed?: {
+            name: string;
+            succeeded: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        sequence?: {
+            steps: string[];
+            satisfied: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+    }>>;
+}, "strict", z.ZodTypeAny, {
+    expectations: {
+        passed: boolean;
+        text: string;
+        evidence?: string | undefined;
+    }[];
+    evalId: string;
+    runNonce: string;
+    arm?: "with" | "without" | undefined;
+    friction?: {
+        message: string;
+        severity: "high" | "medium" | "low";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
+        evidence?: string | undefined;
+        subjectFile?: string | undefined;
+    }[] | undefined;
+    tool?: {
+        passed: boolean;
+        mustRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustNotRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustSucceed?: {
+            name: string;
+            succeeded: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        sequence?: {
+            steps: string[];
+            satisfied: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+    } | undefined;
+}, {
+    expectations: {
+        passed: boolean;
+        text: string;
+        evidence?: string | undefined;
+    }[];
+    evalId: string;
+    runNonce: string;
+    arm?: "with" | "without" | undefined;
+    friction?: {
+        message: string;
+        severity: "high" | "medium" | "low";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
+        evidence?: string | undefined;
+        subjectFile?: string | undefined;
+    }[] | undefined;
+    tool?: {
+        passed: boolean;
+        mustRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustNotRun?: {
+            name: string;
+            ran: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        mustSucceed?: {
+            name: string;
+            succeeded: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+        sequence?: {
+            steps: string[];
+            satisfied: boolean;
+            evidence?: string | undefined;
+        }[] | undefined;
+    } | undefined;
+}>;
+export type EvalFragment = z.infer<typeof EvalFragmentSchema>;
+/**
+ * Thrown when a single grader subagent's fragment output does not match
+ * {@link EvalFragmentSchema}. Deliberately distinct from `GradingSkewError`
+ * (grading-adapter.ts): that error's message is specific to the AGGREGATE
+ * `grading.json` shape (skill-creator's flat report) and is wrong when the
+ * thing that failed to parse is one grader's per-eval fragment — a fragment
+ * shape bug is a GRADER bug, not an aggregate-assembly bug, and the message
+ * should point at the offending eval, not at "Re-sync the vendored
+ * skill-creator".
+ */
+export declare class EvalFragmentError extends Error {
+    constructor(message: string);
+}
+/**
+ * Parse + validate one grader's fragment output. Throws {@link EvalFragmentError}
+ * — never {@link GradingSkewError} — because this is a per-eval fragment, not
+ * the assembled aggregate `grading.json`.
+ *
+ * The verdict channels are strict. The auxiliary `friction` field is sanitized
+ * leniently first (see {@link sanitizeFrictionField}); when items are dropped,
+ * `onWarn` (if given) is called so the operator sees that friction was partial —
+ * grading itself is never affected.
+ */
+export declare function parseEvalFragment(raw: unknown, onWarn?: (message: string) => void): EvalFragment;
+//# sourceMappingURL=eval-fragment.d.ts.map

package/dist/skill-test/eval-fragment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-fragment.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-fragment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,gEAAgE;AAChE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;EAI/B,CAAC;AAEZ,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOpB,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;;;GASG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAI5B;AA+CD;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,YAAY,CAkBhG"}

package/dist/skill-test/eval-fragment.js

@@ -0,0 +1,128 @@
+import { z } from 'zod';
+import { FrictionItemSchema } from './friction-schema.js';
+import { ToolVerdictBodySchema } from './tool-eval-schema.js';
+/** One graded expectation inside a per-eval grader fragment. */
+export const EvalFragmentExpectationSchema = z.object({
+    text: z.string(),
+    passed: z.boolean(),
+    evidence: z.string().optional(),
+}).strict();
+/**
+ * Strict shape of a single grader subagent's output (issue #145 Task 4). One
+ * GRADER spawn grades ONE eval's expectations from that eval's captured
+ * transcript and writes exactly one fragment matching this schema; vat reads
+ * every eval's fragment back and merges them into the run's aggregate grading
+ * result (skill-creator's flat `grading.json` shape — see grading-adapter.ts).
+ *
+ * `.strict()`: this is vat's OWN output artifact contract, not external data
+ * we're auditing (see CLAUDE.md Postel's Law) — an unrecognized field is a
+ * grader bug we want surfaced immediately, not silently dropped.
+ *
+ * `arm` distinguishes a WITH/WITHOUT baseline run's fragment; absent means the
+ * default 'with' (skill present) arm.
+ *
+ * `tool` (issue #145 Phase T) carries this eval's tool-expectation verdict —
+ * a SEPARATE channel from `expectations[]` and from `friction` (see
+ * tool-eval-schema.ts). It is {@link ToolVerdictBodySchema}, i.e. a
+ * {@link import('./tool-eval-schema.js').ToolVerdict} WITHOUT `evalId`: the
+ * fragment already carries `evalId` at its top level, so the merge step
+ * (`mergeFragmentsToToolEval` in fragment-merge.ts) re-attaches it from the
+ * fragment when assembling vat's `tool-eval.json`.
+ */
+export const EvalFragmentSchema = z.object({
+    runNonce: z.string().min(1),
+    evalId: z.string().min(1),
+    arm: z.enum(['with', 'without']).optional(),
+    expectations: z.array(EvalFragmentExpectationSchema).min(1),
+    friction: z.array(FrictionItemSchema).optional(),
+    tool: ToolVerdictBodySchema.optional(),
+}).strict();
+/**
+ * Thrown when a single grader subagent's fragment output does not match
+ * {@link EvalFragmentSchema}. Deliberately distinct from `GradingSkewError`
+ * (grading-adapter.ts): that error's message is specific to the AGGREGATE
+ * `grading.json` shape (skill-creator's flat report) and is wrong when the
+ * thing that failed to parse is one grader's per-eval fragment — a fragment
+ * shape bug is a GRADER bug, not an aggregate-assembly bug, and the message
+ * should point at the offending eval, not at "Re-sync the vendored
+ * skill-creator".
+ */
+export class EvalFragmentError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'EvalFragmentError';
+    }
+}
+/** Best-effort extraction of `evalId` from raw input for error messages, tolerating any shape. */
+function extractRawEvalId(raw) {
+    if (typeof raw === 'object' && raw !== null && 'evalId' in raw) {
+        const value = raw['evalId'];
+        if (typeof value === 'string' && value.length > 0)
+            return `"${value}"`;
+    }
+    return '(unknown)';
+}
+/**
+ * Sanitize ONLY the (non-verdict-bearing) `friction` field before the strict
+ * fragment parse. Friction is auxiliary advisory data, not a verdict channel —
+ * so a grader that wobbles on the friction shape (e.g. emits bare strings, the
+ * common failure mode — PR #147) must NOT be allowed to discard the
+ * verdict-bearing grading for the whole run. We drop friction items that don't
+ * match {@link FrictionItemSchema} (and a `friction` that isn't an array at
+ * all), then hand the rest to the STRICT fragment parse — so the verdict
+ * channels (`runNonce`/`evalId`/`expectations`/`tool`) stay fully fail-closed
+ * and an unknown field elsewhere still surfaces as a grader bug.
+ *
+ * Returns the possibly-rewritten raw value and how many friction items were
+ * dropped (0 when friction is absent or already well-shaped — the input is then
+ * returned untouched).
+ */
+function sanitizeFrictionField(raw) {
+    if (typeof raw !== 'object' || raw === null || !('friction' in raw)) {
+        return { value: raw, dropped: 0 };
+    }
+    const obj = raw;
+    const friction = obj['friction'];
+    if (friction === undefined)
+        return { value: raw, dropped: 0 };
+    if (!Array.isArray(friction)) {
+        // A non-array `friction` (e.g. a bare string) is unusable auxiliary noise —
+        // drop the whole field rather than fail the run over it.
+        const rest = {};
+        for (const [key, val] of Object.entries(obj)) {
+            if (key !== 'friction')
+                rest[key] = val;
+        }
+        return { value: rest, dropped: 1 };
+    }
+    const kept = friction.filter((item) => FrictionItemSchema.safeParse(item).success);
+    if (kept.length === friction.length)
+        return { value: raw, dropped: 0 };
+    return { value: { ...obj, friction: kept }, dropped: friction.length - kept.length };
+}
+/**
+ * Parse + validate one grader's fragment output. Throws {@link EvalFragmentError}
+ * — never {@link GradingSkewError} — because this is a per-eval fragment, not
+ * the assembled aggregate `grading.json`.
+ *
+ * The verdict channels are strict. The auxiliary `friction` field is sanitized
+ * leniently first (see {@link sanitizeFrictionField}); when items are dropped,
+ * `onWarn` (if given) is called so the operator sees that friction was partial —
+ * grading itself is never affected.
+ */
+export function parseEvalFragment(raw, onWarn) {
+    const { value, dropped } = sanitizeFrictionField(raw);
+    if (dropped > 0) {
+        onWarn?.(`grader fragment for eval ${extractRawEvalId(raw)} had ${dropped} malformed friction item(s) — ` +
+            `dropped them (friction is advisory; grading is unaffected).`);
+    }
+    const result = EvalFragmentSchema.safeParse(value);
+    if (!result.success) {
+        const firstIssue = result.error.issues[0];
+        const path = firstIssue?.path.join('.') ?? '(root)';
+        const evalId = extractRawEvalId(raw);
+        throw new EvalFragmentError(`grader fragment for eval ${evalId} has an invalid shape: invalid field at "${path}" (${firstIssue?.message ?? 'unknown'})`);
+    }
+    return result.data;
+}
+//# sourceMappingURL=eval-fragment.js.map

package/dist/skill-test/eval-fragment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-fragment.js","sourceRoot":"","sources":["../../src/skill-test/eval-fragment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,gEAAgE;AAChE,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC,MAAM,EAAE,CAAC;AAIZ;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC3C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE;IAChD,IAAI,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC,MAAM,EAAE,CAAC;AAIZ;;;;;;;;;GASG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,kGAAkG;AAClG,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAI,GAA+B,CAAC,QAAQ,CAAC,CAAC;QACzD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,KAAK,GAAG,CAAC;IACzE,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,qBAAqB,CAAC,GAAY;IACzC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IACjC,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC9D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,4EAA4E;QAC5E,yDAAyD;QACzD,MAAM,IAAI,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,GAAG,KAAK,UAAU;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QAC1C,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACrC,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;IACnF,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACvE,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;AACvF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAY,EAAE,MAAkC;IAChF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,EAAE,CACN,4BAA4B,gBAAgB,CAAC,GAAG,CAAC,QAAQ,OAAO,gCAAgC;YAC9F,6DAA6D,CAChE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,iBAAiB,CACzB,4BAA4B,MAAM,4CAA4C,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAC5H,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}

package/dist/skill-test/eval-grader.d.ts

@@ -0,0 +1,54 @@
+import { spawnHeadlessClaude } from '@vibe-agent-toolkit/utils';
+import { type EvalFragment } from './eval-fragment.js';
+import type { ToolExpectations } from './eval-inputs.js';
+export interface RunGraderInput {
+    evalId: string;
+    /** The FULL captured transcript (stream-json) for this eval's executor run. */
+    transcript: string;
+    expectations: string[];
+    expectedOutput?: string;
+    /** Path to the vendored skill-creator grader rubric (references/grader.md). */
+    rubricPath: string;
+    /**
+     * The eval's declared tool expectations (issue #145 Phase T). Optional — when
+     * absent, `buildGraderPrompt` emits no tool-verdict instruction and the
+     * returned fragment omits `tool`. See `grader-prompt.ts`'s
+     * `BuildGraderPromptOptions.toolExpectations`.
+     */
+    toolExpectations?: ToolExpectations;
+    /** Name → invocation hint for declared executables, a recognition aid alongside `toolExpectations`. */
+    declaredExecutables?: Array<{
+        name: string;
+        howInvoked: string;
+        kind: string;
+    }>;
+    /**
+     * VAT-ONLY directory the grader fragment is written to — deliberately
+     * OUTSIDE the skill-writable sandbox (never the executor's staged/workspace
+     * dir) so untrusted skill code from Task 7's executor run cannot forge or
+     * delete the fragment this run relies on.
+     */
+    graderOutDir: string;
+    graderModel: string;
+    /** Per-run integrity nonce the grader must echo back in the fragment. */
+    nonce: string;
+    maxTurns: number;
+    maxBudgetUsd: number;
+    timeoutMs: number;
+    stallMs?: number;
+    /** Grader auth env — no skill secrets are needed beyond auth. */
+    env: NodeJS.ProcessEnv;
+    /** Injectable seam for tests; defaults to the real {@link spawnHeadlessClaude}. */
+    spawn?: typeof spawnHeadlessClaude;
+    /** Called with each stdout chunk as it streams (for progress echo). */
+    onProgress?: (chunk: string) => void;
+    /**
+     * Reports this grader session's `total_cost_usd` (parsed from its stream-json
+     * transcript's terminal result; `undefined` when the transcript carried none,
+     * e.g. a mock spawn). Called once after a successful grader spawn so the run can
+     * aggregate total spend across all executor+grader sessions (adopter follow-up).
+     */
+    costSink?: (totalCostUsd: number | undefined) => void;
+}
+export declare function runGraderForEval(input: RunGraderInput): Promise<EvalFragment>;
+//# sourceMappingURL=eval-grader.d.ts.map

package/dist/skill-test/eval-grader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-grader.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAsD,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEpH,OAAO,EAAwC,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC7F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMzD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,uGAAuG;IACvG,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChF;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,mFAAmF;IACnF,KAAK,CAAC,EAAE,OAAO,mBAAmB,CAAC;IACnC,uEAAuE;IACvE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC;CACvD;AAsCD,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC,CAoGnF"}