@skill-harness/core 0.2.1 → 0.3.1

package/dist/run.js

@@ -1,8 +1,8 @@
-import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
-import { createHash } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
+import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -10,34 +10,6 @@ import { createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
-/** sha256 of a file, or null when it doesn't exist — missing sources are lint's problem, not run's. */
-function sha256(path) {
-    try {
-        return createHash("sha256").update(readFileSync(path)).digest("hex");
-    }
-    catch {
-        return null;
-    }
-}
-/**
- * Hash every source file this run measures: SKILL.md + each distinct
- * system_prompt_file (agents/<name>.md). Recorded in results.yaml so lint can prove
- * a published result still describes the current text.
- */
-function sourceHashes(skillDir, specPath, scenarios) {
-    const hashes = {};
-    const skillMd = sha256(resolve(skillDir, "SKILL.md"));
-    if (skillMd)
-        hashes["SKILL.md"] = skillMd;
-    for (const s of scenarios) {
-        if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
-            const h = sha256(resolve(dirname(specPath), s.systemPromptFile));
-            if (h)
-                hashes[s.systemPromptFile] = h;
-        }
-    }
-    return hashes;
-}
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -99,7 +71,9 @@ export async function runSkillModel(opts) {
         label: opts.label ?? null,
         mode,
         ...(partial ? { partial: true } : {}),
-        source_hashes: sourceHashes(skillDir, opts.specPath, scenarios),
+        // Only the scenarios this run actually measured: a --only run must not claim
+        // coverage of scenarios it skipped.
+        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios }),
         scenarios: scenarioResults,
     }, ctx);
     if (ctx) {
@@ -136,6 +110,10 @@ async function runRep(scenario, rep, repCount, ctx) {
     let ws = null;
     let transcript = "";
     let gatePrefix = null;
+    // Null until a seeded rep actually reaches its gates: a workspace-setup failure
+    // produces no diff, and writing an empty artifact there would misreport "the
+    // model changed nothing" for a rep that never ran.
+    let stagedDiff = null;
     try {
         try {
             ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
@@ -160,9 +138,11 @@ async function runRep(scenario, rep, repCount, ctx) {
                 if (scenario.mode === "seeded") {
                     const r = await runSeeded(scenario, {
                         skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
+                        specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
                     });
                     transcript = r.transcript;
                     gatePrefix = r.gateFailure;
+                    stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
                 }
                 else {
                     transcript = await ctx.adapter.run({
@@ -178,8 +158,16 @@ async function runRep(scenario, rep, repCount, ctx) {
                     break;
             }
         }
-        writeFileSync(transcriptPath(runDir, scenario.id, mode, repCount > 1 ? rep : undefined), transcript, "utf8");
+        const repSuffix = repCount > 1 ? rep : undefined;
+        writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
         if (scenario.mode === "seeded") {
+            // The workspace is torn down in the `finally` below, so this is the only
+            // chance to keep what the model actually wrote. Persisted uncapped (the
+            // transcript's copy is capped for the judge) and for every rep, pass or
+            // fail — a gate failure is exactly when you want to read the diff.
+            if (stagedDiff !== null) {
+                writeFileSync(diffPath(runDir, scenario.id, mode, repSuffix), stagedDiff, "utf8");
+            }
             appendJournal(runDir, { event: "gate-result", ts: now(), id: scenario.id, ok: !gatePrefix, detail: gatePrefix ?? "", ...repField });
         }
         let verdict;

package/dist/seeded.d.ts

@@ -1,21 +1,101 @@
 import type { Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+import { type ExecResult } from "./util/exec.js";
 interface SeededOpts {
     skillDir: string;
     adapter: HarnessAdapter;
     model: ModelRef;
     mode: RunMode;
     cwd: string;
+    specDir: string;
+    /**
+     * How the vitest gates shell out. Defaults to the real `npx vitest run`.
+     *
+     * A seam, not a mock: a workspace is a bare temp dir, so a test that exercised
+     * the real runner would resolve vitest off the network and be slow and flaky.
+     * Injecting it lets the gate LOGIC — pass, fail, nothing-collected — be tested
+     * deterministically.
+     */
+    runVitest?: (args: string[], cwd: string) => Promise<VitestRun>;
 }
+/**
+ * Result of one vitest invocation — deliberately `ExecResult`, not a narrower
+ * shape of its own.
+ *
+ * An earlier version declared `code: number`. That narrowing was a lie the
+ * compiler happened not to catch (the default's inferred type silently widened
+ * it back), and it is unrepresentable in practice: `exec` SIGKILLs on timeout
+ * and a signal-killed child closes with `code === null`. Declaring non-null
+ * would have made the vitest gate's timeout path — the one an injected double
+ * most needs to reproduce — impossible to express in a test.
+ */
+export type VitestRun = ExecResult;
 export interface SeededOutcome {
     transcript: string;
     gateFailure: string | null;
+    diff: string;
 }
+/**
+ * The added/removed lines of a unified diff — what the model actually *changed*,
+ * with context lines and file headers dropped.
+ *
+ * This is the difference between "the diff mentions `lastIndex`" and "the model
+ * touched `lastIndex`". A unified diff carries three lines of context around every
+ * hunk, so an untouched function sitting near the edit site appears in the diff
+ * verbatim. `build` A2 is exactly that shape — its checklist notes that `lastIndex`
+ * "sits two lines from the edit site" — so a naive substring test against the whole
+ * diff would fail the scenario for every model that fixed the right thing, which is
+ * worse than the prose-dependent item it replaces.
+ *
+ * Classification is HUNK-AWARE rather than prefix-based, because `+++`/`---` are
+ * only headers *outside* a hunk. Filtering on those prefixes anywhere would eat a
+ * changed line whose own source text starts with `++` or `--` — `++counter;` at
+ * column zero, a removed SQL/Lua `-- comment`, a YAML `---` separator. Those
+ * became `+++counter;` and `--- comment` once the diff marker was prepended, were
+ * read as headers, and vanished: `diff_excludes` then reported OK for a diff that
+ * touched the forbidden symbol. A false PASS on an objective gate is worse than
+ * the subjective check it replaced, so the parse follows the format instead of
+ * guessing from prefixes.
+ */
+export declare function changedLines(diff: string): string;
+/**
+ * Cut a diff to a byte budget on a line boundary, appending an explicit marker
+ * naming how much was dropped.
+ *
+ * The marker is not decoration: a silently truncated diff would let the judge
+ * grade "the function is missing" when it was merely cut off, which is the exact
+ * class of false-FAIL this whole change exists to remove. Truncation is reported
+ * as a fact about the transcript, and the untruncated diff is always on disk.
+ */
+export declare function capDiff(diff: string, maxBytes?: number): string;
 /**
  * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
- * the repo, then evaluate objective gates (staged-diff contains + optional vitest
- * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
- * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ * the repo, then evaluate the objective gates it declares — `diff_contains`,
+ * `diff_excludes`, `vitest` and `post_test`. Every gate that is configured runs;
+ * the FIRST failure is what `gateFailure` reports, and a non-null `gateFailure`
+ * makes the scenario an auto-FAIL that never reaches the judge.
+ *
+ * Returns the full staged diff alongside the transcript: the caller persists it
+ * as a run artifact, and a size-capped copy is appended to the transcript under
+ * `=== STAGED DIFF ===` so the judge grades the code rather than the model's
+ * description of it. Workspace creation (fixture copy + git baseline) and
+ * teardown are the caller's responsibility (run.ts).
  */
 export declare function runSeeded(scenario: Scenario, opts: SeededOpts): Promise<SeededOutcome>;
+export interface VitestTally {
+    passed: number;
+    failed: number;
+    skipped: number;
+    todo: number;
+}
+/**
+ * Parse vitest's `Tests  N passed | M skipped (T)` summary line.
+ *
+ * Used to require positive evidence that a hidden `post_test` actually executed
+ * assertions, rather than trusting a zero exit code — which vitest also returns
+ * when every test in the file is skipped. Returns null when no summary line is
+ * present, which the caller treats as "cannot confirm it ran" rather than as a
+ * pass.
+ */
+export declare function vitestTally(out: string): VitestTally | null;
 export {};

package/dist/seeded.js

@@ -1,11 +1,126 @@
+import { copyFileSync, statSync } from "node:fs";
+import { extname, isAbsolute, join, resolve } from "node:path";
 import { exec } from "./util/exec.js";
 import { envNum } from "./util/env.js";
+/**
+ * Filename STEM the post-test is copied to at the workspace root; the extension
+ * (`.test.ts`) is appended at the copy site, which is the part that decides
+ * whether vitest collects it at all.
+ *
+ * Harness-owned rather than the author's basename, for two reasons: it cannot
+ * collide with a fixture file by accident, and a model cannot shadow the check by
+ * creating a *file* at the path it guesses we will use — the copy happens after
+ * the model is done and overwrites unconditionally. (A *directory* there makes
+ * the copy fail; that is handled as an infrastructure error, not a model FAIL.)
+ */
+const POST_TEST_BASE = "skill-harness.post";
 const VITEST_TIMEOUT_MS = envNum("VITEST_TIMEOUT_MS", 120_000);
+/**
+ * The added/removed lines of a unified diff — what the model actually *changed*,
+ * with context lines and file headers dropped.
+ *
+ * This is the difference between "the diff mentions `lastIndex`" and "the model
+ * touched `lastIndex`". A unified diff carries three lines of context around every
+ * hunk, so an untouched function sitting near the edit site appears in the diff
+ * verbatim. `build` A2 is exactly that shape — its checklist notes that `lastIndex`
+ * "sits two lines from the edit site" — so a naive substring test against the whole
+ * diff would fail the scenario for every model that fixed the right thing, which is
+ * worse than the prose-dependent item it replaces.
+ *
+ * Classification is HUNK-AWARE rather than prefix-based, because `+++`/`---` are
+ * only headers *outside* a hunk. Filtering on those prefixes anywhere would eat a
+ * changed line whose own source text starts with `++` or `--` — `++counter;` at
+ * column zero, a removed SQL/Lua `-- comment`, a YAML `---` separator. Those
+ * became `+++counter;` and `--- comment` once the diff marker was prepended, were
+ * read as headers, and vanished: `diff_excludes` then reported OK for a diff that
+ * touched the forbidden symbol. A false PASS on an objective gate is worse than
+ * the subjective check it replaced, so the parse follows the format instead of
+ * guessing from prefixes.
+ */
+export function changedLines(diff) {
+    const out = [];
+    let inHunk = false;
+    for (const line of diff.split("\n")) {
+        if (line.startsWith("@@")) {
+            inHunk = true; // a hunk header opens the region where +/- mean "changed"
+            continue;
+        }
+        if (line.startsWith("diff --git ")) {
+            inHunk = false; // a new file section closes it; its ---/+++ are headers again
+            continue;
+        }
+        if (!inHunk)
+            continue; // index/mode/---/+++ preamble lines
+        if (line.startsWith("+") || line.startsWith("-"))
+            out.push(line);
+    }
+    return out.join("\n");
+}
+/**
+ * Byte cap on the diff copy embedded in the judged transcript. The judge prompt
+ * is a single request, so an unbounded diff (a fixture-wide refactor, a
+ * regenerated lockfile) could overflow the context window and turn a gradeable
+ * run into a judge ERROR. The artifact on disk is never capped — only the copy
+ * the judge reads.
+ */
+const DIFF_MAX_BYTES = envNum("DIFF_MAX_BYTES", 64_000);
+/**
+ * Cut a diff to a byte budget on a line boundary, appending an explicit marker
+ * naming how much was dropped.
+ *
+ * The marker is not decoration: a silently truncated diff would let the judge
+ * grade "the function is missing" when it was merely cut off, which is the exact
+ * class of false-FAIL this whole change exists to remove. Truncation is reported
+ * as a fact about the transcript, and the untruncated diff is always on disk.
+ */
+export function capDiff(diff, maxBytes = DIFF_MAX_BYTES) {
+    const total = Buffer.byteLength(diff, "utf8");
+    if (total <= maxBytes)
+        return diff;
+    // Accumulate whole lines until the next one would exceed the budget. `used`
+    // counts the separators actually emitted by join() — n lines carry n-1 of
+    // them — so the omitted figure in the marker is exact rather than one byte
+    // short. This file's whole thesis is telling the judge accurately what it was
+    // not shown, so an off-by-one here is a small lie in the wrong place.
+    const kept = [];
+    let used = 0;
+    for (const line of diff.split("\n")) {
+        const cost = Buffer.byteLength(line, "utf8") + (kept.length > 0 ? 1 : 0);
+        if (used + cost > maxBytes)
+            break;
+        kept.push(line);
+        used += cost;
+    }
+    // A single line longer than the whole budget (a minified bundle, a lockfile,
+    // a generated blob) would otherwise keep nothing at all and hand the judge a
+    // marker with zero code under it. Show a byte-safe prefix instead: some
+    // evidence beats none, and the marker still says what was cut.
+    if (kept.length === 0) {
+        const head = Buffer.from(diff, "utf8").subarray(0, maxBytes).toString("utf8");
+        // toString() on a boundary-split multibyte sequence yields U+FFFD; drop a
+        // trailing one rather than show the judge a corrupted character.
+        const clean = head.endsWith("�") ? head.slice(0, -1) : head;
+        kept.push(clean);
+        used = Buffer.byteLength(clean, "utf8");
+    }
+    const omitted = total - used;
+    return (kept.join("\n") +
+        `\n[… diff truncated: ${omitted} of ${total} bytes omitted (cap ${maxBytes}). ` +
+        `The complete diff is saved beside this transcript as this scenario's .diff.txt artifact. ` +
+        `Do not treat anything below the cut as absent — it was not shown to you. …]`);
+}
 /**
  * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
- * the repo, then evaluate objective gates (staged-diff contains + optional vitest
- * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
- * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ * the repo, then evaluate the objective gates it declares — `diff_contains`,
+ * `diff_excludes`, `vitest` and `post_test`. Every gate that is configured runs;
+ * the FIRST failure is what `gateFailure` reports, and a non-null `gateFailure`
+ * makes the scenario an auto-FAIL that never reaches the judge.
+ *
+ * Returns the full staged diff alongside the transcript: the caller persists it
+ * as a run artifact, and a size-capped copy is appended to the transcript under
+ * `=== STAGED DIFF ===` so the judge grades the code rather than the model's
+ * description of it. Workspace creation (fixture copy + git baseline) and
+ * teardown are the caller's responsibility (run.ts).
  */
 export async function runSeeded(scenario, opts) {
     const repo = opts.cwd;
@@ -16,26 +131,157 @@ export async function runSeeded(scenario, opts) {
         turns: scenario.turns,
         cwd: repo,
     });
-    await git(repo, ["add", "-A"]);
-    const diff = (await git(repo, ["diff", "--cached"])).stdout;
     const parts = [harnessOut, "", "=== SEEDED GATES ==="];
     let gateFailure = null;
-    const wantDiff = scenario.assert?.diff_contains ?? [];
-    for (const needle of wantDiff) {
-        const ok = diff.includes(needle);
+    const runVitest = opts.runVitest ??
+        ((args, cwd) => exec("npx", ["vitest", "run", ...args], { cwd, timeoutMs: VITEST_TIMEOUT_MS }));
+    // `exec` never throws on a non-zero exit and, on timeout, SIGKILLs the child and
+    // resolves with whatever stdout accumulated (code === null). Ignoring that here
+    // let a truncated or failed capture flow into every downstream consumer as
+    // evidence: diff_contains FAILs blamed on the model, diff_excludes silently
+    // passing, the transcript asserting "the model left no staged changes", and a
+    // partial diff persisted as the "complete" artifact — under the cap, so the
+    // truncation marker never fires. A capture we cannot trust is infrastructure,
+    // and must say so instead of being graded.
+    const add = await git(repo, ["add", "-A"]);
+    const show = await git(repo, ["diff", "--cached"]);
+    const diff = show.stdout;
+    const gitFailure = [add, show].find((r) => r.code !== 0);
+    if (gitFailure) {
+        const why = gitFailure.code === null ? "timed out and was killed" : `exited ${gitFailure.code}`;
+        const msg = `staged diff could not be captured — git ${why} — infrastructure, not model behavior` +
+            (gitFailure.stderr.trim() ? `: ${gitFailure.stderr.trim().split("\n")[0]}` : "");
+        parts.push(`  staged diff: ERROR (${msg})`);
+        gateFailure = msg;
+        return finish(parts, gateFailure, diff);
+    }
+    // BOTH needle gates read the changed lines only, never context. A unified diff
+    // carries three lines of context per hunk, so an untouched symbol near the edit
+    // site appears verbatim in the diff text — and matching that means neither gate
+    // is answering the question it was asked.
+    //
+    // The positive gate is the one this was measured on. `build` A4 asserts
+    // diff_contains ["divide", "ok"], and its fixture's baseline already contains
+    // both — `ok` in `{ ok: true; value: T }`, `divide` inside `divideByZero` — so
+    // any edit near those lines satisfied the gate whether or not the model wrote
+    // either token. Every published A4 result recorded that as an objective pass.
+    // Read against changed lines it means what the checklist means: the model
+    // returned a Result.
+    const changed = changedLines(diff);
+    for (const needle of scenario.assert?.diff_contains ?? []) {
+        const ok = changed.includes(needle);
         parts.push(`  diff_contains ${JSON.stringify(needle)}: ${ok ? "OK" : "MISSING"}`);
         if (!ok && !gateFailure)
             gateFailure = `staged diff missing ${JSON.stringify(needle)}`;
     }
+    // Scope discipline, stated as a fact about the diff rather than inferred from
+    // whether the model remembered to say "I left lastIndex alone".
+    for (const needle of scenario.assert?.diff_excludes ?? []) {
+        const ok = !changed.includes(needle);
+        parts.push(`  diff_excludes ${JSON.stringify(needle)}: ${ok ? "OK" : "PRESENT"}`);
+        if (!ok && !gateFailure)
+            gateFailure = `staged diff touches forbidden ${JSON.stringify(needle)}`;
+    }
     if (scenario.assert?.vitest) {
-        const v = await exec("npx", ["vitest", "run"], { cwd: repo, timeoutMs: VITEST_TIMEOUT_MS });
+        const v = await runVitest([], repo);
+        // code === null means exec SIGKILLed it at the timeout. That is infrastructure,
+        // not a failing test, and must not be reported as the model's fault.
+        const killed = v.code === null;
         const passed = v.code === 0;
-        parts.push(`  vitest run: ${passed ? "PASS" : `FAIL (exit ${v.code})`}`);
-        parts.push(indent(v.stdout.trim() || v.stderr.trim()));
-        if (!passed && !gateFailure)
-            gateFailure = `vitest failed (exit ${v.code})`;
+        parts.push(killed
+            ? `  vitest run: ERROR (timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior)`
+            : `  vitest run: ${passed ? "PASS" : `FAIL (exit ${v.code})`}`);
+        parts.push(indent(bothStreams(v)));
+        if (!passed && !gateFailure) {
+            gateFailure = killed
+                ? `vitest timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior`
+                : `vitest failed (exit ${v.code})`;
+        }
     }
-    return { transcript: parts.join("\n"), gateFailure };
+    const postTest = scenario.assert?.post_test;
+    if (postTest) {
+        const src = isAbsolute(postTest) ? postTest : resolve(opts.specDir, postTest);
+        // statSync().isFile(), not existsSync: a DIRECTORY "exists", and copyFileSync
+        // then throws EISDIR. That rejection escapes runSeeded, and runRep guards only
+        // workspace setup — so it reaches runSkillModel, writeResults never runs, and a
+        // paid multi-scenario run loses every scenario already completed. One spec typo
+        // (`post_test: post` for `post/A1.test.ts`) must cost a scenario, not a run.
+        if (!isReadableFile(src)) {
+            // Not model behavior. It still fails the scenario — a silently skipped gate is
+            // worse than a loud one — but the message says whose fault it is so nobody
+            // reads it as "the model broke the test".
+            const msg = `post_test is not a readable file: ${postTest} — spec error, not model behavior`;
+            parts.push(`  post_test: ERROR (${msg})`);
+            if (!gateFailure)
+                gateFailure = msg;
+        }
+        else {
+            // `.test` is part of the name we build, never taken from the source: extname()
+            // of "A2.test.ts" is ".ts", so appending only that produced
+            // "skill-harness.post.ts" — a file vitest does not collect as a test at all,
+            // which would have made the gate silently vacuous.
+            const dest = join(repo, `${POST_TEST_BASE}.test${extname(src) || ".ts"}`);
+            try {
+                copyFileSync(src, dest);
+            }
+            catch (e) {
+                // Permissions, a race, a full disk. Same reasoning as the branch above:
+                // degrade this scenario, never take the whole run down with it.
+                const msg = `post_test could not be copied into the workspace ` +
+                    `(${e instanceof Error ? e.message : String(e)}) — infrastructure, not model behavior`;
+                parts.push(`  post_test: ERROR (${msg})`);
+                if (!gateFailure)
+                    gateFailure = msg;
+                return finish(parts, gateFailure, diff);
+            }
+            const v = await runVitest([POST_TEST_BASE], repo);
+            const out = `${v.stdout}\n${v.stderr}`;
+            // This gate must never pass without positive evidence that assertions ran.
+            // Absence-of-failure is not enough: a `.skip`/`.todo` left in the file exits
+            // ZERO and prints "Tests  1 skipped", so keying off the exit code reported
+            // PASS for a hidden gate that executed nothing — the exact vacuous-gate
+            // shape post_test exists to prevent, and one nobody would ever notice
+            // because a passing gate produces output no one reads.
+            const tally = vitestTally(out);
+            // Anchored to line start: the unanchored form also matched this string
+            // appearing anywhere in test output or a model-authored console.log, which
+            // would flip a genuine pass into a phantom "fixture is broken". The exact
+            // wording is vitest's ("No test files found, exiting with code 1", verified
+            // against 2.1.9) — recheck it when the vitest major changes.
+            const notCollected = /^\s*No test files found/im.test(out);
+            const killed = v.code === null;
+            let problem = null;
+            if (killed) {
+                problem = `post_test ${JSON.stringify(postTest)} timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior`;
+            }
+            else if (notCollected) {
+                problem = `post_test ${JSON.stringify(postTest)} was never collected by vitest — spec/fixture error, not model behavior`;
+            }
+            else if (tally === null) {
+                problem = `post_test ${JSON.stringify(postTest)} produced no parseable vitest summary (exit ${v.code}) — cannot confirm it ran`;
+            }
+            else if (v.code !== 0 || tally.failed > 0) {
+                // A real failure is checked BEFORE the vacuity conditions below: a run
+                // where everything failed also has zero passes, and reporting that as
+                // "ran no assertions" would point the author at their spec instead of at
+                // the model's code, which is the actual news.
+                problem = `post_test ${JSON.stringify(postTest)} failed (exit ${v.code})`;
+            }
+            else if (tally.skipped > 0 || tally.todo > 0) {
+                problem = `post_test ${JSON.stringify(postTest)} has ${tally.skipped + tally.todo} skipped/todo test(s) — a hidden gate must actually run; spec error, not model behavior`;
+            }
+            else if (tally.passed === 0) {
+                problem = `post_test ${JSON.stringify(postTest)} ran no assertions — spec error, not model behavior`;
+            }
+            parts.push(problem === null
+                ? `  post_test ${JSON.stringify(postTest)}: PASS (${tally.passed} assertion-bearing test(s))`
+                : `  post_test ${JSON.stringify(postTest)}: ${v.code === 0 && !killed ? "ERROR" : "FAIL"} (${problem})`);
+            parts.push(indent(bothStreams(v)));
+            if (problem && !gateFailure)
+                gateFailure = problem;
+        }
+    }
+    return finish(parts, gateFailure, diff);
 }
 function git(cwd, args) {
     return exec("git", args, { cwd, timeoutMs: 30_000 });
@@ -43,4 +289,57 @@ function git(cwd, args) {
 function indent(s) {
     return s.split("\n").map((l) => `    ${l}`).join("\n");
 }
+/** True only for a regular file we can stat. Never throws — a directory, a dangling symlink or EACCES all read as "not usable". */
+function isReadableFile(p) {
+    try {
+        return statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Both streams, labelled.
+ *
+ * The previous `stdout.trim() || stderr.trim()` dropped stderr entirely whenever
+ * stdout was non-empty — which it always is once vitest prints its banner. That
+ * discarded exactly the diagnostics worth keeping, including exec's own
+ * `[skill-harness] killed after …ms timeout` notice, leaving a transcript that
+ * showed partial test output and an unexplained failure.
+ */
+function bothStreams(v) {
+    const o = v.stdout.trim();
+    const e = v.stderr.trim();
+    if (o && e)
+        return `${o}\n[stderr]\n${e}`;
+    return o || e;
+}
+/** Append the staged diff and return the outcome. Every exit path goes through here, so the judge always sees the same sections in the same order. */
+function finish(parts, gateFailure, diff) {
+    // The code itself, last — the gates above only prove that keywords appeared.
+    // Without this section a seeded checklist item about what the code *does* is
+    // graded from the model's own description of its work.
+    parts.push("", "=== STAGED DIFF ===");
+    parts.push(diff.trim() === "" ? "  (empty — the model left no staged changes)" : capDiff(diff));
+    return { transcript: parts.join("\n"), gateFailure, diff };
+}
+/**
+ * Parse vitest's `Tests  N passed | M skipped (T)` summary line.
+ *
+ * Used to require positive evidence that a hidden `post_test` actually executed
+ * assertions, rather than trusting a zero exit code — which vitest also returns
+ * when every test in the file is skipped. Returns null when no summary line is
+ * present, which the caller treats as "cannot confirm it ran" rather than as a
+ * pass.
+ */
+export function vitestTally(out) {
+    const line = /^\s*Tests\s+(.+)$/m.exec(out);
+    if (!line)
+        return null;
+    const read = (word) => {
+        const m = new RegExp(`(\\d+)\\s+${word}`).exec(line[1]);
+        return m ? Number(m[1]) : 0;
+    };
+    return { passed: read("passed"), failed: read("failed"), skipped: read("skipped"), todo: read("todo") };
+}
 //# sourceMappingURL=seeded.js.map