@skill-harness/core 0.2.1 → 0.3.0

package/dist/grade.d.ts

@@ -7,6 +7,8 @@ export interface JudgePromptInput {
     scenario: Scenario;
     transcript: string;
 }
+/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
+export declare const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /** Build the LLM-judge prompt for one transcript (ported from the old grade.sh). */
 export declare function buildJudgePrompt(input: JudgePromptInput): string;
 export interface ParsedVerdict {

package/dist/grade.js

@@ -1,8 +1,33 @@
 import { createWorkspace } from "./workspace.js";
+/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
+export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
+/**
+ * Addendum pointing the judge at the code, added only when the code is actually there.
+ *
+ * A seeded transcript ends with the staged diff, and without this the judge
+ * weighs the model's prose about its work equally with the work itself — which
+ * is how six reps that all passed the objective gates split PASS/FAIL purely on
+ * whether the model wrote "rejects overdrafts" or "subtracts amount".
+ *
+ * Gated on the transcript CONTAINING the diff section, not on `scenario.mode`.
+ * The first sentence is a factual claim about the transcript, and every seeded
+ * transcript saved before this feature existed lacks that section — so keying
+ * off the mode would tell the judge its primary evidence is at the end of a
+ * transcript that has none, while also telling it the gate lines prove nothing.
+ * That is a sweep of FAILs justified by absent evidence, and it would land
+ * squarely on `grade`, the command AGENTS.md rule 4 recommends as the *cheap*
+ * de-confounding step before re-running. Inline scenarios have no diff either,
+ * so their prompt stays byte-identical and every published inline verdict
+ * remains comparable.
+ */
+const SEEDED_DIFF_GUIDANCE = `
+This transcript ends with a "=== STAGED DIFF ===" section: the actual code the assistant wrote, as \`git diff --cached\`. It is the primary evidence. Grade what the diff shows the code DOES, not what the assistant's prose claims it does — a confident description of behavior the diff does not implement is a FAIL, and behavior the diff plainly implements passes even if the assistant described it poorly or not at all. The "=== SEEDED GATES ===" lines above it are keyword and test-run checks only; they do not establish that the required behavior exists. If the diff is marked truncated, judge only what you can see and never infer that cut-off code is missing.
+`;
 /** Build the LLM-judge prompt for one transcript (ported from the old grade.sh). */
 export function buildJudgePrompt(input) {
     const { skill, persona, scenario, transcript } = input;
     const numbered = scenario.checklist.map((c, i) => `${i + 1}. ${c}`).join("\n");
+    const diffGuidance = scenario.mode === "seeded" && transcript.includes(STAGED_DIFF_HEADING) ? SEEDED_DIFF_GUIDANCE : "";
     return `You are grading ONE response from an AI assistant using a "${skill}" skill — ${persona} Judge it ONLY against the checklist below — do not add requirements beyond it.
 
 CHECKLIST (every numbered item must hold for a PASS):
@@ -10,7 +35,7 @@ ${numbered}
 
 TRANSCRIPT (the assistant is the model under test):
 ${transcript}
-
+${diffGuidance}
 Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Then output exactly these two lines:
 VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
 REASON: <15 words or fewer>`;

package/dist/index.d.ts

@@ -14,6 +14,7 @@ export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";

package/dist/index.js

@@ -14,6 +14,7 @@ export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";

package/dist/lint.d.ts

@@ -1,4 +1,4 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "consistency" | "stale" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "lint-error";
 export interface LintFinding {
     readonly skill: string;
     readonly scenario?: string;

package/dist/lint.js

@@ -1,9 +1,10 @@
 import { existsSync, statSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, resolve } from "node:path";
-import { createHash } from "node:crypto";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
+import { currentHashFor, describeSourceKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, UNREADABLE } from "./sources.js";
+import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
 function isDir(p) {
@@ -65,12 +66,58 @@ export function lintSkill(skillDir) {
     // relative to the spec's dir, matching workspace.ts resolve(specDir, fixture) where specDir = <skillDir>/tests.
     const specDir = dirname(specPath);
     for (const s of spec.scenarios) {
-        const fx = typeof s.workspace === "object" && s.workspace !== null ? s.workspace.fixture : undefined;
+        const fx = effectiveFixture(s); // shared with sources.ts so hashing and linting can't drift
         if (fx) {
             const abs = isAbsolute(fx) ? fx : resolve(specDir, fx);
             if (!isDir(abs)) {
                 findings.push({ skill, scenario: s.id, code: "fixture", message: `fixture not found: ${fx}` });
             }
+            else {
+                // A mistyped fixture marker (`_uncommited/`) is rejected at run time by
+                // createWorkspace — but only once a run has been started, where it surfaces as
+                // a scenario FAIL among real results. lint is free, offline and runs in CI, so
+                // it is where an author should meet this. The set flagged here is exactly the
+                // set workspace.ts refuses, so lint can never bless a fixture the runtime then
+                // rejects.
+                // try/catch because lintSkill's contract is "never throws": isDir() proves the
+                // path stats, not that it can be read, and an EACCES on readdir would escape
+                // to cli.ts, which replaces this skill's ENTIRE finding list with one
+                // lint-error — silently discarding the staleness and consistency checks below.
+                let markers = [];
+                try {
+                    markers = unknownMarkerDirs(abs);
+                }
+                catch (e) {
+                    findings.push({
+                        skill, scenario: s.id, code: "fixture",
+                        message: `fixture ${fx} could not be read: ${e instanceof Error ? e.message : String(e)}`,
+                    });
+                }
+                for (const dir of markers) {
+                    const guess = suggestMarker(dir);
+                    findings.push({
+                        skill,
+                        scenario: s.id,
+                        code: "fixture-marker",
+                        message: `fixture ${fx} has unknown top-level marker directory \`${dir}/\`` +
+                            (guess ? ` — did you mean \`${guess}/\`?` : "") +
+                            ` Known markers are ${MARKERS.map((m) => `\`${m}/\``).join(" and ")};` +
+                            ` rename it, or move it deeper if it is ordinary content.`,
+                    });
+                }
+            }
+        }
+    }
+    // assert.post_test must exist. A post-test that isn't there fails its scenario at
+    // run time with a "spec error" message, but that costs a full model run to discover
+    // — and lint is the free, offline gate that exists to catch it first.
+    for (const s of spec.scenarios) {
+        const pt = s.assert?.post_test;
+        if (!pt)
+            continue;
+        const abs = isAbsolute(pt) ? pt : resolve(specDir, pt);
+        if (!isFile(abs)) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: `assert.post_test not found: ${pt}` });
         }
     }
     // system_prompt_file must exist — an agent-file scenario silently falling back to
@@ -124,10 +171,12 @@ export function lintSkill(skillDir) {
         }
     }
     // staleness — the newest FULL (non-partial) run per model tag recorded sha256 hashes of
-    // every source file it measured. If any of those files has changed since, the committed
-    // result describes text that no longer exists — exactly how three regressions hid behind
-    // a 100%-SHIP table for four weeks. Runs predating source_hashes are skipped silently
-    // (no retroactive noise); partial runs never count as coverage.
+    // every source it measured: SKILL.md, agent files, each scenario's definition and each
+    // fixture tree. If any has changed since, the committed result describes inputs that no
+    // longer exist — exactly how three regressions hid behind a 100%-SHIP table for four
+    // weeks, and how a swapped fixture went unreported by a "7 skills, 0 findings" lint.
+    // Runs predating source_hashes (or predating a given key kind) are skipped silently — no
+    // retroactive noise; partial runs never count as coverage.
     for (const tagDir of enumerateTagDirs(resultsRoot)) {
         // Newest FULL run: partial (--only) runs are iteration artifacts and never count as
         // coverage — a fresh partial must not silence a stale full run underneath it.
@@ -149,28 +198,46 @@ export function lintSkill(skillDir) {
             continue; // predates source_hashes → silent
         {
             const newest = full.runDir;
+            const ctx = { skillDir, specDir, scenarios: spec.scenarios };
             for (const [key, recorded] of Object.entries(hashes)) {
-                const abs = key === "SKILL.md" ? join(skillDir, "SKILL.md") : resolve(specDir, key);
-                const current = fileSha256(abs);
+                const what = describeSourceKey(key);
+                const scenario = scenarioIdForKey(key, spec.scenarios);
+                // The run itself failed to hash this source, so it was never verified and
+                // no comparison here can establish anything about it.
+                if (recorded === UNREADABLE) {
+                    findings.push({ skill, scenario, code: "stale", message: `${what} could not be read when the newest ${basename(tagDir)} run was recorded (${newest}) — that source was never verified; re-run` });
+                    continue;
+                }
+                const current = currentHashFor(key, ctx);
+                // undefined = not comparable: a scenario the spec no longer has (a reshape,
+                // per the scenario-set check above), or a key kind written by a newer
+                // skill-harness than this one.
+                if (current === undefined)
+                    continue;
                 if (current === null) {
-                    findings.push({ skill, code: "stale", message: `${key} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
+                    findings.push({ skill, scenario, code: "stale", message: `${what} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
                 }
                 else if (current !== recorded) {
-                    findings.push({ skill, code: "stale", message: `${key} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                }
+            }
+            // Coverage: a scenario the spec defines that the newest full run never
+            // measured. Without this, RENAMING a scenario is invisible — the old key
+            // resolves to "not comparable" and the new one was never recorded — so a
+            // 100%/SHIP scorecard survives an arbitrary spec rewrite reporting zero
+            // findings. Gated on the run having recorded scenario keys at all, so runs
+            // predating the key kind stay silent like every other pre-existing run.
+            if (Object.keys(hashes).some((k) => k.startsWith(SCENARIO_PREFIX))) {
+                for (const s of spec.scenarios) {
+                    if (!(SCENARIO_PREFIX + s.id in hashes)) {
+                        findings.push({ skill, scenario: s.id, code: "stale", message: `the newest ${basename(tagDir)} run (${newest}) did not measure scenario \`${s.id}\` — the published result covers a different scenario set than the spec; re-run before publishing` });
+                    }
                 }
             }
         }
     }
     return findings;
 }
-function fileSha256(p) {
-    try {
-        return createHash("sha256").update(readFileSync(p)).digest("hex");
-    }
-    catch {
-        return null;
-    }
-}
 /** Model-tag dirs under tests/results (each holds timestamped run dirs). */
 function enumerateTagDirs(resultsRoot) {
     if (!existsSync(resultsRoot))

package/dist/results.d.ts

@@ -38,10 +38,13 @@ export interface ResultsFile {
     /** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
     partial?: boolean;
     /**
-     * sha256 of every source file this run measured: SKILL.md plus each distinct
-     * system_prompt_file. Lint compares the newest run's hashes against the current
-     * files — a mismatch means the published result describes text that no longer
-     * exists (the stale-scorecard class this field exists to kill).
+     * sha256 of every source this run measured: SKILL.md, each distinct
+     * system_prompt_file, each scenario's definition (`scenario:<id>`) and each
+     * fixture tree (`fixture:<path>`). Lint compares the newest run's hashes against
+     * the current sources — a mismatch means the published result describes inputs
+     * that no longer exist (the stale-scorecard class this field exists to kill).
+     * See sources.ts for the key scheme; runs recorded before a key kind existed
+     * simply don't carry it, and are never retroactively flagged.
      */
     source_hashes?: Record<string, string>;
     effective_grade: GradeSummary;
@@ -83,7 +86,7 @@ export declare function applyOverride(results: ResultsFile, scenarioId: string,
  * `!…` preservation lines added by preserveTranscript.
  */
 export declare function ensureResultsGitignore(resultsRoot: string): void;
-/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+/** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export declare function repIndexOf(filename: string): number | null;
 /**
  * ALL transcript files for a scenario in a run dir, sorted deterministically:
@@ -95,23 +98,43 @@ export declare function repIndexOf(filename: string): number | null;
  * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
  * false positives from a red/force transcript of the same scenario. Omitted,
  * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
- * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ * scenario's sibling artifacts, which share the `.txt` extension deliberately
+ * (so `results/.gitignore`'s `*.txt` covers them all): judge-raw output
+ * (`<id>.*.judge.txt` — see judgeRawPath) and the staged diff
+ * (`<id>.*.diff.txt` — see diffPath).
  */
 export declare function findTranscriptFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
 export declare function judgeRawPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 /** A scenario's raw judge-output files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findJudgeRawFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/**
+ * Path of a seeded scenario's staged-diff artifact within a run dir (rep-suffixed
+ * for reps).
+ *
+ * The diff is the only record of what the model actually *did* — the workspace is
+ * torn down after every rep, so without this a seeded verdict cannot be audited
+ * after the fact. Named `<id>.<mode>[.rep<k>].diff.txt` so it sorts beside its
+ * transcript and is covered by the `*.txt` rule in results/.gitignore: diffs are
+ * generated evidence, ignored like transcripts, not committed like results.yaml.
+ */
+export declare function diffPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export declare function findTranscriptFile(runDir: string, scenarioId: string): string | null;
 /**
- * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
- * (audit trail for an override — a --reps run has one transcript (and one
- * judge-raw file) per rep, and every rep that drove the verdict must survive
- * a commit, not just an arbitrary one).
+ * Un-gitignore ALL of a scenario's transcript, judge-raw AND staged-diff
+ * artifact files (audit trail for an override — a --reps run has one of each
+ * per rep, and every rep that drove the verdict must survive a commit, not just
+ * an arbitrary one).
  * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
- * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
- * separators so the negation matches on Windows too (git ignore patterns are
- * always forward-slashed).
+ * `.judge.txt` / `.diff.txt`) to results/.gitignore for each, once. The path
+ * uses POSIX separators so the negation matches on Windows too (git ignore
+ * patterns are always forward-slashed).
+ *
+ * The diff belongs here for the same reason the judge-raw output does: an
+ * override says the judge got it wrong, and on a seeded scenario the evidence
+ * for that claim is the code the model wrote.
  */
 export declare function preserveTranscript(resultsRoot: string, runDir: string, scenarioId: string): void;

package/dist/results.js

@@ -150,9 +150,10 @@ export function ensureResultsGitignore(resultsRoot) {
         .filter((l) => l.startsWith("!") && l.trim() !== "!results.yaml");
     writeFileSync(giPath, GITIGNORE_BODY + preserved.map((l) => l + "\n").join(""), "utf8");
 }
-// Matches both transcript (`.rep<k>.txt`) and judge-raw (`.rep<k>.judge.txt`) rep suffixes.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.)?txt$/;
-/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+// Matches transcript (`.rep<k>.txt`), judge-raw (`.rep<k>.judge.txt`) and
+// staged-diff (`.rep<k>.diff.txt`) rep suffixes.
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?txt$/;
+/** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
     return m ? Number(m[1]) : null;
@@ -181,7 +182,10 @@ function sortByRep(files) {
  * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
  * false positives from a red/force transcript of the same scenario. Omitted,
  * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
- * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ * scenario's sibling artifacts, which share the `.txt` extension deliberately
+ * (so `results/.gitignore`'s `*.txt` covers them all): judge-raw output
+ * (`<id>.*.judge.txt` — see judgeRawPath) and the staged diff
+ * (`<id>.*.diff.txt` — see diffPath).
  */
 export function findTranscriptFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -190,7 +194,12 @@ export function findTranscriptFiles(runDir, scenarioId, mode) {
     const matcher = mode !== undefined
         ? new RegExp(`^${escapedId}\\.${mode}(\\.rep\\d+)?\\.txt$`)
         : null;
-    const files = readdirSync(runDir).filter((f) => matcher ? matcher.test(f) : f.startsWith(`${scenarioId}.`) && f.endsWith(".txt") && !f.endsWith(".judge.txt"));
+    const files = readdirSync(runDir).filter((f) => matcher
+        ? matcher.test(f)
+        : f.startsWith(`${scenarioId}.`) &&
+            f.endsWith(".txt") &&
+            !f.endsWith(".judge.txt") &&
+            !f.endsWith(".diff.txt"));
     return sortByRep(files);
 }
 /** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
@@ -208,22 +217,54 @@ export function findJudgeRawFiles(runDir, scenarioId, mode) {
         : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
+/**
+ * Path of a seeded scenario's staged-diff artifact within a run dir (rep-suffixed
+ * for reps).
+ *
+ * The diff is the only record of what the model actually *did* — the workspace is
+ * torn down after every rep, so without this a seeded verdict cannot be audited
+ * after the fact. Named `<id>.<mode>[.rep<k>].diff.txt` so it sorts beside its
+ * transcript and is covered by the `*.txt` rule in results/.gitignore: diffs are
+ * generated evidence, ignored like transcripts, not committed like results.yaml.
+ */
+export function diffPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.diff.txt`);
+}
+/** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export function findDiffFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.diff\\.txt$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.diff\\.txt$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export function findTranscriptFile(runDir, scenarioId) {
     return findTranscriptFiles(runDir, scenarioId)[0] ?? null;
 }
 /**
- * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
- * (audit trail for an override — a --reps run has one transcript (and one
- * judge-raw file) per rep, and every rep that drove the verdict must survive
- * a commit, not just an arbitrary one).
+ * Un-gitignore ALL of a scenario's transcript, judge-raw AND staged-diff
+ * artifact files (audit trail for an override — a --reps run has one of each
+ * per rep, and every rep that drove the verdict must survive a commit, not just
+ * an arbitrary one).
  * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
- * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
- * separators so the negation matches on Windows too (git ignore patterns are
- * always forward-slashed).
+ * `.judge.txt` / `.diff.txt`) to results/.gitignore for each, once. The path
+ * uses POSIX separators so the negation matches on Windows too (git ignore
+ * patterns are always forward-slashed).
+ *
+ * The diff belongs here for the same reason the judge-raw output does: an
+ * override says the judge got it wrong, and on a seeded scenario the evidence
+ * for that claim is the code the model wrote.
  */
 export function preserveTranscript(resultsRoot, runDir, scenarioId) {
-    const files = [...findTranscriptFiles(runDir, scenarioId), ...findJudgeRawFiles(runDir, scenarioId)];
+    const files = [
+        ...findTranscriptFiles(runDir, scenarioId),
+        ...findJudgeRawFiles(runDir, scenarioId),
+        ...findDiffFiles(runDir, scenarioId),
+    ];
     if (files.length === 0)
         return;
     ensureResultsGitignore(resultsRoot);

package/dist/run.js

@@ -1,8 +1,8 @@
-import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
-import { createHash } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
+import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -10,34 +10,6 @@ import { createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
-/** sha256 of a file, or null when it doesn't exist — missing sources are lint's problem, not run's. */
-function sha256(path) {
-    try {
-        return createHash("sha256").update(readFileSync(path)).digest("hex");
-    }
-    catch {
-        return null;
-    }
-}
-/**
- * Hash every source file this run measures: SKILL.md + each distinct
- * system_prompt_file (agents/<name>.md). Recorded in results.yaml so lint can prove
- * a published result still describes the current text.
- */
-function sourceHashes(skillDir, specPath, scenarios) {
-    const hashes = {};
-    const skillMd = sha256(resolve(skillDir, "SKILL.md"));
-    if (skillMd)
-        hashes["SKILL.md"] = skillMd;
-    for (const s of scenarios) {
-        if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
-            const h = sha256(resolve(dirname(specPath), s.systemPromptFile));
-            if (h)
-                hashes[s.systemPromptFile] = h;
-        }
-    }
-    return hashes;
-}
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -99,7 +71,9 @@ export async function runSkillModel(opts) {
         label: opts.label ?? null,
         mode,
         ...(partial ? { partial: true } : {}),
-        source_hashes: sourceHashes(skillDir, opts.specPath, scenarios),
+        // Only the scenarios this run actually measured: a --only run must not claim
+        // coverage of scenarios it skipped.
+        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios }),
         scenarios: scenarioResults,
     }, ctx);
     if (ctx) {
@@ -136,6 +110,10 @@ async function runRep(scenario, rep, repCount, ctx) {
     let ws = null;
     let transcript = "";
     let gatePrefix = null;
+    // Null until a seeded rep actually reaches its gates: a workspace-setup failure
+    // produces no diff, and writing an empty artifact there would misreport "the
+    // model changed nothing" for a rep that never ran.
+    let stagedDiff = null;
     try {
         try {
             ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
@@ -160,9 +138,11 @@ async function runRep(scenario, rep, repCount, ctx) {
                 if (scenario.mode === "seeded") {
                     const r = await runSeeded(scenario, {
                         skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
+                        specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
                     });
                     transcript = r.transcript;
                     gatePrefix = r.gateFailure;
+                    stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
                 }
                 else {
                     transcript = await ctx.adapter.run({
@@ -178,8 +158,16 @@ async function runRep(scenario, rep, repCount, ctx) {
                     break;
             }
         }
-        writeFileSync(transcriptPath(runDir, scenario.id, mode, repCount > 1 ? rep : undefined), transcript, "utf8");
+        const repSuffix = repCount > 1 ? rep : undefined;
+        writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
         if (scenario.mode === "seeded") {
+            // The workspace is torn down in the `finally` below, so this is the only
+            // chance to keep what the model actually wrote. Persisted uncapped (the
+            // transcript's copy is capped for the judge) and for every rep, pass or
+            // fail — a gate failure is exactly when you want to read the diff.
+            if (stagedDiff !== null) {
+                writeFileSync(diffPath(runDir, scenario.id, mode, repSuffix), stagedDiff, "utf8");
+            }
             appendJournal(runDir, { event: "gate-result", ts: now(), id: scenario.id, ok: !gatePrefix, detail: gatePrefix ?? "", ...repField });
         }
         let verdict;

package/dist/seeded.d.ts

@@ -1,21 +1,101 @@
 import type { Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+import { type ExecResult } from "./util/exec.js";
 interface SeededOpts {
     skillDir: string;
     adapter: HarnessAdapter;
     model: ModelRef;
     mode: RunMode;
     cwd: string;
+    specDir: string;
+    /**
+     * How the vitest gates shell out. Defaults to the real `npx vitest run`.
+     *
+     * A seam, not a mock: a workspace is a bare temp dir, so a test that exercised
+     * the real runner would resolve vitest off the network and be slow and flaky.
+     * Injecting it lets the gate LOGIC — pass, fail, nothing-collected — be tested
+     * deterministically.
+     */
+    runVitest?: (args: string[], cwd: string) => Promise<VitestRun>;
 }
+/**
+ * Result of one vitest invocation — deliberately `ExecResult`, not a narrower
+ * shape of its own.
+ *
+ * An earlier version declared `code: number`. That narrowing was a lie the
+ * compiler happened not to catch (the default's inferred type silently widened
+ * it back), and it is unrepresentable in practice: `exec` SIGKILLs on timeout
+ * and a signal-killed child closes with `code === null`. Declaring non-null
+ * would have made the vitest gate's timeout path — the one an injected double
+ * most needs to reproduce — impossible to express in a test.
+ */
+export type VitestRun = ExecResult;
 export interface SeededOutcome {
     transcript: string;
     gateFailure: string | null;
+    diff: string;
 }
+/**
+ * The added/removed lines of a unified diff — what the model actually *changed*,
+ * with context lines and file headers dropped.
+ *
+ * This is the difference between "the diff mentions `lastIndex`" and "the model
+ * touched `lastIndex`". A unified diff carries three lines of context around every
+ * hunk, so an untouched function sitting near the edit site appears in the diff
+ * verbatim. `build` A2 is exactly that shape — its checklist notes that `lastIndex`
+ * "sits two lines from the edit site" — so a naive substring test against the whole
+ * diff would fail the scenario for every model that fixed the right thing, which is
+ * worse than the prose-dependent item it replaces.
+ *
+ * Classification is HUNK-AWARE rather than prefix-based, because `+++`/`---` are
+ * only headers *outside* a hunk. Filtering on those prefixes anywhere would eat a
+ * changed line whose own source text starts with `++` or `--` — `++counter;` at
+ * column zero, a removed SQL/Lua `-- comment`, a YAML `---` separator. Those
+ * became `+++counter;` and `--- comment` once the diff marker was prepended, were
+ * read as headers, and vanished: `diff_excludes` then reported OK for a diff that
+ * touched the forbidden symbol. A false PASS on an objective gate is worse than
+ * the subjective check it replaced, so the parse follows the format instead of
+ * guessing from prefixes.
+ */
+export declare function changedLines(diff: string): string;
+/**
+ * Cut a diff to a byte budget on a line boundary, appending an explicit marker
+ * naming how much was dropped.
+ *
+ * The marker is not decoration: a silently truncated diff would let the judge
+ * grade "the function is missing" when it was merely cut off, which is the exact
+ * class of false-FAIL this whole change exists to remove. Truncation is reported
+ * as a fact about the transcript, and the untruncated diff is always on disk.
+ */
+export declare function capDiff(diff: string, maxBytes?: number): string;
 /**
  * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
- * the repo, then evaluate objective gates (staged-diff contains + optional vitest
- * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
- * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ * the repo, then evaluate the objective gates it declares — `diff_contains`,
+ * `diff_excludes`, `vitest` and `post_test`. Every gate that is configured runs;
+ * the FIRST failure is what `gateFailure` reports, and a non-null `gateFailure`
+ * makes the scenario an auto-FAIL that never reaches the judge.
+ *
+ * Returns the full staged diff alongside the transcript: the caller persists it
+ * as a run artifact, and a size-capped copy is appended to the transcript under
+ * `=== STAGED DIFF ===` so the judge grades the code rather than the model's
+ * description of it. Workspace creation (fixture copy + git baseline) and
+ * teardown are the caller's responsibility (run.ts).
  */
 export declare function runSeeded(scenario: Scenario, opts: SeededOpts): Promise<SeededOutcome>;
+export interface VitestTally {
+    passed: number;
+    failed: number;
+    skipped: number;
+    todo: number;
+}
+/**
+ * Parse vitest's `Tests  N passed | M skipped (T)` summary line.
+ *
+ * Used to require positive evidence that a hidden `post_test` actually executed
+ * assertions, rather than trusting a zero exit code — which vitest also returns
+ * when every test in the file is skipped. Returns null when no summary line is
+ * present, which the caller treats as "cannot confirm it ran" rather than as a
+ * pass.
+ */
+export declare function vitestTally(out: string): VitestTally | null;
 export {};