@skill-harness/core 0.2.1 → 0.3.1

package/dist/grade.d.ts

@@ -7,6 +7,8 @@ export interface JudgePromptInput {
     scenario: Scenario;
     transcript: string;
 }
+/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
+export declare const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /** Build the LLM-judge prompt for one transcript (ported from the old grade.sh). */
 export declare function buildJudgePrompt(input: JudgePromptInput): string;
 export interface ParsedVerdict {

package/dist/grade.js

@@ -1,8 +1,33 @@
 import { createWorkspace } from "./workspace.js";
+/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
+export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
+/**
+ * Addendum pointing the judge at the code, added only when the code is actually there.
+ *
+ * A seeded transcript ends with the staged diff, and without this the judge
+ * weighs the model's prose about its work equally with the work itself — which
+ * is how six reps that all passed the objective gates split PASS/FAIL purely on
+ * whether the model wrote "rejects overdrafts" or "subtracts amount".
+ *
+ * Gated on the transcript CONTAINING the diff section, not on `scenario.mode`.
+ * The first sentence is a factual claim about the transcript, and every seeded
+ * transcript saved before this feature existed lacks that section — so keying
+ * off the mode would tell the judge its primary evidence is at the end of a
+ * transcript that has none, while also telling it the gate lines prove nothing.
+ * That is a sweep of FAILs justified by absent evidence, and it would land
+ * squarely on `grade`, the command AGENTS.md rule 4 recommends as the *cheap*
+ * de-confounding step before re-running. Inline scenarios have no diff either,
+ * so their prompt stays byte-identical and every published inline verdict
+ * remains comparable.
+ */
+const SEEDED_DIFF_GUIDANCE = `
+This transcript ends with a "=== STAGED DIFF ===" section: the actual code the assistant wrote, as \`git diff --cached\`. It is the primary evidence. Grade what the diff shows the code DOES, not what the assistant's prose claims it does — a confident description of behavior the diff does not implement is a FAIL, and behavior the diff plainly implements passes even if the assistant described it poorly or not at all. The "=== SEEDED GATES ===" lines above it are keyword and test-run checks only; they do not establish that the required behavior exists. If the diff is marked truncated, judge only what you can see and never infer that cut-off code is missing.
+`;
 /** Build the LLM-judge prompt for one transcript (ported from the old grade.sh). */
 export function buildJudgePrompt(input) {
     const { skill, persona, scenario, transcript } = input;
     const numbered = scenario.checklist.map((c, i) => `${i + 1}. ${c}`).join("\n");
+    const diffGuidance = scenario.mode === "seeded" && transcript.includes(STAGED_DIFF_HEADING) ? SEEDED_DIFF_GUIDANCE : "";
     return `You are grading ONE response from an AI assistant using a "${skill}" skill — ${persona} Judge it ONLY against the checklist below — do not add requirements beyond it.
 
 CHECKLIST (every numbered item must hold for a PASS):
@@ -10,7 +35,7 @@ ${numbered}
 
 TRANSCRIPT (the assistant is the model under test):
 ${transcript}
-
+${diffGuidance}
 Grade each checklist item PASS or FAIL with a <=12-word justification quoting the transcript. Be skeptical: if an item is not clearly satisfied, mark it FAIL. Then output exactly these two lines:
 VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
 REASON: <15 words or fewer>`;

package/dist/index.d.ts

@@ -14,6 +14,7 @@ export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";

package/dist/index.js

@@ -14,6 +14,7 @@ export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";

package/dist/lift.d.ts

@@ -37,10 +37,30 @@ export interface Lift {
     /** Ids the green run covered that the red baseline did not (and vice versa). */
     greenOnly: string[];
     redOnly: string[];
+    /**
+     * Ids both runs covered that a lift cannot speak to, because the harness runs
+     * them identically in red and green. Reported rather than compared: folding them
+     * in would credit the red side with passes the skill itself produced. See
+     * `LiftOptions.modeInsensitive`.
+     */
+    modeInsensitive: string[];
     /** True when either side was an `--only` run, so coverage is a subset by construction. */
     partial: boolean;
     cells: Record<string, LiftCell>;
 }
+export interface LiftOptions {
+    /**
+     * Scenario ids whose red and green runs are the same run by construction, so
+     * comparing them measures nothing.
+     *
+     * The case that exists today is `system_prompt_file`: the pi adapter treats an
+     * agent-file scenario's file AS the system prompt and passes `--no-skills`
+     * *whatever the mode*, so the skill is loaded on both sides. Left in, such a
+     * cell lands in `kept` (or `both-fail`) and drags the denominator down —
+     * understating lift with evidence that the skill worked.
+     */
+    modeInsensitive?: Iterable<string>;
+}
 /**
  * Compare a red (baseline, skill off) run against a green (skill active) run of
  * the same model: the "does this skill actually do anything?" measurement.
@@ -48,22 +68,10 @@ export interface Lift {
  * Both sides go through `effectiveVerdicts`, so an author override is what
  * counts and an override resolves a misfire — the same rule scoring uses. Only
  * the intersection of scenario ids is compared; a lift cannot speak to a
- * scenario one side never ran.
+ * scenario one side never ran, nor to one the harness ran identically in both
+ * modes (`opts.modeInsensitive`).
  */
-export declare function computeLift(red: ResultsFile, green: ResultsFile): Lift;
+export declare function computeLift(red: ResultsFile, green: ResultsFile, opts?: LiftOptions): Lift;
 /** One line for a human: what the skill did, and what it cost. */
 export declare function liftHeadline(lift: Lift): string;
-/**
- * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
- * with the most recent green run and compute the lift.
- *
- * Deliberately derived on read rather than persisted into results.yaml: a lift
- * is a fact about a *pair* of runs, so caching it inside one run's file would go
- * stale the moment a new baseline lands — the stale-scorecard failure mode
- * `source_hashes` exists to prevent. Deriving also means lift works
- * retroactively on results already committed by 0.1.x/0.2.0 users.
- *
- * A tag with no red baseline is omitted entirely rather than reported as a zero
- * lift: "not measured" and "measured no effect" are different claims.
- */
 export declare function collectLift(skillDir: string): Lift[];

package/dist/lift.js

@@ -1,6 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { readResults, effectiveVerdicts } from "./results.js";
+import { loadSpec } from "./spec.js";
 /** A verdict that carries real evidence about the task, rather than about the harness or the judge. */
 function conclusive(verdict, suspect) {
     // ERROR is a harness failure (timeout, empty reply) — it says nothing about
@@ -30,9 +31,11 @@ function classify(red, green) {
  * Both sides go through `effectiveVerdicts`, so an author override is what
  * counts and an override resolves a misfire — the same rule scoring uses. Only
  * the intersection of scenario ids is compared; a lift cannot speak to a
- * scenario one side never ran.
+ * scenario one side never ran, nor to one the harness ran identically in both
+ * modes (`opts.modeInsensitive`).
  */
-export function computeLift(red, green) {
+export function computeLift(red, green, opts = {}) {
+    const insensitive = new Set(opts.modeInsensitive ?? []);
     const redV = new Map(effectiveVerdicts(red.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
     const greenV = new Map(effectiveVerdicts(green.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
     const cells = {};
@@ -41,10 +44,15 @@ export function computeLift(red, green) {
     let greenPassed = 0;
     // Green order drives display order (it is the run the author is looking at),
     // restricted to ids the red baseline also covered.
+    const modeInsensitive = [];
     for (const [id, g] of greenV) {
         const r = redV.get(id);
         if (!r)
             continue;
+        if (insensitive.has(id)) {
+            modeInsensitive.push(id);
+            continue;
+        }
         const cls = classify(r, g);
         cells[id] = { red: r.verdict, redSuspect: r.suspect, green: g.verdict, class: cls };
         counts[cls]++;
@@ -71,14 +79,21 @@ export function computeLift(red, green) {
         delta: greenPassed - redPassed,
         greenOnly: [...greenV.keys()].filter((id) => !redV.has(id)),
         redOnly: [...redV.keys()].filter((id) => !greenV.has(id)),
+        modeInsensitive,
         partial: Boolean(red.partial || green.partial),
         cells,
     };
 }
 /** One line for a human: what the skill did, and what it cost. */
 export function liftHeadline(lift) {
-    if (lift.compared === 0)
+    if (lift.compared === 0) {
+        // Excluded-but-shared is not the same as never-shared. Claiming the runs had
+        // no scenario in common would hide the reason the lift is empty.
+        if (lift.modeInsensitive.length > 0) {
+            return `nothing comparable (${lift.modeInsensitive.length} shared, all run identically in both modes)`;
+        }
         return "no shared scenarios to compare";
+    }
     // Everything inconclusive is NOT "no effect" — it is no measurement. Saying
     // "no measured effect" here would be the same not-measured/measured-no-effect
     // conflation this module refuses to make when a red baseline is missing.
@@ -98,6 +113,9 @@ export function liftHeadline(lift) {
     }
     if (lift.inconclusive > 0)
         segments.push(`${lift.inconclusive} inconclusive`);
+    if (lift.modeInsensitive.length > 0) {
+        segments.push(`${lift.modeInsensitive.length} not comparable (same run in both modes)`);
+    }
     if (lift.partial)
         segments.push("partial run");
     return segments.join(" · ");
@@ -124,10 +142,31 @@ function isDir(p) {
  * A tag with no red baseline is omitted entirely rather than reported as a zero
  * lift: "not measured" and "measured no effect" are different claims.
  */
+/**
+ * Scenario ids the harness runs identically in red and green, read from the spec
+ * rather than from results.yaml: a lift is derived on read, so this has to work
+ * on runs recorded before the field existed — and `scenario:<id>` source hashes
+ * fold the value in without preserving it.
+ *
+ * Never throws: an unparseable spec must degrade to "nothing excluded" rather
+ * than take down a view that is otherwise readable from the results alone.
+ */
+function modeInsensitiveIds(skillDir) {
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    if (!existsSync(specPath))
+        return [];
+    try {
+        return loadSpec(specPath).scenarios.filter((s) => s.systemPromptFile).map((s) => s.id);
+    }
+    catch {
+        return [];
+    }
+}
 export function collectLift(skillDir) {
     const resultsRoot = join(skillDir, "tests", "results");
     if (!existsSync(resultsRoot))
         return [];
+    const modeInsensitive = modeInsensitiveIds(skillDir);
     const lifts = [];
     for (const tag of readdirSync(resultsRoot).filter((n) => isDir(join(resultsRoot, n))).sort()) {
         const tagDir = join(resultsRoot, tag);
@@ -156,7 +195,7 @@ export function collectLift(skillDir) {
         }
         if (!red || !green)
             continue;
-        lifts.push({ ...computeLift(red, green), tag });
+        lifts.push({ ...computeLift(red, green, { modeInsensitive }), tag });
     }
     return lifts;
 }

package/dist/lint.d.ts

@@ -1,4 +1,4 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "consistency" | "stale" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "lint-error";
 export interface LintFinding {
     readonly skill: string;
     readonly scenario?: string;

package/dist/lint.js

@@ -1,9 +1,10 @@
 import { existsSync, statSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, resolve } from "node:path";
-import { createHash } from "node:crypto";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
+import { currentHashFor, describeSourceKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, UNREADABLE } from "./sources.js";
+import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
 function isDir(p) {
@@ -65,12 +66,58 @@ export function lintSkill(skillDir) {
     // relative to the spec's dir, matching workspace.ts resolve(specDir, fixture) where specDir = <skillDir>/tests.
     const specDir = dirname(specPath);
     for (const s of spec.scenarios) {
-        const fx = typeof s.workspace === "object" && s.workspace !== null ? s.workspace.fixture : undefined;
+        const fx = effectiveFixture(s); // shared with sources.ts so hashing and linting can't drift
         if (fx) {
             const abs = isAbsolute(fx) ? fx : resolve(specDir, fx);
             if (!isDir(abs)) {
                 findings.push({ skill, scenario: s.id, code: "fixture", message: `fixture not found: ${fx}` });
             }
+            else {
+                // A mistyped fixture marker (`_uncommited/`) is rejected at run time by
+                // createWorkspace — but only once a run has been started, where it surfaces as
+                // a scenario FAIL among real results. lint is free, offline and runs in CI, so
+                // it is where an author should meet this. The set flagged here is exactly the
+                // set workspace.ts refuses, so lint can never bless a fixture the runtime then
+                // rejects.
+                // try/catch because lintSkill's contract is "never throws": isDir() proves the
+                // path stats, not that it can be read, and an EACCES on readdir would escape
+                // to cli.ts, which replaces this skill's ENTIRE finding list with one
+                // lint-error — silently discarding the staleness and consistency checks below.
+                let markers = [];
+                try {
+                    markers = unknownMarkerDirs(abs);
+                }
+                catch (e) {
+                    findings.push({
+                        skill, scenario: s.id, code: "fixture",
+                        message: `fixture ${fx} could not be read: ${e instanceof Error ? e.message : String(e)}`,
+                    });
+                }
+                for (const dir of markers) {
+                    const guess = suggestMarker(dir);
+                    findings.push({
+                        skill,
+                        scenario: s.id,
+                        code: "fixture-marker",
+                        message: `fixture ${fx} has unknown top-level marker directory \`${dir}/\`` +
+                            (guess ? ` — did you mean \`${guess}/\`?` : "") +
+                            ` Known markers are ${MARKERS.map((m) => `\`${m}/\``).join(" and ")};` +
+                            ` rename it, or move it deeper if it is ordinary content.`,
+                    });
+                }
+            }
+        }
+    }
+    // assert.post_test must exist. A post-test that isn't there fails its scenario at
+    // run time with a "spec error" message, but that costs a full model run to discover
+    // — and lint is the free, offline gate that exists to catch it first.
+    for (const s of spec.scenarios) {
+        const pt = s.assert?.post_test;
+        if (!pt)
+            continue;
+        const abs = isAbsolute(pt) ? pt : resolve(specDir, pt);
+        if (!isFile(abs)) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: `assert.post_test not found: ${pt}` });
         }
     }
     // system_prompt_file must exist — an agent-file scenario silently falling back to
@@ -124,10 +171,12 @@ export function lintSkill(skillDir) {
         }
     }
     // staleness — the newest FULL (non-partial) run per model tag recorded sha256 hashes of
-    // every source file it measured. If any of those files has changed since, the committed
-    // result describes text that no longer exists — exactly how three regressions hid behind
-    // a 100%-SHIP table for four weeks. Runs predating source_hashes are skipped silently
-    // (no retroactive noise); partial runs never count as coverage.
+    // every source it measured: SKILL.md, agent files, each scenario's definition and each
+    // fixture tree. If any has changed since, the committed result describes inputs that no
+    // longer exist — exactly how three regressions hid behind a 100%-SHIP table for four
+    // weeks, and how a swapped fixture went unreported by a "7 skills, 0 findings" lint.
+    // Runs predating source_hashes (or predating a given key kind) are skipped silently — no
+    // retroactive noise; partial runs never count as coverage.
     for (const tagDir of enumerateTagDirs(resultsRoot)) {
         // Newest FULL run: partial (--only) runs are iteration artifacts and never count as
         // coverage — a fresh partial must not silence a stale full run underneath it.
@@ -149,28 +198,46 @@ export function lintSkill(skillDir) {
             continue; // predates source_hashes → silent
         {
             const newest = full.runDir;
+            const ctx = { skillDir, specDir, scenarios: spec.scenarios };
             for (const [key, recorded] of Object.entries(hashes)) {
-                const abs = key === "SKILL.md" ? join(skillDir, "SKILL.md") : resolve(specDir, key);
-                const current = fileSha256(abs);
+                const what = describeSourceKey(key);
+                const scenario = scenarioIdForKey(key, spec.scenarios);
+                // The run itself failed to hash this source, so it was never verified and
+                // no comparison here can establish anything about it.
+                if (recorded === UNREADABLE) {
+                    findings.push({ skill, scenario, code: "stale", message: `${what} could not be read when the newest ${basename(tagDir)} run was recorded (${newest}) — that source was never verified; re-run` });
+                    continue;
+                }
+                const current = currentHashFor(key, ctx);
+                // undefined = not comparable: a scenario the spec no longer has (a reshape,
+                // per the scenario-set check above), or a key kind written by a newer
+                // skill-harness than this one.
+                if (current === undefined)
+                    continue;
                 if (current === null) {
-                    findings.push({ skill, code: "stale", message: `${key} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
+                    findings.push({ skill, scenario, code: "stale", message: `${what} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
                 }
                 else if (current !== recorded) {
-                    findings.push({ skill, code: "stale", message: `${key} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                }
+            }
+            // Coverage: a scenario the spec defines that the newest full run never
+            // measured. Without this, RENAMING a scenario is invisible — the old key
+            // resolves to "not comparable" and the new one was never recorded — so a
+            // 100%/SHIP scorecard survives an arbitrary spec rewrite reporting zero
+            // findings. Gated on the run having recorded scenario keys at all, so runs
+            // predating the key kind stay silent like every other pre-existing run.
+            if (Object.keys(hashes).some((k) => k.startsWith(SCENARIO_PREFIX))) {
+                for (const s of spec.scenarios) {
+                    if (!(SCENARIO_PREFIX + s.id in hashes)) {
+                        findings.push({ skill, scenario: s.id, code: "stale", message: `the newest ${basename(tagDir)} run (${newest}) did not measure scenario \`${s.id}\` — the published result covers a different scenario set than the spec; re-run before publishing` });
+                    }
                 }
             }
         }
     }
     return findings;
 }
-function fileSha256(p) {
-    try {
-        return createHash("sha256").update(readFileSync(p)).digest("hex");
-    }
-    catch {
-        return null;
-    }
-}
 /** Model-tag dirs under tests/results (each holds timestamped run dirs). */
 function enumerateTagDirs(resultsRoot) {
     if (!existsSync(resultsRoot))

package/dist/results.d.ts

@@ -38,10 +38,13 @@ export interface ResultsFile {
     /** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
     partial?: boolean;
     /**
-     * sha256 of every source file this run measured: SKILL.md plus each distinct
-     * system_prompt_file. Lint compares the newest run's hashes against the current
-     * files — a mismatch means the published result describes text that no longer
-     * exists (the stale-scorecard class this field exists to kill).
+     * sha256 of every source this run measured: SKILL.md, each distinct
+     * system_prompt_file, each scenario's definition (`scenario:<id>`) and each
+     * fixture tree (`fixture:<path>`). Lint compares the newest run's hashes against
+     * the current sources — a mismatch means the published result describes inputs
+     * that no longer exist (the stale-scorecard class this field exists to kill).
+     * See sources.ts for the key scheme; runs recorded before a key kind existed
+     * simply don't carry it, and are never retroactively flagged.
      */
     source_hashes?: Record<string, string>;
     effective_grade: GradeSummary;
@@ -83,7 +86,7 @@ export declare function applyOverride(results: ResultsFile, scenarioId: string,
  * `!…` preservation lines added by preserveTranscript.
  */
 export declare function ensureResultsGitignore(resultsRoot: string): void;
-/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+/** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export declare function repIndexOf(filename: string): number | null;
 /**
  * ALL transcript files for a scenario in a run dir, sorted deterministically:
@@ -95,23 +98,43 @@ export declare function repIndexOf(filename: string): number | null;
  * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
  * false positives from a red/force transcript of the same scenario. Omitted,
  * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
- * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ * scenario's sibling artifacts, which share the `.txt` extension deliberately
+ * (so `results/.gitignore`'s `*.txt` covers them all): judge-raw output
+ * (`<id>.*.judge.txt` — see judgeRawPath) and the staged diff
+ * (`<id>.*.diff.txt` — see diffPath).
  */
 export declare function findTranscriptFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
 export declare function judgeRawPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 /** A scenario's raw judge-output files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findJudgeRawFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/**
+ * Path of a seeded scenario's staged-diff artifact within a run dir (rep-suffixed
+ * for reps).
+ *
+ * The diff is the only record of what the model actually *did* — the workspace is
+ * torn down after every rep, so without this a seeded verdict cannot be audited
+ * after the fact. Named `<id>.<mode>[.rep<k>].diff.txt` so it sorts beside its
+ * transcript and is covered by the `*.txt` rule in results/.gitignore: diffs are
+ * generated evidence, ignored like transcripts, not committed like results.yaml.
+ */
+export declare function diffPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export declare function findTranscriptFile(runDir: string, scenarioId: string): string | null;
 /**
- * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
- * (audit trail for an override — a --reps run has one transcript (and one
- * judge-raw file) per rep, and every rep that drove the verdict must survive
- * a commit, not just an arbitrary one).
+ * Un-gitignore ALL of a scenario's transcript, judge-raw AND staged-diff
+ * artifact files (audit trail for an override — a --reps run has one of each
+ * per rep, and every rep that drove the verdict must survive a commit, not just
+ * an arbitrary one).
  * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
- * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
- * separators so the negation matches on Windows too (git ignore patterns are
- * always forward-slashed).
+ * `.judge.txt` / `.diff.txt`) to results/.gitignore for each, once. The path
+ * uses POSIX separators so the negation matches on Windows too (git ignore
+ * patterns are always forward-slashed).
+ *
+ * The diff belongs here for the same reason the judge-raw output does: an
+ * override says the judge got it wrong, and on a seeded scenario the evidence
+ * for that claim is the code the model wrote.
  */
 export declare function preserveTranscript(resultsRoot: string, runDir: string, scenarioId: string): void;

package/dist/results.js

@@ -150,9 +150,10 @@ export function ensureResultsGitignore(resultsRoot) {
         .filter((l) => l.startsWith("!") && l.trim() !== "!results.yaml");
     writeFileSync(giPath, GITIGNORE_BODY + preserved.map((l) => l + "\n").join(""), "utf8");
 }
-// Matches both transcript (`.rep<k>.txt`) and judge-raw (`.rep<k>.judge.txt`) rep suffixes.
-const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.)?txt$/;
-/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+// Matches transcript (`.rep<k>.txt`), judge-raw (`.rep<k>.judge.txt`) and
+// staged-diff (`.rep<k>.diff.txt`) rep suffixes.
+const REP_SUFFIX_RE = /\.rep(\d+)\.(?:judge\.|diff\.)?txt$/;
+/** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export function repIndexOf(filename) {
     const m = REP_SUFFIX_RE.exec(filename);
     return m ? Number(m[1]) : null;
@@ -181,7 +182,10 @@ function sortByRep(files) {
  * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
  * false positives from a red/force transcript of the same scenario. Omitted,
  * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
- * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ * scenario's sibling artifacts, which share the `.txt` extension deliberately
+ * (so `results/.gitignore`'s `*.txt` covers them all): judge-raw output
+ * (`<id>.*.judge.txt` — see judgeRawPath) and the staged diff
+ * (`<id>.*.diff.txt` — see diffPath).
  */
 export function findTranscriptFiles(runDir, scenarioId, mode) {
     if (!existsSync(runDir))
@@ -190,7 +194,12 @@ export function findTranscriptFiles(runDir, scenarioId, mode) {
     const matcher = mode !== undefined
         ? new RegExp(`^${escapedId}\\.${mode}(\\.rep\\d+)?\\.txt$`)
         : null;
-    const files = readdirSync(runDir).filter((f) => matcher ? matcher.test(f) : f.startsWith(`${scenarioId}.`) && f.endsWith(".txt") && !f.endsWith(".judge.txt"));
+    const files = readdirSync(runDir).filter((f) => matcher
+        ? matcher.test(f)
+        : f.startsWith(`${scenarioId}.`) &&
+            f.endsWith(".txt") &&
+            !f.endsWith(".judge.txt") &&
+            !f.endsWith(".diff.txt"));
     return sortByRep(files);
 }
 /** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
@@ -208,22 +217,54 @@ export function findJudgeRawFiles(runDir, scenarioId, mode) {
         : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.judge\\.txt$`);
     return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
 }
+/**
+ * Path of a seeded scenario's staged-diff artifact within a run dir (rep-suffixed
+ * for reps).
+ *
+ * The diff is the only record of what the model actually *did* — the workspace is
+ * torn down after every rep, so without this a seeded verdict cannot be audited
+ * after the fact. Named `<id>.<mode>[.rep<k>].diff.txt` so it sorts beside its
+ * transcript and is covered by the `*.txt` rule in results/.gitignore: diffs are
+ * generated evidence, ignored like transcripts, not committed like results.yaml.
+ */
+export function diffPath(runDir, scenarioId, mode, rep) {
+    const base = rep === undefined ? `${scenarioId}.${mode}` : `${scenarioId}.${mode}.rep${rep}`;
+    return join(runDir, `${base}.diff.txt`);
+}
+/** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export function findDiffFiles(runDir, scenarioId, mode) {
+    if (!existsSync(runDir))
+        return [];
+    const esc = scenarioId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = mode === undefined
+        ? new RegExp(`^${esc}\\..*\\.diff\\.txt$`)
+        : new RegExp(`^${esc}\\.${mode}(\\.rep\\d+)?\\.diff\\.txt$`);
+    return sortByRep(readdirSync(runDir).filter((f) => re.test(f)));
+}
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export function findTranscriptFile(runDir, scenarioId) {
     return findTranscriptFiles(runDir, scenarioId)[0] ?? null;
 }
 /**
- * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
- * (audit trail for an override — a --reps run has one transcript (and one
- * judge-raw file) per rep, and every rep that drove the verdict must survive
- * a commit, not just an arbitrary one).
+ * Un-gitignore ALL of a scenario's transcript, judge-raw AND staged-diff
+ * artifact files (audit trail for an override — a --reps run has one of each
+ * per rep, and every rep that drove the verdict must survive a commit, not just
+ * an arbitrary one).
  * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
- * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
- * separators so the negation matches on Windows too (git ignore patterns are
- * always forward-slashed).
+ * `.judge.txt` / `.diff.txt`) to results/.gitignore for each, once. The path
+ * uses POSIX separators so the negation matches on Windows too (git ignore
+ * patterns are always forward-slashed).
+ *
+ * The diff belongs here for the same reason the judge-raw output does: an
+ * override says the judge got it wrong, and on a seeded scenario the evidence
+ * for that claim is the code the model wrote.
  */
 export function preserveTranscript(resultsRoot, runDir, scenarioId) {
-    const files = [...findTranscriptFiles(runDir, scenarioId), ...findJudgeRawFiles(runDir, scenarioId)];
+    const files = [
+        ...findTranscriptFiles(runDir, scenarioId),
+        ...findJudgeRawFiles(runDir, scenarioId),
+        ...findDiffFiles(runDir, scenarioId),
+    ];
     if (files.length === 0)
         return;
     ensureResultsGitignore(resultsRoot);