@skill-harness/core 0.1.2 → 0.3.0

package/dist/lint.js

@@ -3,6 +3,8 @@ import { basename, dirname, isAbsolute, join, resolve } from "node:path";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
+import { currentHashFor, describeSourceKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, UNREADABLE } from "./sources.js";
+import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
 function isDir(p) {
@@ -13,6 +15,14 @@ function isDir(p) {
         return false;
     }
 }
+function isFile(p) {
+    try {
+        return statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * Validate one skill's spec + fixtures statically (and results-consistency when
  * committed results exist — see the consistency block). Never throws: a bad spec
@@ -56,12 +66,68 @@ export function lintSkill(skillDir) {
     // relative to the spec's dir, matching workspace.ts resolve(specDir, fixture) where specDir = <skillDir>/tests.
     const specDir = dirname(specPath);
     for (const s of spec.scenarios) {
-        const fx = typeof s.workspace === "object" && s.workspace !== null ? s.workspace.fixture : undefined;
+        const fx = effectiveFixture(s); // shared with sources.ts so hashing and linting can't drift
         if (fx) {
             const abs = isAbsolute(fx) ? fx : resolve(specDir, fx);
             if (!isDir(abs)) {
                 findings.push({ skill, scenario: s.id, code: "fixture", message: `fixture not found: ${fx}` });
             }
+            else {
+                // A mistyped fixture marker (`_uncommited/`) is rejected at run time by
+                // createWorkspace — but only once a run has been started, where it surfaces as
+                // a scenario FAIL among real results. lint is free, offline and runs in CI, so
+                // it is where an author should meet this. The set flagged here is exactly the
+                // set workspace.ts refuses, so lint can never bless a fixture the runtime then
+                // rejects.
+                // try/catch because lintSkill's contract is "never throws": isDir() proves the
+                // path stats, not that it can be read, and an EACCES on readdir would escape
+                // to cli.ts, which replaces this skill's ENTIRE finding list with one
+                // lint-error — silently discarding the staleness and consistency checks below.
+                let markers = [];
+                try {
+                    markers = unknownMarkerDirs(abs);
+                }
+                catch (e) {
+                    findings.push({
+                        skill, scenario: s.id, code: "fixture",
+                        message: `fixture ${fx} could not be read: ${e instanceof Error ? e.message : String(e)}`,
+                    });
+                }
+                for (const dir of markers) {
+                    const guess = suggestMarker(dir);
+                    findings.push({
+                        skill,
+                        scenario: s.id,
+                        code: "fixture-marker",
+                        message: `fixture ${fx} has unknown top-level marker directory \`${dir}/\`` +
+                            (guess ? ` — did you mean \`${guess}/\`?` : "") +
+                            ` Known markers are ${MARKERS.map((m) => `\`${m}/\``).join(" and ")};` +
+                            ` rename it, or move it deeper if it is ordinary content.`,
+                    });
+                }
+            }
+        }
+    }
+    // assert.post_test must exist. A post-test that isn't there fails its scenario at
+    // run time with a "spec error" message, but that costs a full model run to discover
+    // — and lint is the free, offline gate that exists to catch it first.
+    for (const s of spec.scenarios) {
+        const pt = s.assert?.post_test;
+        if (!pt)
+            continue;
+        const abs = isAbsolute(pt) ? pt : resolve(specDir, pt);
+        if (!isFile(abs)) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: `assert.post_test not found: ${pt}` });
+        }
+    }
+    // system_prompt_file must exist — an agent-file scenario silently falling back to
+    // skill activation would measure the wrong artifact entirely.
+    for (const s of spec.scenarios) {
+        if (!s.systemPromptFile)
+            continue;
+        const abs = isAbsolute(s.systemPromptFile) ? s.systemPromptFile : resolve(specDir, s.systemPromptFile);
+        if (!isFile(abs)) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: `system_prompt_file not found: ${s.systemPromptFile}` });
         }
     }
     // results-consistency — only for committed results.yaml (skipped silently otherwise).
@@ -78,9 +144,17 @@ export function lintSkill(skillDir) {
             if (raw?.schema !== 2)
                 continue; // schema-1 intentionally skipped — no finding
             const r = readResults(runDir);
-            const ctx = r.mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
-            const recomputed = finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, scenarios: r.scenarios }, ctx).effective_grade;
-            if (JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
+            // A run whose scenario set no longer matches the spec predates a spec reshape
+            // (scenarios added/removed). Its grade was computed against the OLD ship bar and
+            // cannot be meaningfully recomputed against the new one — recomputing would flag
+            // every historical run each time a spec grows. Staleness (source_hashes) is the
+            // mechanism that says "re-run"; consistency only polices runs the current spec
+            // can actually re-score. Override/transcript rules below still apply.
+            const specIds = new Set(spec.scenarios.map((sc) => sc.id));
+            const sameSet = r.scenarios.length === specIds.size && r.scenarios.every((sc) => specIds.has(sc.id));
+            const ctx = r.mode === "green" && !r.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+            const recomputed = !sameSet ? null : finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, partial: r.partial, source_hashes: r.source_hashes, scenarios: r.scenarios }, ctx).effective_grade;
+            if (recomputed && JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
                 findings.push({ skill, code: "consistency", message: `results.yaml effective_grade is stale in ${runDir} (recompute differs)` });
             }
             for (const s of r.scenarios) {
@@ -96,8 +170,98 @@ export function lintSkill(skillDir) {
             findings.push({ skill, code: "consistency", message: `results.yaml unreadable or malformed in ${runDir}: ${e instanceof Error ? e.message : String(e)}` });
         }
     }
+    // staleness — the newest FULL (non-partial) run per model tag recorded sha256 hashes of
+    // every source it measured: SKILL.md, agent files, each scenario's definition and each
+    // fixture tree. If any has changed since, the committed result describes inputs that no
+    // longer exist — exactly how three regressions hid behind a 100%-SHIP table for four
+    // weeks, and how a swapped fixture went unreported by a "7 skills, 0 findings" lint.
+    // Runs predating source_hashes (or predating a given key kind) are skipped silently — no
+    // retroactive noise; partial runs never count as coverage.
+    for (const tagDir of enumerateTagDirs(resultsRoot)) {
+        // Newest FULL run: partial (--only) runs are iteration artifacts and never count as
+        // coverage — a fresh partial must not silence a stale full run underneath it.
+        let full = null;
+        for (const runDir of runDirsNewestFirst(tagDir)) {
+            try {
+                const r = readResults(runDir);
+                if (r.partial)
+                    continue;
+                full = { runDir, r };
+                break;
+            }
+            catch {
+                break;
+            } // unreadable → the consistency block already reports it
+        }
+        const hashes = full?.r.source_hashes;
+        if (!full || !hashes)
+            continue; // predates source_hashes → silent
+        {
+            const newest = full.runDir;
+            const ctx = { skillDir, specDir, scenarios: spec.scenarios };
+            for (const [key, recorded] of Object.entries(hashes)) {
+                const what = describeSourceKey(key);
+                const scenario = scenarioIdForKey(key, spec.scenarios);
+                // The run itself failed to hash this source, so it was never verified and
+                // no comparison here can establish anything about it.
+                if (recorded === UNREADABLE) {
+                    findings.push({ skill, scenario, code: "stale", message: `${what} could not be read when the newest ${basename(tagDir)} run was recorded (${newest}) — that source was never verified; re-run` });
+                    continue;
+                }
+                const current = currentHashFor(key, ctx);
+                // undefined = not comparable: a scenario the spec no longer has (a reshape,
+                // per the scenario-set check above), or a key kind written by a newer
+                // skill-harness than this one.
+                if (current === undefined)
+                    continue;
+                if (current === null) {
+                    findings.push({ skill, scenario, code: "stale", message: `${what} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
+                }
+                else if (current !== recorded) {
+                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                }
+            }
+            // Coverage: a scenario the spec defines that the newest full run never
+            // measured. Without this, RENAMING a scenario is invisible — the old key
+            // resolves to "not comparable" and the new one was never recorded — so a
+            // 100%/SHIP scorecard survives an arbitrary spec rewrite reporting zero
+            // findings. Gated on the run having recorded scenario keys at all, so runs
+            // predating the key kind stay silent like every other pre-existing run.
+            if (Object.keys(hashes).some((k) => k.startsWith(SCENARIO_PREFIX))) {
+                for (const s of spec.scenarios) {
+                    if (!(SCENARIO_PREFIX + s.id in hashes)) {
+                        findings.push({ skill, scenario: s.id, code: "stale", message: `the newest ${basename(tagDir)} run (${newest}) did not measure scenario \`${s.id}\` — the published result covers a different scenario set than the spec; re-run before publishing` });
+                    }
+                }
+            }
+        }
+    }
     return findings;
 }
+/** Model-tag dirs under tests/results (each holds timestamped run dirs). */
+function enumerateTagDirs(resultsRoot) {
+    if (!existsSync(resultsRoot))
+        return [];
+    try {
+        return readdirSync(resultsRoot).map((t) => join(resultsRoot, t)).filter(isDir);
+    }
+    catch {
+        return [];
+    }
+}
+/** Timestamped run dirs holding a results.yaml, newest first (ISO slugs sort lexicographically). */
+function runDirsNewestFirst(tagDir) {
+    let timestamps;
+    try {
+        timestamps = readdirSync(tagDir);
+    }
+    catch {
+        return [];
+    }
+    return timestamps.sort().reverse()
+        .map((ts) => join(tagDir, ts))
+        .filter((d) => isDir(d) && existsSync(join(d, "results.yaml")));
+}
 /**
  * All committed run dirs under a skill's tests/results (<tag>/<timestamp>/results.yaml).
  * Empty if none. Never throws: unreadable/dangling entries (e.g. a broken symlink, or a

package/dist/regrade.d.ts

@@ -39,6 +39,12 @@ export interface RegradeRunOptions {
     judge: ModelRef;
     specDir: string;
     now?: () => string;
+    /**
+     * Re-judge ONLY scenarios whose stored verdict is untrustworthy — suspect (misfire) or
+     * JUDGE-AMBIGUOUS. Everything else is carried verbatim: their transcripts were judged
+     * cleanly, so spending judge calls on them buys nothing.
+     */
+    onlySuspect?: boolean;
 }
 /**
  * Re-judge every green-transcript scenario in a run dir with `judge` — no

package/dist/regrade.js

@@ -65,7 +65,19 @@ export async function regradeRun(opts) {
     // a recorded verdict or shrink the grade denominator. Fail fast, before
     // spending any judge calls.
     const specById = new Map(spec.scenarios.map((s) => [s.id, s]));
-    const targets = (prev?.scenarios ?? spec.scenarios).map((s) => s.id);
+    const recorded = prev?.scenarios ?? spec.scenarios.map((s) => ({ id: s.id }));
+    let targets = recorded.map((s) => s.id);
+    if (opts.onlySuspect) {
+        if (!prev)
+            throw new Error(`--suspect-only needs a prior results.yaml in ${runDir}`);
+        targets = prev.scenarios
+            .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
+            .map((s) => s.id);
+        if (targets.length === 0) {
+            // Nothing untrustworthy — a no-op, not an error. Return the file as-is.
+            return prev;
+        }
+    }
     const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, "green").length === 0);
     if (missing.length === targets.length) {
         throw new Error(`no green transcripts in ${runDir} — nothing to re-grade`);
@@ -73,8 +85,14 @@ export async function regradeRun(opts) {
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
     }
+    const targetSet = new Set(targets);
     const scenarioResults = [];
-    for (const id of targets) {
+    for (const rec of recorded) {
+        const id = rec.id;
+        if (!targetSet.has(id)) {
+            scenarioResults.push(rec); // clean verdict, carried verbatim (onlySuspect)
+            continue;
+        }
         const scenario = specById.get(id); // guaranteed present by the guard above
         const prevScenario = prev?.scenarios.find((s) => s.id === id);
         const threshold = effectiveThreshold(prevScenario, scenario);
@@ -84,7 +102,7 @@ export async function regradeRun(opts) {
         const carry = overrides.get(id);
         scenarioResults.push({ ...rr, override: carry?.override ?? null, note: carry?.note ?? "" });
     }
-    const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = mode === "green" && !prev?.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: prev?.harness ?? "pi",
@@ -93,6 +111,10 @@ export async function regradeRun(opts) {
         timestamp: prev?.timestamp ?? now(),
         label: prev?.label ?? null,
         mode,
+        // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
+        // the recorded hashes stay, keeping an honestly-stale run honestly stale.
+        partial: prev?.partial,
+        source_hashes: prev?.source_hashes,
         scenarios: scenarioResults,
     }, ctx);
     const g = results.effective_grade;

package/dist/report.d.ts

@@ -1,5 +1,6 @@
 import { type ShipBar } from "./spec.js";
 import { type ResultsFile } from "./results.js";
+import { type Lift } from "./lift.js";
 export interface RunColumn {
     index: number;
     label: string;
@@ -20,6 +21,17 @@ export interface RunColumn {
         override: string | null;
         note: string;
     }>;
+    /**
+     * Red-vs-green lift for this model, when the tag has both a red baseline and a
+     * green run. Undefined means "never measured" — which is not the same claim as
+     * a zero lift, so the report must not render a 0 for it.
+     *
+     * Only set when THIS column is the green run the lift was computed from (see
+     * collectReport): the review UI recomputes lift from the column's live cells,
+     * which is only valid if those cells are the green side of the comparison.
+     */
+    lift?: Lift;
+    liftHeadline?: string;
 }
 export interface ReportData {
     skill: string;
@@ -48,6 +60,8 @@ export declare function publicView(data: ReportData): {
         critical: boolean;
     }[];
     columns: {
+        lift?: Lift | undefined;
+        liftHeadline?: string | undefined;
         index: number;
         label: string;
         tag: string;

package/dist/report.js

@@ -2,6 +2,7 @@ import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
 import { readResults } from "./results.js";
+import { collectLift, liftHeadline } from "./lift.js";
 /** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
 function latestRunDir(tagDir) {
     if (!statSync(tagDir).isDirectory())
@@ -21,6 +22,8 @@ export function collectReport(skillDir) {
     const spec = loadSpec(specPath);
     const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
     const resultsRoot = join(skillDir, "tests", "results");
+    // Lift is keyed by model tag, the same key columns are built from.
+    const liftByTag = new Map(collectLift(skillDir).map((l) => [l.tag, l]));
     const columns = [];
     if (existsSync(resultsRoot)) {
         const tags = readdirSync(resultsRoot)
@@ -46,16 +49,27 @@ export function collectReport(skillDir) {
                     note: s.note,
                 };
             }
+            const tag = tagDir.split("/").pop();
+            // A column is the tag's LATEST run, which is not necessarily the green one —
+            // record a red baseline after a green run and the newest run in the tag is
+            // red. The review UI recomputes lift from `cells` (so author overrides move
+            // it live), so attaching a lift to a column whose cells are the RED run
+            // would have it compare red against red and report "no effect" for a skill
+            // that in fact gained every scenario. Attach only when this column IS the
+            // green side of the comparison.
+            const tagLift = liftByTag.get(tag);
+            const lift = tagLift && tagLift.greenTimestamp === r.timestamp ? tagLift : undefined;
             columns.push({
                 index: columns.length,
                 label: r.model,
-                tag: tagDir.split("/").pop(),
+                tag,
                 runDir,
                 timestamp: r.timestamp,
                 mode: r.mode,
                 grade: r.effective_grade,
                 judge: r.judge,
                 cells,
+                ...(lift ? { lift, liftHeadline: liftHeadline(lift) } : {}),
             });
         }
     }
@@ -77,6 +91,7 @@ export function publicView(data) {
             grade: c.grade,
             judge: c.judge,
             cells: c.cells,
+            ...(c.lift ? { lift: c.lift, liftHeadline: c.liftHeadline } : {}),
         })),
     };
 }

package/dist/rescore.d.ts

@@ -0,0 +1,34 @@
+import type { Spec } from "./spec.js";
+import { type ResultsFile, type ScenarioResult } from "./results.js";
+export interface RescoreOptions {
+    runDir: string;
+    spec: Spec;
+    now?: () => string;
+}
+export interface RescoreChange {
+    id: string;
+    from: ScenarioResult["judge_verdict"];
+    to: ScenarioResult["judge_verdict"];
+    passes: number;
+    clean: number;
+    fromThreshold: number;
+    toThreshold: number;
+}
+export interface RescoreResult {
+    results: ResultsFile;
+    changes: RescoreChange[];
+}
+/**
+ * Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
+ * calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
+ * When the policy changes, the honest move is to recompute the old measurements under it
+ * rather than reconcile two numbers in prose — and to record what moved.
+ *
+ * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
+ * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
+ * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
+ * changes the collapse rule, nothing about what the judge said.
+ */
+export declare function rescoreRun(opts: RescoreOptions): RescoreResult;
+/** Locate the spec for a run dir: results/<tag>/<ts> → ../../../specification.yaml */
+export declare function specPathForRunDir(runDir: string): string;

package/dist/rescore.js

@@ -0,0 +1,66 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { readResults, writeResults } from "./results.js";
+import { appendJournal } from "./journal.js";
+/**
+ * Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
+ * calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
+ * When the policy changes, the honest move is to recompute the old measurements under it
+ * rather than reconcile two numbers in prose — and to record what moved.
+ *
+ * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
+ * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
+ * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
+ * changes the collapse rule, nothing about what the judge said.
+ */
+export function rescoreRun(opts) {
+    const now = opts.now ?? (() => new Date().toISOString());
+    const prev = readResults(opts.runDir);
+    if (!prev)
+        throw new Error(`no results.yaml in ${opts.runDir}`);
+    const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
+    const changes = [];
+    const scenarios = prev.scenarios.map((s) => {
+        const scenario = specById.get(s.id);
+        // no rate to re-apply, or an untrustworthy verdict → carry verbatim
+        if (!scenario || s.reps === undefined || s.clean === undefined || s.passes === undefined)
+            return s;
+        if (s.judge_verdict === "ERROR" || s.judge_verdict === "JUDGE-AMBIGUOUS")
+            return s;
+        if (s.clean === 0)
+            return s;
+        const toThreshold = scenario.passThreshold ?? 0.5;
+        const fromThreshold = s.pass_threshold ?? toThreshold;
+        if (toThreshold === fromThreshold)
+            return s;
+        const rate = s.passes / s.clean;
+        const verdict = rate >= toThreshold ? "PASS" : "FAIL";
+        if (verdict !== s.judge_verdict) {
+            changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
+        }
+        return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
+    });
+    const ctx = prev.mode === "green" && !prev.partial
+        ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
+        : null;
+    const results = writeResults(opts.runDir, {
+        skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
+        timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
+        partial: prev.partial, source_hashes: prev.source_hashes, scenarios,
+    }, ctx);
+    appendJournal(opts.runDir, {
+        event: "rescore", ts: now(),
+        changed: changes.map((c) => `${c.id}: ${c.from}->${c.to} (${c.passes}/${c.clean} @ ${c.toThreshold})`),
+        passed: results.effective_grade.passed, total: results.effective_grade.total,
+        pct: results.effective_grade.pct, ship: results.effective_grade.ship,
+    });
+    return { results, changes };
+}
+/** Locate the spec for a run dir: results/<tag>/<ts> → ../../../specification.yaml */
+export function specPathForRunDir(runDir) {
+    const p = join(runDir, "..", "..", "..", "specification.yaml");
+    if (!existsSync(p))
+        throw new Error(`no specification.yaml above ${runDir}`);
+    return p;
+}
+//# sourceMappingURL=rescore.js.map

package/dist/results.d.ts

@@ -35,6 +35,18 @@ export interface ResultsFile {
     timestamp: string;
     label: string | null;
     mode: string;
+    /** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
+    partial?: boolean;
+    /**
+     * sha256 of every source this run measured: SKILL.md, each distinct
+     * system_prompt_file, each scenario's definition (`scenario:<id>`) and each
+     * fixture tree (`fixture:<path>`). Lint compares the newest run's hashes against
+     * the current sources — a mismatch means the published result describes inputs
+     * that no longer exist (the stale-scorecard class this field exists to kill).
+     * See sources.ts for the key scheme; runs recorded before a key kind existed
+     * simply don't carry it, and are never retroactively flagged.
+     */
+    source_hashes?: Record<string, string>;
     effective_grade: GradeSummary;
     scenarios: ScenarioResult[];
 }
@@ -74,7 +86,7 @@ export declare function applyOverride(results: ResultsFile, scenarioId: string,
  * `!…` preservation lines added by preserveTranscript.
  */
 export declare function ensureResultsGitignore(resultsRoot: string): void;
-/** The rep index embedded in a transcript/judge-raw filename (`.rep<k>.`), or null for a plain (non-rep) file. */
+/** The rep index embedded in a transcript / judge-raw / staged-diff filename (`.rep<k>.`), or null for a plain (non-rep) file. */
 export declare function repIndexOf(filename: string): number | null;
 /**
  * ALL transcript files for a scenario in a run dir, sorted deterministically:
@@ -86,23 +98,43 @@ export declare function repIndexOf(filename: string): number | null;
  * `<id>.<mode>.rep<k>.txt`) — e.g. to detect a green-only condition without
  * false positives from a red/force transcript of the same scenario. Omitted,
  * behavior is unchanged: any `<id>.*.txt` regardless of mode, excluding this
- * scenario's judge-raw artifacts (`<id>.*.judge.txt` — see judgeRawPath).
+ * scenario's sibling artifacts, which share the `.txt` extension deliberately
+ * (so `results/.gitignore`'s `*.txt` covers them all): judge-raw output
+ * (`<id>.*.judge.txt` — see judgeRawPath) and the staged diff
+ * (`<id>.*.diff.txt` — see diffPath).
  */
 export declare function findTranscriptFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** Path of a scenario's raw judge-output artifact within a run dir (rep-suffixed for reps). */
 export declare function judgeRawPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 /** A scenario's raw judge-output files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findJudgeRawFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/**
+ * Path of a seeded scenario's staged-diff artifact within a run dir (rep-suffixed
+ * for reps).
+ *
+ * The diff is the only record of what the model actually *did* — the workspace is
+ * torn down after every rep, so without this a seeded verdict cannot be audited
+ * after the fact. Named `<id>.<mode>[.rep<k>].diff.txt` so it sorts beside its
+ * transcript and is covered by the `*.txt` rule in results/.gitignore: diffs are
+ * generated evidence, ignored like transcripts, not committed like results.yaml.
+ */
+export declare function diffPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export declare function findTranscriptFile(runDir: string, scenarioId: string): string | null;
 /**
- * Un-gitignore ALL of a scenario's transcript AND judge-raw artifact files
- * (audit trail for an override — a --reps run has one transcript (and one
- * judge-raw file) per rep, and every rep that drove the verdict must survive
- * a commit, not just an arbitrary one).
+ * Un-gitignore ALL of a scenario's transcript, judge-raw AND staged-diff
+ * artifact files (audit trail for an override — a --reps run has one of each
+ * per rep, and every rep that drove the verdict must survive a commit, not just
+ * an arbitrary one).
  * Appends `!<tag>/<ts>/<id>.<mode>[.rep<k>].txt` (and the matching
- * `.judge.txt`) to results/.gitignore for each, once. The path uses POSIX
- * separators so the negation matches on Windows too (git ignore patterns are
- * always forward-slashed).
+ * `.judge.txt` / `.diff.txt`) to results/.gitignore for each, once. The path
+ * uses POSIX separators so the negation matches on Windows too (git ignore
+ * patterns are always forward-slashed).
+ *
+ * The diff belongs here for the same reason the judge-raw output does: an
+ * override says the judge got it wrong, and on a seeded scenario the evidence
+ * for that claim is the code the model wrote.
  */
 export declare function preserveTranscript(resultsRoot: string, runDir: string, scenarioId: string): void;