@skill-harness/core 0.1.1 → 0.2.1

package/dist/adapters/types.d.ts

@@ -14,6 +14,12 @@ export interface RunReq {
     mode: RunMode;
     turns: string[];
     cwd: string;
+    /**
+     * Abs path to a markdown file to use AS the system prompt, instead of activating
+     * skillDir as a skill. Used to test a subagent definition (agents/<name>.md) in the
+     * single-shot shape it actually runs in; overrides `mode`'s skill flags.
+     */
+    systemPromptFile?: string;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {

package/dist/discover.d.ts

@@ -10,5 +10,9 @@ export interface DiscoveredSkill {
  * Returns skills sorted by name (testable or not).
  */
 export declare function discover(root: string): DiscoveredSkill[];
-/** Resolve a single skill by name; throws a helpful error if absent or specless. */
+/**
+ * Resolve a single skill by name; throws a helpful error if absent or specless.
+ * A directory that exists but lacks a SKILL.md gets a specific error (rather than
+ * the generic "no skill") so callers don't reimplement the SKILL.md existence check.
+ */
 export declare function resolveSkill(root: string, name: string): DiscoveredSkill;

package/dist/discover.js

@@ -24,10 +24,18 @@ export function discover(root) {
     skills.sort((a, b) => a.name.localeCompare(b.name));
     return skills;
 }
-/** Resolve a single skill by name; throws a helpful error if absent or specless. */
+/**
+ * Resolve a single skill by name; throws a helpful error if absent or specless.
+ * A directory that exists but lacks a SKILL.md gets a specific error (rather than
+ * the generic "no skill") so callers don't reimplement the SKILL.md existence check.
+ */
 export function resolveSkill(root, name) {
     const skill = discover(root).find((s) => s.name === name);
     if (!skill) {
+        const dir = join(root, name);
+        if (existsSync(dir) && statSync(dir).isDirectory() && !existsSync(join(dir, "SKILL.md"))) {
+            throw new Error(`skill \`${name}\` has no SKILL.md (looked in ${dir})`);
+        }
         throw new Error(`no skill \`${name}\` under ${root}`);
     }
     return skill;

package/dist/grade.d.ts

@@ -13,7 +13,18 @@ export interface ParsedVerdict {
     verdict: Verdict;
     reason: string;
 }
-/** Parse a judge's raw output into a verdict + reason. Unparseable → ERROR. */
+/**
+ * Parse a judge's raw output into a verdict + reason.
+ *
+ * Judges sometimes emit MORE than one verdict block (a first pass, then a restated
+ * conclusion). Every block is read, never just the first:
+ *   - all blocks agree  → that verdict, with the reason from the LAST block (the
+ *     judge's final word) in full
+ *   - blocks disagree   → JUDGE-AMBIGUOUS, which counts as a non-pass and carries both
+ *     verdicts in the reason so a rejudge can be queued. Silently taking either one
+ *     would be inventing a grade the judge did not give.
+ * Unparseable → ERROR.
+ */
 export declare function parseVerdict(out: string): ParsedVerdict;
 /**
  * Judge-≠-subject de-confound guard. True when the judge resembles the model

package/dist/grade.js

@@ -15,18 +15,39 @@ Grade each checklist item PASS or FAIL with a <=12-word justification quoting th
 VERDICT: PASS      (only if EVERY item passed)   — or —   VERDICT: FAIL
 REASON: <15 words or fewer>`;
 }
-const VERDICT_RE = /VERDICT\**\s*:?\s*\**\s*(PASS|FAIL)/i;
-const REASON_RE = /REASON\**\s*:?\s*\**\s*(.*)$/im;
-/** Parse a judge's raw output into a verdict + reason. Unparseable → ERROR. */
+// Both anchor to the start of a line and REQUIRE the colon. Without those anchors the
+// reason pattern matched any word containing "reason" — git-ops GLM C1 stored
+// "able given no repo present.", a fragment of "Reasonable" in the judge's prose, which
+// then read as a FAIL-verdict-with-passing-reason misfire that never happened.
+const VERDICT_RE = /^\s*\**\s*VERDICT\**\s*:\s*\**\s*(PASS|FAIL)/gim;
+const REASON_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/gim;
+/**
+ * Parse a judge's raw output into a verdict + reason.
+ *
+ * Judges sometimes emit MORE than one verdict block (a first pass, then a restated
+ * conclusion). Every block is read, never just the first:
+ *   - all blocks agree  → that verdict, with the reason from the LAST block (the
+ *     judge's final word) in full
+ *   - blocks disagree   → JUDGE-AMBIGUOUS, which counts as a non-pass and carries both
+ *     verdicts in the reason so a rejudge can be queued. Silently taking either one
+ *     would be inventing a grade the judge did not give.
+ * Unparseable → ERROR.
+ */
 export function parseVerdict(out) {
-    const vm = out.match(VERDICT_RE);
-    if (!vm) {
+    const verdicts = [...out.matchAll(VERDICT_RE)].map((m) => m[1].toUpperCase());
+    if (verdicts.length === 0) {
         return { verdict: "ERROR", reason: "judge produced no parseable verdict" };
     }
-    const verdict = vm[1].toUpperCase();
-    const rm = out.match(REASON_RE);
-    const reason = rm ? rm[1].trim() : "";
-    return { verdict, reason };
+    const reasons = [...out.matchAll(REASON_RE)].map((m) => m[1].trim());
+    const reason = reasons.length > 0 ? reasons[reasons.length - 1] : "";
+    const unique = [...new Set(verdicts)];
+    if (unique.length > 1) {
+        return {
+            verdict: "JUDGE-AMBIGUOUS",
+            reason: `judge emitted conflicting verdicts (${verdicts.join(", ")}) — needs rejudge; last reason: ${reason}`,
+        };
+    }
+    return { verdict: unique[0], reason };
 }
 /**
  * Judge-≠-subject de-confound guard. True when the judge resembles the model
@@ -51,13 +72,30 @@ const ITEM_RE = /^\s*\d+[.)]\s*\**\s*(PASS|FAIL)\b/gim;
 export function detectMisfire(raw, verdict) {
     if (verdict === "ERROR")
         return false;
+    // Conflicting verdicts are suspect by construction — there is no consistent grade.
+    if (verdict === "JUDGE-AMBIGUOUS")
+        return true;
     const items = [...raw.matchAll(ITEM_RE)].map((m) => m[1].toUpperCase() === "PASS");
-    if (items.length === 0)
+    if (items.length === 0) {
+        // No item lines to cross-check, so fall back to the verdict-vs-reason shape: a FAIL
+        // whose reason says everything passed is the misfire class from REVIEW-FINDINGS
+        // finding 2. Deliberately narrow — an earlier version of this tripwire fired on
+        // terse genuine FAILs, so it requires an explicitly total claim ("all items ...
+        // pass", "every item ... satisfied") and no negation anywhere in the reason.
+        if (verdict === "FAIL") {
+            const reason = (raw.match(REASON_LINE_RE)?.[1] ?? "").trim();
+            const totalPass = /\b(all|every)\b[^.]*\b(pass(es|ed)?|satisf(y|ies|ied)|hold(s)?|met)\b/i.test(reason);
+            const negated = /\b(not|no|n't|fails?|failed|missing|except|but|however)\b/i.test(reason);
+            return totalPass && !negated;
+        }
         return false; // fail-open
+    }
     const andItems = items.every((ok) => ok);
     const verdictBool = verdict === "PASS";
     return verdictBool !== andItems;
 }
+// Non-global twin of REASON_RE: matchAll needs /g, a single .match() must not have it.
+const REASON_LINE_RE = /^\s*\**\s*REASON\**\s*:\s*\**\s*(.*)$/im;
 /** Drive the judge for one transcript and parse the result. */
 export async function gradeTranscript(adapter, judge, prompt, cwd) {
     const raw = await adapter.judge({ model: judge, prompt, cwd });

package/dist/index.d.ts

@@ -8,10 +8,14 @@ export * from "./journal.js";
 export * from "./scheduler.js";
 export * from "./reps.js";
 export * from "./regrade.js";
+export * from "./rescore.js";
 export * from "./workspace.js";
 export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";
+export * from "./util/env.js";
+export * from "./scaffold.js";

package/dist/index.js

@@ -8,11 +8,15 @@ export * from "./journal.js";
 export * from "./scheduler.js";
 export * from "./reps.js";
 export * from "./regrade.js";
+export * from "./rescore.js";
 export * from "./workspace.js";
 export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./lift.js";
 export * from "./adapters/types.js";
 export * from "./util/exec.js";
+export * from "./util/env.js";
+export * from "./scaffold.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -54,6 +54,20 @@ export type JournalEvent = {
     id: string;
     reason: string;
     rep?: number;
+} | {
+    event: "empty-response-retry";
+    ts: string;
+    id: string;
+    attempt: number;
+    rep?: number;
+} | {
+    event: "rescore";
+    ts: string;
+    changed: string[];
+    passed: number;
+    total: number;
+    pct: number;
+    ship: boolean;
 } | {
     event: "score";
     ts: string;

package/dist/lift.d.ts

@@ -0,0 +1,69 @@
+import { type ResultsFile } from "./results.js";
+import type { Verdict } from "./score.js";
+/**
+ * What a skill did to one scenario, red (skill off) vs green (skill active).
+ *
+ * `inconclusive` is load-bearing: it is what stops lift from becoming a number
+ * that only ever goes up. See `classify`.
+ */
+export type LiftClass = "gained" | "regressed" | "kept" | "both-fail" | "inconclusive";
+export interface LiftCell {
+    red: Verdict;
+    /**
+     * Whether the red side's verdict was an unresolved misfire. Carried (not just
+     * folded into `class`) because the review UI recomputes the class live as the
+     * author overrides green verdicts — it needs the red evidence, not a frozen
+     * conclusion. See assets/report.grade.js's liftClass.
+     */
+    redSuspect: boolean;
+    green: Verdict;
+    class: LiftClass;
+}
+export interface Lift {
+    tag: string;
+    model: string;
+    redTimestamp: string;
+    greenTimestamp: string;
+    /** Scenario ids present in both runs — the only ones a lift can speak to. */
+    compared: number;
+    gained: number;
+    regressed: number;
+    kept: number;
+    bothFail: number;
+    inconclusive: number;
+    redPassed: number;
+    greenPassed: number;
+    delta: number;
+    /** Ids the green run covered that the red baseline did not (and vice versa). */
+    greenOnly: string[];
+    redOnly: string[];
+    /** True when either side was an `--only` run, so coverage is a subset by construction. */
+    partial: boolean;
+    cells: Record<string, LiftCell>;
+}
+/**
+ * Compare a red (baseline, skill off) run against a green (skill active) run of
+ * the same model: the "does this skill actually do anything?" measurement.
+ *
+ * Both sides go through `effectiveVerdicts`, so an author override is what
+ * counts and an override resolves a misfire — the same rule scoring uses. Only
+ * the intersection of scenario ids is compared; a lift cannot speak to a
+ * scenario one side never ran.
+ */
+export declare function computeLift(red: ResultsFile, green: ResultsFile): Lift;
+/** One line for a human: what the skill did, and what it cost. */
+export declare function liftHeadline(lift: Lift): string;
+/**
+ * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
+ * with the most recent green run and compute the lift.
+ *
+ * Deliberately derived on read rather than persisted into results.yaml: a lift
+ * is a fact about a *pair* of runs, so caching it inside one run's file would go
+ * stale the moment a new baseline lands — the stale-scorecard failure mode
+ * `source_hashes` exists to prevent. Deriving also means lift works
+ * retroactively on results already committed by 0.1.x/0.2.0 users.
+ *
+ * A tag with no red baseline is omitted entirely rather than reported as a zero
+ * lift: "not measured" and "measured no effect" are different claims.
+ */
+export declare function collectLift(skillDir: string): Lift[];

package/dist/lift.js

@@ -0,0 +1,163 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { readResults, effectiveVerdicts } from "./results.js";
+/** A verdict that carries real evidence about the task, rather than about the harness or the judge. */
+function conclusive(verdict, suspect) {
+    // ERROR is a harness failure (timeout, empty reply) — it says nothing about
+    // whether the agent could do the task. JUDGE-AMBIGUOUS and an unresolved
+    // misfire are judge failures. Treating any of them as a FAIL would make
+    // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
+    // measure flakiness while looking like skill value.
+    return !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
+}
+function classify(red, green) {
+    if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))
+        return "inconclusive";
+    const redPass = red.verdict === "PASS";
+    const greenPass = green.verdict === "PASS";
+    if (redPass && greenPass)
+        return "kept";
+    if (!redPass && greenPass)
+        return "gained";
+    if (redPass && !greenPass)
+        return "regressed";
+    return "both-fail";
+}
+/**
+ * Compare a red (baseline, skill off) run against a green (skill active) run of
+ * the same model: the "does this skill actually do anything?" measurement.
+ *
+ * Both sides go through `effectiveVerdicts`, so an author override is what
+ * counts and an override resolves a misfire — the same rule scoring uses. Only
+ * the intersection of scenario ids is compared; a lift cannot speak to a
+ * scenario one side never ran.
+ */
+export function computeLift(red, green) {
+    const redV = new Map(effectiveVerdicts(red.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
+    const greenV = new Map(effectiveVerdicts(green.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
+    const cells = {};
+    const counts = { gained: 0, regressed: 0, kept: 0, "both-fail": 0, inconclusive: 0 };
+    let redPassed = 0;
+    let greenPassed = 0;
+    // Green order drives display order (it is the run the author is looking at),
+    // restricted to ids the red baseline also covered.
+    for (const [id, g] of greenV) {
+        const r = redV.get(id);
+        if (!r)
+            continue;
+        const cls = classify(r, g);
+        cells[id] = { red: r.verdict, redSuspect: r.suspect, green: g.verdict, class: cls };
+        counts[cls]++;
+        if (cls !== "inconclusive") {
+            if (r.verdict === "PASS")
+                redPassed++;
+            if (g.verdict === "PASS")
+                greenPassed++;
+        }
+    }
+    return {
+        tag: "",
+        model: green.model,
+        redTimestamp: red.timestamp,
+        greenTimestamp: green.timestamp,
+        compared: Object.keys(cells).length,
+        gained: counts.gained,
+        regressed: counts.regressed,
+        kept: counts.kept,
+        bothFail: counts["both-fail"],
+        inconclusive: counts.inconclusive,
+        redPassed,
+        greenPassed,
+        delta: greenPassed - redPassed,
+        greenOnly: [...greenV.keys()].filter((id) => !redV.has(id)),
+        redOnly: [...redV.keys()].filter((id) => !greenV.has(id)),
+        partial: Boolean(red.partial || green.partial),
+        cells,
+    };
+}
+/** One line for a human: what the skill did, and what it cost. */
+export function liftHeadline(lift) {
+    if (lift.compared === 0)
+        return "no shared scenarios to compare";
+    // Everything inconclusive is NOT "no effect" — it is no measurement. Saying
+    // "no measured effect" here would be the same not-measured/measured-no-effect
+    // conflation this module refuses to make when a red baseline is missing.
+    const conclusive = lift.compared - lift.inconclusive;
+    if (conclusive === 0) {
+        return `nothing conclusive to compare (${lift.inconclusive} inconclusive — fix the harness/judge, then re-run)`;
+    }
+    const segments = [];
+    if (lift.gained === 0 && lift.regressed === 0) {
+        segments.push(lift.kept > 0
+            ? `no measured effect (${lift.kept} passed without the skill too)`
+            : "no measured effect");
+    }
+    else {
+        const sign = lift.delta > 0 ? `+${lift.delta}` : String(lift.delta);
+        segments.push(`${sign} net (${lift.gained} gained, ${lift.regressed} regressed)`);
+    }
+    if (lift.inconclusive > 0)
+        segments.push(`${lift.inconclusive} inconclusive`);
+    if (lift.partial)
+        segments.push("partial run");
+    return segments.join(" · ");
+}
+/** A directory that exists right now; false (never throws) if it vanished concurrently. */
+function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
+ * with the most recent green run and compute the lift.
+ *
+ * Deliberately derived on read rather than persisted into results.yaml: a lift
+ * is a fact about a *pair* of runs, so caching it inside one run's file would go
+ * stale the moment a new baseline lands — the stale-scorecard failure mode
+ * `source_hashes` exists to prevent. Deriving also means lift works
+ * retroactively on results already committed by 0.1.x/0.2.0 users.
+ *
+ * A tag with no red baseline is omitted entirely rather than reported as a zero
+ * lift: "not measured" and "measured no effect" are different claims.
+ */
+export function collectLift(skillDir) {
+    const resultsRoot = join(skillDir, "tests", "results");
+    if (!existsSync(resultsRoot))
+        return [];
+    const lifts = [];
+    for (const tag of readdirSync(resultsRoot).filter((n) => isDir(join(resultsRoot, n))).sort()) {
+        const tagDir = join(resultsRoot, tag);
+        const runDirs = readdirSync(tagDir)
+            .map((n) => join(tagDir, n))
+            .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
+            .sort(); // timestamp-slug names ⇒ chronological ascending
+        // Mode is only knowable after reading results.yaml, so every run in the tag
+        // is read; last-wins gives the most recent of each mode.
+        let red;
+        let green;
+        for (const rd of runDirs) {
+            let r;
+            try {
+                r = readResults(rd);
+            }
+            catch (e) {
+                // A corrupt/truncated results.yaml must not take down the whole view.
+                console.warn(`skill-harness lift: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
+                continue;
+            }
+            if (r.mode === "red")
+                red = r;
+            else if (r.mode === "green")
+                green = r;
+        }
+        if (!red || !green)
+            continue;
+        lifts.push({ ...computeLift(red, green), tag });
+    }
+    return lifts;
+}
+//# sourceMappingURL=lift.js.map

package/dist/lint.d.ts

@@ -1,4 +1,4 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "consistency" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "consistency" | "stale" | "lint-error";
 export interface LintFinding {
     readonly skill: string;
     readonly scenario?: string;

package/dist/lint.js

@@ -1,5 +1,6 @@
 import { existsSync, statSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, resolve } from "node:path";
+import { createHash } from "node:crypto";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
@@ -13,6 +14,14 @@ function isDir(p) {
         return false;
     }
 }
+function isFile(p) {
+    try {
+        return statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * Validate one skill's spec + fixtures statically (and results-consistency when
  * committed results exist — see the consistency block). Never throws: a bad spec
@@ -64,6 +73,16 @@ export function lintSkill(skillDir) {
             }
         }
     }
+    // system_prompt_file must exist — an agent-file scenario silently falling back to
+    // skill activation would measure the wrong artifact entirely.
+    for (const s of spec.scenarios) {
+        if (!s.systemPromptFile)
+            continue;
+        const abs = isAbsolute(s.systemPromptFile) ? s.systemPromptFile : resolve(specDir, s.systemPromptFile);
+        if (!isFile(abs)) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: `system_prompt_file not found: ${s.systemPromptFile}` });
+        }
+    }
     // results-consistency — only for committed results.yaml (skipped silently otherwise).
     // Each run dir gets ONE try: schema-1 is intentionally skipped (continue, no finding —
     // migrateResults carries a schema-1 grade verbatim, so recomputing it would false-flag).
@@ -78,9 +97,17 @@ export function lintSkill(skillDir) {
             if (raw?.schema !== 2)
                 continue; // schema-1 intentionally skipped — no finding
             const r = readResults(runDir);
-            const ctx = r.mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
-            const recomputed = finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, scenarios: r.scenarios }, ctx).effective_grade;
-            if (JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
+            // A run whose scenario set no longer matches the spec predates a spec reshape
+            // (scenarios added/removed). Its grade was computed against the OLD ship bar and
+            // cannot be meaningfully recomputed against the new one — recomputing would flag
+            // every historical run each time a spec grows. Staleness (source_hashes) is the
+            // mechanism that says "re-run"; consistency only polices runs the current spec
+            // can actually re-score. Override/transcript rules below still apply.
+            const specIds = new Set(spec.scenarios.map((sc) => sc.id));
+            const sameSet = r.scenarios.length === specIds.size && r.scenarios.every((sc) => specIds.has(sc.id));
+            const ctx = r.mode === "green" && !r.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+            const recomputed = !sameSet ? null : finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, partial: r.partial, source_hashes: r.source_hashes, scenarios: r.scenarios }, ctx).effective_grade;
+            if (recomputed && JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
                 findings.push({ skill, code: "consistency", message: `results.yaml effective_grade is stale in ${runDir} (recompute differs)` });
             }
             for (const s of r.scenarios) {
@@ -96,8 +123,78 @@ export function lintSkill(skillDir) {
             findings.push({ skill, code: "consistency", message: `results.yaml unreadable or malformed in ${runDir}: ${e instanceof Error ? e.message : String(e)}` });
         }
     }
+    // staleness — the newest FULL (non-partial) run per model tag recorded sha256 hashes of
+    // every source file it measured. If any of those files has changed since, the committed
+    // result describes text that no longer exists — exactly how three regressions hid behind
+    // a 100%-SHIP table for four weeks. Runs predating source_hashes are skipped silently
+    // (no retroactive noise); partial runs never count as coverage.
+    for (const tagDir of enumerateTagDirs(resultsRoot)) {
+        // Newest FULL run: partial (--only) runs are iteration artifacts and never count as
+        // coverage — a fresh partial must not silence a stale full run underneath it.
+        let full = null;
+        for (const runDir of runDirsNewestFirst(tagDir)) {
+            try {
+                const r = readResults(runDir);
+                if (r.partial)
+                    continue;
+                full = { runDir, r };
+                break;
+            }
+            catch {
+                break;
+            } // unreadable → the consistency block already reports it
+        }
+        const hashes = full?.r.source_hashes;
+        if (!full || !hashes)
+            continue; // predates source_hashes → silent
+        {
+            const newest = full.runDir;
+            for (const [key, recorded] of Object.entries(hashes)) {
+                const abs = key === "SKILL.md" ? join(skillDir, "SKILL.md") : resolve(specDir, key);
+                const current = fileSha256(abs);
+                if (current === null) {
+                    findings.push({ skill, code: "stale", message: `${key} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
+                }
+                else if (current !== recorded) {
+                    findings.push({ skill, code: "stale", message: `${key} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                }
+            }
+        }
+    }
     return findings;
 }
+function fileSha256(p) {
+    try {
+        return createHash("sha256").update(readFileSync(p)).digest("hex");
+    }
+    catch {
+        return null;
+    }
+}
+/** Model-tag dirs under tests/results (each holds timestamped run dirs). */
+function enumerateTagDirs(resultsRoot) {
+    if (!existsSync(resultsRoot))
+        return [];
+    try {
+        return readdirSync(resultsRoot).map((t) => join(resultsRoot, t)).filter(isDir);
+    }
+    catch {
+        return [];
+    }
+}
+/** Timestamped run dirs holding a results.yaml, newest first (ISO slugs sort lexicographically). */
+function runDirsNewestFirst(tagDir) {
+    let timestamps;
+    try {
+        timestamps = readdirSync(tagDir);
+    }
+    catch {
+        return [];
+    }
+    return timestamps.sort().reverse()
+        .map((ts) => join(tagDir, ts))
+        .filter((d) => isDir(d) && existsSync(join(d, "results.yaml")));
+}
 /**
  * All committed run dirs under a skill's tests/results (<tag>/<timestamp>/results.yaml).
  * Empty if none. Never throws: unreadable/dangling entries (e.g. a broken symlink, or a

package/dist/regrade.d.ts

@@ -39,6 +39,12 @@ export interface RegradeRunOptions {
     judge: ModelRef;
     specDir: string;
     now?: () => string;
+    /**
+     * Re-judge ONLY scenarios whose stored verdict is untrustworthy — suspect (misfire) or
+     * JUDGE-AMBIGUOUS. Everything else is carried verbatim: their transcripts were judged
+     * cleanly, so spending judge calls on them buys nothing.
+     */
+    onlySuspect?: boolean;
 }
 /**
  * Re-judge every green-transcript scenario in a run dir with `judge` — no

package/dist/regrade.js

@@ -65,7 +65,19 @@ export async function regradeRun(opts) {
     // a recorded verdict or shrink the grade denominator. Fail fast, before
     // spending any judge calls.
     const specById = new Map(spec.scenarios.map((s) => [s.id, s]));
-    const targets = (prev?.scenarios ?? spec.scenarios).map((s) => s.id);
+    const recorded = prev?.scenarios ?? spec.scenarios.map((s) => ({ id: s.id }));
+    let targets = recorded.map((s) => s.id);
+    if (opts.onlySuspect) {
+        if (!prev)
+            throw new Error(`--suspect-only needs a prior results.yaml in ${runDir}`);
+        targets = prev.scenarios
+            .filter((s) => s.suspect || s.judge_verdict === "JUDGE-AMBIGUOUS")
+            .map((s) => s.id);
+        if (targets.length === 0) {
+            // Nothing untrustworthy — a no-op, not an error. Return the file as-is.
+            return prev;
+        }
+    }
     const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, "green").length === 0);
     if (missing.length === targets.length) {
         throw new Error(`no green transcripts in ${runDir} — nothing to re-grade`);
@@ -73,8 +85,14 @@ export async function regradeRun(opts) {
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
     }
+    const targetSet = new Set(targets);
     const scenarioResults = [];
-    for (const id of targets) {
+    for (const rec of recorded) {
+        const id = rec.id;
+        if (!targetSet.has(id)) {
+            scenarioResults.push(rec); // clean verdict, carried verbatim (onlySuspect)
+            continue;
+        }
         const scenario = specById.get(id); // guaranteed present by the guard above
         const prevScenario = prev?.scenarios.find((s) => s.id === id);
         const threshold = effectiveThreshold(prevScenario, scenario);
@@ -84,7 +102,7 @@ export async function regradeRun(opts) {
         const carry = overrides.get(id);
         scenarioResults.push({ ...rr, override: carry?.override ?? null, note: carry?.note ?? "" });
     }
-    const ctx = mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = mode === "green" && !prev?.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: prev?.harness ?? "pi",
@@ -93,6 +111,10 @@ export async function regradeRun(opts) {
         timestamp: prev?.timestamp ?? now(),
         label: prev?.label ?? null,
         mode,
+        // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
+        // the recorded hashes stay, keeping an honestly-stale run honestly stale.
+        partial: prev?.partial,
+        source_hashes: prev?.source_hashes,
         scenarios: scenarioResults,
     }, ctx);
     const g = results.effective_grade;