@skill-harness/core 0.4.0 → 0.6.0

package/dist/adapters/types.d.ts

@@ -32,4 +32,14 @@ export interface HarnessAdapter {
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     judge(req: JudgeReq): Promise<string>;
+    /**
+     * The harness CLI's own version, recorded in `results.yaml` as
+     * `harness_cli_version`. Null when it cannot be determined — a version this
+     * adapter had to guess at is worse than none, because the whole point of the
+     * field is to identify which CLI produced a transcript.
+     *
+     * Optional so a test double or a future adapter need not implement it; callers
+     * treat a missing method exactly like a null answer.
+     */
+    version?(): Promise<string | null>;
 }

package/dist/canary.d.ts

@@ -0,0 +1,44 @@
+import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+/**
+ * The probe target: the longest `## ` heading in the skill's body.
+ *
+ * Body-only by construction — the frontmatter `description` is always in context
+ * under progressive disclosure, so anything quotable from it would pass against a
+ * model that never read the instructions. Longest rather than first because the
+ * check is "did you see this text", and `## Overview` is guessable while
+ * `## Refuse a metered judge, whatever chose it` is not.
+ *
+ * Null when the body has no `## ` heading: there is then nothing to ask for that a
+ * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
+ * worse than none.
+ */
+export declare function deliveryAnchor(skillMd: string): string | null;
+export interface CanaryResult {
+    /** `pass` = the anchor came back; `fail` = it did not; `skipped` = nothing safe to probe for. */
+    status: "pass" | "fail" | "skipped";
+    /** What the probe looked for (null when skipped). */
+    anchor: string | null;
+    /** Why it was skipped, or what the model said instead (trimmed) — for the log and the journal. */
+    detail: string;
+}
+export declare function canaryPrompt(skillName: string, anchor: string): string;
+export interface CanaryOptions {
+    adapter: HarnessAdapter;
+    model: ModelRef;
+    skillDir: string;
+    skillName: string;
+    /** Neutral cwd, same as a scenario gets — the probe must not see a repo either. */
+    cwd: string;
+}
+/**
+ * Run the probe in green mode and report whether the skill body was reachable.
+ *
+ * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
+ * with the reply in `detail`, because "the harness answered without the skill" and
+ * "the model said something odd" are both reasons not to spend a wave. An adapter
+ * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
+ * setup errors with their own messages.
+ */
+export declare function runDeliveryCanary(opts: CanaryOptions): Promise<CanaryResult>;
+/** The abort message for a failed canary: what was measured, and what to do instead. */
+export declare function canaryFailure(skillName: string, result: CanaryResult, cliVersion: string | null): string;

package/dist/canary.js

@@ -0,0 +1,123 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * The delivery canary: one cheap probe, before any scenario runs, that the skill
+ * under test actually reached the model.
+ *
+ * Why this exists. `--mode green` asks the harness to activate the skill, and the
+ * harness can decline without saying so. Measured on pi: 0.80.x wrapped the prompt
+ * with the skill body; 0.83.0 switched to progressive disclosure (only the
+ * description is in context, the body loads on demand — "models don't always do
+ * this", pi's own docs); and a nonexistent `--skill` path is accepted silently,
+ * exit 0 with a normal answer. The reference corpus ran two full waves in that
+ * state: `architect` came back 7/14, ≈ its no-skill baseline, and looked entirely
+ * plausible — the only tell was a contradictory failure mix (over-ceremony AND
+ * capitulation at once) that no single skill edit produces.
+ *
+ * What it can and cannot prove. A pass means the skill's body is *reachable* in
+ * this exact invocation — which is what kills the whole silent-non-delivery class:
+ * a dropped flag, a wrong path, a harness that stopped honoring the mode. It does
+ * NOT prove the body entered context for every later scenario; under progressive
+ * disclosure that is the model's choice per turn, and no probe can promise it. The
+ * mode whose delivery needs no promise is `force` (SKILL.md as system prompt),
+ * which is why `run` recommends it rather than pretending the canary is equivalent.
+ *
+ * Cost: exactly one subject call per run, and only when asked for (`--canary`).
+ * A run that aborts here has spent one rep instead of a wave.
+ */
+/** Frontmatter-stripped body of a SKILL.md. */
+function skillBody(text) {
+    const m = /^---\r?\n[\s\S]*?\r?\n---\r?\n/.exec(text);
+    return m ? text.slice(m[0].length) : text;
+}
+/**
+ * The probe target: the longest `## ` heading in the skill's body.
+ *
+ * Body-only by construction — the frontmatter `description` is always in context
+ * under progressive disclosure, so anything quotable from it would pass against a
+ * model that never read the instructions. Longest rather than first because the
+ * check is "did you see this text", and `## Overview` is guessable while
+ * `## Refuse a metered judge, whatever chose it` is not.
+ *
+ * Null when the body has no `## ` heading: there is then nothing to ask for that a
+ * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
+ * worse than none.
+ */
+export function deliveryAnchor(skillMd) {
+    const headings = [...skillBody(skillMd).matchAll(/^##[ \t]+(.+?)[ \t]*$/gm)].map((m) => m[1].trim());
+    if (headings.length === 0)
+        return null;
+    return headings.reduce((a, b) => (b.length > a.length ? b : a));
+}
+/** Normalize for comparison: case, whitespace runs, and markdown emphasis/backticks. */
+function normalize(s) {
+    return s.toLowerCase().replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
+}
+export function canaryPrompt(skillName, anchor) {
+    // Asks for every heading, not just the anchor, because that is a question a model
+    // WITH the instructions answers naturally and a model without them can only
+    // invent. The instruction to say NOT_AVAILABLE keeps an honest miss from looking
+    // like a refusal to follow format.
+    return (`Answer from the instructions you have loaded — do not perform any task.\n\n` +
+        `List every level-2 markdown heading (lines starting with "## ") in the instructions of ` +
+        `the skill named "${skillName}", verbatim, one per line, with no other text.\n` +
+        `If you have no such instructions available, reply exactly: NOT_AVAILABLE\n\n` +
+        `(The heading text is what matters; keep it exact.)`);
+}
+/**
+ * What the model actually said, per the adapters' shared transcript convention
+ * (`>>> USER:` / `<<< ASSISTANT:`). Falls back to the whole text for an adapter
+ * that doesn't use the markers.
+ */
+function assistantReply(transcript) {
+    const parts = transcript.split(/^<<< ASSISTANT:\s*$/m);
+    return (parts.length > 1 ? parts[parts.length - 1] : transcript).trim();
+}
+/**
+ * Run the probe in green mode and report whether the skill body was reachable.
+ *
+ * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
+ * with the reply in `detail`, because "the harness answered without the skill" and
+ * "the model said something odd" are both reasons not to spend a wave. An adapter
+ * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
+ * setup errors with their own messages.
+ */
+export async function runDeliveryCanary(opts) {
+    const skillMd = readFileSync(join(opts.skillDir, "SKILL.md"), "utf8");
+    const anchor = deliveryAnchor(skillMd);
+    if (!anchor) {
+        return {
+            status: "skipped",
+            anchor: null,
+            detail: `${opts.skillName}/SKILL.md has no \`## \` heading to probe for — nothing a reply could prove`,
+        };
+    }
+    const transcript = await opts.adapter.run({
+        skillDir: opts.skillDir,
+        model: opts.model,
+        mode: "green",
+        turns: [canaryPrompt(opts.skillName, anchor)],
+        cwd: opts.cwd,
+    });
+    const ok = normalize(transcript).includes(normalize(anchor));
+    return {
+        status: ok ? "pass" : "fail",
+        anchor,
+        // The reply, not the transcript: the transcript opens with our own prompt, and a
+        // failure report whose first 400 characters are the question is useless.
+        detail: ok ? "" : assistantReply(transcript).slice(0, 400),
+    };
+}
+/** The abort message for a failed canary: what was measured, and what to do instead. */
+export function canaryFailure(skillName, result, cliVersion) {
+    return (`delivery canary FAILED for ${skillName}: the model could not quote its own skill instructions ` +
+        `(looked for the heading \`${result.anchor}\`).\n` +
+        `  The skill is not reaching the model, so every scenario in this run would measure a naked ` +
+        `model and score like a result. Nothing has been spent beyond this one probe.\n` +
+        `  harness CLI: ${cliVersion ?? "unknown"}. On pi ≥ 0.83.0 \`--skill\` is progressive disclosure ` +
+        `(description in context, body on demand) and a nonexistent path is accepted silently.\n` +
+        `  Fix: re-run with \`--mode force\` (SKILL.md as the system prompt — delivery no version has made ` +
+        `conditional), or check that the skill dir is the one you meant.\n` +
+        `  What the model said instead: ${result.detail || "(nothing)"}`);
+}
+//# sourceMappingURL=canary.js.map

package/dist/discover.d.ts

@@ -8,6 +8,13 @@ export interface DiscoveredSkill {
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
  * SKILL.md. It is testable iff `<skill>/tests/specification.yaml` exists.
  * Returns skills sorted by name (testable or not).
+ *
+ * `dir` and `specPath` are ABSOLUTE, whatever `root` was. They are handed to child
+ * processes that run in a neutral cwd of the harness's choosing (`pi --skill
+ * <dir>`), so a relative `--skills .` used to produce a path that resolved to
+ * nothing over there — and pi accepts a nonexistent `--skill` path silently, exit 0
+ * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
+ * but the honest fix is here, where the path is built.
  */
 export declare function discover(root: string): DiscoveredSkill[];
 /**

package/dist/discover.js

@@ -1,19 +1,27 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 /**
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
  * SKILL.md. It is testable iff `<skill>/tests/specification.yaml` exists.
  * Returns skills sorted by name (testable or not).
+ *
+ * `dir` and `specPath` are ABSOLUTE, whatever `root` was. They are handed to child
+ * processes that run in a neutral cwd of the harness's choosing (`pi --skill
+ * <dir>`), so a relative `--skills .` used to produce a path that resolved to
+ * nothing over there — and pi accepts a nonexistent `--skill` path silently, exit 0
+ * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
+ * but the honest fix is here, where the path is built.
  */
 export function discover(root) {
-    if (!existsSync(root) || !statSync(root).isDirectory()) {
+    const absRoot = resolve(root);
+    if (!existsSync(absRoot) || !statSync(absRoot).isDirectory()) {
         throw new Error(`skills root is not a directory: ${root}`);
     }
     const skills = [];
-    for (const name of readdirSync(root)) {
+    for (const name of readdirSync(absRoot)) {
         if (name.startsWith("."))
             continue;
-        const dir = join(root, name);
+        const dir = join(absRoot, name);
         if (!statSync(dir).isDirectory())
             continue;
         if (!existsSync(join(dir, "SKILL.md")))
@@ -32,7 +40,7 @@ export function discover(root) {
 export function resolveSkill(root, name) {
     const skill = discover(root).find((s) => s.name === name);
     if (!skill) {
-        const dir = join(root, name);
+        const dir = join(resolve(root), name);
         if (existsSync(dir) && statSync(dir).isDirectory() && !existsSync(join(dir, "SKILL.md"))) {
             throw new Error(`skill \`${name}\` has no SKILL.md (looked in ${dir})`);
         }

package/dist/index.d.ts

@@ -25,3 +25,5 @@ export * from "./defaults.js";
 export * from "./judge-policy.js";
 export * from "./regate.js";
 export * from "./downgrade.js";
+export * from "./canary.js";
+export * from "./stability.js";

package/dist/index.js

@@ -25,4 +25,6 @@ export * from "./defaults.js";
 export * from "./judge-policy.js";
 export * from "./regate.js";
 export * from "./downgrade.js";
+export * from "./canary.js";
+export * from "./stability.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -22,12 +22,26 @@ export type JournalEvent = {
     skill: string;
     harness: string;
     model: string;
+    /** The harness CLI's own version (`pi --version`), or null when it could not be asked. */
+    harness_cli_version?: string | null;
     judge: {
         provider: string;
         model: string;
     };
     mode: string;
     label: string | null;
+}
+/**
+ * The pre-flight delivery probe (green mode, `--canary`): did the model quote a
+ * body-only heading of its own skill back? `fail` aborts the run, so a journal
+ * carrying a failed canary is the record of a wave that was NOT spent.
+ */
+ | {
+    event: "delivery-canary";
+    ts: string;
+    status: "pass" | "fail" | "skipped";
+    anchor: string | null;
+    detail: string;
 } | {
     event: "scenario-started";
     ts: string;

package/dist/lift.d.ts

@@ -24,6 +24,19 @@ export interface Lift {
     model: string;
     redTimestamp: string;
     greenTimestamp: string;
+    /**
+     * How the skill was delivered on the non-baseline side: `green` (the harness
+     * activated it) or `force` (SKILL.md as the system prompt).
+     *
+     * The `green*` field names are kept as the *skill-active side*, not as a claim
+     * about the mode — they are the wire format the committed report assets and the
+     * review UI read, and renaming them would break every published report to say
+     * something the `mode` field already says. A red baseline is mode-independent
+     * (`--no-skills` either way), so red-vs-force is as valid a comparison as
+     * red-vs-green — but which one you are looking at changes what the number means,
+     * so it is recorded rather than implied.
+     */
+    mode: string;
     /** Scenario ids present in both runs — the only ones a lift can speak to. */
     compared: number;
     gained: number;

package/dist/lift.js

@@ -1,6 +1,6 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, effectiveVerdicts } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
 import { loadSpec } from "./spec.js";
 function aggregationShape(s) {
     const reps = s.reps ?? 1;
@@ -98,6 +98,7 @@ export function computeLift(red, green, opts = {}) {
     return {
         tag: "",
         model: green.model,
+        mode: green.mode,
         redTimestamp: red.timestamp,
         greenTimestamp: green.timestamp,
         compared: Object.keys(cells).length,
@@ -193,7 +194,7 @@ function isDir(p) {
 }
 /**
  * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
- * with the most recent green run and compute the lift.
+ * with the most recent skill-delivered run (green or force) and compute the lift.
  *
  * Deliberately derived on read rather than persisted into results.yaml: a lift
  * is a fact about a *pair* of runs, so caching it inside one run's file would go
@@ -237,9 +238,13 @@ export function collectLift(skillDir) {
             .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
             .sort(); // timestamp-slug names ⇒ chronological ascending
         // Mode is only knowable after reading results.yaml, so every run in the tag
-        // is read; last-wins gives the most recent of each mode.
+        // is read; last-wins gives the most recent baseline and the most recent
+        // skill-delivered run. The skill side is whichever scored mode ran most
+        // recently — a corpus that moved from green to force delivery should see its
+        // lift follow, and the baseline it is measured against is the same either way
+        // (`--no-skills` in both).
         let red;
-        let green;
+        let skillOn;
         for (const rd of runDirs) {
             let r;
             try {
@@ -252,12 +257,12 @@ export function collectLift(skillDir) {
             }
             if (r.mode === "red")
                 red = r;
-            else if (r.mode === "green")
-                green = r;
+            else if (isScoredMode(r.mode))
+                skillOn = r;
         }
-        if (!red || !green)
+        if (!red || !skillOn)
             continue;
-        lifts.push({ ...computeLift(red, green, { modeInsensitive }), tag });
+        lifts.push({ ...computeLift(red, skillOn, { modeInsensitive }), tag });
     }
     return lifts;
 }

package/dist/lint.d.ts

@@ -1,10 +1,25 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "stability" | "lint-error";
+/**
+ * How much a finding means.
+ *
+ * `error` (the default, and every code that existed through 0.5.0) fails the gate.
+ * `info` reports something a reader should know that is NOT a defect: a boundary cell
+ * is a statement about how much one run of a scenario is worth, not a broken spec, and
+ * turning it red would make "this cell needs more reps" indistinguishable from "your
+ * fixture is missing". Omitted rather than written on every finding so the shape stays
+ * backward-compatible for anything already reading this list.
+ */
+export type LintSeverity = "error" | "info";
 export interface LintFinding {
     readonly skill: string;
     readonly scenario?: string;
     readonly code: LintCode;
     readonly message: string;
+    /** Absent means `error` — only findings that must not fail CI carry this. */
+    readonly severity?: LintSeverity;
 }
+/** True when a finding fails the gate. The single place the exit-code rule lives. */
+export declare function failsGate(f: LintFinding): boolean;
 /**
  * Validate one skill's spec + fixtures statically (and results-consistency when
  * committed results exist — see the consistency block). Never throws: a bad spec

package/dist/lint.js

@@ -2,10 +2,16 @@ import { existsSync, statSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, resolve } from "node:path";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
-import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
+import { readResults, finalizeResults, findTranscriptFiles, resultsPath, scoreContextFor } from "./results.js";
 import { currentHashFor, describeSourceKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
 import { downgradeWarning } from "./downgrade.js";
+import { collectScoredRuns } from "./trends.js";
+import { boundaryCells, stabilityFrom, stabilityNote } from "./stability.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
+/** True when a finding fails the gate. The single place the exit-code rule lives. */
+export function failsGate(f) {
+    return (f.severity ?? "error") === "error";
+}
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
 function isDir(p) {
@@ -163,10 +169,17 @@ export function lintSkill(skillDir) {
             // can actually re-score. Override/transcript rules below still apply.
             const specIds = new Set(spec.scenarios.map((sc) => sc.id));
             const sameSet = r.scenarios.length === specIds.size && r.scenarios.every((sc) => specIds.has(sc.id));
-            const ctx = r.mode === "green" && !r.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+            const ctx = scoreContextFor(r, spec);
             const recomputed = !sameSet ? null : finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, partial: r.partial, source_hashes: r.source_hashes, scenarios: r.scenarios }, ctx).effective_grade;
             if (recomputed && JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
-                findings.push({ skill, code: "consistency", message: `results.yaml effective_grade is stale in ${runDir} (recompute differs)` });
+                // The remedy is named because this finding now has a benign, expected cause
+                // as well as a suspicious one: a force run recorded before 0.5.0 carries a
+                // "not scored" placeholder, and today's policy scores it (see SCORED_MODES).
+                // `rescore` is free and offline, so the fix is never a reason to re-measure.
+                findings.push({
+                    skill, code: "consistency",
+                    message: `results.yaml effective_grade is stale in ${runDir} (recompute differs) — re-apply the current scoring policy: rescore (free, offline)`,
+                });
             }
             for (const s of r.scenarios) {
                 if (s.override != null) {
@@ -258,6 +271,29 @@ export function lintSkill(skillDir) {
             }
         }
     }
+    // Run-over-run stability — INFO, never a gate failure. Derived from committed
+    // history at zero cost, and it answers a question no single results.yaml can: a
+    // scenario can be internally unanimous in every run and still land on a different
+    // side each time. Measured in the reference corpus: two consecutive full runs, one
+    // 3/3 PASS and the next 0/3 FAIL, each `flakiness 0.00`.
+    //
+    // In lint because that is where a repo already looks, and free because it reads what
+    // is on disk. Wrapped: lintSkill must never throw, and a stability read touches every
+    // run file in the tree.
+    try {
+        for (const cell of boundaryCells(stabilityFrom(collectScoredRuns(skillDir), spec))) {
+            findings.push({
+                skill, scenario: cell.id, code: "stability", severity: "info",
+                message: `${cell.tag} mode=${cell.mode}: ${stabilityNote(cell)}`,
+            });
+        }
+    }
+    catch (e) {
+        findings.push({
+            skill, code: "stability", severity: "info",
+            message: `run-over-run stability could not be derived: ${e instanceof Error ? e.message : String(e)}`,
+        });
+    }
     return findings;
 }
 /** Model-tag dirs under tests/results (each holds timestamped run dirs). */

package/dist/regate.js

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import { parseVerdict, detectMisfire } from "./grade.js";
 import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
 import { judgeOneRep } from "./regrade.js";
-import { readResults, writeResults, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, effectiveThreshold, } from "./results.js";
+import { readResults, writeResults, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, effectiveThreshold, scoreContextFor, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -32,8 +32,8 @@ function rewriteTranscript(path, gateLines) {
     writeFileSync(path, `${head}${TRAILER}\n${gateLines.join("\n")}\n\n${tail}`, "utf8");
 }
 /** Recover a rep's judge verdict from its saved judge-raw artifact — free, and exact. */
-function verdictFromSavedJudgement(runDir, id, rep) {
-    const path = judgeRawPath(runDir, id, "green", rep);
+function verdictFromSavedJudgement(runDir, id, mode, rep) {
+    const path = judgeRawPath(runDir, id, mode, rep);
     if (!existsSync(path))
         return null;
     const raw = readFileSync(path, "utf8");
@@ -72,6 +72,10 @@ function verdictFromSavedJudgement(runDir, id, rep) {
 export async function regateRun(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = readResults(opts.runDir);
+    // The run's own mode names its artifacts (`<id>.<mode>[.rep<k>].diff.txt`). Read
+    // from the record rather than assumed green: force runs are scored measurements
+    // too, and looking for green artifacts under a force run finds nothing at all.
+    const mode = prev.mode;
     const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
     // Why a scenario cannot be regated, collected rather than thrown one at a time: a
     // mixed spec (needles here, vitest there) should regate what it can, and only a run
@@ -87,7 +91,7 @@ export async function regateRun(opts) {
                 `no saved artifact can stand in for it, so this scenario needs a re-run`);
             continue;
         }
-        if (findDiffFiles(opts.runDir, s.id, "green").length === 0) {
+        if (findDiffFiles(opts.runDir, s.id, mode).length === 0) {
             blocked.push(`${s.id}: no staged-diff artifact on disk (\`.diff.txt\` is gitignored — regate needs the run dir that produced it)`);
             continue;
         }
@@ -106,7 +110,7 @@ export async function regateRun(opts) {
             scenarios.push(rec); // untouched: not regatable, or no gates
             continue;
         }
-        const diffFiles = findDiffFiles(opts.runDir, scenario.id, "green");
+        const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
         const outcomes = [];
         // Per scenario, not run-wide: with several regated scenarios, a global counter
         // would report every change as "re-judged" because some other scenario was.
@@ -116,7 +120,7 @@ export async function regateRun(opts) {
             const rep = repIndexOf(file) ?? undefined;
             const diff = readFileSync(join(opts.runDir, file), "utf8");
             const gate = evaluateNeedleGates(scenario, diff);
-            const tPath = transcriptPath(opts.runDir, scenario.id, "green", rep);
+            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
             const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
             const oldGateFailed = GATE_FAILED_RE.test(before.slice(before.indexOf(TRAILER)));
             // The trailer is regenerated whatever the outcome: leaving a stale
@@ -132,7 +136,7 @@ export async function regateRun(opts) {
             if (!oldGateFailed) {
                 // The judge already saw this rep. Its verdict is on disk — re-read it rather
                 // than paying to ask the same question again.
-                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, rep);
+                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep);
                 outcomes.push(saved ?? { verdict: rec.judge_verdict, reason: rec.judge_reason, suspect: rec.suspect });
                 continue;
             }
@@ -141,7 +145,7 @@ export async function regateRun(opts) {
             outcomes.push(await judgeOneRep({
                 runDir: opts.runDir, spec: opts.spec, scenario, transcript,
                 adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                mode: "green", rep, now,
+                mode, rep, now,
             }));
             judgeCalls++;
             judgedHere++;
@@ -160,11 +164,11 @@ export async function regateRun(opts) {
             });
         }
     }
-    const ctx = prev.mode === "green" && !prev.partial
-        ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
-        : null;
+    const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         skill: prev.skill, harness: prev.harness, model: prev.model,
+        // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
+        harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
         judge: { provider: opts.judge.provider, model: opts.judge.model },
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode, partial: prev.partial,
         // Only the `gates:` keys of the scenarios actually re-evaluated. Stimulus, rubric

package/dist/regrade.d.ts

@@ -23,6 +23,15 @@ export interface RegradeOptions {
     judge: ModelRef;
     specDir: string;
     threshold: number;
+    /**
+     * Which mode's saved transcripts to re-judge — the run's own mode, since
+     * transcript filenames are `<id>.<mode>[.rep<k>].txt`.
+     *
+     * Defaults to `green` for callers that predate force being a scored mode. A
+     * force-mode run whose transcripts were looked up as green found none and failed
+     * with "nothing to re-grade", which is how ten scorable runs stayed ungraded.
+     */
+    mode?: string;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -39,10 +48,11 @@ export declare function judgeOneRep(opts: {
     now: () => string;
 }): Promise<RepOutcome>;
 /**
- * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
- * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
- * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
- * (override/note empty; the caller merges any prior override + persists).
+ * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
+ * harness re-run. Rewrites the judge-raw artifact per rep, emits per-rep
+ * judge-verdict (+ misfire-flag) journal events, and returns the aggregated
+ * ScenarioResult (override/note empty; the caller merges any prior override +
+ * persists).
  */
 export declare function regradeScenario(opts: RegradeOptions): Promise<ScenarioResult>;
 export interface RegradeRunOptions {
@@ -60,13 +70,13 @@ export interface RegradeRunOptions {
     onlySuspect?: boolean;
 }
 /**
- * Re-judge every green-transcript scenario in a run dir with `judge` — no
- * harness re-run. Targets are the run's RECORDED scenarios (falling back to
- * the spec for a run with no prior results.yaml), so re-grading rewrites the
- * whole results.yaml consistently with what the run actually recorded. Each
- * target must still exist in the spec (for its checklist) AND have a green
- * transcript on disk; anything missing fails fast before spending any judge
- * calls. Preserves each prior scenario's override/note, rewrites
+ * Re-judge every scenario in a run dir that has a transcript for the run's own
+ * mode, with `judge` — no harness re-run. Targets are the run's RECORDED scenarios
+ * (falling back to the spec for a run with no prior results.yaml), so re-grading
+ * rewrites the whole results.yaml consistently with what the run actually recorded.
+ * Each target must still exist in the spec (for its checklist) AND have a
+ * transcript on disk for that mode; anything missing fails fast before spending
+ * any judge calls. Preserves each prior scenario's override/note, rewrites
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */