@skill-harness/core 0.3.2 → 0.5.0

package/dist/lift.js

@@ -1,6 +1,6 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, effectiveVerdicts } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
 import { loadSpec } from "./spec.js";
 function aggregationShape(s) {
     const reps = s.reps ?? 1;
@@ -98,6 +98,7 @@ export function computeLift(red, green, opts = {}) {
     return {
         tag: "",
         model: green.model,
+        mode: green.mode,
         redTimestamp: red.timestamp,
         greenTimestamp: green.timestamp,
         compared: Object.keys(cells).length,
@@ -193,7 +194,7 @@ function isDir(p) {
 }
 /**
  * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
- * with the most recent green run and compute the lift.
+ * with the most recent skill-delivered run (green or force) and compute the lift.
  *
  * Deliberately derived on read rather than persisted into results.yaml: a lift
  * is a fact about a *pair* of runs, so caching it inside one run's file would go
@@ -237,9 +238,13 @@ export function collectLift(skillDir) {
             .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
             .sort(); // timestamp-slug names ⇒ chronological ascending
         // Mode is only knowable after reading results.yaml, so every run in the tag
-        // is read; last-wins gives the most recent of each mode.
+        // is read; last-wins gives the most recent baseline and the most recent
+        // skill-delivered run. The skill side is whichever scored mode ran most
+        // recently — a corpus that moved from green to force delivery should see its
+        // lift follow, and the baseline it is measured against is the same either way
+        // (`--no-skills` in both).
         let red;
-        let green;
+        let skillOn;
         for (const rd of runDirs) {
             let r;
             try {
@@ -252,12 +257,12 @@ export function collectLift(skillDir) {
             }
             if (r.mode === "red")
                 red = r;
-            else if (r.mode === "green")
-                green = r;
+            else if (isScoredMode(r.mode))
+                skillOn = r;
         }
-        if (!red || !green)
+        if (!red || !skillOn)
             continue;
-        lifts.push({ ...computeLift(red, green, { modeInsensitive }), tag });
+        lifts.push({ ...computeLift(red, skillOn, { modeInsensitive }), tag });
     }
     return lifts;
 }

package/dist/lint.js

@@ -2,8 +2,9 @@ import { existsSync, statSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, resolve } from "node:path";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
-import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
-import { currentHashFor, describeSourceKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, UNREADABLE } from "./sources.js";
+import { readResults, finalizeResults, findTranscriptFiles, resultsPath, scoreContextFor } from "./results.js";
+import { currentHashFor, describeSourceKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
+import { downgradeWarning } from "./downgrade.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
@@ -41,6 +42,16 @@ export function lintSkill(skillDir) {
         return [{ skill: basename(skillDir), code: "spec", message }];
     }
     const skill = spec.skill;
+    // A lint from an older tool than the records it is checking cannot be trusted to
+    // check them: a 0.1.0 binary reported 38 spurious `consistency` findings against
+    // partial runs that 0.3.x handles correctly, and it cannot read source-hash key
+    // kinds added after it shipped. Reported as a finding (so CI sees it) rather than
+    // thrown, because refusing to lint is worse than linting with a caveat, and it is
+    // one of the few ways this situation announces itself at all.
+    const stale = downgradeWarning(skillDir);
+    if (stale) {
+        findings.push({ skill, code: "consistency", message: stale.replace(/^warning: /, "") });
+    }
     // ship_bar sanity
     if (spec.ship_bar.total < 1) {
         findings.push({ skill, code: "ship_bar", message: "ship_bar.total must be >= 1" });
@@ -152,10 +163,17 @@ export function lintSkill(skillDir) {
             // can actually re-score. Override/transcript rules below still apply.
             const specIds = new Set(spec.scenarios.map((sc) => sc.id));
             const sameSet = r.scenarios.length === specIds.size && r.scenarios.every((sc) => specIds.has(sc.id));
-            const ctx = r.mode === "green" && !r.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+            const ctx = scoreContextFor(r, spec);
             const recomputed = !sameSet ? null : finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, partial: r.partial, source_hashes: r.source_hashes, scenarios: r.scenarios }, ctx).effective_grade;
             if (recomputed && JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
-                findings.push({ skill, code: "consistency", message: `results.yaml effective_grade is stale in ${runDir} (recompute differs)` });
+                // The remedy is named because this finding now has a benign, expected cause
+                // as well as a suspicious one: a force run recorded before 0.5.0 carries a
+                // "not scored" placeholder, and today's policy scores it (see SCORED_MODES).
+                // `rescore` is free and offline, so the fix is never a reason to re-measure.
+                findings.push({
+                    skill, code: "consistency",
+                    message: `results.yaml effective_grade is stale in ${runDir} (recompute differs) — re-apply the current scoring policy: rescore (free, offline)`,
+                });
             }
             for (const s of r.scenarios) {
                 if (s.override != null) {
@@ -198,7 +216,7 @@ export function lintSkill(skillDir) {
             continue; // predates source_hashes → silent
         {
             const newest = full.runDir;
-            const ctx = { skillDir, specDir, scenarios: spec.scenarios };
+            const ctx = { skillDir, specDir, scenarios: spec.scenarios, judgePersona: spec.judge_persona };
             for (const [key, recorded] of Object.entries(hashes)) {
                 const what = describeSourceKey(key);
                 const scenario = scenarioIdForKey(key, spec.scenarios);
@@ -218,7 +236,12 @@ export function lintSkill(skillDir) {
                     findings.push({ skill, scenario, code: "stale", message: `${what} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
                 }
                 else if (current !== recorded) {
-                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                    // The remedy is per key kind, and it is the whole point of the split: a
+                    // rubric edit is re-gradeable from saved transcripts, a policy edit is a
+                    // free rescore, a needle edit is a free regate. Only stimulus drift costs
+                    // model spend. One message saying "re-run" for all four is what made
+                    // correcting a known-bad rubric expensive enough to skip.
+                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; ${remedyForKey(key)}` });
                 }
             }
             // Coverage: a scenario the spec defines that the newest full run never
@@ -227,9 +250,15 @@ export function lintSkill(skillDir) {
             // 100%/SHIP scorecard survives an arbitrary spec rewrite reporting zero
             // findings. Gated on the run having recorded scenario keys at all, so runs
             // predating the key kind stay silent like every other pre-existing run.
-            if (Object.keys(hashes).some((k) => k.startsWith(SCENARIO_PREFIX))) {
+            // Either key kind counts as "this run recorded per-scenario hashes": 0.4.0+ runs
+            // carry `stimulus:<id>`, older ones the combined `scenario:<id>`. Checking only
+            // the legacy prefix would silently drop coverage checking for every new run.
+            const scenarioKeyPrefix = Object.keys(hashes).some((k) => k.startsWith(STIMULUS_PREFIX))
+                ? STIMULUS_PREFIX
+                : SCENARIO_PREFIX;
+            if (Object.keys(hashes).some((k) => k.startsWith(scenarioKeyPrefix))) {
                 for (const s of spec.scenarios) {
-                    if (!(SCENARIO_PREFIX + s.id in hashes)) {
+                    if (!(scenarioKeyPrefix + s.id in hashes)) {
                         findings.push({ skill, scenario: s.id, code: "stale", message: `the newest ${basename(tagDir)} run (${newest}) did not measure scenario \`${s.id}\` — the published result covers a different scenario set than the spec; re-run before publishing` });
                     }
                 }

package/dist/regate.d.ts

@@ -0,0 +1,57 @@
+import type { Spec } from "./spec.js";
+import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+import { type ResultsFile } from "./results.js";
+import type { Verdict } from "./score.js";
+export interface RegateOptions {
+    runDir: string;
+    spec: Spec;
+    specDir: string;
+    adapter: HarnessAdapter;
+    judge: ModelRef;
+    now?: () => string;
+}
+export interface RegateChange {
+    id: string;
+    from: Verdict;
+    to: Verdict;
+    /** What the corrected needles say about this scenario now. */
+    gate: "pass" | "fail";
+    /** Whether re-deciding it required a judge call (only a gate-FAIL → gate-pass flip does). */
+    judged: boolean;
+}
+export interface RegateResult {
+    results: ResultsFile;
+    changes: RegateChange[];
+    /** Judge calls actually made, for the cost line the CLI prints. */
+    judgeCalls: number;
+}
+/**
+ * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
+ * verdicts they determined — without re-running the model.
+ *
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
+ * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
+ * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
+ * a re-run. Measured on the C2 needle fix: **9 judge calls instead of 81
+ * rep-executions across three models.**
+ *
+ * Per rep, exactly one of four things happens:
+ *
+ * | old gate | new gate | outcome | cost |
+ * |---|---|---|---|
+ * | fail | fail | FAIL, with the corrected reason | free |
+ * | pass | fail | FAIL — the gate is objective and it says no | free |
+ * | pass | pass | the rep's saved judgement, re-parsed from its judge-raw artifact | free |
+ * | fail | pass | judged now: the judge never saw this rep, because the gate blocked it | 1 judge call |
+ *
+ * That third row is what keeps this cheap without guessing: a rep the judge already
+ * saw has its verdict on disk, so regate re-reads it rather than re-asking.
+ *
+ * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
+ * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
+ * so a scenario carrying either is not regatable. Diffs and judge-raw files are
+ * gitignored, so this works for whoever holds the run dirs — the repo owner, or CI that
+ * just ran — which is exactly the situation it is needed in.
+ */
+export declare function regateRun(opts: RegateOptions): Promise<RegateResult>;

package/dist/regate.js

@@ -0,0 +1,199 @@
+import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseVerdict, detectMisfire } from "./grade.js";
+import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
+import { judgeOneRep } from "./regrade.js";
+import { readResults, writeResults, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, effectiveThreshold, scoreContextFor, } from "./results.js";
+import { outcomesToResult } from "./reps.js";
+import { appendJournal } from "./journal.js";
+import { gatesDigest, GATES_PREFIX } from "./sources.js";
+/** Marker in a saved transcript that a needle gate reported a failure. */
+const GATE_FAILED_RE = /: (MISSING|PRESENT)$/m;
+const TRAILER = "=== SEEDED GATES ===";
+const DIFF_HEADER = "=== STAGED DIFF ===";
+/**
+ * Rebuild a transcript with a fresh gates trailer, preserving the model's turns and
+ * the embedded diff exactly.
+ *
+ * The trailer is harness-generated annotation appended *after* the model's output, so
+ * regenerating it corrects our own note rather than falsifying a transcript. The old
+ * file is still kept beside the new one (`.pre-regate.txt`) so the audit trail never
+ * depends on the reader accepting that distinction.
+ */
+function rewriteTranscript(path, gateLines) {
+    const original = readFileSync(path, "utf8");
+    const trailerAt = original.indexOf(TRAILER);
+    if (trailerAt === -1)
+        return; // no trailer to correct (non-seeded shape); leave it alone
+    const diffAt = original.indexOf(DIFF_HEADER);
+    const head = original.slice(0, trailerAt);
+    const tail = diffAt === -1 ? "" : original.slice(diffAt);
+    renameSync(path, path.replace(/\.txt$/, ".pre-regate.txt"));
+    writeFileSync(path, `${head}${TRAILER}\n${gateLines.join("\n")}\n\n${tail}`, "utf8");
+}
+/** Recover a rep's judge verdict from its saved judge-raw artifact — free, and exact. */
+function verdictFromSavedJudgement(runDir, id, mode, rep) {
+    const path = judgeRawPath(runDir, id, mode, rep);
+    if (!existsSync(path))
+        return null;
+    const raw = readFileSync(path, "utf8");
+    const parsed = parseVerdict(raw);
+    return { verdict: parsed.verdict, reason: parsed.reason, suspect: detectMisfire(raw, parsed.verdict) };
+}
+/**
+ * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
+ * verdicts they determined — without re-running the model.
+ *
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
+ * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
+ * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
+ * a re-run. Measured on the C2 needle fix: **9 judge calls instead of 81
+ * rep-executions across three models.**
+ *
+ * Per rep, exactly one of four things happens:
+ *
+ * | old gate | new gate | outcome | cost |
+ * |---|---|---|---|
+ * | fail | fail | FAIL, with the corrected reason | free |
+ * | pass | fail | FAIL — the gate is objective and it says no | free |
+ * | pass | pass | the rep's saved judgement, re-parsed from its judge-raw artifact | free |
+ * | fail | pass | judged now: the judge never saw this rep, because the gate blocked it | 1 judge call |
+ *
+ * That third row is what keeps this cheap without guessing: a rep the judge already
+ * saw has its verdict on disk, so regate re-reads it rather than re-asking.
+ *
+ * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
+ * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
+ * so a scenario carrying either is not regatable. Diffs and judge-raw files are
+ * gitignored, so this works for whoever holds the run dirs — the repo owner, or CI that
+ * just ran — which is exactly the situation it is needed in.
+ */
+export async function regateRun(opts) {
+    const now = opts.now ?? (() => new Date().toISOString());
+    const prev = readResults(opts.runDir);
+    // The run's own mode names its artifacts (`<id>.<mode>[.rep<k>].diff.txt`). Read
+    // from the record rather than assumed green: force runs are scored measurements
+    // too, and looking for green artifacts under a force run finds nothing at all.
+    const mode = prev.mode;
+    const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
+    // Why a scenario cannot be regated, collected rather than thrown one at a time: a
+    // mixed spec (needles here, vitest there) should regate what it can, and only a run
+    // with nothing regatable is an error worth refusing.
+    const blocked = [];
+    const targets = [];
+    for (const rec of prev.scenarios) {
+        const s = specById.get(rec.id);
+        if (!s || !hasNeedleGates(s))
+            continue; // nothing for regate to re-decide
+        if (s.assert?.vitest || s.assert?.post_test) {
+            blocked.push(`${s.id}: declares ${s.assert.vitest ? "assert.vitest" : "assert.post_test"}, which needs the workspace — ` +
+                `no saved artifact can stand in for it, so this scenario needs a re-run`);
+            continue;
+        }
+        if (findDiffFiles(opts.runDir, s.id, mode).length === 0) {
+            blocked.push(`${s.id}: no staged-diff artifact on disk (\`.diff.txt\` is gitignored — regate needs the run dir that produced it)`);
+            continue;
+        }
+        targets.push(s);
+    }
+    if (targets.length === 0) {
+        throw new Error(`nothing to regate in ${opts.runDir}` +
+            (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff_contains/diff_excludes"));
+    }
+    const changes = [];
+    let judgeCalls = 0;
+    const scenarios = [];
+    for (const rec of prev.scenarios) {
+        const scenario = targets.find((s) => s.id === rec.id);
+        if (!scenario) {
+            scenarios.push(rec); // untouched: not regatable, or no gates
+            continue;
+        }
+        const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
+        const outcomes = [];
+        // Per scenario, not run-wide: with several regated scenarios, a global counter
+        // would report every change as "re-judged" because some other scenario was.
+        let judgedHere = 0;
+        let gateFailedHere = false;
+        for (const file of diffFiles) {
+            const rep = repIndexOf(file) ?? undefined;
+            const diff = readFileSync(join(opts.runDir, file), "utf8");
+            const gate = evaluateNeedleGates(scenario, diff);
+            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
+            const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
+            const oldGateFailed = GATE_FAILED_RE.test(before.slice(before.indexOf(TRAILER)));
+            // The trailer is regenerated whatever the outcome: leaving a stale
+            // `MISSING` note beside a corrected verdict would misinform the next reader
+            // (and the next judge, which reads this transcript).
+            if (existsSync(tPath))
+                rewriteTranscript(tPath, gate.lines);
+            if (gate.failure) {
+                gateFailedHere = true;
+                outcomes.push({ verdict: "FAIL", reason: gate.failure, suspect: false });
+                continue;
+            }
+            if (!oldGateFailed) {
+                // The judge already saw this rep. Its verdict is on disk — re-read it rather
+                // than paying to ask the same question again.
+                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep);
+                outcomes.push(saved ?? { verdict: rec.judge_verdict, reason: rec.judge_reason, suspect: rec.suspect });
+                continue;
+            }
+            // The gate blocked this rep before, so no judgement of it exists anywhere.
+            const transcript = readFileSync(tPath, "utf8");
+            outcomes.push(await judgeOneRep({
+                runDir: opts.runDir, spec: opts.spec, scenario, transcript,
+                adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
+                mode, rep, now,
+            }));
+            judgeCalls++;
+            judgedHere++;
+        }
+        const threshold = effectiveThreshold(rec, scenario);
+        const next = outcomesToResult(scenario.id, outcomes, outcomes.length, threshold);
+        // Overrides and their notes survive: a regate re-decides the gate, and an author
+        // override is a statement about the judge, not about the needle.
+        scenarios.push({ ...next, override: rec.override, note: rec.note });
+        const to = next.judge_verdict;
+        if (to !== rec.judge_verdict) {
+            changes.push({
+                id: scenario.id, from: rec.judge_verdict, to,
+                gate: gateFailedHere ? "fail" : "pass",
+                judged: judgedHere > 0,
+            });
+        }
+    }
+    const ctx = scoreContextFor(prev, opts.spec);
+    const results = writeResults(opts.runDir, {
+        skill: prev.skill, harness: prev.harness, model: prev.model,
+        // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
+        harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
+        judge: { provider: opts.judge.provider, model: opts.judge.model },
+        timestamp: prev.timestamp, label: prev.label, mode: prev.mode, partial: prev.partial,
+        // Only the `gates:` keys of the scenarios actually re-evaluated. Stimulus, rubric
+        // and policy were not re-decided here, so their hashes stay exactly as recorded.
+        source_hashes: refreshGateHashes(prev.source_hashes, targets),
+        scenarios,
+    }, ctx);
+    appendJournal(opts.runDir, {
+        event: "regate", ts: now(),
+        scenarios: targets.map((s) => s.id),
+        changed: changes.map((c) => `${c.id}: ${c.from}->${c.to} (gate ${c.gate}${c.judged ? ", re-judged" : ""})`),
+        judge_calls: judgeCalls,
+        ...(blocked.length > 0 ? { skipped: blocked } : {}),
+    });
+    return { results, changes, judgeCalls };
+}
+function refreshGateHashes(recorded, regated) {
+    if (!recorded)
+        return undefined;
+    const next = { ...recorded };
+    for (const s of regated) {
+        const digest = gatesDigest(s);
+        if (digest !== null && GATES_PREFIX + s.id in next)
+            next[GATES_PREFIX + s.id] = digest;
+    }
+    return next;
+}
+//# sourceMappingURL=regate.js.map

package/dist/regrade.d.ts

@@ -2,6 +2,19 @@ import type { Spec, Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
 import { type ScenarioResult, type ResultsFile } from "./results.js";
 import { type RepOutcome } from "./reps.js";
+/**
+ * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
+ * re-grade actually judged under (plus the persona, which applies to all of them).
+ *
+ * Everything else is preserved deliberately: the transcripts were produced by the old
+ * stimulus, so a stimulus hash must stay stale until someone re-runs. `--suspect-only`
+ * is why this takes an id list rather than refreshing every rubric key — a re-grade
+ * that touched two scenarios must not certify the rubric of the twelve it skipped.
+ *
+ * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
+ * a run that never recorded any would claim a coverage it does not have.
+ */
+export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[]): Record<string, string> | undefined;
 export interface RegradeOptions {
     runDir: string;
     spec: Spec;
@@ -10,6 +23,15 @@ export interface RegradeOptions {
     judge: ModelRef;
     specDir: string;
     threshold: number;
+    /**
+     * Which mode's saved transcripts to re-judge — the run's own mode, since
+     * transcript filenames are `<id>.<mode>[.rep<k>].txt`.
+     *
+     * Defaults to `green` for callers that predate force being a scored mode. A
+     * force-mode run whose transcripts were looked up as green found none and failed
+     * with "nothing to re-grade", which is how ten scorable runs stayed ungraded.
+     */
+    mode?: string;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -26,10 +48,11 @@ export declare function judgeOneRep(opts: {
     now: () => string;
 }): Promise<RepOutcome>;
 /**
- * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
- * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
- * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
- * (override/note empty; the caller merges any prior override + persists).
+ * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
+ * harness re-run. Rewrites the judge-raw artifact per rep, emits per-rep
+ * judge-verdict (+ misfire-flag) journal events, and returns the aggregated
+ * ScenarioResult (override/note empty; the caller merges any prior override +
+ * persists).
  */
 export declare function regradeScenario(opts: RegradeOptions): Promise<ScenarioResult>;
 export interface RegradeRunOptions {
@@ -47,13 +70,13 @@ export interface RegradeRunOptions {
     onlySuspect?: boolean;
 }
 /**
- * Re-judge every green-transcript scenario in a run dir with `judge` — no
- * harness re-run. Targets are the run's RECORDED scenarios (falling back to
- * the spec for a run with no prior results.yaml), so re-grading rewrites the
- * whole results.yaml consistently with what the run actually recorded. Each
- * target must still exist in the spec (for its checklist) AND have a green
- * transcript on disk; anything missing fails fast before spending any judge
- * calls. Preserves each prior scenario's override/note, rewrites
+ * Re-judge every scenario in a run dir that has a transcript for the run's own
+ * mode, with `judge` — no harness re-run. Targets are the run's RECORDED scenarios
+ * (falling back to the spec for a run with no prior results.yaml), so re-grading
+ * rewrites the whole results.yaml consistently with what the run actually recorded.
+ * Each target must still exist in the spec (for its checklist) AND have a
+ * transcript on disk for that mode; anything missing fails fast before spending
+ * any judge calls. Preserves each prior scenario's override/note, rewrites
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */

package/dist/regrade.js

@@ -1,9 +1,38 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, } from "./results.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
+import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
+/**
+ * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
+ * re-grade actually judged under (plus the persona, which applies to all of them).
+ *
+ * Everything else is preserved deliberately: the transcripts were produced by the old
+ * stimulus, so a stimulus hash must stay stale until someone re-runs. `--suspect-only`
+ * is why this takes an id list rather than refreshing every rubric key — a re-grade
+ * that touched two scenarios must not certify the rubric of the twelve it skipped.
+ *
+ * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
+ * a run that never recorded any would claim a coverage it does not have.
+ */
+export function refreshRubricHashes(recorded, spec, judgedIds) {
+    if (!recorded)
+        return undefined;
+    const next = { ...recorded };
+    const specById = new Map(spec.scenarios.map((s) => [s.id, s]));
+    for (const id of judgedIds) {
+        const s = specById.get(id);
+        // Only refresh a key the run already carried: adding one for a scenario whose
+        // rubric was never hashed would fabricate coverage.
+        if (s && RUBRIC_PREFIX + id in next)
+            next[RUBRIC_PREFIX + id] = rubricDigest(s);
+    }
+    if (PERSONA_KEY in next)
+        next[PERSONA_KEY] = personaDigest(spec.judge_persona);
+    return next;
+}
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
 export async function judgeOneRep(opts) {
     const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
@@ -17,16 +46,18 @@ export async function judgeOneRep(opts) {
     return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
 }
 /**
- * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
- * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
- * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
- * (override/note empty; the caller merges any prior override + persists).
+ * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
+ * harness re-run. Rewrites the judge-raw artifact per rep, emits per-rep
+ * judge-verdict (+ misfire-flag) journal events, and returns the aggregated
+ * ScenarioResult (override/note empty; the caller merges any prior override +
+ * persists).
  */
 export async function regradeScenario(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
-    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, "green");
+    const mode = opts.mode ?? "green";
+    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
     if (files.length === 0)
-        throw new Error(`no green transcripts for ${opts.scenario.id} in ${opts.runDir}`);
+        throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
     const repCount = files.length;
     const outcomes = [];
     for (const file of files) {
@@ -34,19 +65,19 @@ export async function regradeScenario(opts) {
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
-            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode: "green", rep, now,
+            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now,
         }));
     }
     return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
 }
 /**
- * Re-judge every green-transcript scenario in a run dir with `judge` — no
- * harness re-run. Targets are the run's RECORDED scenarios (falling back to
- * the spec for a run with no prior results.yaml), so re-grading rewrites the
- * whole results.yaml consistently with what the run actually recorded. Each
- * target must still exist in the spec (for its checklist) AND have a green
- * transcript on disk; anything missing fails fast before spending any judge
- * calls. Preserves each prior scenario's override/note, rewrites
+ * Re-judge every scenario in a run dir that has a transcript for the run's own
+ * mode, with `judge` — no harness re-run. Targets are the run's RECORDED scenarios
+ * (falling back to the spec for a run with no prior results.yaml), so re-grading
+ * rewrites the whole results.yaml consistently with what the run actually recorded.
+ * Each target must still exist in the spec (for its checklist) AND have a
+ * transcript on disk for that mode; anything missing fails fast before spending
+ * any judge calls. Preserves each prior scenario's override/note, rewrites
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
@@ -78,9 +109,9 @@ export async function regradeRun(opts) {
             return prev;
         }
     }
-    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, "green").length === 0);
+    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, mode).length === 0);
     if (missing.length === targets.length) {
-        throw new Error(`no green transcripts in ${runDir} — nothing to re-grade`);
+        throw new Error(`no ${mode} transcripts in ${runDir} — nothing to re-grade`);
     }
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
@@ -97,24 +128,34 @@ export async function regradeRun(opts) {
         const prevScenario = prev?.scenarios.find((s) => s.id === id);
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
-            runDir, spec, scenario, adapter, judge, specDir, threshold, now,
+            runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
         });
         const carry = overrides.get(id);
         scenarioResults.push({ ...rr, override: carry?.override ?? null, note: carry?.note ?? "" });
     }
-    const ctx = mode === "green" && !prev?.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: prev?.harness ?? "pi",
+        // The harness CLI that produced these transcripts, carried verbatim: a re-grade
+        // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
+        // here would credit the old transcripts to a version that never ran them.
+        harness_cli_version: prev?.harness_cli_version,
+        delivery_canary: prev?.delivery_canary,
         model: prev?.model ?? "unknown",
         judge: { provider: judge.provider, model: judge.model },
         timestamp: prev?.timestamp ?? now(),
         label: prev?.label ?? null,
         mode,
         // A re-grade judges the SAVED transcripts, which were produced by the OLD text —
-        // the recorded hashes stay, keeping an honestly-stale run honestly stale.
+        // the recorded **stimulus** hashes stay, keeping an honestly-stale run honestly
+        // stale. The rubric hashes are a different matter: this re-grade applied the
+        // CURRENT checklist and persona to those transcripts, so "the verdicts reflect
+        // today's rubric" is now a true statement about the record, and the hashes should
+        // say so. Doctrine narrowed 0.4.0, from "recorded hashes stay" to "recorded
+        // *stimulus* hashes stay" — see refreshRubricHashes.
         partial: prev?.partial,
-        source_hashes: prev?.source_hashes,
+        source_hashes: refreshRubricHashes(prev?.source_hashes, spec, targets),
         scenarios: scenarioResults,
     }, ctx);
     const g = results.effective_grade;

package/dist/report.d.ts

@@ -22,13 +22,14 @@ export interface RunColumn {
         note: string;
     }>;
     /**
-     * Red-vs-green lift for this model, when the tag has both a red baseline and a
-     * green run. Undefined means "never measured" — which is not the same claim as
-     * a zero lift, so the report must not render a 0 for it.
+     * Baseline-vs-skill lift for this model, when the tag has both a red baseline and
+     * a skill-delivered run (green or force). Undefined means "never measured" —
+     * which is not the same claim as a zero lift, so the report must not render a 0
+     * for it.
      *
-     * Only set when THIS column is the green run the lift was computed from (see
+     * Only set when THIS column is the skill-side run the lift was computed from (see
      * collectReport): the review UI recomputes lift from the column's live cells,
-     * which is only valid if those cells are the green side of the comparison.
+     * which is only valid if those cells are the skill side of the comparison.
      */
     lift?: Lift;
     liftHeadline?: string;