@skill-harness/core 0.3.1 → 0.4.0

package/dist/defaults.d.ts

@@ -0,0 +1,30 @@
+/**
+ * The judge used when nothing else says otherwise: Opus through the **`claude-code`
+ * provider**, which authenticates with the user's Claude subscription (OAuth via
+ * `claude -p`) rather than a metered API key.
+ *
+ * The model is deliberately the strongest available — judging is the one place
+ * where a weak model silently corrupts every number in a scorecard. The *provider*
+ * is what changed in 0.3.3: the default was `anthropic:claude-opus-4-8`, a metered
+ * API, and it billed a corpus once by accident because nothing in the tool surface
+ * distinguishes "the flag I forgot" from "the flag I meant". A default must not be
+ * able to spend money that was not asked for. The metered path is still one flag
+ * away (`--judge anthropic:claude-opus-4-8`) for anyone who wants it — an API key
+ * scales past a subscription's rate limits, which matters for a large `--reps` run.
+ */
+export declare const BAKED_DEFAULT_JUDGE = "claude-code:claude-opus-4-8";
+/**
+ * Resolve the default judge: `SKILL_HARNESS_JUDGE` if set, else the baked value.
+ * An explicit `--judge` always wins over both — this is only the default.
+ *
+ * The env layer exists because this harness is built for someone who steers a
+ * process rather than typing every flag: judge policy belongs to the repo or the
+ * shell, set once. It is also how you opt *into* the metered API deliberately
+ * (`SKILL_HARNESS_JUDGE=anthropic:claude-opus-4-8`) instead of by forgetting a
+ * flag. Read through `readEnv`, so the pre-rename `SKILL_CHECK_JUDGE` keeps working
+ * with the usual one-time notice.
+ *
+ * Resolved per call, not at module load: tests and long-lived processes (the pi
+ * extension) must see an env change without a reload.
+ */
+export declare function defaultJudge(): string;

package/dist/defaults.js

@@ -0,0 +1,34 @@
+import { readEnv } from "./util/env.js";
+/**
+ * The judge used when nothing else says otherwise: Opus through the **`claude-code`
+ * provider**, which authenticates with the user's Claude subscription (OAuth via
+ * `claude -p`) rather than a metered API key.
+ *
+ * The model is deliberately the strongest available — judging is the one place
+ * where a weak model silently corrupts every number in a scorecard. The *provider*
+ * is what changed in 0.3.3: the default was `anthropic:claude-opus-4-8`, a metered
+ * API, and it billed a corpus once by accident because nothing in the tool surface
+ * distinguishes "the flag I forgot" from "the flag I meant". A default must not be
+ * able to spend money that was not asked for. The metered path is still one flag
+ * away (`--judge anthropic:claude-opus-4-8`) for anyone who wants it — an API key
+ * scales past a subscription's rate limits, which matters for a large `--reps` run.
+ */
+export const BAKED_DEFAULT_JUDGE = "claude-code:claude-opus-4-8";
+/**
+ * Resolve the default judge: `SKILL_HARNESS_JUDGE` if set, else the baked value.
+ * An explicit `--judge` always wins over both — this is only the default.
+ *
+ * The env layer exists because this harness is built for someone who steers a
+ * process rather than typing every flag: judge policy belongs to the repo or the
+ * shell, set once. It is also how you opt *into* the metered API deliberately
+ * (`SKILL_HARNESS_JUDGE=anthropic:claude-opus-4-8`) instead of by forgetting a
+ * flag. Read through `readEnv`, so the pre-rename `SKILL_CHECK_JUDGE` keeps working
+ * with the usual one-time notice.
+ *
+ * Resolved per call, not at module load: tests and long-lived processes (the pi
+ * extension) must see an env change without a reload.
+ */
+export function defaultJudge() {
+    return readEnv("JUDGE") ?? BAKED_DEFAULT_JUDGE;
+}
+//# sourceMappingURL=defaults.js.map

package/dist/downgrade.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Compare two semver-ish version strings by numeric component.
+ *
+ * String comparison is the trap this exists to avoid: `"0.10.0" < "0.9.0"` is true
+ * lexically and false in fact, which would make the tripwire fire backwards exactly
+ * once — on the release where it mattered most. A prerelease suffix is dropped rather
+ * than ordered: `0.4.0-rc.1` vs `0.4.0` is not a distinction worth a refusal, and
+ * getting it wrong in either direction is worse than treating them as equal.
+ */
+export declare function compareVersions(a: string, b: string): number;
+/**
+ * The newest `harness_version` recorded anywhere under `<skillDir>/tests/results/`, or
+ * null when no run records one.
+ *
+ * Null is the honest answer for a pre-0.3.3 tree and must stay silent: those runs
+ * carry no version, so there is nothing to compare and no basis for a warning. The
+ * tripwire therefore only sharpens as fresh runs land — the same forward-looking
+ * bargain `source_hashes` made.
+ */
+export declare function newestRecordedVersion(skillDir: string): string | null;
+/**
+ * Refuse to write a fresh measurement with an older tool than the one that produced
+ * the records already in the tree.
+ *
+ * The failure this kills, measured on the reference corpus: a stale global **0.1.0**
+ * install would have spent ~102 rep-executions grading *without showing the judge the
+ * staged diff* — the exact defect the run was meant to correct — and every resulting
+ * number would have looked entirely plausible. It also emitted 38 spurious findings
+ * that the current version does not. Nothing announced any of it.
+ *
+ * `schema` cannot serve here: 0.2.1 → 0.3.0 kept `schema: 2` while changing what a
+ * verdict *means*. Only the writing version distinguishes those measurements.
+ *
+ * Refusal is for `run` alone, because only `run` mints a new measurement that would sit
+ * beside newer ones as if comparable. `grade` and `lint` warn (see `downgradeWarning`):
+ * both are how someone diagnoses this in the first place, and blocking diagnosis is a
+ * bad trade.
+ */
+export declare function assertNotDowngraded(skillDir: string, command: "run" | "grade" | "lint"): void;
+/** The loud-but-not-fatal version, for `grade` and `lint`. Null when nothing is newer. */
+export declare function downgradeWarning(skillDir: string): string | null;

package/dist/downgrade.js

@@ -0,0 +1,100 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { readResults } from "./results.js";
+import { HARNESS_VERSION } from "./version.js";
+/**
+ * Compare two semver-ish version strings by numeric component.
+ *
+ * String comparison is the trap this exists to avoid: `"0.10.0" < "0.9.0"` is true
+ * lexically and false in fact, which would make the tripwire fire backwards exactly
+ * once — on the release where it mattered most. A prerelease suffix is dropped rather
+ * than ordered: `0.4.0-rc.1` vs `0.4.0` is not a distinction worth a refusal, and
+ * getting it wrong in either direction is worse than treating them as equal.
+ */
+export function compareVersions(a, b) {
+    const parts = (v) => v.split("-")[0].split(".").map((n) => Number(n) || 0);
+    const [pa, pb] = [parts(a), parts(b)];
+    for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+        const d = (pa[i] ?? 0) - (pb[i] ?? 0);
+        if (d !== 0)
+            return d > 0 ? 1 : -1;
+    }
+    return 0;
+}
+function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * The newest `harness_version` recorded anywhere under `<skillDir>/tests/results/`, or
+ * null when no run records one.
+ *
+ * Null is the honest answer for a pre-0.3.3 tree and must stay silent: those runs
+ * carry no version, so there is nothing to compare and no basis for a warning. The
+ * tripwire therefore only sharpens as fresh runs land — the same forward-looking
+ * bargain `source_hashes` made.
+ */
+export function newestRecordedVersion(skillDir) {
+    const root = join(skillDir, "tests", "results");
+    if (!existsSync(root))
+        return null;
+    let newest = null;
+    for (const tag of readdirSync(root).filter((n) => isDir(join(root, n)))) {
+        const tagDir = join(root, tag);
+        for (const run of readdirSync(tagDir).filter((n) => isDir(join(tagDir, n)))) {
+            try {
+                const v = readResults(join(tagDir, run)).harness_version;
+                if (v && (newest === null || compareVersions(v, newest) > 0))
+                    newest = v;
+            }
+            catch {
+                // Unreadable results are the consistency check's problem, not this one's.
+            }
+        }
+    }
+    return newest;
+}
+function upgradeAdvice(recorded) {
+    return (`You are running skill-harness ${HARNESS_VERSION}; this tree holds results recorded by ${recorded}.\n` +
+        `  A global install goes stale silently — check with \`skill-harness --version\`, and prefer\n` +
+        `  \`npx skill-harness@${recorded}\` (or \`npm i -g skill-harness@latest\`) so the tool matches the records.`);
+}
+/**
+ * Refuse to write a fresh measurement with an older tool than the one that produced
+ * the records already in the tree.
+ *
+ * The failure this kills, measured on the reference corpus: a stale global **0.1.0**
+ * install would have spent ~102 rep-executions grading *without showing the judge the
+ * staged diff* — the exact defect the run was meant to correct — and every resulting
+ * number would have looked entirely plausible. It also emitted 38 spurious findings
+ * that the current version does not. Nothing announced any of it.
+ *
+ * `schema` cannot serve here: 0.2.1 → 0.3.0 kept `schema: 2` while changing what a
+ * verdict *means*. Only the writing version distinguishes those measurements.
+ *
+ * Refusal is for `run` alone, because only `run` mints a new measurement that would sit
+ * beside newer ones as if comparable. `grade` and `lint` warn (see `downgradeWarning`):
+ * both are how someone diagnoses this in the first place, and blocking diagnosis is a
+ * bad trade.
+ */
+export function assertNotDowngraded(skillDir, command) {
+    if (command !== "run")
+        return;
+    const recorded = newestRecordedVersion(skillDir);
+    if (!recorded || compareVersions(recorded, HARNESS_VERSION) <= 0)
+        return;
+    throw new Error(`refusing to run: these results were recorded by a NEWER skill-harness, so a run from this one would not be comparable.\n  ` +
+        upgradeAdvice(recorded));
+}
+/** The loud-but-not-fatal version, for `grade` and `lint`. Null when nothing is newer. */
+export function downgradeWarning(skillDir) {
+    const recorded = newestRecordedVersion(skillDir);
+    if (!recorded || compareVersions(recorded, HARNESS_VERSION) <= 0)
+        return null;
+    return `warning: this skill-harness is older than the tool that recorded these results.\n  ${upgradeAdvice(recorded)}`;
+}
+//# sourceMappingURL=downgrade.js.map

package/dist/index.d.ts

@@ -20,3 +20,8 @@ export * from "./adapters/types.js";
 export * from "./util/exec.js";
 export * from "./util/env.js";
 export * from "./scaffold.js";
+export * from "./version.js";
+export * from "./defaults.js";
+export * from "./judge-policy.js";
+export * from "./regate.js";
+export * from "./downgrade.js";

package/dist/index.js

@@ -20,4 +20,9 @@ export * from "./adapters/types.js";
 export * from "./util/exec.js";
 export * from "./util/env.js";
 export * from "./scaffold.js";
+export * from "./version.js";
+export * from "./defaults.js";
+export * from "./judge-policy.js";
+export * from "./regate.js";
+export * from "./downgrade.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -68,6 +68,20 @@ export type JournalEvent = {
     total: number;
     pct: number;
     ship: boolean;
+}
+/**
+ * A regate: needle gates re-evaluated against the saved staged diffs. `judge_calls`
+ * is on the record because regate is advertised as free apart from the reps whose
+ * gate verdict flipped — a claim the journal should be able to settle. `skipped`
+ * names scenarios it could not regate (vitest/post_test, or missing diff artifacts).
+ */
+ | {
+    event: "regate";
+    ts: string;
+    scenarios: string[];
+    changed: string[];
+    judge_calls: number;
+    skipped?: string[];
 } | {
     event: "score";
     ts: string;

package/dist/judge-policy.d.ts

@@ -0,0 +1,42 @@
+import type { ModelRef } from "./adapters/types.js";
+/** Whether judging with this ref can charge a per-token API. */
+export declare function isMeteredJudge(judge: ModelRef): boolean;
+/** Whether the user has explicitly accepted metered judging for this repo/shell. */
+export declare function allowMeteredJudge(): boolean;
+export interface JudgeAllowOpts {
+    /**
+     * Where the judge came from, in the user's own vocabulary — `--judge`,
+     * `SKILL_HARNESS_JUDGE`, `the run's recorded judge`. A refusal has to say which
+     * knob to turn, and for a regrade the answer is not the one the user expects:
+     * nobody typed anything, the run's own `results.yaml` supplied it.
+     */
+    source: string;
+    /** Per-invocation opt-in (`--allow-metered-judge`); the env var is read here. */
+    allowMetered?: boolean;
+}
+/**
+ * Refuse to judge through a metered API unless the user explicitly asked for it.
+ *
+ * Why a hard refusal and not a warning: judging is the only cost in this tool that
+ * happens *without* a decision. The subject model is chosen per run and is the point
+ * of the exercise; the judge is a default, and a default that bills is a bug — it
+ * already billed a corpus once. A warning scrolls past inside a run's progress
+ * output, and by then the money is spent.
+ *
+ * Three paths could reach a metered API, and only one of them involves typing a
+ * flag: `--judge anthropic:…`; a `SKILL_HARNESS_JUDGE` set (or mistyped) to a
+ * metered provider; and `grade`, which re-judges with the judge the run *recorded* —
+ * so a run whose `results.yaml` names a metered judge bills on every later regrade,
+ * with no flag involved at all.
+ *
+ * That third path is latent rather than live in the corpus this was built against:
+ * checked 2026-08-05, all ~140 committed `results.yaml` in `principal-pi-skills`
+ * record `provider: claude-code`, because its owner always passed the subscription
+ * judge explicitly. The old default was reachable, not taken. Worth stating
+ * precisely — "your whole archive bills on regrade" would have been a scarier claim
+ * than the evidence supports.
+ *
+ * Deliberately not applied to the subject model: paying to run the model under test
+ * is what a run *is*.
+ */
+export declare function assertJudgeAllowed(judge: ModelRef, opts: JudgeAllowOpts): void;

package/dist/judge-policy.js

@@ -0,0 +1,61 @@
+import { envFlag } from "./util/env.js";
+import { BAKED_DEFAULT_JUDGE } from "./defaults.js";
+/**
+ * Judge providers that cannot bill a per-token API.
+ *
+ * `claude-code` shells out to the `claude` CLI, authenticated with the user's
+ * Claude subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local` are local
+ * runtimes. Everything else is assumed to charge.
+ *
+ * An **allow-list**, deliberately, not a deny-list of known-paid providers: a
+ * provider nobody has classified yet should be treated as able to bill, because
+ * being wrong in that direction produces a surprise invoice while being wrong in
+ * this direction produces one extra flag.
+ */
+const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "ollama", "lmstudio", "llamacpp", "local"]);
+/** Whether judging with this ref can charge a per-token API. */
+export function isMeteredJudge(judge) {
+    return !FREE_JUDGE_PROVIDERS.has(judge.provider);
+}
+/** Whether the user has explicitly accepted metered judging for this repo/shell. */
+export function allowMeteredJudge() {
+    return envFlag("ALLOW_METERED_JUDGE");
+}
+/**
+ * Refuse to judge through a metered API unless the user explicitly asked for it.
+ *
+ * Why a hard refusal and not a warning: judging is the only cost in this tool that
+ * happens *without* a decision. The subject model is chosen per run and is the point
+ * of the exercise; the judge is a default, and a default that bills is a bug — it
+ * already billed a corpus once. A warning scrolls past inside a run's progress
+ * output, and by then the money is spent.
+ *
+ * Three paths could reach a metered API, and only one of them involves typing a
+ * flag: `--judge anthropic:…`; a `SKILL_HARNESS_JUDGE` set (or mistyped) to a
+ * metered provider; and `grade`, which re-judges with the judge the run *recorded* —
+ * so a run whose `results.yaml` names a metered judge bills on every later regrade,
+ * with no flag involved at all.
+ *
+ * That third path is latent rather than live in the corpus this was built against:
+ * checked 2026-08-05, all ~140 committed `results.yaml` in `principal-pi-skills`
+ * record `provider: claude-code`, because its owner always passed the subscription
+ * judge explicitly. The old default was reachable, not taken. Worth stating
+ * precisely — "your whole archive bills on regrade" would have been a scarier claim
+ * than the evidence supports.
+ *
+ * Deliberately not applied to the subject model: paying to run the model under test
+ * is what a run *is*.
+ */
+export function assertJudgeAllowed(judge, opts) {
+    if (!isMeteredJudge(judge))
+        return;
+    if (opts.allowMetered || allowMeteredJudge())
+        return;
+    const token = `${judge.provider}:${judge.model}`;
+    throw new Error(`refusing to judge with ${token}: \`${judge.provider}\` bills a per-token API key, and it came from ${opts.source}.\n` +
+        `  Judging is meant to cost nothing you did not ask for.\n` +
+        `  • judge on your Claude subscription instead:  --judge ${BAKED_DEFAULT_JUDGE}\n` +
+        `  • allow the metered API for this command:     --allow-metered-judge\n` +
+        `  • allow it for this repo or shell:            export SKILL_HARNESS_ALLOW_METERED_JUDGE=1`);
+}
+//# sourceMappingURL=judge-policy.js.map

package/dist/lift.d.ts

@@ -44,10 +44,29 @@ export interface Lift {
      * `LiftOptions.modeInsensitive`.
      */
     modeInsensitive: string[];
+    /**
+     * Ids both runs covered whose two verdicts were produced by different
+     * aggregations, so the comparison is not like-for-like. Excluded rather than
+     * compared: see `comparableAggregation`.
+     */
+    aggregationMismatch: LiftAggregationMismatch[];
     /** True when either side was an `--only` run, so coverage is a subset by construction. */
     partial: boolean;
     cells: Record<string, LiftCell>;
 }
+/**
+ * How one side produced a scenario's verdict: over how many reps, and under which
+ * majority threshold (null when no aggregation happened).
+ */
+export interface AggregationShape {
+    reps: number;
+    threshold: number | null;
+}
+export interface LiftAggregationMismatch {
+    id: string;
+    red: AggregationShape;
+    green: AggregationShape;
+}
 export interface LiftOptions {
     /**
      * Scenario ids whose red and green runs are the same run by construction, so

package/dist/lift.js

@@ -2,6 +2,26 @@ import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { readResults, effectiveVerdicts } from "./results.js";
 import { loadSpec } from "./spec.js";
+function aggregationShape(s) {
+    const reps = s.reps ?? 1;
+    // At one rep `outcomesToResult` keeps the single judge verdict and never calls
+    // `aggregateReps`, so a `pass_threshold` sitting beside it was applied to
+    // nothing. Normalizing it away keeps a stray field from faking a mismatch.
+    return { reps, threshold: reps > 1 ? s.pass_threshold ?? null : null };
+}
+/**
+ * Whether two verdicts were produced the same way, and so mean the same thing.
+ *
+ * A one-rep verdict is a single draw; a three-rep verdict is a majority over
+ * three. Across that gap `red FAIL -> green PASS` can be sampling alone, and
+ * `gained` would be reporting the harness's own asymmetry as skill value — the
+ * inverse of the `modeInsensitive` error, and pointing the number the *other*
+ * way. The threshold counts too: 1-of-3 versus 3-of-3 is a different majority
+ * policy at the same N, so the aggregate is not the same measurement.
+ */
+function comparableAggregation(red, green) {
+    return red.reps === green.reps && red.threshold === green.threshold;
+}
 /** A verdict that carries real evidence about the task, rather than about the harness or the judge. */
 function conclusive(verdict, suspect) {
     // ERROR is a harness failure (timeout, empty reply) — it says nothing about
@@ -38,6 +58,8 @@ export function computeLift(red, green, opts = {}) {
     const insensitive = new Set(opts.modeInsensitive ?? []);
     const redV = new Map(effectiveVerdicts(red.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
     const greenV = new Map(effectiveVerdicts(green.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
+    const redShape = new Map(red.scenarios.map((s) => [s.id, aggregationShape(s)]));
+    const greenShape = new Map(green.scenarios.map((s) => [s.id, aggregationShape(s)]));
     const cells = {};
     const counts = { gained: 0, regressed: 0, kept: 0, "both-fail": 0, inconclusive: 0 };
     let redPassed = 0;
@@ -45,6 +67,7 @@ export function computeLift(red, green, opts = {}) {
     // Green order drives display order (it is the run the author is looking at),
     // restricted to ids the red baseline also covered.
     const modeInsensitive = [];
+    const aggregationMismatch = [];
     for (const [id, g] of greenV) {
         const r = redV.get(id);
         if (!r)
@@ -53,6 +76,15 @@ export function computeLift(red, green, opts = {}) {
             modeInsensitive.push(id);
             continue;
         }
+        // Checked before classification, and reported separately, for the reason
+        // modeInsensitive is: there is no honest bucket for two verdicts that were
+        // not measured the same way.
+        const rShape = redShape.get(id) ?? { reps: 1, threshold: null };
+        const gShape = greenShape.get(id) ?? { reps: 1, threshold: null };
+        if (!comparableAggregation(rShape, gShape)) {
+            aggregationMismatch.push({ id, red: rShape, green: gShape });
+            continue;
+        }
         const cls = classify(r, g);
         cells[id] = { red: r.verdict, redSuspect: r.suspect, green: g.verdict, class: cls };
         counts[cls]++;
@@ -80,17 +112,44 @@ export function computeLift(red, green, opts = {}) {
         greenOnly: [...greenV.keys()].filter((id) => !redV.has(id)),
         redOnly: [...redV.keys()].filter((id) => !greenV.has(id)),
         modeInsensitive,
+        aggregationMismatch,
         partial: Boolean(red.partial || green.partial),
         cells,
     };
 }
+function reps(n) {
+    return n === 1 ? "1 rep" : `${n} reps`;
+}
+/** What differs between the two sides, in the words of the flag that caused it. */
+function describeMismatch(ms) {
+    const distinct = new Set(ms.map((m) => m.red.reps !== m.green.reps
+        ? `red ${reps(m.red.reps)} vs ${reps(m.green.reps)}`
+        : `red pass threshold ${m.red.threshold} vs ${m.green.threshold}`));
+    return distinct.size === 1 ? [...distinct][0] : "red and green aggregated differently";
+}
+/** The one command that would make the comparison measurable. */
+function mismatchRemedy(ms) {
+    const greenReps = new Set(ms.map((m) => m.green.reps));
+    if (greenReps.size === 1 && ms.every((m) => m.red.reps !== m.green.reps)) {
+        return `re-run the baseline with --reps ${[...greenReps][0]}`;
+    }
+    return "re-measure both sides the same way";
+}
 /** One line for a human: what the skill did, and what it cost. */
 export function liftHeadline(lift) {
     if (lift.compared === 0) {
         // Excluded-but-shared is not the same as never-shared. Claiming the runs had
         // no scenario in common would hide the reason the lift is empty.
-        if (lift.modeInsensitive.length > 0) {
-            return `nothing comparable (${lift.modeInsensitive.length} shared, all run identically in both modes)`;
+        const mismatched = lift.aggregationMismatch.length;
+        const insensitive = lift.modeInsensitive.length;
+        if (mismatched > 0 && insensitive > 0) {
+            return `nothing comparable (${insensitive} run identically in both modes, ${mismatched} ${describeMismatch(lift.aggregationMismatch)})`;
+        }
+        if (mismatched > 0) {
+            return `nothing comparable (${mismatched} shared, ${describeMismatch(lift.aggregationMismatch)} — ${mismatchRemedy(lift.aggregationMismatch)})`;
+        }
+        if (insensitive > 0) {
+            return `nothing comparable (${insensitive} shared, all run identically in both modes)`;
         }
         return "no shared scenarios to compare";
     }
@@ -116,6 +175,9 @@ export function liftHeadline(lift) {
     if (lift.modeInsensitive.length > 0) {
         segments.push(`${lift.modeInsensitive.length} not comparable (same run in both modes)`);
     }
+    if (lift.aggregationMismatch.length > 0) {
+        segments.push(`${lift.aggregationMismatch.length} not comparable (${describeMismatch(lift.aggregationMismatch)})`);
+    }
     if (lift.partial)
         segments.push("partial run");
     return segments.join(" · ");

package/dist/lint.js

@@ -3,7 +3,8 @@ import { basename, dirname, isAbsolute, join, resolve } from "node:path";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
-import { currentHashFor, describeSourceKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, UNREADABLE } from "./sources.js";
+import { currentHashFor, describeSourceKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
+import { downgradeWarning } from "./downgrade.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
@@ -41,6 +42,16 @@ export function lintSkill(skillDir) {
         return [{ skill: basename(skillDir), code: "spec", message }];
     }
     const skill = spec.skill;
+    // A lint from an older tool than the records it is checking cannot be trusted to
+    // check them: a 0.1.0 binary reported 38 spurious `consistency` findings against
+    // partial runs that 0.3.x handles correctly, and it cannot read source-hash key
+    // kinds added after it shipped. Reported as a finding (so CI sees it) rather than
+    // thrown, because refusing to lint is worse than linting with a caveat, and it is
+    // one of the few ways this situation announces itself at all.
+    const stale = downgradeWarning(skillDir);
+    if (stale) {
+        findings.push({ skill, code: "consistency", message: stale.replace(/^warning: /, "") });
+    }
     // ship_bar sanity
     if (spec.ship_bar.total < 1) {
         findings.push({ skill, code: "ship_bar", message: "ship_bar.total must be >= 1" });
@@ -198,7 +209,7 @@ export function lintSkill(skillDir) {
             continue; // predates source_hashes → silent
         {
             const newest = full.runDir;
-            const ctx = { skillDir, specDir, scenarios: spec.scenarios };
+            const ctx = { skillDir, specDir, scenarios: spec.scenarios, judgePersona: spec.judge_persona };
             for (const [key, recorded] of Object.entries(hashes)) {
                 const what = describeSourceKey(key);
                 const scenario = scenarioIdForKey(key, spec.scenarios);
@@ -218,7 +229,12 @@ export function lintSkill(skillDir) {
                     findings.push({ skill, scenario, code: "stale", message: `${what} no longer exists but the newest ${basename(tagDir)} run measured it (${newest})` });
                 }
                 else if (current !== recorded) {
-                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; re-run before publishing` });
+                    // The remedy is per key kind, and it is the whole point of the split: a
+                    // rubric edit is re-gradeable from saved transcripts, a policy edit is a
+                    // free rescore, a needle edit is a free regate. Only stimulus drift costs
+                    // model spend. One message saying "re-run" for all four is what made
+                    // correcting a known-bad rubric expensive enough to skip.
+                    findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; ${remedyForKey(key)}` });
                 }
             }
             // Coverage: a scenario the spec defines that the newest full run never
@@ -227,9 +243,15 @@ export function lintSkill(skillDir) {
             // 100%/SHIP scorecard survives an arbitrary spec rewrite reporting zero
             // findings. Gated on the run having recorded scenario keys at all, so runs
             // predating the key kind stay silent like every other pre-existing run.
-            if (Object.keys(hashes).some((k) => k.startsWith(SCENARIO_PREFIX))) {
+            // Either key kind counts as "this run recorded per-scenario hashes": 0.4.0+ runs
+            // carry `stimulus:<id>`, older ones the combined `scenario:<id>`. Checking only
+            // the legacy prefix would silently drop coverage checking for every new run.
+            const scenarioKeyPrefix = Object.keys(hashes).some((k) => k.startsWith(STIMULUS_PREFIX))
+                ? STIMULUS_PREFIX
+                : SCENARIO_PREFIX;
+            if (Object.keys(hashes).some((k) => k.startsWith(scenarioKeyPrefix))) {
                 for (const s of spec.scenarios) {
-                    if (!(SCENARIO_PREFIX + s.id in hashes)) {
+                    if (!(scenarioKeyPrefix + s.id in hashes)) {
                         findings.push({ skill, scenario: s.id, code: "stale", message: `the newest ${basename(tagDir)} run (${newest}) did not measure scenario \`${s.id}\` — the published result covers a different scenario set than the spec; re-run before publishing` });
                     }
                 }

package/dist/regate.d.ts

@@ -0,0 +1,57 @@
+import type { Spec } from "./spec.js";
+import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+import { type ResultsFile } from "./results.js";
+import type { Verdict } from "./score.js";
+export interface RegateOptions {
+    runDir: string;
+    spec: Spec;
+    specDir: string;
+    adapter: HarnessAdapter;
+    judge: ModelRef;
+    now?: () => string;
+}
+export interface RegateChange {
+    id: string;
+    from: Verdict;
+    to: Verdict;
+    /** What the corrected needles say about this scenario now. */
+    gate: "pass" | "fail";
+    /** Whether re-deciding it required a judge call (only a gate-FAIL → gate-pass flip does). */
+    judged: boolean;
+}
+export interface RegateResult {
+    results: ResultsFile;
+    changes: RegateChange[];
+    /** Judge calls actually made, for the cost line the CLI prints. */
+    judgeCalls: number;
+}
+/**
+ * Re-evaluate needle gates against a run's **saved staged diffs** and re-decide the
+ * verdicts they determined — without re-running the model.
+ *
+ * `diff_contains` / `diff_excludes` are pure functions of the diff, and since
+ * `f6a5f6c` every seeded rep persists its diff as a run artifact. So the defect class
+ * "the gate was wrong, the behavior wasn't" — hit three times in the reference corpus
+ * (a context needle, a baseline-satisfied needle, a filename needle) — no longer costs
+ * a re-run. Measured on the C2 needle fix: **9 judge calls instead of 81
+ * rep-executions across three models.**
+ *
+ * Per rep, exactly one of four things happens:
+ *
+ * | old gate | new gate | outcome | cost |
+ * |---|---|---|---|
+ * | fail | fail | FAIL, with the corrected reason | free |
+ * | pass | fail | FAIL — the gate is objective and it says no | free |
+ * | pass | pass | the rep's saved judgement, re-parsed from its judge-raw artifact | free |
+ * | fail | pass | judged now: the judge never saw this rep, because the gate blocked it | 1 judge call |
+ *
+ * That third row is what keeps this cheap without guessing: a rep the judge already
+ * saw has its verdict on disk, so regate re-reads it rather than re-asking.
+ *
+ * **Limits, deliberately hard failures rather than partial work:** `assert.vitest` and
+ * `assert.post_test` need the workspace and cannot be re-evaluated from any artifact,
+ * so a scenario carrying either is not regatable. Diffs and judge-raw files are
+ * gitignored, so this works for whoever holds the run dirs — the repo owner, or CI that
+ * just ran — which is exactly the situation it is needed in.
+ */
+export declare function regateRun(opts: RegateOptions): Promise<RegateResult>;