@skill-harness/core 0.3.2 → 0.5.0

package/dist/report.js

@@ -50,13 +50,14 @@ export function collectReport(skillDir) {
                 };
             }
             const tag = tagDir.split("/").pop();
-            // A column is the tag's LATEST run, which is not necessarily the green one —
-            // record a red baseline after a green run and the newest run in the tag is
-            // red. The review UI recomputes lift from `cells` (so author overrides move
+            // A column is the tag's LATEST run, which is not necessarily the skill-side
+            // one — record a red baseline after a green run and the newest run in the tag
+            // is red. The review UI recomputes lift from `cells` (so author overrides move
             // it live), so attaching a lift to a column whose cells are the RED run
             // would have it compare red against red and report "no effect" for a skill
             // that in fact gained every scenario. Attach only when this column IS the
-            // green side of the comparison.
+            // skill side of the comparison — matched on the timestamp, which also keeps a
+            // green column from borrowing a force run's lift and vice versa.
             const tagLift = liftByTag.get(tag);
             const lift = tagLift && tagLift.greenTimestamp === r.timestamp ? tagLift : undefined;
             columns.push({

package/dist/rescore.d.ts

@@ -24,6 +24,12 @@ export interface RescoreResult {
  * When the policy changes, the honest move is to recompute the old measurements under it
  * rather than reconcile two numbers in prose — and to record what moved.
  *
+ * It is also how a run gets a grade it never had: every rescore recomputes
+ * `effective_grade` under the current scoring policy, and since 0.5.0 that policy
+ * scores force runs too (see SCORED_MODES). A corpus holding force-mode runs
+ * recorded as "not scored" turns them into real scorecards with `rescore`, at zero
+ * model and zero judge spend — verdict changes are then genuinely optional output.
+ *
  * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
  * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
  * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this

package/dist/rescore.js

@@ -1,13 +1,40 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults } from "./results.js";
+import { readResults, writeResults, scoreContextFor } from "./results.js";
 import { appendJournal } from "./journal.js";
+import { policyDigest, POLICY_PREFIX } from "./sources.js";
+/**
+ * Refresh the `policy:` keys a run recorded, since a rescore has just re-applied the
+ * current policy to every rep it holds.
+ *
+ * Unlike a re-grade, this needs no id list: `rescoreRun` walks every recorded
+ * scenario, and the ones it carries verbatim (single-rep, ERROR, JUDGE-AMBIGUOUS) are
+ * carried *because the current threshold cannot change their verdict* — they are
+ * scored under today's policy too. Only keys the run already carried are touched, so
+ * this never fabricates coverage.
+ */
+function refreshPolicyHashes(recorded, spec) {
+    if (!recorded)
+        return undefined;
+    const next = { ...recorded };
+    for (const s of spec.scenarios) {
+        if (POLICY_PREFIX + s.id in next)
+            next[POLICY_PREFIX + s.id] = policyDigest(s);
+    }
+    return next;
+}
 /**
  * Re-score a run against the CURRENT spec's pass thresholds — no model calls, no judge
  * calls. Reps are the raw measurement (`passes` of `clean`); a threshold is *policy*.
  * When the policy changes, the honest move is to recompute the old measurements under it
  * rather than reconcile two numbers in prose — and to record what moved.
  *
+ * It is also how a run gets a grade it never had: every rescore recomputes
+ * `effective_grade` under the current scoring policy, and since 0.5.0 that policy
+ * scores force runs too (see SCORED_MODES). A corpus holding force-mode runs
+ * recorded as "not scored" turns them into real scorecards with `rescore`, at zero
+ * model and zero judge spend — verdict changes are then genuinely optional output.
+ *
  * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
  * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
  * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
@@ -40,13 +67,21 @@ export function rescoreRun(opts) {
         }
         return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
     });
-    const ctx = prev.mode === "green" && !prev.partial
-        ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
-        : null;
+    const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
-        partial: prev.partial, source_hashes: prev.source_hashes, scenarios,
+        partial: prev.partial,
+        // Provenance of the measurement, not of this rewrite: a rescore re-applies a
+        // threshold to reps the recorded harness already produced.
+        harness_cli_version: prev.harness_cli_version,
+        delivery_canary: prev.delivery_canary,
+        // A rescore re-applies the CURRENT policy (thresholds, critical set) to the
+        // recorded reps, so `policy:` drift is genuinely resolved by having run this —
+        // that is what makes `rescore` the honest remedy lint names for it. Stimulus,
+        // rubric and gate hashes are untouched: none of them was re-evaluated here.
+        source_hashes: refreshPolicyHashes(prev.source_hashes, opts.spec),
+        scenarios,
     }, ctx);
     appendJournal(opts.runDir, {
         event: "rescore", ts: now(),

package/dist/results.d.ts

@@ -25,6 +25,47 @@ export interface GradeSummary {
 }
 export interface ResultsFile {
     schema: 2;
+    /**
+     * The harness version that wrote this file — provenance for the numbers in it.
+     *
+     * Optional because runs recorded before the field existed do not have one, and
+     * inventing a version for them would fabricate provenance. Not part of `schema`:
+     * 0.2.1 → 0.3.0 kept `schema: 2` while changing what a verdict means, which is
+     * exactly the drift `schema` cannot express. Written by `finalizeResults`, so no
+     * writer can forget it.
+     */
+    harness_version?: string;
+    /**
+     * The version of the harness CLI that produced the transcripts — `pi --version`
+     * for `harness: pi`.
+     *
+     * Provenance for the *delivery*, not for the tool: pi 0.80.x wrapped a `--skill`
+     * prompt with the skill body, pi 0.83.0 switched to progressive disclosure, and
+     * that upgrade silently changed what `--mode green` measured. Two waves of runs
+     * in the reference corpus are indistinguishable from a naked-model baseline, and
+     * the incident is invisible in the artifacts precisely because nothing recorded
+     * which pi ran.
+     *
+     * Written only by `run` (the command that actually invokes the harness) and
+     * carried verbatim by every rewriter — `grade`/`rescore`/`regate` re-decide
+     * verdicts, they do not re-deliver the skill, so re-stamping this field with
+     * today's pi would attribute the old transcripts to a version that never
+     * produced them. Optional: runs recorded before the field existed have none, and
+     * an adapter that cannot report a version writes none rather than guessing.
+     */
+    harness_cli_version?: string;
+    /**
+     * `pass` when this run proved, before spending the wave, that the skill body was
+     * reachable in the model's context (see canary.ts). Absent means the probe was
+     * not asked for — never that it failed, because a failed canary aborts the run
+     * and no results.yaml is written.
+     *
+     * Only green runs can carry it: red delivers nothing by design and force delivers
+     * through the system prompt. It is provenance for the *validity* of a green run,
+     * which is why it lives here rather than only in the journal — `journal.jsonl` is
+     * gitignored, and this claim has to survive a commit.
+     */
+    delivery_canary?: "pass";
     skill: string;
     harness: string;
     model: string;
@@ -50,6 +91,53 @@ export interface ResultsFile {
     effective_grade: GradeSummary;
     scenarios: ScenarioResult[];
 }
+/**
+ * The run modes whose results carry a real grade: the ones where the skill under
+ * test was actually delivered to the model.
+ *
+ * `green` activates the skill through the harness's own mechanism (`pi --skill`);
+ * `force` puts SKILL.md in the system prompt. Both are measurements OF THE SKILL,
+ * so both are scored against the ship bar. `red` is the control — the model with
+ * no skill — and scoring it would produce a ship grade for the thing the skill is
+ * measured against.
+ *
+ * Force was unscored until 0.5.0, when it stopped being an escape hatch and became
+ * a deployment: on pi 0.83.0 `--skill` switched to progressive disclosure (the
+ * description is in context, the body loads on demand — "models don't always do
+ * this"), so skill-as-system-prompt is the delivery a corpus can actually rely on.
+ * Ten committed force runs in the reference corpus read `not scored` for exactly
+ * that reason. Scored directly rather than behind a spec flag or a `--score-force`
+ * opt-in: "was the skill in front of the model?" is a property of the mode, not a
+ * per-repo preference, and a second knob would just be a second thing to forget.
+ *
+ * Consequence to expect, and it is the intended one: a force run recorded before
+ * 0.5.0 carries a "not scored" placeholder grade that a recompute now disagrees
+ * with, so `lint` flags it as stale and `rescore` (free) writes the real grade.
+ *
+ * The two modes are NOT interchangeable measurements of the same thing — placement
+ * changes behavior in both directions (measured on identical skill text: `build` A1
+ * 0/3 → 3/3, `plan` C2 3/3 → 0/3). Anything that plots or compares runs over time
+ * therefore keeps the epochs apart rather than pooling them; see trends.ts.
+ */
+export declare const SCORED_MODES: readonly string[];
+/** Whether a run in this mode delivered the skill, and so has a grade worth computing. */
+export declare function isScoredMode(mode: string): boolean;
+/**
+ * The one place "does this run get a grade?" is decided: the scoring mode gate plus
+ * the `--only` partial gate, in one predicate every writer shares.
+ *
+ * Before 0.5.0 this ternary was open-coded in seven places (run, grade, rescore,
+ * regate, lint, and both review-server writers) — which is how force runs came to
+ * be unscored in all seven at once, and how any future mode would have had to be
+ * remembered seven times.
+ */
+export declare function scoreContextFor(run: {
+    mode: string;
+    partial?: boolean;
+}, spec: {
+    ship_bar: ShipBar;
+    critical: string[];
+}): ScoreContext | null;
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export declare function effectiveThreshold(prevScenario: ScenarioResult | undefined, scenario: Scenario): number;
 /** Everything a caller may set. The grade is computed, never supplied. */

package/dist/results.js

@@ -3,6 +3,54 @@ import { join, relative, sep } from "node:path";
 import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
+import { HARNESS_VERSION } from "./version.js";
+/**
+ * The run modes whose results carry a real grade: the ones where the skill under
+ * test was actually delivered to the model.
+ *
+ * `green` activates the skill through the harness's own mechanism (`pi --skill`);
+ * `force` puts SKILL.md in the system prompt. Both are measurements OF THE SKILL,
+ * so both are scored against the ship bar. `red` is the control — the model with
+ * no skill — and scoring it would produce a ship grade for the thing the skill is
+ * measured against.
+ *
+ * Force was unscored until 0.5.0, when it stopped being an escape hatch and became
+ * a deployment: on pi 0.83.0 `--skill` switched to progressive disclosure (the
+ * description is in context, the body loads on demand — "models don't always do
+ * this"), so skill-as-system-prompt is the delivery a corpus can actually rely on.
+ * Ten committed force runs in the reference corpus read `not scored` for exactly
+ * that reason. Scored directly rather than behind a spec flag or a `--score-force`
+ * opt-in: "was the skill in front of the model?" is a property of the mode, not a
+ * per-repo preference, and a second knob would just be a second thing to forget.
+ *
+ * Consequence to expect, and it is the intended one: a force run recorded before
+ * 0.5.0 carries a "not scored" placeholder grade that a recompute now disagrees
+ * with, so `lint` flags it as stale and `rescore` (free) writes the real grade.
+ *
+ * The two modes are NOT interchangeable measurements of the same thing — placement
+ * changes behavior in both directions (measured on identical skill text: `build` A1
+ * 0/3 → 3/3, `plan` C2 3/3 → 0/3). Anything that plots or compares runs over time
+ * therefore keeps the epochs apart rather than pooling them; see trends.ts.
+ */
+export const SCORED_MODES = ["green", "force"];
+/** Whether a run in this mode delivered the skill, and so has a grade worth computing. */
+export function isScoredMode(mode) {
+    return SCORED_MODES.includes(mode);
+}
+/**
+ * The one place "does this run get a grade?" is decided: the scoring mode gate plus
+ * the `--only` partial gate, in one predicate every writer shares.
+ *
+ * Before 0.5.0 this ternary was open-coded in seven places (run, grade, rescore,
+ * regate, lint, and both review-server writers) — which is how force runs came to
+ * be unscored in all seven at once, and how any future mode would have had to be
+ * remembered seven times.
+ */
+export function scoreContextFor(run, spec) {
+    if (!isScoredMode(run.mode) || run.partial)
+        return null;
+    return { shipBar: spec.ship_bar, critical: spec.critical };
+}
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export function effectiveThreshold(prevScenario, scenario) {
     return prevScenario?.pass_threshold ?? scenario.passThreshold ?? 0.5;
@@ -51,6 +99,14 @@ export function finalizeResults(draft, ctx) {
     }
     return {
         schema: 2,
+        // Stamped here, the single place every writer passes through, so `run`,
+        // `grade`, `rescore` and the review UI's override save all record which tool
+        // produced the record they leave behind.
+        harness_version: HARNESS_VERSION,
+        // Omitted rather than written as null when absent: a run whose adapter could
+        // not report a version must not look like one that reported "nothing".
+        ...(draft.harness_cli_version ? { harness_cli_version: draft.harness_cli_version } : {}),
+        ...(draft.delivery_canary ? { delivery_canary: draft.delivery_canary } : {}),
         skill: draft.skill,
         harness: draft.harness,
         model: draft.model,

package/dist/run.d.ts

@@ -24,6 +24,13 @@ export interface RunOptions {
      * ship-graded: a subset passing says nothing about the ship bar.
      */
     only?: string[];
+    /**
+     * Green mode only: spend ONE probe up front proving the skill reaches the model,
+     * and abort the run if it doesn't (see canary.ts). Off by default — it costs a
+     * rep, and the deterministic half of this failure class (a skill dir that isn't
+     * there) is already refused by the adapter for free.
+     */
+    canary?: boolean;
 }
 export interface RunSummary {
     runDir: string;

package/dist/run.js

@@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -10,6 +10,7 @@ import { createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
+import { runDeliveryCanary, canaryFailure } from "./canary.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -36,12 +37,52 @@ export async function runSkillModel(opts) {
     const runDir = runDirFor(skillDir, adapter.name, model, timestamp);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
+    // Which harness CLI delivered the skill, asked once per run and recorded with the
+    // numbers. A pi upgrade (0.80.x → 0.83.0) silently changed what green mode
+    // measures, and the incident was invisible in the artifacts because nothing wrote
+    // this down. Never fatal: `null` means the adapter couldn't say.
+    const harnessCliVersion = (await adapter.version?.()) ?? null;
     appendJournal(runDir, {
         event: "run-started", ts: now(),
         skill: spec.skill, harness: adapter.name, model: opts.modelToken,
+        harness_cli_version: harnessCliVersion,
         judge: { provider: judge.provider, model: judge.model },
         mode, label: opts.label ?? null,
     });
+    // The canary spends one probe before the wave, so a run that isn't measuring the
+    // skill dies for the price of a rep instead of producing a plausible scorecard.
+    // Green only: red delivers nothing by design, and force delivers through the
+    // system prompt, which needs no probe.
+    let canaryStatus = null;
+    if (opts.canary && mode !== "green") {
+        // Ignoring a flag silently is a small version of the bug this whole feature is
+        // about. Say it, and say why it isn't needed.
+        log(`  --canary ignored in mode=${mode} — ${mode === "force" ? "the system prompt delivers the skill unconditionally" : "a baseline delivers no skill by design"}`);
+    }
+    if (opts.canary && mode === "green") {
+        const probeCwd = createWorkspace("none", { specDir: dirname(opts.specPath) });
+        let canary;
+        try {
+            canary = await runDeliveryCanary({
+                adapter, model, skillDir, skillName: spec.skill, cwd: probeCwd.cwd,
+            });
+        }
+        finally {
+            probeCwd.cleanup();
+        }
+        appendJournal(runDir, {
+            event: "delivery-canary", ts: now(),
+            status: canary.status, anchor: canary.anchor, detail: canary.detail,
+        });
+        if (canary.status === "fail")
+            throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
+        if (canary.status === "skipped")
+            log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+        else {
+            canaryStatus = "pass";
+            log(`  ✓ delivery canary — the model quoted its skill instructions back (\`${canary.anchor}\`)`);
+        }
+    }
     // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
     const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
     const owners = [];
@@ -61,10 +102,12 @@ export async function runSkillModel(opts) {
         const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
-    const ctx = mode === "green" && !partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = scoreContextFor({ mode, partial }, spec);
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: adapter.name,
+        harness_cli_version: harnessCliVersion ?? undefined,
+        delivery_canary: canaryStatus ?? undefined,
         model: opts.modelToken,
         judge: { provider: judge.provider, model: judge.model },
         timestamp,
@@ -73,7 +116,7 @@ export async function runSkillModel(opts) {
         ...(partial ? { partial: true } : {}),
         // Only the scenarios this run actually measured: a --only run must not claim
         // coverage of scenarios it skipped.
-        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios }),
+        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
         scenarios: scenarioResults,
     }, ctx);
     if (ctx) {
@@ -226,16 +269,26 @@ export function formatScorecard(summary, lift) {
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";
     lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
-    // Lift is a statement about a green run. On a red run the caller may still have
-    // a lift in hand (a green run exists in the same tag), but printing it under a
-    // baseline scorecard reads as if the baseline itself gained something.
-    if (lift && results.mode === "green") {
+    // Lift is a statement about a skill-delivered run (green or force). On a red run
+    // the caller may still have a lift in hand (a scored run exists in the same tag),
+    // but printing it under a baseline scorecard reads as if the baseline itself
+    // gained something.
+    if (lift && isScoredMode(results.mode)) {
         lines.push(`  LIFT:  ${liftHeadline(lift)}  (vs red baseline ${lift.redTimestamp})`);
     }
-    else if (results.mode === "green") {
+    else if (isScoredMode(results.mode)) {
         // The grade alone can't answer "does this skill do anything?", so say how.
         lines.push(`  LIFT:  no red baseline — run with --mode red to measure what the skill adds`);
     }
+    // Said on the scorecard, not just in the docs: the one thing that can invalidate
+    // a green number is invisible in the number. `harness_cli_version` is recorded
+    // beside the verdicts so a reader can tell which pi produced them.
+    if (results.mode === "green" && !results.delivery_canary) {
+        lines.push(`  NOTE:  green delivery is harness-version-dependent` +
+            (results.harness_cli_version ? ` (${results.harness} ${results.harness_cli_version})` : "") +
+            ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +
+            ` Use --mode force for delivery that cannot silently degrade, or --canary to prove it per run.`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run.js.map

package/dist/seeded.d.ts

@@ -58,6 +58,24 @@ export interface SeededOutcome {
  * guessing from prefixes.
  */
 export declare function changedLines(diff: string): string;
+/** Whether a scenario declares any needle gate at all. */
+export declare function hasNeedleGates(scenario: Scenario): boolean;
+/**
+ * Evaluate `diff_contains` / `diff_excludes` against a staged diff: the trailer lines
+ * to report, and the first failure (null when every needle is satisfied).
+ *
+ * A **pure function of the diff**, which is what makes `regate` possible — the saved
+ * `.diff.txt` artifact holds everything these gates need, so correcting a needle never
+ * requires re-running the model. Shared with `regate` deliberately: two copies of this
+ * loop would let a regated verdict disagree with what a fresh run would have produced,
+ * which is the same drift the fixture-marker check refuses between lint and runtime.
+ *
+ * Both gates read the CHANGED lines only, never context — see `changedLines`.
+ */
+export declare function evaluateNeedleGates(scenario: Scenario, diff: string): {
+    lines: string[];
+    failure: string | null;
+};
 /**
  * Cut a diff to a byte budget on a line boundary, appending an explicit marker
  * naming how much was dropped.

package/dist/seeded.js

@@ -56,6 +56,42 @@ export function changedLines(diff) {
     }
     return out.join("\n");
 }
+/** Whether a scenario declares any needle gate at all. */
+export function hasNeedleGates(scenario) {
+    return (scenario.assert?.diff_contains?.length ?? 0) > 0 || (scenario.assert?.diff_excludes?.length ?? 0) > 0;
+}
+/**
+ * Evaluate `diff_contains` / `diff_excludes` against a staged diff: the trailer lines
+ * to report, and the first failure (null when every needle is satisfied).
+ *
+ * A **pure function of the diff**, which is what makes `regate` possible — the saved
+ * `.diff.txt` artifact holds everything these gates need, so correcting a needle never
+ * requires re-running the model. Shared with `regate` deliberately: two copies of this
+ * loop would let a regated verdict disagree with what a fresh run would have produced,
+ * which is the same drift the fixture-marker check refuses between lint and runtime.
+ *
+ * Both gates read the CHANGED lines only, never context — see `changedLines`.
+ */
+export function evaluateNeedleGates(scenario, diff) {
+    const changed = changedLines(diff);
+    const lines = [];
+    let failure = null;
+    for (const needle of scenario.assert?.diff_contains ?? []) {
+        const ok = changed.includes(needle);
+        lines.push(`  diff_contains ${JSON.stringify(needle)}: ${ok ? "OK" : "MISSING"}`);
+        if (!ok && !failure)
+            failure = `staged diff missing ${JSON.stringify(needle)}`;
+    }
+    // Scope discipline, stated as a fact about the diff rather than inferred from
+    // whether the model remembered to say "I left lastIndex alone".
+    for (const needle of scenario.assert?.diff_excludes ?? []) {
+        const ok = !changed.includes(needle);
+        lines.push(`  diff_excludes ${JSON.stringify(needle)}: ${ok ? "OK" : "PRESENT"}`);
+        if (!ok && !failure)
+            failure = `staged diff touches forbidden ${JSON.stringify(needle)}`;
+    }
+    return { lines, failure };
+}
 /**
  * Byte cap on the diff copy embedded in the judged transcript. The judge prompt
  * is a single request, so an unbounded diff (a fixture-wide refactor, a
@@ -167,21 +203,10 @@ export async function runSeeded(scenario, opts) {
     // either token. Every published A4 result recorded that as an objective pass.
     // Read against changed lines it means what the checklist means: the model
     // returned a Result.
-    const changed = changedLines(diff);
-    for (const needle of scenario.assert?.diff_contains ?? []) {
-        const ok = changed.includes(needle);
-        parts.push(`  diff_contains ${JSON.stringify(needle)}: ${ok ? "OK" : "MISSING"}`);
-        if (!ok && !gateFailure)
-            gateFailure = `staged diff missing ${JSON.stringify(needle)}`;
-    }
-    // Scope discipline, stated as a fact about the diff rather than inferred from
-    // whether the model remembered to say "I left lastIndex alone".
-    for (const needle of scenario.assert?.diff_excludes ?? []) {
-        const ok = !changed.includes(needle);
-        parts.push(`  diff_excludes ${JSON.stringify(needle)}: ${ok ? "OK" : "PRESENT"}`);
-        if (!ok && !gateFailure)
-            gateFailure = `staged diff touches forbidden ${JSON.stringify(needle)}`;
-    }
+    const needles = evaluateNeedleGates(scenario, diff);
+    parts.push(...needles.lines);
+    if (needles.failure && !gateFailure)
+        gateFailure = needles.failure;
     if (scenario.assert?.vitest) {
         const v = await runVitest([], repo);
         // code === null means exec SIGKILLed it at the timeout. That is infrastructure,

package/dist/sources.d.ts

@@ -37,8 +37,44 @@ import type { Scenario } from "./spec.js";
  * reindenting the YAML or reordering scenarios is correctly a no-op, while
  * changing a single checklist word is correctly a change.
  */
+/**
+ * The pre-0.4.0 combined key: one digest over a scenario's stimulus, rubric, policy
+ * and gates together. Still read (runs recorded with it must keep comparing), never
+ * written. See `scenarioDigest`.
+ */
 export declare const SCENARIO_PREFIX = "scenario:";
 export declare const FIXTURE_PREFIX = "fixture:";
+/**
+ * The split: three (four, with gates) digests per scenario, each mapped to the
+ * cheapest tool that can honestly restore freshness.
+ *
+ * | key | contents | drift means | remedy |
+ * |---|---|---|---|
+ * | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
+ * | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
+ * | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
+ * | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (free; judges only flipped reps) |
+ * | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
+ *
+ * Why this matters more than it looks: with one key, lint had exactly one remedy for
+ * any drift — "re-run" — so **correcting a rubric cost model spend**. Measured on the
+ * reference corpus, two parked branches (one needle, one checklist rewrite) demanded
+ * 135 rep-executions to restore freshness while producing zero new information about
+ * the models. A gate that charges that much to fix a known-bad rubric is pressure to
+ * leave the rubric in place, which inverts the point of having a gate.
+ *
+ * The strictness is unchanged: every edit still marks something stale. Only the price
+ * of getting back to fresh changed.
+ */
+export declare const STIMULUS_PREFIX = "stimulus:";
+export declare const RUBRIC_PREFIX = "rubric:";
+export declare const POLICY_PREFIX = "policy:";
+export declare const GATES_PREFIX = "gates:";
+/**
+ * The spec-level rubric key. `__persona` cannot collide with a scenario id: ids are
+ * validated as `[A-Za-z][A-Za-z0-9_-]*`, so none can begin with an underscore.
+ */
+export declare const PERSONA_KEY = "rubric:__persona";
 /**
  * Recorded in place of a hash when a source existed but could not be read.
  *
@@ -65,14 +101,24 @@ export declare function fileSha256(path: string): string | null;
  * normalised so a Linux-recorded hash still matches on Windows.
  */
 export declare function dirSha256(dir: string): string | null;
+export declare function stimulusDigest(s: Scenario): string;
+export declare function rubricDigest(s: Scenario): string;
+export declare function policyDigest(s: Scenario): string;
+/** Null when the scenario declares no needle gates — no key is recorded for it. */
+export declare function gatesDigest(s: Scenario): string | null;
+/** The spec-level judge persona, which is rubric for every scenario at once. */
+export declare function personaDigest(persona: string): string;
 /**
- * A scenario's semantic digest: everything that changes what the scenario
- * measures, and nothing that doesn't.
+ * The pre-0.4.0 combined digest: everything about a scenario in one hash.
+ *
+ * **Read-only now** — `sourceHashes` writes the four split keys instead. Kept because
+ * `lint` must still compare runs that recorded `scenario:<id>`, and those runs are
+ * every scorecard published before 0.4.0. Deleting it would turn "no findings" into
+ * "no comparison" for the entire existing corpus, silently.
  *
- * Built from the parsed scenario rather than its YAML text, so formatting is
- * irrelevant. `critical` is included because it changes whether the scenario can
- * block a ship; `title` is included because it is what a reader of the scorecard
- * believes was tested.
+ * Its bytes must therefore never change again: this is a stored-hash format, not an
+ * implementation detail. The facet digests take new fields; this one is frozen at the
+ * 0.3.x field set, which is why it does not go through `facets()`.
  */
 export declare function scenarioDigest(s: Scenario): string;
 /**
@@ -89,6 +135,13 @@ export interface SourceContext {
     skillDir: string;
     specDir: string;
     scenarios: Scenario[];
+    /**
+     * The spec's `judge_persona`. Required rather than optional: an optional field that
+     * silently disables the `rubric:__persona` comparison is the blind-spot shape this
+     * module exists to prevent, and making it required lets the compiler find every
+     * caller instead of leaving one quietly un-checked.
+     */
+    judgePersona: string;
 }
 /**
  * Hash every source this run measures: SKILL.md, each distinct
@@ -111,5 +164,13 @@ export declare function sourceHashes(ctx: SourceContext): Record<string, string>
 export declare function currentHashFor(key: string, ctx: SourceContext): string | null | undefined;
 /** Human label for a recorded key, used in lint messages. */
 export declare function describeSourceKey(key: string): string;
+/**
+ * The cheapest command that honestly restores freshness for this key kind.
+ *
+ * This string is the feature. Before the split, lint's only remedy was "re-run", so a
+ * one-word checklist fix cost a full model pass — pressure to leave a known-bad rubric
+ * in place. Naming the actual remedy is what converts that into a free command.
+ */
+export declare function remedyForKey(key: string): string;
 /** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
 export declare function scenarioIdForKey(key: string, scenarios: Scenario[]): string | undefined;