@skill-harness/core 0.4.0 → 0.6.0

package/dist/regrade.js

@@ -1,7 +1,7 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, } from "./results.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
@@ -46,16 +46,18 @@ export async function judgeOneRep(opts) {
     return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
 }
 /**
- * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
- * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
- * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
- * (override/note empty; the caller merges any prior override + persists).
+ * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
+ * harness re-run. Rewrites the judge-raw artifact per rep, emits per-rep
+ * judge-verdict (+ misfire-flag) journal events, and returns the aggregated
+ * ScenarioResult (override/note empty; the caller merges any prior override +
+ * persists).
  */
 export async function regradeScenario(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
-    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, "green");
+    const mode = opts.mode ?? "green";
+    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
     if (files.length === 0)
-        throw new Error(`no green transcripts for ${opts.scenario.id} in ${opts.runDir}`);
+        throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
     const repCount = files.length;
     const outcomes = [];
     for (const file of files) {
@@ -63,19 +65,19 @@ export async function regradeScenario(opts) {
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
-            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode: "green", rep, now,
+            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now,
         }));
     }
     return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
 }
 /**
- * Re-judge every green-transcript scenario in a run dir with `judge` — no
- * harness re-run. Targets are the run's RECORDED scenarios (falling back to
- * the spec for a run with no prior results.yaml), so re-grading rewrites the
- * whole results.yaml consistently with what the run actually recorded. Each
- * target must still exist in the spec (for its checklist) AND have a green
- * transcript on disk; anything missing fails fast before spending any judge
- * calls. Preserves each prior scenario's override/note, rewrites
+ * Re-judge every scenario in a run dir that has a transcript for the run's own
+ * mode, with `judge` — no harness re-run. Targets are the run's RECORDED scenarios
+ * (falling back to the spec for a run with no prior results.yaml), so re-grading
+ * rewrites the whole results.yaml consistently with what the run actually recorded.
+ * Each target must still exist in the spec (for its checklist) AND have a
+ * transcript on disk for that mode; anything missing fails fast before spending
+ * any judge calls. Preserves each prior scenario's override/note, rewrites
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
@@ -107,9 +109,9 @@ export async function regradeRun(opts) {
             return prev;
         }
     }
-    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, "green").length === 0);
+    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, mode).length === 0);
     if (missing.length === targets.length) {
-        throw new Error(`no green transcripts in ${runDir} — nothing to re-grade`);
+        throw new Error(`no ${mode} transcripts in ${runDir} — nothing to re-grade`);
     }
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
@@ -126,15 +128,20 @@ export async function regradeRun(opts) {
         const prevScenario = prev?.scenarios.find((s) => s.id === id);
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
-            runDir, spec, scenario, adapter, judge, specDir, threshold, now,
+            runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
         });
         const carry = overrides.get(id);
         scenarioResults.push({ ...rr, override: carry?.override ?? null, note: carry?.note ?? "" });
     }
-    const ctx = mode === "green" && !prev?.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: prev?.harness ?? "pi",
+        // The harness CLI that produced these transcripts, carried verbatim: a re-grade
+        // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
+        // here would credit the old transcripts to a version that never ran them.
+        harness_cli_version: prev?.harness_cli_version,
+        delivery_canary: prev?.delivery_canary,
         model: prev?.model ?? "unknown",
         judge: { provider: judge.provider, model: judge.model },
         timestamp: prev?.timestamp ?? now(),

package/dist/report.d.ts

@@ -20,15 +20,28 @@ export interface RunColumn {
         flakiness?: number;
         override: string | null;
         note: string;
+        /**
+         * Run-over-run history for this cell, derived from the tag's other runs in the
+         * SAME mode. Present only for a cell that flipped (`state: "boundary"`) — a
+         * marker on every cell would bury the one signal it exists to show, and the
+         * per-cell `flakiness` beside it is a within-run number that cannot see this.
+         */
+        stability?: {
+            flips: number;
+            compared: number;
+            volatility: number | null;
+            note: string;
+        };
     }>;
     /**
-     * Red-vs-green lift for this model, when the tag has both a red baseline and a
-     * green run. Undefined means "never measured" — which is not the same claim as
-     * a zero lift, so the report must not render a 0 for it.
+     * Baseline-vs-skill lift for this model, when the tag has both a red baseline and
+     * a skill-delivered run (green or force). Undefined means "never measured" —
+     * which is not the same claim as a zero lift, so the report must not render a 0
+     * for it.
      *
-     * Only set when THIS column is the green run the lift was computed from (see
+     * Only set when THIS column is the skill-side run the lift was computed from (see
      * collectReport): the review UI recomputes lift from the column's live cells,
-     * which is only valid if those cells are the green side of the comparison.
+     * which is only valid if those cells are the skill side of the comparison.
      */
     lift?: Lift;
     liftHeadline?: string;
@@ -82,6 +95,18 @@ export declare function publicView(data: ReportData): {
             flakiness?: number;
             override: string | null;
             note: string;
+            /**
+             * Run-over-run history for this cell, derived from the tag's other runs in the
+             * SAME mode. Present only for a cell that flipped (`state: "boundary"`) — a
+             * marker on every cell would bury the one signal it exists to show, and the
+             * per-cell `flakiness` beside it is a within-run number that cannot see this.
+             */
+            stability?: {
+                flips: number;
+                compared: number;
+                volatility: number | null;
+                note: string;
+            };
         }>;
     }[];
 };

package/dist/report.js

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import { loadSpec } from "./spec.js";
 import { readResults } from "./results.js";
 import { collectLift, liftHeadline } from "./lift.js";
+import { boundaryCells, collectStability, stabilityNote } from "./stability.js";
 /** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
 function latestRunDir(tagDir) {
     if (!statSync(tagDir).isDirectory())
@@ -24,6 +25,9 @@ export function collectReport(skillDir) {
     const resultsRoot = join(skillDir, "tests", "results");
     // Lift is keyed by model tag, the same key columns are built from.
     const liftByTag = new Map(collectLift(skillDir).map((l) => [l.tag, l]));
+    // Stability is keyed by tag + mode + scenario: a column shows one delivery mode, and
+    // green and force histories are never one series (placement moves verdicts).
+    const boundaryByCell = new Map(boundaryCells(collectStability(skillDir)).map((c) => [`${c.tag}\u0000${c.mode}\u0000${c.id}`, c]));
     const columns = [];
     if (existsSync(resultsRoot)) {
         const tags = readdirSync(resultsRoot)
@@ -35,9 +39,19 @@ export function collectReport(skillDir) {
             if (!runDir)
                 continue;
             const r = readResults(runDir);
+            const tagName = tagDir.split("/").pop();
             const cells = {};
             for (const s of r.scenarios) {
+                const boundary = boundaryByCell.get(`${tagName}\u0000${r.mode}\u0000${s.id}`);
                 cells[s.id] = {
+                    ...(boundary
+                        ? {
+                            stability: {
+                                flips: boundary.flips, compared: boundary.compared,
+                                volatility: boundary.volatility, note: stabilityNote(boundary),
+                            },
+                        }
+                        : {}),
                     judge_verdict: s.judge_verdict,
                     judge_reason: s.judge_reason,
                     suspect: s.suspect ?? false, // suspect defaults false for older results that predate the field
@@ -49,14 +63,15 @@ export function collectReport(skillDir) {
                     note: s.note,
                 };
             }
-            const tag = tagDir.split("/").pop();
-            // A column is the tag's LATEST run, which is not necessarily the green one —
-            // record a red baseline after a green run and the newest run in the tag is
-            // red. The review UI recomputes lift from `cells` (so author overrides move
+            const tag = tagName;
+            // A column is the tag's LATEST run, which is not necessarily the skill-side
+            // one — record a red baseline after a green run and the newest run in the tag
+            // is red. The review UI recomputes lift from `cells` (so author overrides move
             // it live), so attaching a lift to a column whose cells are the RED run
             // would have it compare red against red and report "no effect" for a skill
             // that in fact gained every scenario. Attach only when this column IS the
-            // green side of the comparison.
+            // skill side of the comparison — matched on the timestamp, which also keeps a
+            // green column from borrowing a force run's lift and vice versa.
             const tagLift = liftByTag.get(tag);
             const lift = tagLift && tagLift.greenTimestamp === r.timestamp ? tagLift : undefined;
             columns.push({

package/dist/rescore.d.ts

@@ -24,6 +24,12 @@ export interface RescoreResult {
  * When the policy changes, the honest move is to recompute the old measurements under it
  * rather than reconcile two numbers in prose — and to record what moved.
  *
+ * It is also how a run gets a grade it never had: every rescore recomputes
+ * `effective_grade` under the current scoring policy, and since 0.5.0 that policy
+ * scores force runs too (see SCORED_MODES). A corpus holding force-mode runs
+ * recorded as "not scored" turns them into real scorecards with `rescore`, at zero
+ * model and zero judge spend — verdict changes are then genuinely optional output.
+ *
  * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
  * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
  * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults } from "./results.js";
+import { readResults, writeResults, scoreContextFor } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -29,6 +29,12 @@ function refreshPolicyHashes(recorded, spec) {
  * When the policy changes, the honest move is to recompute the old measurements under it
  * rather than reconcile two numbers in prose — and to record what moved.
  *
+ * It is also how a run gets a grade it never had: every rescore recomputes
+ * `effective_grade` under the current scoring policy, and since 0.5.0 that policy
+ * scores force runs too (see SCORED_MODES). A corpus holding force-mode runs
+ * recorded as "not scored" turns them into real scorecards with `rescore`, at zero
+ * model and zero judge spend — verdict changes are then genuinely optional output.
+ *
  * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
  * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
  * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
@@ -61,13 +67,15 @@ export function rescoreRun(opts) {
         }
         return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
     });
-    const ctx = prev.mode === "green" && !prev.partial
-        ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
-        : null;
+    const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
         partial: prev.partial,
+        // Provenance of the measurement, not of this rewrite: a rescore re-applies a
+        // threshold to reps the recorded harness already produced.
+        harness_cli_version: prev.harness_cli_version,
+        delivery_canary: prev.delivery_canary,
         // A rescore re-applies the CURRENT policy (thresholds, critical set) to the
         // recorded reps, so `policy:` drift is genuinely resolved by having run this —
         // that is what makes `rescore` the honest remedy lint names for it. Stimulus,

package/dist/results.d.ts

@@ -35,6 +35,37 @@ export interface ResultsFile {
      * writer can forget it.
      */
     harness_version?: string;
+    /**
+     * The version of the harness CLI that produced the transcripts — `pi --version`
+     * for `harness: pi`.
+     *
+     * Provenance for the *delivery*, not for the tool: pi 0.80.x wrapped a `--skill`
+     * prompt with the skill body, pi 0.83.0 switched to progressive disclosure, and
+     * that upgrade silently changed what `--mode green` measured. Two waves of runs
+     * in the reference corpus are indistinguishable from a naked-model baseline, and
+     * the incident is invisible in the artifacts precisely because nothing recorded
+     * which pi ran.
+     *
+     * Written only by `run` (the command that actually invokes the harness) and
+     * carried verbatim by every rewriter — `grade`/`rescore`/`regate` re-decide
+     * verdicts, they do not re-deliver the skill, so re-stamping this field with
+     * today's pi would attribute the old transcripts to a version that never
+     * produced them. Optional: runs recorded before the field existed have none, and
+     * an adapter that cannot report a version writes none rather than guessing.
+     */
+    harness_cli_version?: string;
+    /**
+     * `pass` when this run proved, before spending the wave, that the skill body was
+     * reachable in the model's context (see canary.ts). Absent means the probe was
+     * not asked for — never that it failed, because a failed canary aborts the run
+     * and no results.yaml is written.
+     *
+     * Only green runs can carry it: red delivers nothing by design and force delivers
+     * through the system prompt. It is provenance for the *validity* of a green run,
+     * which is why it lives here rather than only in the journal — `journal.jsonl` is
+     * gitignored, and this claim has to survive a commit.
+     */
+    delivery_canary?: "pass";
     skill: string;
     harness: string;
     model: string;
@@ -60,6 +91,53 @@ export interface ResultsFile {
     effective_grade: GradeSummary;
     scenarios: ScenarioResult[];
 }
+/**
+ * The run modes whose results carry a real grade: the ones where the skill under
+ * test was actually delivered to the model.
+ *
+ * `green` activates the skill through the harness's own mechanism (`pi --skill`);
+ * `force` puts SKILL.md in the system prompt. Both are measurements OF THE SKILL,
+ * so both are scored against the ship bar. `red` is the control — the model with
+ * no skill — and scoring it would produce a ship grade for the thing the skill is
+ * measured against.
+ *
+ * Force was unscored until 0.5.0, when it stopped being an escape hatch and became
+ * a deployment: on pi 0.83.0 `--skill` switched to progressive disclosure (the
+ * description is in context, the body loads on demand — "models don't always do
+ * this"), so skill-as-system-prompt is the delivery a corpus can actually rely on.
+ * Ten committed force runs in the reference corpus read `not scored` for exactly
+ * that reason. Scored directly rather than behind a spec flag or a `--score-force`
+ * opt-in: "was the skill in front of the model?" is a property of the mode, not a
+ * per-repo preference, and a second knob would just be a second thing to forget.
+ *
+ * Consequence to expect, and it is the intended one: a force run recorded before
+ * 0.5.0 carries a "not scored" placeholder grade that a recompute now disagrees
+ * with, so `lint` flags it as stale and `rescore` (free) writes the real grade.
+ *
+ * The two modes are NOT interchangeable measurements of the same thing — placement
+ * changes behavior in both directions (measured on identical skill text: `build` A1
+ * 0/3 → 3/3, `plan` C2 3/3 → 0/3). Anything that plots or compares runs over time
+ * therefore keeps the epochs apart rather than pooling them; see trends.ts.
+ */
+export declare const SCORED_MODES: readonly string[];
+/** Whether a run in this mode delivered the skill, and so has a grade worth computing. */
+export declare function isScoredMode(mode: string): boolean;
+/**
+ * The one place "does this run get a grade?" is decided: the scoring mode gate plus
+ * the `--only` partial gate, in one predicate every writer shares.
+ *
+ * Before 0.5.0 this ternary was open-coded in seven places (run, grade, rescore,
+ * regate, lint, and both review-server writers) — which is how force runs came to
+ * be unscored in all seven at once, and how any future mode would have had to be
+ * remembered seven times.
+ */
+export declare function scoreContextFor(run: {
+    mode: string;
+    partial?: boolean;
+}, spec: {
+    ship_bar: ShipBar;
+    critical: string[];
+}): ScoreContext | null;
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export declare function effectiveThreshold(prevScenario: ScenarioResult | undefined, scenario: Scenario): number;
 /** Everything a caller may set. The grade is computed, never supplied. */

package/dist/results.js

@@ -4,6 +4,53 @@ import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
+/**
+ * The run modes whose results carry a real grade: the ones where the skill under
+ * test was actually delivered to the model.
+ *
+ * `green` activates the skill through the harness's own mechanism (`pi --skill`);
+ * `force` puts SKILL.md in the system prompt. Both are measurements OF THE SKILL,
+ * so both are scored against the ship bar. `red` is the control — the model with
+ * no skill — and scoring it would produce a ship grade for the thing the skill is
+ * measured against.
+ *
+ * Force was unscored until 0.5.0, when it stopped being an escape hatch and became
+ * a deployment: on pi 0.83.0 `--skill` switched to progressive disclosure (the
+ * description is in context, the body loads on demand — "models don't always do
+ * this"), so skill-as-system-prompt is the delivery a corpus can actually rely on.
+ * Ten committed force runs in the reference corpus read `not scored` for exactly
+ * that reason. Scored directly rather than behind a spec flag or a `--score-force`
+ * opt-in: "was the skill in front of the model?" is a property of the mode, not a
+ * per-repo preference, and a second knob would just be a second thing to forget.
+ *
+ * Consequence to expect, and it is the intended one: a force run recorded before
+ * 0.5.0 carries a "not scored" placeholder grade that a recompute now disagrees
+ * with, so `lint` flags it as stale and `rescore` (free) writes the real grade.
+ *
+ * The two modes are NOT interchangeable measurements of the same thing — placement
+ * changes behavior in both directions (measured on identical skill text: `build` A1
+ * 0/3 → 3/3, `plan` C2 3/3 → 0/3). Anything that plots or compares runs over time
+ * therefore keeps the epochs apart rather than pooling them; see trends.ts.
+ */
+export const SCORED_MODES = ["green", "force"];
+/** Whether a run in this mode delivered the skill, and so has a grade worth computing. */
+export function isScoredMode(mode) {
+    return SCORED_MODES.includes(mode);
+}
+/**
+ * The one place "does this run get a grade?" is decided: the scoring mode gate plus
+ * the `--only` partial gate, in one predicate every writer shares.
+ *
+ * Before 0.5.0 this ternary was open-coded in seven places (run, grade, rescore,
+ * regate, lint, and both review-server writers) — which is how force runs came to
+ * be unscored in all seven at once, and how any future mode would have had to be
+ * remembered seven times.
+ */
+export function scoreContextFor(run, spec) {
+    if (!isScoredMode(run.mode) || run.partial)
+        return null;
+    return { shipBar: spec.ship_bar, critical: spec.critical };
+}
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export function effectiveThreshold(prevScenario, scenario) {
     return prevScenario?.pass_threshold ?? scenario.passThreshold ?? 0.5;
@@ -56,6 +103,10 @@ export function finalizeResults(draft, ctx) {
         // `grade`, `rescore` and the review UI's override save all record which tool
         // produced the record they leave behind.
         harness_version: HARNESS_VERSION,
+        // Omitted rather than written as null when absent: a run whose adapter could
+        // not report a version must not look like one that reported "nothing".
+        ...(draft.harness_cli_version ? { harness_cli_version: draft.harness_cli_version } : {}),
+        ...(draft.delivery_canary ? { delivery_canary: draft.delivery_canary } : {}),
         skill: draft.skill,
         harness: draft.harness,
         model: draft.model,

package/dist/run.d.ts

@@ -2,6 +2,7 @@ import type { Spec } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ResultsFile } from "./results.js";
 import { type Lift } from "./lift.js";
+import { type ScenarioStability } from "./stability.js";
 export interface RunOptions {
     spec: Spec;
     skillDir: string;
@@ -24,6 +25,13 @@ export interface RunOptions {
      * ship-graded: a subset passing says nothing about the ship bar.
      */
     only?: string[];
+    /**
+     * Green mode only: spend ONE probe up front proving the skill reaches the model,
+     * and abort the run if it doesn't (see canary.ts). Off by default — it costs a
+     * rep, and the deterministic half of this failure class (a skill dir that isn't
+     * there) is already refused by the adapter for free.
+     */
+    canary?: boolean;
 }
 export interface RunSummary {
     runDir: string;
@@ -48,5 +56,12 @@ export declare function hasEmptyAssistantTurn(transcript: string): boolean;
  * exists. Passing it for a green run turns the scorecard from "the skill scored
  * B" into "the skill *did* this much" — without a baseline the grade alone can't
  * distinguish a skill that works from a model that never needed it.
+ *
+ * `stability` is the run-over-run half of the same argument, and it is derived from
+ * history rather than from this run: a cell that flipped its verdict between the last
+ * two runs of this skill × model × mode is worth less than the ✓ beside it suggests,
+ * and this run's own `flaky 0.00` cannot say so — it only ever looked at one run. Pass
+ * the cells for THIS tag and mode; anything else would report another model's history
+ * under this model's scorecard.
  */
-export declare function formatScorecard(summary: RunSummary, lift?: Lift): string;
+export declare function formatScorecard(summary: RunSummary, lift?: Lift, stability?: ScenarioStability[]): string;

package/dist/run.js

@@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -10,6 +10,8 @@ import { createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
+import { runDeliveryCanary, canaryFailure } from "./canary.js";
+import { boundaryCells, stabilityNote } from "./stability.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -36,12 +38,52 @@ export async function runSkillModel(opts) {
     const runDir = runDirFor(skillDir, adapter.name, model, timestamp);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
+    // Which harness CLI delivered the skill, asked once per run and recorded with the
+    // numbers. A pi upgrade (0.80.x → 0.83.0) silently changed what green mode
+    // measures, and the incident was invisible in the artifacts because nothing wrote
+    // this down. Never fatal: `null` means the adapter couldn't say.
+    const harnessCliVersion = (await adapter.version?.()) ?? null;
     appendJournal(runDir, {
         event: "run-started", ts: now(),
         skill: spec.skill, harness: adapter.name, model: opts.modelToken,
+        harness_cli_version: harnessCliVersion,
         judge: { provider: judge.provider, model: judge.model },
         mode, label: opts.label ?? null,
     });
+    // The canary spends one probe before the wave, so a run that isn't measuring the
+    // skill dies for the price of a rep instead of producing a plausible scorecard.
+    // Green only: red delivers nothing by design, and force delivers through the
+    // system prompt, which needs no probe.
+    let canaryStatus = null;
+    if (opts.canary && mode !== "green") {
+        // Ignoring a flag silently is a small version of the bug this whole feature is
+        // about. Say it, and say why it isn't needed.
+        log(`  --canary ignored in mode=${mode} — ${mode === "force" ? "the system prompt delivers the skill unconditionally" : "a baseline delivers no skill by design"}`);
+    }
+    if (opts.canary && mode === "green") {
+        const probeCwd = createWorkspace("none", { specDir: dirname(opts.specPath) });
+        let canary;
+        try {
+            canary = await runDeliveryCanary({
+                adapter, model, skillDir, skillName: spec.skill, cwd: probeCwd.cwd,
+            });
+        }
+        finally {
+            probeCwd.cleanup();
+        }
+        appendJournal(runDir, {
+            event: "delivery-canary", ts: now(),
+            status: canary.status, anchor: canary.anchor, detail: canary.detail,
+        });
+        if (canary.status === "fail")
+            throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
+        if (canary.status === "skipped")
+            log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+        else {
+            canaryStatus = "pass";
+            log(`  ✓ delivery canary — the model quoted its skill instructions back (\`${canary.anchor}\`)`);
+        }
+    }
     // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
     const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
     const owners = [];
@@ -61,10 +103,12 @@ export async function runSkillModel(opts) {
         const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
-    const ctx = mode === "green" && !partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = scoreContextFor({ mode, partial }, spec);
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: adapter.name,
+        harness_cli_version: harnessCliVersion ?? undefined,
+        delivery_canary: canaryStatus ?? undefined,
         model: opts.modelToken,
         judge: { provider: judge.provider, model: judge.model },
         timestamp,
@@ -208,8 +252,15 @@ async function runRep(scenario, rep, repCount, ctx) {
  * exists. Passing it for a green run turns the scorecard from "the skill scored
  * B" into "the skill *did* this much" — without a baseline the grade alone can't
  * distinguish a skill that works from a model that never needed it.
+ *
+ * `stability` is the run-over-run half of the same argument, and it is derived from
+ * history rather than from this run: a cell that flipped its verdict between the last
+ * two runs of this skill × model × mode is worth less than the ✓ beside it suggests,
+ * and this run's own `flaky 0.00` cannot say so — it only ever looked at one run. Pass
+ * the cells for THIS tag and mode; anything else would report another model's history
+ * under this model's scorecard.
  */
-export function formatScorecard(summary, lift) {
+export function formatScorecard(summary, lift, stability) {
     const { results } = summary;
     const g = results.effective_grade;
     const lines = [];
@@ -226,16 +277,32 @@ export function formatScorecard(summary, lift) {
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";
     lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
-    // Lift is a statement about a green run. On a red run the caller may still have
-    // a lift in hand (a green run exists in the same tag), but printing it under a
-    // baseline scorecard reads as if the baseline itself gained something.
-    if (lift && results.mode === "green") {
+    // Lift is a statement about a skill-delivered run (green or force). On a red run
+    // the caller may still have a lift in hand (a scored run exists in the same tag),
+    // but printing it under a baseline scorecard reads as if the baseline itself
+    // gained something.
+    if (lift && isScoredMode(results.mode)) {
         lines.push(`  LIFT:  ${liftHeadline(lift)}  (vs red baseline ${lift.redTimestamp})`);
     }
-    else if (results.mode === "green") {
+    else if (isScoredMode(results.mode)) {
         // The grade alone can't answer "does this skill do anything?", so say how.
         lines.push(`  LIFT:  no red baseline — run with --mode red to measure what the skill adds`);
     }
+    // Said on the scorecard, not just in the docs: the one thing that can invalidate
+    // a green number is invisible in the number. `harness_cli_version` is recorded
+    // beside the verdicts so a reader can tell which pi produced them.
+    if (results.mode === "green" && !results.delivery_canary) {
+        lines.push(`  NOTE:  green delivery is harness-version-dependent` +
+            (results.harness_cli_version ? ` (${results.harness} ${results.harness_cli_version})` : "") +
+            ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +
+            ` Use --mode force for delivery that cannot silently degrade, or --canary to prove it per run.`);
+    }
+    // Boundary cells last, because they qualify the verdicts above: a ✓ on a cell that
+    // flipped between the last two runs is one draw, whatever its rep count said.
+    const ran = new Set(results.scenarios.map((s) => s.id));
+    for (const s of boundaryCells(stability ?? []).filter((c) => ran.has(c.id))) {
+        lines.push(`  ⇄ ${stabilityNote(s)}`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run.js.map

package/dist/sources.d.ts

@@ -172,5 +172,31 @@ export declare function describeSourceKey(key: string): string;
  * in place. Naming the actual remedy is what converts that into a free command.
  */
 export declare function remedyForKey(key: string): string;
+/** The skill-text key. Skill-wide: it belongs to every scenario at once. */
+export declare const SKILL_KEY = "SKILL.md";
+/**
+ * Every recorded key whose drift could change THIS scenario's verdict — excluding the
+ * two skill-wide ones (`SKILL.md`, `rubric:__persona`), which callers handle
+ * separately because they move every scenario at once.
+ *
+ * Written for run-over-run comparison (see stability.ts): "did these two runs ask this
+ * scenario the same question, judged by the same rubric?" is answerable from the
+ * recorded hashes, and only if you know which keys belong to the scenario. Derived
+ * from the spec rather than from the key strings, because the path-shaped keys
+ * (`system_prompt_file`, `post_test`) carry no scenario id at all.
+ *
+ * `policy:<id>` is deliberately NOT here. Its `reps`/`pass_threshold` half is already
+ * compared as an *aggregation* shape (1 draw vs a majority of 3 is the comparison a
+ * hash cannot express), and its `critical` half changes whether a verdict can block a
+ * ship, never what the verdict is. Including it would report a critical-set edit as
+ * "these runs measured different things", which is false.
+ *
+ * Both key generations are returned: 0.4.0+ runs carry the split facet keys, older
+ * ones the combined `scenario:<id>`. A caller comparing two runs must not treat a
+ * combined digest and a split one as comparable — they hash different byte layouts —
+ * so it compares only keys BOTH runs recorded, and treats "no shared key" as
+ * unverifiable rather than unchanged.
+ */
+export declare function scenarioSourceKeys(s: Scenario): string[];
 /** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
 export declare function scenarioIdForKey(key: string, scenarios: Scenario[]): string | undefined;