@skill-harness/core 0.4.0 → 0.5.0

package/dist/adapters/types.d.ts

@@ -32,4 +32,14 @@ export interface HarnessAdapter {
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     judge(req: JudgeReq): Promise<string>;
+    /**
+     * The harness CLI's own version, recorded in `results.yaml` as
+     * `harness_cli_version`. Null when it cannot be determined — a version this
+     * adapter had to guess at is worse than none, because the whole point of the
+     * field is to identify which CLI produced a transcript.
+     *
+     * Optional so a test double or a future adapter need not implement it; callers
+     * treat a missing method exactly like a null answer.
+     */
+    version?(): Promise<string | null>;
 }

package/dist/canary.d.ts

@@ -0,0 +1,44 @@
+import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+/**
+ * The probe target: the longest `## ` heading in the skill's body.
+ *
+ * Body-only by construction — the frontmatter `description` is always in context
+ * under progressive disclosure, so anything quotable from it would pass against a
+ * model that never read the instructions. Longest rather than first because the
+ * check is "did you see this text", and `## Overview` is guessable while
+ * `## Refuse a metered judge, whatever chose it` is not.
+ *
+ * Null when the body has no `## ` heading: there is then nothing to ask for that a
+ * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
+ * worse than none.
+ */
+export declare function deliveryAnchor(skillMd: string): string | null;
+export interface CanaryResult {
+    /** `pass` = the anchor came back; `fail` = it did not; `skipped` = nothing safe to probe for. */
+    status: "pass" | "fail" | "skipped";
+    /** What the probe looked for (null when skipped). */
+    anchor: string | null;
+    /** Why it was skipped, or what the model said instead (trimmed) — for the log and the journal. */
+    detail: string;
+}
+export declare function canaryPrompt(skillName: string, anchor: string): string;
+export interface CanaryOptions {
+    adapter: HarnessAdapter;
+    model: ModelRef;
+    skillDir: string;
+    skillName: string;
+    /** Neutral cwd, same as a scenario gets — the probe must not see a repo either. */
+    cwd: string;
+}
+/**
+ * Run the probe in green mode and report whether the skill body was reachable.
+ *
+ * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
+ * with the reply in `detail`, because "the harness answered without the skill" and
+ * "the model said something odd" are both reasons not to spend a wave. An adapter
+ * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
+ * setup errors with their own messages.
+ */
+export declare function runDeliveryCanary(opts: CanaryOptions): Promise<CanaryResult>;
+/** The abort message for a failed canary: what was measured, and what to do instead. */
+export declare function canaryFailure(skillName: string, result: CanaryResult, cliVersion: string | null): string;

package/dist/canary.js

@@ -0,0 +1,123 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * The delivery canary: one cheap probe, before any scenario runs, that the skill
+ * under test actually reached the model.
+ *
+ * Why this exists. `--mode green` asks the harness to activate the skill, and the
+ * harness can decline without saying so. Measured on pi: 0.80.x wrapped the prompt
+ * with the skill body; 0.83.0 switched to progressive disclosure (only the
+ * description is in context, the body loads on demand — "models don't always do
+ * this", pi's own docs); and a nonexistent `--skill` path is accepted silently,
+ * exit 0 with a normal answer. The reference corpus ran two full waves in that
+ * state: `architect` came back 7/14, ≈ its no-skill baseline, and looked entirely
+ * plausible — the only tell was a contradictory failure mix (over-ceremony AND
+ * capitulation at once) that no single skill edit produces.
+ *
+ * What it can and cannot prove. A pass means the skill's body is *reachable* in
+ * this exact invocation — which is what kills the whole silent-non-delivery class:
+ * a dropped flag, a wrong path, a harness that stopped honoring the mode. It does
+ * NOT prove the body entered context for every later scenario; under progressive
+ * disclosure that is the model's choice per turn, and no probe can promise it. The
+ * mode whose delivery needs no promise is `force` (SKILL.md as system prompt),
+ * which is why `run` recommends it rather than pretending the canary is equivalent.
+ *
+ * Cost: exactly one subject call per run, and only when asked for (`--canary`).
+ * A run that aborts here has spent one rep instead of a wave.
+ */
+/** Frontmatter-stripped body of a SKILL.md. */
+function skillBody(text) {
+    const m = /^---\r?\n[\s\S]*?\r?\n---\r?\n/.exec(text);
+    return m ? text.slice(m[0].length) : text;
+}
+/**
+ * The probe target: the longest `## ` heading in the skill's body.
+ *
+ * Body-only by construction — the frontmatter `description` is always in context
+ * under progressive disclosure, so anything quotable from it would pass against a
+ * model that never read the instructions. Longest rather than first because the
+ * check is "did you see this text", and `## Overview` is guessable while
+ * `## Refuse a metered judge, whatever chose it` is not.
+ *
+ * Null when the body has no `## ` heading: there is then nothing to ask for that a
+ * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
+ * worse than none.
+ */
+export function deliveryAnchor(skillMd) {
+    const headings = [...skillBody(skillMd).matchAll(/^##[ \t]+(.+?)[ \t]*$/gm)].map((m) => m[1].trim());
+    if (headings.length === 0)
+        return null;
+    return headings.reduce((a, b) => (b.length > a.length ? b : a));
+}
+/** Normalize for comparison: case, whitespace runs, and markdown emphasis/backticks. */
+function normalize(s) {
+    return s.toLowerCase().replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
+}
+export function canaryPrompt(skillName, anchor) {
+    // Asks for every heading, not just the anchor, because that is a question a model
+    // WITH the instructions answers naturally and a model without them can only
+    // invent. The instruction to say NOT_AVAILABLE keeps an honest miss from looking
+    // like a refusal to follow format.
+    return (`Answer from the instructions you have loaded — do not perform any task.\n\n` +
+        `List every level-2 markdown heading (lines starting with "## ") in the instructions of ` +
+        `the skill named "${skillName}", verbatim, one per line, with no other text.\n` +
+        `If you have no such instructions available, reply exactly: NOT_AVAILABLE\n\n` +
+        `(The heading text is what matters; keep it exact.)`);
+}
+/**
+ * What the model actually said, per the adapters' shared transcript convention
+ * (`>>> USER:` / `<<< ASSISTANT:`). Falls back to the whole text for an adapter
+ * that doesn't use the markers.
+ */
+function assistantReply(transcript) {
+    const parts = transcript.split(/^<<< ASSISTANT:\s*$/m);
+    return (parts.length > 1 ? parts[parts.length - 1] : transcript).trim();
+}
+/**
+ * Run the probe in green mode and report whether the skill body was reachable.
+ *
+ * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
+ * with the reply in `detail`, because "the harness answered without the skill" and
+ * "the model said something odd" are both reasons not to spend a wave. An adapter
+ * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
+ * setup errors with their own messages.
+ */
+export async function runDeliveryCanary(opts) {
+    const skillMd = readFileSync(join(opts.skillDir, "SKILL.md"), "utf8");
+    const anchor = deliveryAnchor(skillMd);
+    if (!anchor) {
+        return {
+            status: "skipped",
+            anchor: null,
+            detail: `${opts.skillName}/SKILL.md has no \`## \` heading to probe for — nothing a reply could prove`,
+        };
+    }
+    const transcript = await opts.adapter.run({
+        skillDir: opts.skillDir,
+        model: opts.model,
+        mode: "green",
+        turns: [canaryPrompt(opts.skillName, anchor)],
+        cwd: opts.cwd,
+    });
+    const ok = normalize(transcript).includes(normalize(anchor));
+    return {
+        status: ok ? "pass" : "fail",
+        anchor,
+        // The reply, not the transcript: the transcript opens with our own prompt, and a
+        // failure report whose first 400 characters are the question is useless.
+        detail: ok ? "" : assistantReply(transcript).slice(0, 400),
+    };
+}
+/** The abort message for a failed canary: what was measured, and what to do instead. */
+export function canaryFailure(skillName, result, cliVersion) {
+    return (`delivery canary FAILED for ${skillName}: the model could not quote its own skill instructions ` +
+        `(looked for the heading \`${result.anchor}\`).\n` +
+        `  The skill is not reaching the model, so every scenario in this run would measure a naked ` +
+        `model and score like a result. Nothing has been spent beyond this one probe.\n` +
+        `  harness CLI: ${cliVersion ?? "unknown"}. On pi ≥ 0.83.0 \`--skill\` is progressive disclosure ` +
+        `(description in context, body on demand) and a nonexistent path is accepted silently.\n` +
+        `  Fix: re-run with \`--mode force\` (SKILL.md as the system prompt — delivery no version has made ` +
+        `conditional), or check that the skill dir is the one you meant.\n` +
+        `  What the model said instead: ${result.detail || "(nothing)"}`);
+}
+//# sourceMappingURL=canary.js.map

package/dist/discover.d.ts

@@ -8,6 +8,13 @@ export interface DiscoveredSkill {
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
  * SKILL.md. It is testable iff `<skill>/tests/specification.yaml` exists.
  * Returns skills sorted by name (testable or not).
+ *
+ * `dir` and `specPath` are ABSOLUTE, whatever `root` was. They are handed to child
+ * processes that run in a neutral cwd of the harness's choosing (`pi --skill
+ * <dir>`), so a relative `--skills .` used to produce a path that resolved to
+ * nothing over there — and pi accepts a nonexistent `--skill` path silently, exit 0
+ * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
+ * but the honest fix is here, where the path is built.
  */
 export declare function discover(root: string): DiscoveredSkill[];
 /**

package/dist/discover.js

@@ -1,19 +1,27 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 /**
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
  * SKILL.md. It is testable iff `<skill>/tests/specification.yaml` exists.
  * Returns skills sorted by name (testable or not).
+ *
+ * `dir` and `specPath` are ABSOLUTE, whatever `root` was. They are handed to child
+ * processes that run in a neutral cwd of the harness's choosing (`pi --skill
+ * <dir>`), so a relative `--skills .` used to produce a path that resolved to
+ * nothing over there — and pi accepts a nonexistent `--skill` path silently, exit 0
+ * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
+ * but the honest fix is here, where the path is built.
  */
 export function discover(root) {
-    if (!existsSync(root) || !statSync(root).isDirectory()) {
+    const absRoot = resolve(root);
+    if (!existsSync(absRoot) || !statSync(absRoot).isDirectory()) {
         throw new Error(`skills root is not a directory: ${root}`);
     }
     const skills = [];
-    for (const name of readdirSync(root)) {
+    for (const name of readdirSync(absRoot)) {
         if (name.startsWith("."))
             continue;
-        const dir = join(root, name);
+        const dir = join(absRoot, name);
         if (!statSync(dir).isDirectory())
             continue;
         if (!existsSync(join(dir, "SKILL.md")))
@@ -32,7 +40,7 @@ export function discover(root) {
 export function resolveSkill(root, name) {
     const skill = discover(root).find((s) => s.name === name);
     if (!skill) {
-        const dir = join(root, name);
+        const dir = join(resolve(root), name);
         if (existsSync(dir) && statSync(dir).isDirectory() && !existsSync(join(dir, "SKILL.md"))) {
             throw new Error(`skill \`${name}\` has no SKILL.md (looked in ${dir})`);
         }

package/dist/index.d.ts

@@ -25,3 +25,4 @@ export * from "./defaults.js";
 export * from "./judge-policy.js";
 export * from "./regate.js";
 export * from "./downgrade.js";
+export * from "./canary.js";

package/dist/index.js

@@ -25,4 +25,5 @@ export * from "./defaults.js";
 export * from "./judge-policy.js";
 export * from "./regate.js";
 export * from "./downgrade.js";
+export * from "./canary.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -22,12 +22,26 @@ export type JournalEvent = {
     skill: string;
     harness: string;
     model: string;
+    /** The harness CLI's own version (`pi --version`), or null when it could not be asked. */
+    harness_cli_version?: string | null;
     judge: {
         provider: string;
         model: string;
     };
     mode: string;
     label: string | null;
+}
+/**
+ * The pre-flight delivery probe (green mode, `--canary`): did the model quote a
+ * body-only heading of its own skill back? `fail` aborts the run, so a journal
+ * carrying a failed canary is the record of a wave that was NOT spent.
+ */
+ | {
+    event: "delivery-canary";
+    ts: string;
+    status: "pass" | "fail" | "skipped";
+    anchor: string | null;
+    detail: string;
 } | {
     event: "scenario-started";
     ts: string;

package/dist/lift.d.ts

@@ -24,6 +24,19 @@ export interface Lift {
     model: string;
     redTimestamp: string;
     greenTimestamp: string;
+    /**
+     * How the skill was delivered on the non-baseline side: `green` (the harness
+     * activated it) or `force` (SKILL.md as the system prompt).
+     *
+     * The `green*` field names are kept as the *skill-active side*, not as a claim
+     * about the mode — they are the wire format the committed report assets and the
+     * review UI read, and renaming them would break every published report to say
+     * something the `mode` field already says. A red baseline is mode-independent
+     * (`--no-skills` either way), so red-vs-force is as valid a comparison as
+     * red-vs-green — but which one you are looking at changes what the number means,
+     * so it is recorded rather than implied.
+     */
+    mode: string;
     /** Scenario ids present in both runs — the only ones a lift can speak to. */
     compared: number;
     gained: number;

package/dist/lift.js

@@ -1,6 +1,6 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, effectiveVerdicts } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
 import { loadSpec } from "./spec.js";
 function aggregationShape(s) {
     const reps = s.reps ?? 1;
@@ -98,6 +98,7 @@ export function computeLift(red, green, opts = {}) {
     return {
         tag: "",
         model: green.model,
+        mode: green.mode,
         redTimestamp: red.timestamp,
         greenTimestamp: green.timestamp,
         compared: Object.keys(cells).length,
@@ -193,7 +194,7 @@ function isDir(p) {
 }
 /**
  * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
- * with the most recent green run and compute the lift.
+ * with the most recent skill-delivered run (green or force) and compute the lift.
  *
  * Deliberately derived on read rather than persisted into results.yaml: a lift
  * is a fact about a *pair* of runs, so caching it inside one run's file would go
@@ -237,9 +238,13 @@ export function collectLift(skillDir) {
             .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
             .sort(); // timestamp-slug names ⇒ chronological ascending
         // Mode is only knowable after reading results.yaml, so every run in the tag
-        // is read; last-wins gives the most recent of each mode.
+        // is read; last-wins gives the most recent baseline and the most recent
+        // skill-delivered run. The skill side is whichever scored mode ran most
+        // recently — a corpus that moved from green to force delivery should see its
+        // lift follow, and the baseline it is measured against is the same either way
+        // (`--no-skills` in both).
         let red;
-        let green;
+        let skillOn;
         for (const rd of runDirs) {
             let r;
             try {
@@ -252,12 +257,12 @@ export function collectLift(skillDir) {
             }
             if (r.mode === "red")
                 red = r;
-            else if (r.mode === "green")
-                green = r;
+            else if (isScoredMode(r.mode))
+                skillOn = r;
         }
-        if (!red || !green)
+        if (!red || !skillOn)
             continue;
-        lifts.push({ ...computeLift(red, green, { modeInsensitive }), tag });
+        lifts.push({ ...computeLift(red, skillOn, { modeInsensitive }), tag });
     }
     return lifts;
 }

package/dist/lint.js

@@ -2,7 +2,7 @@ import { existsSync, statSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, resolve } from "node:path";
 import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
-import { readResults, finalizeResults, findTranscriptFiles, resultsPath } from "./results.js";
+import { readResults, finalizeResults, findTranscriptFiles, resultsPath, scoreContextFor } from "./results.js";
 import { currentHashFor, describeSourceKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
 import { downgradeWarning } from "./downgrade.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
@@ -163,10 +163,17 @@ export function lintSkill(skillDir) {
             // can actually re-score. Override/transcript rules below still apply.
             const specIds = new Set(spec.scenarios.map((sc) => sc.id));
             const sameSet = r.scenarios.length === specIds.size && r.scenarios.every((sc) => specIds.has(sc.id));
-            const ctx = r.mode === "green" && !r.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+            const ctx = scoreContextFor(r, spec);
             const recomputed = !sameSet ? null : finalizeResults({ skill: r.skill, harness: r.harness, model: r.model, judge: r.judge, timestamp: r.timestamp, label: r.label, mode: r.mode, partial: r.partial, source_hashes: r.source_hashes, scenarios: r.scenarios }, ctx).effective_grade;
             if (recomputed && JSON.stringify(recomputed) !== JSON.stringify(r.effective_grade)) {
-                findings.push({ skill, code: "consistency", message: `results.yaml effective_grade is stale in ${runDir} (recompute differs)` });
+                // The remedy is named because this finding now has a benign, expected cause
+                // as well as a suspicious one: a force run recorded before 0.5.0 carries a
+                // "not scored" placeholder, and today's policy scores it (see SCORED_MODES).
+                // `rescore` is free and offline, so the fix is never a reason to re-measure.
+                findings.push({
+                    skill, code: "consistency",
+                    message: `results.yaml effective_grade is stale in ${runDir} (recompute differs) — re-apply the current scoring policy: rescore (free, offline)`,
+                });
             }
             for (const s of r.scenarios) {
                 if (s.override != null) {

package/dist/regate.js

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import { parseVerdict, detectMisfire } from "./grade.js";
 import { evaluateNeedleGates, hasNeedleGates } from "./seeded.js";
 import { judgeOneRep } from "./regrade.js";
-import { readResults, writeResults, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, effectiveThreshold, } from "./results.js";
+import { readResults, writeResults, transcriptPath, judgeRawPath, repIndexOf, findDiffFiles, effectiveThreshold, scoreContextFor, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { gatesDigest, GATES_PREFIX } from "./sources.js";
@@ -32,8 +32,8 @@ function rewriteTranscript(path, gateLines) {
     writeFileSync(path, `${head}${TRAILER}\n${gateLines.join("\n")}\n\n${tail}`, "utf8");
 }
 /** Recover a rep's judge verdict from its saved judge-raw artifact — free, and exact. */
-function verdictFromSavedJudgement(runDir, id, rep) {
-    const path = judgeRawPath(runDir, id, "green", rep);
+function verdictFromSavedJudgement(runDir, id, mode, rep) {
+    const path = judgeRawPath(runDir, id, mode, rep);
     if (!existsSync(path))
         return null;
     const raw = readFileSync(path, "utf8");
@@ -72,6 +72,10 @@ function verdictFromSavedJudgement(runDir, id, rep) {
 export async function regateRun(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
     const prev = readResults(opts.runDir);
+    // The run's own mode names its artifacts (`<id>.<mode>[.rep<k>].diff.txt`). Read
+    // from the record rather than assumed green: force runs are scored measurements
+    // too, and looking for green artifacts under a force run finds nothing at all.
+    const mode = prev.mode;
     const specById = new Map(opts.spec.scenarios.map((s) => [s.id, s]));
     // Why a scenario cannot be regated, collected rather than thrown one at a time: a
     // mixed spec (needles here, vitest there) should regate what it can, and only a run
@@ -87,7 +91,7 @@ export async function regateRun(opts) {
                 `no saved artifact can stand in for it, so this scenario needs a re-run`);
             continue;
         }
-        if (findDiffFiles(opts.runDir, s.id, "green").length === 0) {
+        if (findDiffFiles(opts.runDir, s.id, mode).length === 0) {
             blocked.push(`${s.id}: no staged-diff artifact on disk (\`.diff.txt\` is gitignored — regate needs the run dir that produced it)`);
             continue;
         }
@@ -106,7 +110,7 @@ export async function regateRun(opts) {
             scenarios.push(rec); // untouched: not regatable, or no gates
             continue;
         }
-        const diffFiles = findDiffFiles(opts.runDir, scenario.id, "green");
+        const diffFiles = findDiffFiles(opts.runDir, scenario.id, mode);
         const outcomes = [];
         // Per scenario, not run-wide: with several regated scenarios, a global counter
         // would report every change as "re-judged" because some other scenario was.
@@ -116,7 +120,7 @@ export async function regateRun(opts) {
             const rep = repIndexOf(file) ?? undefined;
             const diff = readFileSync(join(opts.runDir, file), "utf8");
             const gate = evaluateNeedleGates(scenario, diff);
-            const tPath = transcriptPath(opts.runDir, scenario.id, "green", rep);
+            const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
             const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
             const oldGateFailed = GATE_FAILED_RE.test(before.slice(before.indexOf(TRAILER)));
             // The trailer is regenerated whatever the outcome: leaving a stale
@@ -132,7 +136,7 @@ export async function regateRun(opts) {
             if (!oldGateFailed) {
                 // The judge already saw this rep. Its verdict is on disk — re-read it rather
                 // than paying to ask the same question again.
-                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, rep);
+                const saved = verdictFromSavedJudgement(opts.runDir, scenario.id, mode, rep);
                 outcomes.push(saved ?? { verdict: rec.judge_verdict, reason: rec.judge_reason, suspect: rec.suspect });
                 continue;
             }
@@ -141,7 +145,7 @@ export async function regateRun(opts) {
             outcomes.push(await judgeOneRep({
                 runDir: opts.runDir, spec: opts.spec, scenario, transcript,
                 adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                mode: "green", rep, now,
+                mode, rep, now,
             }));
             judgeCalls++;
             judgedHere++;
@@ -160,11 +164,11 @@ export async function regateRun(opts) {
             });
         }
     }
-    const ctx = prev.mode === "green" && !prev.partial
-        ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
-        : null;
+    const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         skill: prev.skill, harness: prev.harness, model: prev.model,
+        // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
+        harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
         judge: { provider: opts.judge.provider, model: opts.judge.model },
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode, partial: prev.partial,
         // Only the `gates:` keys of the scenarios actually re-evaluated. Stimulus, rubric

package/dist/regrade.d.ts

@@ -23,6 +23,15 @@ export interface RegradeOptions {
     judge: ModelRef;
     specDir: string;
     threshold: number;
+    /**
+     * Which mode's saved transcripts to re-judge — the run's own mode, since
+     * transcript filenames are `<id>.<mode>[.rep<k>].txt`.
+     *
+     * Defaults to `green` for callers that predate force being a scored mode. A
+     * force-mode run whose transcripts were looked up as green found none and failed
+     * with "nothing to re-grade", which is how ten scorable runs stayed ungraded.
+     */
+    mode?: string;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -39,10 +48,11 @@ export declare function judgeOneRep(opts: {
     now: () => string;
 }): Promise<RepOutcome>;
 /**
- * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
- * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
- * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
- * (override/note empty; the caller merges any prior override + persists).
+ * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
+ * harness re-run. Rewrites the judge-raw artifact per rep, emits per-rep
+ * judge-verdict (+ misfire-flag) journal events, and returns the aggregated
+ * ScenarioResult (override/note empty; the caller merges any prior override +
+ * persists).
  */
 export declare function regradeScenario(opts: RegradeOptions): Promise<ScenarioResult>;
 export interface RegradeRunOptions {
@@ -60,13 +70,13 @@ export interface RegradeRunOptions {
     onlySuspect?: boolean;
 }
 /**
- * Re-judge every green-transcript scenario in a run dir with `judge` — no
- * harness re-run. Targets are the run's RECORDED scenarios (falling back to
- * the spec for a run with no prior results.yaml), so re-grading rewrites the
- * whole results.yaml consistently with what the run actually recorded. Each
- * target must still exist in the spec (for its checklist) AND have a green
- * transcript on disk; anything missing fails fast before spending any judge
- * calls. Preserves each prior scenario's override/note, rewrites
+ * Re-judge every scenario in a run dir that has a transcript for the run's own
+ * mode, with `judge` — no harness re-run. Targets are the run's RECORDED scenarios
+ * (falling back to the spec for a run with no prior results.yaml), so re-grading
+ * rewrites the whole results.yaml consistently with what the run actually recorded.
+ * Each target must still exist in the spec (for its checklist) AND have a
+ * transcript on disk for that mode; anything missing fails fast before spending
+ * any judge calls. Preserves each prior scenario's override/note, rewrites
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */

package/dist/regrade.js

@@ -1,7 +1,7 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { buildJudgePrompt, judgeInWorkspace } from "./grade.js";
-import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, } from "./results.js";
+import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResults, effectiveThreshold, scoreContextFor, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
@@ -46,16 +46,18 @@ export async function judgeOneRep(opts) {
     return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
 }
 /**
- * Re-judge a scenario's saved GREEN transcript(s) with `judge` — no harness
- * re-run. Rewrites the judge-raw artifact per rep, emits per-rep judge-verdict
- * (+ misfire-flag) journal events, and returns the aggregated ScenarioResult
- * (override/note empty; the caller merges any prior override + persists).
+ * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
+ * harness re-run. Rewrites the judge-raw artifact per rep, emits per-rep
+ * judge-verdict (+ misfire-flag) journal events, and returns the aggregated
+ * ScenarioResult (override/note empty; the caller merges any prior override +
+ * persists).
  */
 export async function regradeScenario(opts) {
     const now = opts.now ?? (() => new Date().toISOString());
-    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, "green");
+    const mode = opts.mode ?? "green";
+    const files = findTranscriptFiles(opts.runDir, opts.scenario.id, mode);
     if (files.length === 0)
-        throw new Error(`no green transcripts for ${opts.scenario.id} in ${opts.runDir}`);
+        throw new Error(`no ${mode} transcripts for ${opts.scenario.id} in ${opts.runDir}`);
     const repCount = files.length;
     const outcomes = [];
     for (const file of files) {
@@ -63,19 +65,19 @@ export async function regradeScenario(opts) {
         const transcript = readFileSync(join(opts.runDir, file), "utf8");
         outcomes.push(await judgeOneRep({
             runDir: opts.runDir, spec: opts.spec, scenario: opts.scenario, transcript,
-            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode: "green", rep, now,
+            adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir, mode, rep, now,
         }));
     }
     return outcomesToResult(opts.scenario.id, outcomes, repCount, opts.threshold);
 }
 /**
- * Re-judge every green-transcript scenario in a run dir with `judge` — no
- * harness re-run. Targets are the run's RECORDED scenarios (falling back to
- * the spec for a run with no prior results.yaml), so re-grading rewrites the
- * whole results.yaml consistently with what the run actually recorded. Each
- * target must still exist in the spec (for its checklist) AND have a green
- * transcript on disk; anything missing fails fast before spending any judge
- * calls. Preserves each prior scenario's override/note, rewrites
+ * Re-judge every scenario in a run dir that has a transcript for the run's own
+ * mode, with `judge` — no harness re-run. Targets are the run's RECORDED scenarios
+ * (falling back to the spec for a run with no prior results.yaml), so re-grading
+ * rewrites the whole results.yaml consistently with what the run actually recorded.
+ * Each target must still exist in the spec (for its checklist) AND have a
+ * transcript on disk for that mode; anything missing fails fast before spending
+ * any judge calls. Preserves each prior scenario's override/note, rewrites
  * results.yaml, emits the `score` journal event, and returns the new
  * ResultsFile. Shared by `cmdGrade` and the pi-extension's `judge` command.
  */
@@ -107,9 +109,9 @@ export async function regradeRun(opts) {
             return prev;
         }
     }
-    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, "green").length === 0);
+    const missing = targets.filter((id) => !specById.has(id) || findTranscriptFiles(runDir, id, mode).length === 0);
     if (missing.length === targets.length) {
-        throw new Error(`no green transcripts in ${runDir} — nothing to re-grade`);
+        throw new Error(`no ${mode} transcripts in ${runDir} — nothing to re-grade`);
     }
     if (missing.length > 0) {
         throw new Error(`cannot re-grade ${missing.join(", ")} in ${runDir} (transcript missing or scenario no longer in the spec) — re-run instead of grading`);
@@ -126,15 +128,20 @@ export async function regradeRun(opts) {
         const prevScenario = prev?.scenarios.find((s) => s.id === id);
         const threshold = effectiveThreshold(prevScenario, scenario);
         const rr = await regradeScenario({
-            runDir, spec, scenario, adapter, judge, specDir, threshold, now,
+            runDir, spec, scenario, adapter, judge, specDir, threshold, mode, now,
         });
         const carry = overrides.get(id);
         scenarioResults.push({ ...rr, override: carry?.override ?? null, note: carry?.note ?? "" });
     }
-    const ctx = mode === "green" && !prev?.partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = scoreContextFor({ mode, partial: prev?.partial }, spec);
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: prev?.harness ?? "pi",
+        // The harness CLI that produced these transcripts, carried verbatim: a re-grade
+        // re-asks the judge, it does not re-deliver the skill, so stamping today's pi
+        // here would credit the old transcripts to a version that never ran them.
+        harness_cli_version: prev?.harness_cli_version,
+        delivery_canary: prev?.delivery_canary,
         model: prev?.model ?? "unknown",
         judge: { provider: judge.provider, model: judge.model },
         timestamp: prev?.timestamp ?? now(),

package/dist/report.d.ts

@@ -22,13 +22,14 @@ export interface RunColumn {
         note: string;
     }>;
     /**
-     * Red-vs-green lift for this model, when the tag has both a red baseline and a
-     * green run. Undefined means "never measured" — which is not the same claim as
-     * a zero lift, so the report must not render a 0 for it.
+     * Baseline-vs-skill lift for this model, when the tag has both a red baseline and
+     * a skill-delivered run (green or force). Undefined means "never measured" —
+     * which is not the same claim as a zero lift, so the report must not render a 0
+     * for it.
      *
-     * Only set when THIS column is the green run the lift was computed from (see
+     * Only set when THIS column is the skill-side run the lift was computed from (see
      * collectReport): the review UI recomputes lift from the column's live cells,
-     * which is only valid if those cells are the green side of the comparison.
+     * which is only valid if those cells are the skill side of the comparison.
      */
     lift?: Lift;
     liftHeadline?: string;

package/dist/report.js

@@ -50,13 +50,14 @@ export function collectReport(skillDir) {
                 };
             }
             const tag = tagDir.split("/").pop();
-            // A column is the tag's LATEST run, which is not necessarily the green one —
-            // record a red baseline after a green run and the newest run in the tag is
-            // red. The review UI recomputes lift from `cells` (so author overrides move
+            // A column is the tag's LATEST run, which is not necessarily the skill-side
+            // one — record a red baseline after a green run and the newest run in the tag
+            // is red. The review UI recomputes lift from `cells` (so author overrides move
             // it live), so attaching a lift to a column whose cells are the RED run
             // would have it compare red against red and report "no effect" for a skill
             // that in fact gained every scenario. Attach only when this column IS the
-            // green side of the comparison.
+            // skill side of the comparison — matched on the timestamp, which also keeps a
+            // green column from borrowing a force run's lift and vice versa.
             const tagLift = liftByTag.get(tag);
             const lift = tagLift && tagLift.greenTimestamp === r.timestamp ? tagLift : undefined;
             columns.push({

package/dist/rescore.d.ts

@@ -24,6 +24,12 @@ export interface RescoreResult {
  * When the policy changes, the honest move is to recompute the old measurements under it
  * rather than reconcile two numbers in prose — and to record what moved.
  *
+ * It is also how a run gets a grade it never had: every rescore recomputes
+ * `effective_grade` under the current scoring policy, and since 0.5.0 that policy
+ * scores force runs too (see SCORED_MODES). A corpus holding force-mode runs
+ * recorded as "not scored" turns them into real scorecards with `rescore`, at zero
+ * model and zero judge spend — verdict changes are then genuinely optional output.
+ *
  * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
  * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
  * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults } from "./results.js";
+import { readResults, writeResults, scoreContextFor } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -29,6 +29,12 @@ function refreshPolicyHashes(recorded, spec) {
  * When the policy changes, the honest move is to recompute the old measurements under it
  * rather than reconcile two numbers in prose — and to record what moved.
  *
+ * It is also how a run gets a grade it never had: every rescore recomputes
+ * `effective_grade` under the current scoring policy, and since 0.5.0 that policy
+ * scores force runs too (see SCORED_MODES). A corpus holding force-mode runs
+ * recorded as "not scored" turns them into real scorecards with `rescore`, at zero
+ * model and zero judge spend — verdict changes are then genuinely optional output.
+ *
  * Only reps-bearing scenarios can be re-scored: a single-rep verdict has no rate to
  * re-apply a threshold to, and ERROR/JUDGE-AMBIGUOUS carry no trustworthy rate at all —
  * both are carried verbatim. Overrides, notes, and suspect flags are preserved: this
@@ -61,13 +67,15 @@ export function rescoreRun(opts) {
         }
         return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
     });
-    const ctx = prev.mode === "green" && !prev.partial
-        ? { shipBar: opts.spec.ship_bar, critical: opts.spec.critical }
-        : null;
+    const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {
         skill: prev.skill, harness: prev.harness, model: prev.model, judge: prev.judge,
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode,
         partial: prev.partial,
+        // Provenance of the measurement, not of this rewrite: a rescore re-applies a
+        // threshold to reps the recorded harness already produced.
+        harness_cli_version: prev.harness_cli_version,
+        delivery_canary: prev.delivery_canary,
         // A rescore re-applies the CURRENT policy (thresholds, critical set) to the
         // recorded reps, so `policy:` drift is genuinely resolved by having run this —
         // that is what makes `rescore` the honest remedy lint names for it. Stimulus,

package/dist/results.d.ts

@@ -35,6 +35,37 @@ export interface ResultsFile {
      * writer can forget it.
      */
     harness_version?: string;
+    /**
+     * The version of the harness CLI that produced the transcripts — `pi --version`
+     * for `harness: pi`.
+     *
+     * Provenance for the *delivery*, not for the tool: pi 0.80.x wrapped a `--skill`
+     * prompt with the skill body, pi 0.83.0 switched to progressive disclosure, and
+     * that upgrade silently changed what `--mode green` measured. Two waves of runs
+     * in the reference corpus are indistinguishable from a naked-model baseline, and
+     * the incident is invisible in the artifacts precisely because nothing recorded
+     * which pi ran.
+     *
+     * Written only by `run` (the command that actually invokes the harness) and
+     * carried verbatim by every rewriter — `grade`/`rescore`/`regate` re-decide
+     * verdicts, they do not re-deliver the skill, so re-stamping this field with
+     * today's pi would attribute the old transcripts to a version that never
+     * produced them. Optional: runs recorded before the field existed have none, and
+     * an adapter that cannot report a version writes none rather than guessing.
+     */
+    harness_cli_version?: string;
+    /**
+     * `pass` when this run proved, before spending the wave, that the skill body was
+     * reachable in the model's context (see canary.ts). Absent means the probe was
+     * not asked for — never that it failed, because a failed canary aborts the run
+     * and no results.yaml is written.
+     *
+     * Only green runs can carry it: red delivers nothing by design and force delivers
+     * through the system prompt. It is provenance for the *validity* of a green run,
+     * which is why it lives here rather than only in the journal — `journal.jsonl` is
+     * gitignored, and this claim has to survive a commit.
+     */
+    delivery_canary?: "pass";
     skill: string;
     harness: string;
     model: string;
@@ -60,6 +91,53 @@ export interface ResultsFile {
     effective_grade: GradeSummary;
     scenarios: ScenarioResult[];
 }
+/**
+ * The run modes whose results carry a real grade: the ones where the skill under
+ * test was actually delivered to the model.
+ *
+ * `green` activates the skill through the harness's own mechanism (`pi --skill`);
+ * `force` puts SKILL.md in the system prompt. Both are measurements OF THE SKILL,
+ * so both are scored against the ship bar. `red` is the control — the model with
+ * no skill — and scoring it would produce a ship grade for the thing the skill is
+ * measured against.
+ *
+ * Force was unscored until 0.5.0, when it stopped being an escape hatch and became
+ * a deployment: on pi 0.83.0 `--skill` switched to progressive disclosure (the
+ * description is in context, the body loads on demand — "models don't always do
+ * this"), so skill-as-system-prompt is the delivery a corpus can actually rely on.
+ * Ten committed force runs in the reference corpus read `not scored` for exactly
+ * that reason. Scored directly rather than behind a spec flag or a `--score-force`
+ * opt-in: "was the skill in front of the model?" is a property of the mode, not a
+ * per-repo preference, and a second knob would just be a second thing to forget.
+ *
+ * Consequence to expect, and it is the intended one: a force run recorded before
+ * 0.5.0 carries a "not scored" placeholder grade that a recompute now disagrees
+ * with, so `lint` flags it as stale and `rescore` (free) writes the real grade.
+ *
+ * The two modes are NOT interchangeable measurements of the same thing — placement
+ * changes behavior in both directions (measured on identical skill text: `build` A1
+ * 0/3 → 3/3, `plan` C2 3/3 → 0/3). Anything that plots or compares runs over time
+ * therefore keeps the epochs apart rather than pooling them; see trends.ts.
+ */
+export declare const SCORED_MODES: readonly string[];
+/** Whether a run in this mode delivered the skill, and so has a grade worth computing. */
+export declare function isScoredMode(mode: string): boolean;
+/**
+ * The one place "does this run get a grade?" is decided: the scoring mode gate plus
+ * the `--only` partial gate, in one predicate every writer shares.
+ *
+ * Before 0.5.0 this ternary was open-coded in seven places (run, grade, rescore,
+ * regate, lint, and both review-server writers) — which is how force runs came to
+ * be unscored in all seven at once, and how any future mode would have had to be
+ * remembered seven times.
+ */
+export declare function scoreContextFor(run: {
+    mode: string;
+    partial?: boolean;
+}, spec: {
+    ship_bar: ShipBar;
+    critical: string[];
+}): ScoreContext | null;
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export declare function effectiveThreshold(prevScenario: ScenarioResult | undefined, scenario: Scenario): number;
 /** Everything a caller may set. The grade is computed, never supplied. */

package/dist/results.js

@@ -4,6 +4,53 @@ import yaml from "js-yaml";
 import { modelSlug } from "./adapters/types.js";
 import { score } from "./score.js";
 import { HARNESS_VERSION } from "./version.js";
+/**
+ * The run modes whose results carry a real grade: the ones where the skill under
+ * test was actually delivered to the model.
+ *
+ * `green` activates the skill through the harness's own mechanism (`pi --skill`);
+ * `force` puts SKILL.md in the system prompt. Both are measurements OF THE SKILL,
+ * so both are scored against the ship bar. `red` is the control — the model with
+ * no skill — and scoring it would produce a ship grade for the thing the skill is
+ * measured against.
+ *
+ * Force was unscored until 0.5.0, when it stopped being an escape hatch and became
+ * a deployment: on pi 0.83.0 `--skill` switched to progressive disclosure (the
+ * description is in context, the body loads on demand — "models don't always do
+ * this"), so skill-as-system-prompt is the delivery a corpus can actually rely on.
+ * Ten committed force runs in the reference corpus read `not scored` for exactly
+ * that reason. Scored directly rather than behind a spec flag or a `--score-force`
+ * opt-in: "was the skill in front of the model?" is a property of the mode, not a
+ * per-repo preference, and a second knob would just be a second thing to forget.
+ *
+ * Consequence to expect, and it is the intended one: a force run recorded before
+ * 0.5.0 carries a "not scored" placeholder grade that a recompute now disagrees
+ * with, so `lint` flags it as stale and `rescore` (free) writes the real grade.
+ *
+ * The two modes are NOT interchangeable measurements of the same thing — placement
+ * changes behavior in both directions (measured on identical skill text: `build` A1
+ * 0/3 → 3/3, `plan` C2 3/3 → 0/3). Anything that plots or compares runs over time
+ * therefore keeps the epochs apart rather than pooling them; see trends.ts.
+ */
+export const SCORED_MODES = ["green", "force"];
+/** Whether a run in this mode delivered the skill, and so has a grade worth computing. */
+export function isScoredMode(mode) {
+    return SCORED_MODES.includes(mode);
+}
+/**
+ * The one place "does this run get a grade?" is decided: the scoring mode gate plus
+ * the `--only` partial gate, in one predicate every writer shares.
+ *
+ * Before 0.5.0 this ternary was open-coded in seven places (run, grade, rescore,
+ * regate, lint, and both review-server writers) — which is how force runs came to
+ * be unscored in all seven at once, and how any future mode would have had to be
+ * remembered seven times.
+ */
+export function scoreContextFor(run, spec) {
+    if (!isScoredMode(run.mode) || run.partial)
+        return null;
+    return { shipBar: spec.ship_bar, critical: spec.critical };
+}
 /** The pass-threshold a re-grade uses: the run's persisted value, else the spec's per-scenario value, else 0.5. */
 export function effectiveThreshold(prevScenario, scenario) {
     return prevScenario?.pass_threshold ?? scenario.passThreshold ?? 0.5;
@@ -56,6 +103,10 @@ export function finalizeResults(draft, ctx) {
         // `grade`, `rescore` and the review UI's override save all record which tool
         // produced the record they leave behind.
         harness_version: HARNESS_VERSION,
+        // Omitted rather than written as null when absent: a run whose adapter could
+        // not report a version must not look like one that reported "nothing".
+        ...(draft.harness_cli_version ? { harness_cli_version: draft.harness_cli_version } : {}),
+        ...(draft.delivery_canary ? { delivery_canary: draft.delivery_canary } : {}),
         skill: draft.skill,
         harness: draft.harness,
         model: draft.model,

package/dist/run.d.ts

@@ -24,6 +24,13 @@ export interface RunOptions {
      * ship-graded: a subset passing says nothing about the ship bar.
      */
     only?: string[];
+    /**
+     * Green mode only: spend ONE probe up front proving the skill reaches the model,
+     * and abort the run if it doesn't (see canary.ts). Off by default — it costs a
+     * rep, and the deterministic half of this failure class (a skill dir that isn't
+     * there) is already refused by the adapter for free.
+     */
+    canary?: boolean;
 }
 export interface RunSummary {
     runDir: string;

package/dist/run.js

@@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, } from "./results.js";
+import { runDirFor, transcriptPath, diffPath, writeResults, ensureResultsGitignore, scoreContextFor, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -10,6 +10,7 @@ import { createWorkspace } from "./workspace.js";
 import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
+import { runDeliveryCanary, canaryFailure } from "./canary.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -36,12 +37,52 @@ export async function runSkillModel(opts) {
     const runDir = runDirFor(skillDir, adapter.name, model, timestamp);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
+    // Which harness CLI delivered the skill, asked once per run and recorded with the
+    // numbers. A pi upgrade (0.80.x → 0.83.0) silently changed what green mode
+    // measures, and the incident was invisible in the artifacts because nothing wrote
+    // this down. Never fatal: `null` means the adapter couldn't say.
+    const harnessCliVersion = (await adapter.version?.()) ?? null;
     appendJournal(runDir, {
         event: "run-started", ts: now(),
         skill: spec.skill, harness: adapter.name, model: opts.modelToken,
+        harness_cli_version: harnessCliVersion,
         judge: { provider: judge.provider, model: judge.model },
         mode, label: opts.label ?? null,
     });
+    // The canary spends one probe before the wave, so a run that isn't measuring the
+    // skill dies for the price of a rep instead of producing a plausible scorecard.
+    // Green only: red delivers nothing by design, and force delivers through the
+    // system prompt, which needs no probe.
+    let canaryStatus = null;
+    if (opts.canary && mode !== "green") {
+        // Ignoring a flag silently is a small version of the bug this whole feature is
+        // about. Say it, and say why it isn't needed.
+        log(`  --canary ignored in mode=${mode} — ${mode === "force" ? "the system prompt delivers the skill unconditionally" : "a baseline delivers no skill by design"}`);
+    }
+    if (opts.canary && mode === "green") {
+        const probeCwd = createWorkspace("none", { specDir: dirname(opts.specPath) });
+        let canary;
+        try {
+            canary = await runDeliveryCanary({
+                adapter, model, skillDir, skillName: spec.skill, cwd: probeCwd.cwd,
+            });
+        }
+        finally {
+            probeCwd.cleanup();
+        }
+        appendJournal(runDir, {
+            event: "delivery-canary", ts: now(),
+            status: canary.status, anchor: canary.anchor, detail: canary.detail,
+        });
+        if (canary.status === "fail")
+            throw new Error(canaryFailure(spec.skill, canary, harnessCliVersion));
+        if (canary.status === "skipped")
+            log(`  ⚠ delivery canary skipped — ${canary.detail}`);
+        else {
+            canaryStatus = "pass";
+            log(`  ✓ delivery canary — the model quoted its skill instructions back (\`${canary.anchor}\`)`);
+        }
+    }
     // scenario × rep tasks; runPool preserves input order so we can slice per scenario.
     const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
     const owners = [];
@@ -61,10 +102,12 @@ export async function runSkillModel(opts) {
         const threshold = scenario.passThreshold ?? opts.passThreshold ?? 0.5;
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
-    const ctx = mode === "green" && !partial ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
+    const ctx = scoreContextFor({ mode, partial }, spec);
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: adapter.name,
+        harness_cli_version: harnessCliVersion ?? undefined,
+        delivery_canary: canaryStatus ?? undefined,
         model: opts.modelToken,
         judge: { provider: judge.provider, model: judge.model },
         timestamp,
@@ -226,16 +269,26 @@ export function formatScorecard(summary, lift) {
     const ship = g.ship ? "SHIP" : "NOT READY";
     const note = g.note ? ` (${g.note})` : "";
     lines.push(`  GRADE: ${g.letter} (${g.pct}%) — ${g.passed}/${g.total} — ${ship}${note}`);
-    // Lift is a statement about a green run. On a red run the caller may still have
-    // a lift in hand (a green run exists in the same tag), but printing it under a
-    // baseline scorecard reads as if the baseline itself gained something.
-    if (lift && results.mode === "green") {
+    // Lift is a statement about a skill-delivered run (green or force). On a red run
+    // the caller may still have a lift in hand (a scored run exists in the same tag),
+    // but printing it under a baseline scorecard reads as if the baseline itself
+    // gained something.
+    if (lift && isScoredMode(results.mode)) {
         lines.push(`  LIFT:  ${liftHeadline(lift)}  (vs red baseline ${lift.redTimestamp})`);
     }
-    else if (results.mode === "green") {
+    else if (isScoredMode(results.mode)) {
         // The grade alone can't answer "does this skill do anything?", so say how.
         lines.push(`  LIFT:  no red baseline — run with --mode red to measure what the skill adds`);
     }
+    // Said on the scorecard, not just in the docs: the one thing that can invalidate
+    // a green number is invisible in the number. `harness_cli_version` is recorded
+    // beside the verdicts so a reader can tell which pi produced them.
+    if (results.mode === "green" && !results.delivery_canary) {
+        lines.push(`  NOTE:  green delivery is harness-version-dependent` +
+            (results.harness_cli_version ? ` (${results.harness} ${results.harness_cli_version})` : "") +
+            ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +
+            ` Use --mode force for delivery that cannot silently degrade, or --canary to prove it per run.`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run.js.map

package/dist/trends.d.ts

@@ -14,6 +14,17 @@ export interface TrendRun {
 export interface TrendModel {
     model: string;
     tag: string;
+    /**
+     * The delivery mode every run in this series shares (`green` or `force`).
+     *
+     * A series is per tag AND per mode, never pooled: the two modes are different
+     * deliveries of the same text, and placement moves verdicts in both directions at
+     * once (measured on identical skill text: `build` A1 0/3 → 3/3 with force, `plan`
+     * C2 3/3 → 0/3). A sparkline that ran green then force would draw that epoch
+     * change as skill progress — or regression — which is the one thing a trend line
+     * must not invent.
+     */
+    mode: string;
     runs: TrendRun[];
     truncated: boolean;
     skipped: number;
@@ -35,15 +46,18 @@ export interface TrendData {
  * rule: an override resolves a misfire) + reps flakiness. Read-only; no
  * absolute paths in the result.
  *
- * Only scored (mode === "green") runs are included in the history — a
- * red/force run has no real grade (`effective_grade` is a "not scored"
- * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
- * the sparkline/grid. Non-green runs are deliberately excluded, which is
- * distinct from `skipped`: a run's mode can only be known after reading its
- * results.yaml, so every candidate run-dir in the tag is read (not just the
- * most recent `limit`) before filtering to green and applying the `limit`
- * window — trends is a bounded, on-demand, local view, so this extra read
- * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ * Only scored runs are included in the history — a red baseline has no real grade
+ * (`effective_grade` is a "not scored" placeholder; see run.ts) and would otherwise
+ * plot as a misleading 0% dip in the sparkline/grid. Red runs are deliberately
+ * excluded, which is distinct from `skipped`: a run's mode can only be known after
+ * reading its results.yaml, so every candidate run-dir in the tag is read (not just
+ * the most recent `limit`) before filtering and applying the `limit` window —
+ * trends is a bounded, on-demand, local view, so this extra read cost is
+ * acceptable.
+ *
+ * Green and force runs both count, but never in the same series: a tag with both
+ * yields one TrendModel per mode (see `TrendModel.mode`), each with its own
+ * `limit` window. A tag with no scored run at all is omitted entirely.
  *
  * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
  * write) is logged via `console.warn` and skipped — never surfaced or thrown —

package/dist/trends.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults, effectiveVerdicts } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
 /** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
 function isDir(p) {
     try {
@@ -19,15 +19,18 @@ function isDir(p) {
  * rule: an override resolves a misfire) + reps flakiness. Read-only; no
  * absolute paths in the result.
  *
- * Only scored (mode === "green") runs are included in the history — a
- * red/force run has no real grade (`effective_grade` is a "not scored"
- * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
- * the sparkline/grid. Non-green runs are deliberately excluded, which is
- * distinct from `skipped`: a run's mode can only be known after reading its
- * results.yaml, so every candidate run-dir in the tag is read (not just the
- * most recent `limit`) before filtering to green and applying the `limit`
- * window — trends is a bounded, on-demand, local view, so this extra read
- * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ * Only scored runs are included in the history — a red baseline has no real grade
+ * (`effective_grade` is a "not scored" placeholder; see run.ts) and would otherwise
+ * plot as a misleading 0% dip in the sparkline/grid. Red runs are deliberately
+ * excluded, which is distinct from `skipped`: a run's mode can only be known after
+ * reading its results.yaml, so every candidate run-dir in the tag is read (not just
+ * the most recent `limit`) before filtering and applying the `limit` window —
+ * trends is a bounded, on-demand, local view, so this extra read cost is
+ * acceptable.
+ *
+ * Green and force runs both count, but never in the same series: a tag with both
+ * yields one TrendModel per mode (see `TrendModel.mode`), each with its own
+ * `limit` window. A tag with no scored run at all is omitted entirely.
  *
  * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
  * write) is logged via `console.warn` and skipped — never surfaced or thrown —
@@ -54,10 +57,11 @@ export function collectTrends(skillDir, limit = 20) {
             if (runDirs.length === 0)
                 continue;
             // Read every candidate run (mode isn't knowable from the dir name) and
-            // filter to green (scored) runs before applying the `limit` window —
-            // filtering after the slice would let red/force runs consume window
-            // slots, undercounting the green history even when more exists.
-            const greenRuns = [];
+            // filter to scored runs before applying the `limit` window — filtering
+            // after the slice would let red runs consume window slots, undercounting
+            // the history even when more exists. Bucketed by mode, in first-seen
+            // order, so each delivery epoch gets its own series and its own window.
+            const byMode = new Map();
             let skipped = 0;
             for (const rd of runDirs) {
                 let r;
@@ -71,30 +75,35 @@ export function collectTrends(skillDir, limit = 20) {
                     skipped++;
                     continue;
                 }
-                if (r.mode !== "green")
-                    continue; // not scored — deliberate exclusion, not a skip
-                greenRuns.push(r);
+                if (!isScoredMode(r.mode))
+                    continue; // baseline — deliberate exclusion, not a skip
+                (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
             }
-            if (greenRuns.length === 0)
+            if (byMode.size === 0)
                 continue;
-            const truncated = greenRuns.length > limit;
-            const kept = greenRuns.slice(-limit); // most recent `limit`, newest last
-            const runs = [];
-            let model = "";
-            for (const r of kept) {
-                // effectiveVerdicts is the single source of truth for the
-                // override-aware verdict/suspect rule (suspect = s.suspect &&
-                // s.override == null — an override resolves the misfire); zip in
-                // flakiness from the matching ScenarioResult.
-                const verdicts = effectiveVerdicts(r.scenarios);
-                const cells = {};
-                r.scenarios.forEach((s, i) => {
-                    cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
-                });
-                runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
-                model = r.model; // last successfully-read run (kept is ascending) wins
+            for (const [mode, scoredRuns] of byMode) {
+                const truncated = scoredRuns.length > limit;
+                const kept = scoredRuns.slice(-limit); // most recent `limit`, newest last
+                const runs = [];
+                let model = "";
+                for (const r of kept) {
+                    // effectiveVerdicts is the single source of truth for the
+                    // override-aware verdict/suspect rule (suspect = s.suspect &&
+                    // s.override == null — an override resolves the misfire); zip in
+                    // flakiness from the matching ScenarioResult.
+                    const verdicts = effectiveVerdicts(r.scenarios);
+                    const cells = {};
+                    r.scenarios.forEach((s, i) => {
+                        cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
+                    });
+                    runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
+                    model = r.model; // last successfully-read run (kept is ascending) wins
+                }
+                // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
+                // two series reports the same count on both — the alternative is attributing a
+                // parse failure to a mode nobody could read.
+                models.push({ model, tag, mode, runs, truncated, skipped });
             }
-            models.push({ model, tag, runs, truncated, skipped });
         }
     }
     return { skill: spec.skill, scenarios, models };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.4.0",
+    "version": "0.5.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",