@skill-harness/core 0.7.0 → 0.8.0

package/dist/index.d.ts

@@ -14,6 +14,7 @@ export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./restamp.js";
 export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";

package/dist/index.js

@@ -14,6 +14,7 @@ export * from "./seeded.js";
 export * from "./report.js";
 export * from "./trends.js";
 export * from "./lint.js";
+export * from "./restamp.js";
 export * from "./sources.js";
 export * from "./lift.js";
 export * from "./adapters/types.js";

package/dist/lint.d.ts

@@ -27,3 +27,10 @@ export declare function failsGate(f: LintFinding): boolean;
  * report every problem across every skill.
  */
 export declare function lintSkill(skillDir: string): LintFinding[];
+/**
+ * All committed run dirs under a skill's tests/results (<tag>/<timestamp>/results.yaml).
+ * Empty if none. Never throws: unreadable/dangling entries (e.g. a broken symlink, or a
+ * TOCTOU removal between readdir and statSync) are skipped rather than propagated, so a
+ * single bad entry can't abort lintSkill's "never throws" contract.
+ */
+export declare function enumerateRunDirs(resultsRoot: string): string[];

package/dist/lint.js

@@ -4,7 +4,7 @@ import yaml from "js-yaml";
 import { loadSpec, SpecError } from "./spec.js";
 import { computeCoverage } from "./instruction-coverage.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath, scoreContextFor } from "./results.js";
-import { currentHashFor, describeSourceKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
+import { currentHashFor, describeSourceKey, isSupersededKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
 import { downgradeWarning } from "./downgrade.js";
 import { collectScoredRuns } from "./trends.js";
 import { boundaryCells, stabilityFrom, stabilityNote } from "./stability.js";
@@ -247,6 +247,12 @@ export function lintSkill(skillDir) {
             const newest = full.runDir;
             const ctx = { skillDir, specDir, scenarios: spec.scenarios, judgePersona: spec.judge_persona };
             for (const [key, recorded] of Object.entries(hashes)) {
+                // A raw-bytes key whose model-visible digest this run also recorded is not the
+                // honest comparison: it reports a `description`-reword or an `allowed-tools:`
+                // line — text no graded run could observe — as a reason to re-run. The pair is
+                // recorded together and exactly one of them is compared.
+                if (isSupersededKey(key, hashes))
+                    continue;
                 const what = describeSourceKey(key);
                 const scenario = scenarioIdForKey(key, spec.scenarios);
                 // The run itself failed to hash this source, so it was never verified and
@@ -349,7 +355,7 @@ function runDirsNewestFirst(tagDir) {
  * TOCTOU removal between readdir and statSync) are skipped rather than propagated, so a
  * single bad entry can't abort lintSkill's "never throws" contract.
  */
-function enumerateRunDirs(resultsRoot) {
+export function enumerateRunDirs(resultsRoot) {
     if (!existsSync(resultsRoot))
         return [];
     const out = [];

package/dist/restamp.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Upgrade already-committed runs to the model-visible digests, without re-running anything.
+ *
+ * ## Why a command and not a rule inside lint
+ *
+ * A stored hash is one-way. A run recorded before 0.8.0 carries only sha256 of SKILL.md's
+ * RAW BYTES, so once that file has been edited there is nothing in the record — and
+ * nothing saved beside it; run dirs hold transcripts, not a copy of the skill — that can
+ * say whether the edit touched the body or only the frontmatter. Lint therefore cannot
+ * retroactively forgive a frontmatter-only edit on a legacy record, and pretending
+ * otherwise would mean dropping the check that catches real body edits across the whole
+ * published corpus.
+ *
+ * What CAN be established, and is the entire proof this command rests on: when the
+ * recorded raw-bytes hash still equals the file's current raw-bytes hash, that run
+ * measured the exact bytes on disk right now. The model-visible digest of those same
+ * bytes is then a true statement about what that run measured, and recording it is
+ * bookkeeping rather than a claim. Any record that fails the check is left untouched —
+ * it stays honestly stale, and `run` is the only thing that can clear it.
+ *
+ * So the migration is: upgrade while the board is green, and every later
+ * frontmatter-only edit is free. Skipping it costs nothing either — a legacy record with
+ * an unedited skill keeps matching on the raw-bytes key exactly as before.
+ *
+ * ## `--from <ref>`, for when the edit already landed
+ *
+ * Nobody discovers a gate over-fires until it has over-fired, so the green worktree the
+ * paragraph above assumes is usually already gone. The pre-edit bytes are still in git,
+ * and they carry the same proof in two steps: the recorded hash matches `<ref>`'s bytes
+ * (so the run measured THOSE), and `<ref>`'s model-visible digest equals today's (so the
+ * text the model receives has not moved since). Only then is the digest recorded.
+ *
+ * It is the same proof, not a weaker one — with no `--from`, "the ref" is just the
+ * working tree, which is why both go through one code path.
+ *
+ * Free and offline: no model, no judge, no network. It rewrites only `source_hashes`.
+ */
+/**
+ * Three exclusive buckets that SUM to `runs`, plus `partial` as a subset of `upgraded`.
+ *
+ * Summing is the point. These counters are the only thing an operator can judge a
+ * migration by, and "18 upgraded, 20 left alone" out of 140 leaves 102 records
+ * unaccounted for — which reads like the command quietly skipped them rather than like
+ * the 102 having nothing to upgrade.
+ */
+export interface RestampReport {
+    /** Run records examined. `upgraded + unprovable + unchanged`. */
+    runs: number;
+    /** Records that gained at least one model-visible digest, and were rewritten. */
+    upgraded: number;
+    /**
+     * Records that gained NOTHING because a document they measured has already moved —
+     * honestly stale, and `run` is the only remedy.
+     */
+    unprovable: number;
+    /** Records with nothing to do: no `source_hashes`, unreadable, or already upgraded. */
+    unchanged: number;
+    /** Of `upgraded`, those still carrying a document that could not be proven. */
+    partial: number;
+    /** `<runDir>: <key>` for each digest added, for the CLI to print. */
+    added: string[];
+}
+export interface RestampOptions {
+    /** Git ref holding the bytes the runs measured, for a tree that has already been edited. */
+    from?: string;
+}
+/** Upgrade every provable record under one skill's tests/results. */
+export declare function restampSkill(skillDir: string, opts?: RestampOptions): RestampReport;

package/dist/restamp.js

@@ -0,0 +1,194 @@
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import yaml from "js-yaml";
+import { loadSpec } from "./spec.js";
+import { resultsPath } from "./results.js";
+import { enumerateRunDirs } from "./lint.js";
+import { promptDocDigest, SKILL_KEY, SKILL_PROMPT_KEY, PROMPT_PREFIX, UNREADABLE, } from "./sources.js";
+/**
+ * The `source_hashes` map of a raw-loaded results doc, or null when the doc has none.
+ * Read from the RAW yaml rather than through `readResults` on purpose: this rewrites the
+ * file, and `readResults` migrates schema 1 → 2 in memory, so writing its output back
+ * would silently upgrade the schema of records nobody asked to touch.
+ */
+function rawHashes(doc) {
+    if (!doc || typeof doc !== "object")
+        return null;
+    const h = doc.source_hashes;
+    if (!h || typeof h !== "object" || Array.isArray(h))
+        return null;
+    return h;
+}
+/**
+ * `git show <ref>:<path>` as BYTES, or null when the ref, the repo, or the path isn't there.
+ *
+ * `execFileSync` with a buffer encoding rather than the shared `exec` helper: that one
+ * accumulates stdout as decoded strings, so a multi-byte character landing on a chunk
+ * boundary would come back with a replacement character and hash to something the run
+ * never recorded. This comparison is byte-exact by definition, so it reads bytes.
+ */
+function bytesAtRef(abs, ref) {
+    const cwd = dirname(abs);
+    try {
+        const top = execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd, encoding: "utf8" }).trim();
+        const rel = relative(top, abs).split("\\").join("/");
+        return execFileSync("git", ["show", `${ref}:${rel}`], { cwd, maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return null; // no git, no repo, no such ref, or the file did not exist there
+    }
+}
+/**
+ * The bytes a run is claimed to have measured: `<ref>`'s copy, or the working tree's when
+ * no ref was given. Null when they cannot be read at all, which proves nothing and so
+ * upgrades nothing.
+ */
+function measuredBytes(abs, from) {
+    if (from !== undefined)
+        return bytesAtRef(abs, from);
+    try {
+        return readFileSync(abs);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Fail loudly when `--from` names something git cannot resolve.
+ *
+ * Without this the whole migration degrades into "0 upgraded, everything left alone",
+ * which is indistinguishable from a corpus that genuinely cannot be proven — measured on
+ * a clone whose only local branch was the feature branch, where `--from main` silently
+ * did nothing and read as the tool being broken. A ref the operator got wrong is their
+ * mistake to see, not one to absorb.
+ */
+function assertRefResolves(dir, ref) {
+    try {
+        execFileSync("git", ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], { cwd: dir, stdio: "ignore" });
+    }
+    catch {
+        throw new Error(`--from \`${ref}\` does not resolve to a commit in the repo holding ${dir} — ` +
+            `check the ref exists locally (a fresh clone often has no local \`main\`; try \`origin/main\`)`);
+    }
+}
+/**
+ * Write via a temp file in the same directory, then rename.
+ *
+ * `rename(2)` is atomic within a filesystem, so a reader sees the old file or the new one
+ * and never a half-written one. It matters more here than in the writers that produce a
+ * run: this loops over an entire published corpus rewriting records that already hold
+ * measurements nobody can reproduce without spending model tokens, so an interrupt or a
+ * full disk partway through would truncate exactly the artifacts the command exists to
+ * preserve. The temp file is `.tmp`-suffixed beside the target rather than in the system
+ * temp dir, because a cross-device rename is not atomic — and is not even permitted.
+ */
+function writeAtomically(path, text) {
+    const tmp = `${path}.restamp.tmp`;
+    writeFileSync(tmp, text, "utf8");
+    try {
+        renameSync(tmp, path);
+    }
+    catch (e) {
+        try {
+            rmSync(tmp, { force: true });
+        }
+        catch { /* the rename failure is what matters */ }
+        throw e;
+    }
+}
+/** Upgrade every provable record under one skill's tests/results. */
+export function restampSkill(skillDir, opts = {}) {
+    const report = { runs: 0, upgraded: 0, unprovable: 0, unchanged: 0, partial: 0, added: [] };
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    const spec = loadSpec(specPath);
+    const specDir = dirname(specPath);
+    // Only `system_prompt_file` paths, resolved from the CURRENT spec. The other
+    // path-shaped keys (`post_test`, `env.extensions`) name code, not prompt documents:
+    // every byte of those reaches the run, so they keep hashing raw and are left alone.
+    const promptDocs = new Map([[SKILL_KEY, resolve(skillDir, "SKILL.md")]]);
+    for (const s of spec.scenarios) {
+        if (s.systemPromptFile)
+            promptDocs.set(s.systemPromptFile, resolve(specDir, s.systemPromptFile));
+    }
+    if (opts.from !== undefined)
+        assertRefResolves(skillDir, opts.from);
+    // Resolved once per document, not once per run: with `--from`, each resolution is two
+    // git invocations, and a corpus of 140 runs would otherwise pay for them 140 times over.
+    const provable = new Map();
+    for (const [legacyKey, abs] of promptDocs) {
+        const measured = measuredBytes(abs, opts.from);
+        let currentPrompt = null;
+        try {
+            currentPrompt = promptDocDigest(readFileSync(abs, "utf8"));
+        }
+        catch {
+            currentPrompt = null;
+        }
+        provable.set(legacyKey, {
+            measuredRaw: measured === null ? null : createHash("sha256").update(measured).digest("hex"),
+            measuredPrompt: measured === null ? null : promptDocDigest(measured.toString("utf8")),
+            currentPrompt,
+        });
+    }
+    for (const runDir of enumerateRunDirs(join(skillDir, "tests", "results"))) {
+        report.runs++;
+        let doc;
+        try {
+            doc = yaml.load(readFileSync(resultsPath(runDir), "utf8"));
+        }
+        catch {
+            // Unreadable/malformed → lint already reports it; never rewrite what we can't read.
+            // Still bucketed: every record counted in `runs` has to be accounted for somewhere.
+            report.unchanged++;
+            continue;
+        }
+        const hashes = rawHashes(doc);
+        if (!hashes) {
+            report.unchanged++; // predates source_hashes → nothing to upgrade from
+            continue;
+        }
+        let blocked = false;
+        const additions = [];
+        for (const legacyKey of promptDocs.keys()) {
+            const recorded = hashes[legacyKey];
+            if (recorded === undefined || recorded === UNREADABLE)
+                continue; // never measured → nothing proven
+            const upgradedKey = legacyKey === SKILL_KEY ? SKILL_PROMPT_KEY : PROMPT_PREFIX + legacyKey;
+            if (hashes[upgradedKey] !== undefined)
+                continue; // already upgraded
+            const p = provable.get(legacyKey);
+            if (p.currentPrompt === null)
+                continue; // the file is gone; lint reports that itself
+            // Two steps, and both are required. First: this run measured the reference bytes —
+            // otherwise the reference says nothing about THIS run. Second: those bytes and
+            // today's carry the same model-visible text — otherwise what the model reads has
+            // genuinely moved and only a re-run can speak for it.
+            if (p.measuredRaw === null || recorded !== p.measuredRaw || p.measuredPrompt !== p.currentPrompt) {
+                blocked = true;
+                continue;
+            }
+            additions.push([upgradedKey, p.currentPrompt]);
+        }
+        if (additions.length === 0) {
+            // Exclusive: a record that gained nothing is `unprovable` only if something
+            // actually blocked it, and `unchanged` otherwise.
+            if (blocked)
+                report.unprovable++;
+            else
+                report.unchanged++;
+            continue;
+        }
+        for (const [k, v] of additions) {
+            hashes[k] = v;
+            report.added.push(`${runDir}: ${k}`);
+        }
+        writeAtomically(resultsPath(runDir), yaml.dump(doc, { lineWidth: 100 }));
+        report.upgraded++;
+        if (blocked)
+            report.partial++;
+    }
+    return report;
+}
+//# sourceMappingURL=restamp.js.map

package/dist/sources.d.ts

@@ -174,6 +174,56 @@ export declare function describeSourceKey(key: string): string;
 export declare function remedyForKey(key: string): string;
 /** The skill-text key. Skill-wide: it belongs to every scenario at once. */
 export declare const SKILL_KEY = "SKILL.md";
+/**
+ * The model-visible digests of the two prompt documents a run delivers: the skill text
+ * and any `system_prompt_file`. They measure what the model RECEIVES, where the bare
+ * keys above (`SKILL.md`, `<agent path>`) measure the file's raw bytes.
+ *
+ * | key | contents |
+ * |---|---|
+ * | `skill:prompt` | SKILL.md's body + its model-visible frontmatter |
+ * | `prompt:<path>` | the same, for one `system_prompt_file` |
+ *
+ * Both are prefixed, so a skill-harness OLDER than this one reads them through
+ * `currentHashFor`'s unknown-prefix guard and reports "not comparable" rather than a
+ * confident wrong finding about a source that is fine.
+ */
+export declare const SKILL_PROMPT_KEY = "skill:prompt";
+export declare const PROMPT_PREFIX = "prompt:";
+/**
+ * The digest of a prompt document: its body verbatim, plus every frontmatter field the
+ * model can actually receive.
+ *
+ * The gate exists to protect published claims, so "the current text" has to mean the text
+ * the model was given. Hashing the raw bytes — which is what shipped through 0.7.0 —
+ * charged a full re-run for edits no graded run could observe: measured on the reference
+ * corpus, adding one `allowed-tools:` key to seven SKILL.md files with byte-identical
+ * bodies took lint from 2 findings to 22, and 20 of those demanded paid re-waves.
+ *
+ * Versioned in the hashed tuple (`prompt-doc/1`) because this is a stored-hash format: if
+ * the rule for "model-visible" ever changes, the tag changes with it and old digests stay
+ * legible as what they were.
+ */
+export declare function promptDocDigest(text: string): string;
+/** `promptDocDigest` of a file, or null when it doesn't exist / isn't readable. */
+export declare function promptDocDigestOfFile(path: string): string | null;
+/**
+ * True when `key` is a raw-bytes key whose model-visible counterpart THIS RUN also
+ * recorded — in which case the counterpart is the honest comparison and the raw-bytes one
+ * must be skipped.
+ *
+ * Both keys keep being written, and this is why: an older skill-harness reading a newer
+ * results.yaml still finds the bare `SKILL.md` key and keeps checking it, and `restamp`
+ * needs the raw-bytes hash as the proof it upgrades from. Superseding at COMPARISON time
+ * rather than dropping the key at record time is what buys both.
+ *
+ * `UNREADABLE` on EITHER side supersedes nothing. If the counterpart is unreadable there
+ * is nothing better to compare against; and if the raw-bytes key itself is unreadable, the
+ * run never verified that document at all — a fact lint must keep reporting, since the
+ * one thing this module refuses to do is let an unhashed source drop out of checking for
+ * the life of a result.
+ */
+export declare function isSupersededKey(key: string, recorded: Record<string, string>): boolean;
 /**
  * Every recorded key whose drift could change THIS scenario's verdict — excluding the
  * two skill-wide ones (`SKILL.md`, `rubric:__persona`), which callers handle

package/dist/sources.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { readFileSync, readdirSync } from "node:fs";
 import { isAbsolute, join, resolve } from "node:path";
+import yaml from "js-yaml";
 /**
  * What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
  * lint can prove a published result still describes the current inputs.
@@ -292,7 +293,12 @@ export function sourceHashes(ctx) {
     const hashes = {};
     // UNREADABLE rather than omission on every branch below: a source we failed to
     // hash must stay visible to lint, not vanish from the record. See UNREADABLE.
-    hashes["SKILL.md"] = fileSha256(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
+    // Two digests of one file, and both are load-bearing. `SKILL.md` is the raw bytes,
+    // kept so an older skill-harness reading this record still has the key it knows and so
+    // `restamp` has a proof to upgrade from; `skill:prompt` is the text the model actually
+    // receives, and is the one lint compares (see `isSupersededKey`).
+    hashes[SKILL_KEY] = fileSha256(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
+    hashes[SKILL_PROMPT_KEY] = promptDocDigestOfFile(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
     hashes[PERSONA_KEY] = personaDigest(ctx.judgePersona);
     for (const s of ctx.scenarios) {
         // Split, not combined: each facet's drift has a different cheapest remedy, and a
@@ -304,8 +310,13 @@ export function sourceHashes(ctx) {
         const gates = gatesDigest(s);
         if (gates !== null)
             hashes[GATES_PREFIX + s.id] = gates;
+        // An agent file is a prompt document too, delivered with `--append-system-prompt`,
+        // and its `tools:` header is read by the harness rather than the model — so it gets
+        // the same pair of digests, for the same reasons, as SKILL.md above.
         if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
-            hashes[s.systemPromptFile] = fileSha256(resolve(ctx.specDir, s.systemPromptFile)) ?? UNREADABLE;
+            const abs = resolve(ctx.specDir, s.systemPromptFile);
+            hashes[s.systemPromptFile] = fileSha256(abs) ?? UNREADABLE;
+            hashes[PROMPT_PREFIX + s.systemPromptFile] = promptDocDigestOfFile(abs) ?? UNREADABLE;
         }
         // Extension CONTENTS, not just the paths the stimulus digest already covers.
         // An orchestration scenario's subagent tool lives in these files: editing one
@@ -344,8 +355,16 @@ export function sourceHashes(ctx) {
  * from drifting: a new key kind is defined once, for both sides.
  */
 export function currentHashFor(key, ctx) {
-    if (key === "SKILL.md")
+    if (key === SKILL_KEY)
         return fileSha256(resolve(ctx.skillDir, "SKILL.md"));
+    if (key === SKILL_PROMPT_KEY)
+        return promptDocDigestOfFile(resolve(ctx.skillDir, "SKILL.md"));
+    // Before the unknown-prefix guard below, which would otherwise swallow these as keys
+    // from a newer skill-harness. A deleted agent file must still resolve to null ("gone"),
+    // not undefined ("not comparable").
+    if (key.startsWith(PROMPT_PREFIX)) {
+        return promptDocDigestOfFile(resolve(ctx.specDir, key.slice(PROMPT_PREFIX.length)));
+    }
     if (key === PERSONA_KEY)
         return personaDigest(ctx.judgePersona);
     // Facet keys, and the legacy combined key, all resolve per scenario id. A removed
@@ -383,6 +402,13 @@ export function currentHashFor(key, ctx) {
 export function describeSourceKey(key) {
     if (key === PERSONA_KEY)
         return "the judge persona";
+    // Deliberately the same label as the raw-bytes key: only one of the pair is ever
+    // compared (see `isSupersededKey`), and a reader of a lint finding cares which FILE
+    // moved, not which of two digests of it was the one consulted.
+    if (key === SKILL_PROMPT_KEY)
+        return SKILL_KEY;
+    if (key.startsWith(PROMPT_PREFIX))
+        return key.slice(PROMPT_PREFIX.length);
     if (key.startsWith(STIMULUS_PREFIX))
         return `the stimulus for \`${key.slice(STIMULUS_PREFIX.length)}\``;
     if (key.startsWith(RUBRIC_PREFIX))
@@ -427,6 +453,149 @@ export function remedyForKey(key) {
 }
 /** The skill-text key. Skill-wide: it belongs to every scenario at once. */
 export const SKILL_KEY = "SKILL.md";
+/**
+ * The model-visible digests of the two prompt documents a run delivers: the skill text
+ * and any `system_prompt_file`. They measure what the model RECEIVES, where the bare
+ * keys above (`SKILL.md`, `<agent path>`) measure the file's raw bytes.
+ *
+ * | key | contents |
+ * |---|---|
+ * | `skill:prompt` | SKILL.md's body + its model-visible frontmatter |
+ * | `prompt:<path>` | the same, for one `system_prompt_file` |
+ *
+ * Both are prefixed, so a skill-harness OLDER than this one reads them through
+ * `currentHashFor`'s unknown-prefix guard and reports "not comparable" rather than a
+ * confident wrong finding about a source that is fine.
+ */
+export const SKILL_PROMPT_KEY = "skill:prompt";
+export const PROMPT_PREFIX = "prompt:";
+/**
+ * Frontmatter keys the model never receives: capability declarations the HARNESS
+ * consumes to build a tool allowlist.
+ *
+ * Why a denylist and not an allowlist of "body + description". An allowlist makes the
+ * safe-looking choice the dangerous one: the day a harness starts putting some new
+ * frontmatter key in context, every published number silently stops being protected and
+ * nothing says so. A denylist fails the other way — a newly-invented inert key charges
+ * one unnecessary re-run until it is named here, which is a one-line patch and a visible
+ * complaint. This module already takes that side everywhere else (see `UNREADABLE`, and
+ * the unknown-prefix guard in `currentHashFor`): a blind spot is the worst outcome, and
+ * a false alarm is merely expensive.
+ *
+ * `tools` and `allowed-tools` are the two spellings of the same ceiling in the reference
+ * corpus. Both are read by pi/pi-daddy to build a `--tools` allowlist; neither is ever
+ * rendered into the model's context.
+ */
+const CAPABILITY_KEYS = new Set(["allowed-tools", "tools"]);
+/** Opening `---` on line 1 only — a `---` further down is a horizontal rule. */
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/;
+/** A prompt document split into its frontmatter text (null when it has none) and its body. */
+function splitPromptDoc(text) {
+    const m = FRONTMATTER_RE.exec(text);
+    // Unterminated frontmatter falls through to "all body", the same call
+    // instruction-coverage.ts makes: treat the whole file as content rather than lose it.
+    return m ? { frontmatter: m[1], body: text.slice(m[0].length) } : { frontmatter: null, body: text };
+}
+/**
+ * Parsed YAML in a stable shape: object keys sorted, string scalars trimmed.
+ *
+ * Trimmed because YAML block style is not stimulus — `description: >` clips a trailing
+ * newline that `description: <text>` does not, so an author refolding a paragraph the
+ * model receives identically would otherwise be charged a re-run.
+ */
+function canonicalValue(v) {
+    if (typeof v === "string")
+        return v.trim();
+    if (Array.isArray(v))
+        return v.map(canonicalValue);
+    if (v && typeof v === "object") {
+        return Object.entries(v)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([k, val]) => [k, canonicalValue(val)]);
+    }
+    return v;
+}
+/**
+ * The frontmatter a run can observe, canonically. Built from the PARSED YAML for the
+ * same reason the scenario digests are built from the parsed scenario: reindenting or
+ * reordering keys changes no stimulus and must not read as a change.
+ *
+ * Unparseable frontmatter is hashed VERBATIM rather than dropped. Dropping it would mean
+ * a file whose frontmatter is malformed quietly loses that half of its protection — and
+ * a malformed header is exactly when an author is most likely to be mid-edit.
+ */
+function modelVisibleFrontmatter(fm) {
+    if (fm === null)
+        return null;
+    let parsed;
+    try {
+        parsed = yaml.load(fm);
+    }
+    catch {
+        return ["unparsed", fm];
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
+        return ["unparsed", fm];
+    return [
+        "parsed",
+        Object.entries(parsed)
+            .filter(([k]) => !CAPABILITY_KEYS.has(k))
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([k, v]) => [k, canonicalValue(v)]),
+    ];
+}
+/**
+ * The digest of a prompt document: its body verbatim, plus every frontmatter field the
+ * model can actually receive.
+ *
+ * The gate exists to protect published claims, so "the current text" has to mean the text
+ * the model was given. Hashing the raw bytes — which is what shipped through 0.7.0 —
+ * charged a full re-run for edits no graded run could observe: measured on the reference
+ * corpus, adding one `allowed-tools:` key to seven SKILL.md files with byte-identical
+ * bodies took lint from 2 findings to 22, and 20 of those demanded paid re-waves.
+ *
+ * Versioned in the hashed tuple (`prompt-doc/1`) because this is a stored-hash format: if
+ * the rule for "model-visible" ever changes, the tag changes with it and old digests stay
+ * legible as what they were.
+ */
+export function promptDocDigest(text) {
+    const { frontmatter, body } = splitPromptDoc(text);
+    return sha(JSON.stringify(["prompt-doc/1", modelVisibleFrontmatter(frontmatter), body]));
+}
+/** `promptDocDigest` of a file, or null when it doesn't exist / isn't readable. */
+export function promptDocDigestOfFile(path) {
+    try {
+        return promptDocDigest(readFileSync(path, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * True when `key` is a raw-bytes key whose model-visible counterpart THIS RUN also
+ * recorded — in which case the counterpart is the honest comparison and the raw-bytes one
+ * must be skipped.
+ *
+ * Both keys keep being written, and this is why: an older skill-harness reading a newer
+ * results.yaml still finds the bare `SKILL.md` key and keeps checking it, and `restamp`
+ * needs the raw-bytes hash as the proof it upgrades from. Superseding at COMPARISON time
+ * rather than dropping the key at record time is what buys both.
+ *
+ * `UNREADABLE` on EITHER side supersedes nothing. If the counterpart is unreadable there
+ * is nothing better to compare against; and if the raw-bytes key itself is unreadable, the
+ * run never verified that document at all — a fact lint must keep reporting, since the
+ * one thing this module refuses to do is let an unhashed source drop out of checking for
+ * the life of a result.
+ */
+export function isSupersededKey(key, recorded) {
+    if (key === SKILL_PROMPT_KEY || key.startsWith(PROMPT_PREFIX))
+        return false;
+    if (recorded[key] === UNREADABLE)
+        return false;
+    const upgraded = key === SKILL_KEY ? SKILL_PROMPT_KEY : PROMPT_PREFIX + key;
+    const v = recorded[upgraded];
+    return v !== undefined && v !== UNREADABLE;
+}
 /**
  * Every recorded key whose drift could change THIS scenario's verdict — excluding the
  * two skill-wide ones (`SKILL.md`, `rubric:__persona`), which callers handle
@@ -458,8 +627,10 @@ export function scenarioSourceKeys(s) {
     ];
     if (gatesDigest(s) !== null)
         keys.push(GATES_PREFIX + s.id);
-    if (s.systemPromptFile)
+    if (s.systemPromptFile) {
         keys.push(s.systemPromptFile); // the agent file IS the stimulus
+        keys.push(PROMPT_PREFIX + s.systemPromptFile); // ...as the model receives it
+    }
     for (const ext of s.extensions ?? [])
         keys.push(ext); // an edited extension is new stimulus
     if (s.assert?.post_test)

package/dist/stability.js

@@ -2,7 +2,7 @@ import { join } from "node:path";
 import { loadSpec } from "./spec.js";
 import { effectiveVerdicts } from "./results.js";
 import { collectScoredRuns } from "./trends.js";
-import { describeSourceKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY } from "./sources.js";
+import { describeSourceKey, isSupersededKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY, SKILL_PROMPT_KEY } from "./sources.js";
 const DEFAULT_WINDOW = 5;
 /** A verdict that carries evidence about the task rather than about the harness or judge. */
 function conclusive(v) {
@@ -23,6 +23,25 @@ function pointFor(r, s, verdict) {
         unanimous: reps > 1 && s.flakiness === 0,
     };
 }
+/**
+ * Did the skill text the MODEL RECEIVES differ between two runs?
+ *
+ * Prefers `skill:prompt` when both runs recorded it, and falls back to the raw-bytes
+ * `SKILL.md` key for runs that predate it. A frontmatter-only edit moves the raw bytes and
+ * not the prompt, and labelling a flip "SKILL.md changed across that step" on that basis
+ * would point a reader at an edit no model could have seen — the note's whole job is to
+ * say which of two explanations the record supports, so it must not invent a third.
+ */
+function skillTextChanged(prev, cur) {
+    for (const key of [SKILL_PROMPT_KEY, SKILL_KEY]) {
+        const a = prev?.[key];
+        const b = cur?.[key];
+        if (a === undefined || b === undefined)
+            continue;
+        return a !== b;
+    }
+    return false;
+}
 /** reps + threshold, normalised the way lift.ts normalises it (a lone rep has no threshold). */
 function shapeOf(s) {
     const reps = s.reps ?? 1;
@@ -43,6 +62,11 @@ function compareSources(a, b, keys) {
     let shared = 0;
     const changed = [];
     for (const key of keys) {
+        // Same rule as lint's: when both runs recorded the model-visible digest of a prompt
+        // document, THAT is the comparison. Leaving the raw-bytes key in would call two runs
+        // "different questions" over an `allowed-tools:` line neither model could read.
+        if (isSupersededKey(key, a) && isSupersededKey(key, b))
+            continue;
         const va = a[key];
         const vb = b[key];
         if (va === undefined || vb === undefined)
@@ -77,9 +101,7 @@ function stabilityForScenario(group, scenario, window) {
         const cur = raw[i];
         const from = points[i - 1];
         const to = points[i];
-        const skillChanged = prev.r.source_hashes?.[SKILL_KEY] !== undefined &&
-            cur.r.source_hashes?.[SKILL_KEY] !== undefined &&
-            prev.r.source_hashes[SKILL_KEY] !== cur.r.source_hashes[SKILL_KEY];
+        const skillChanged = skillTextChanged(prev.r.source_hashes, cur.r.source_hashes);
         const base = { from, to, flipped: false, skillChanged, changedSources: [], unanimousFlip: false };
         if (!prev.ok || !cur.ok) {
             pairs.push({ ...base, status: "inconclusive" });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.7.0",
+    "version": "0.8.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",