@skill-harness/core 0.6.0 → 0.8.0

package/dist/sources.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { readFileSync, readdirSync } from "node:fs";
 import { isAbsolute, join, resolve } from "node:path";
+import yaml from "js-yaml";
 /**
  * What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
  * lint can prove a published result still describes the current inputs.
@@ -167,24 +168,50 @@ function walk(dir, prefix = "") {
  * transcripts are actually invalid.
  */
 function facets(s) {
-    const { id, title, critical, mode, turns, checklist, fixture, assert, workspace, remote, systemPromptFile, reps, passThreshold, ...restScenario } = s;
+    const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
     const _scenarioExhaustive = restScenario;
     void _scenarioExhaustive;
+    // `covers` is destructured into a discard on purpose, and this comment is the
+    // decision the guard demanded: it belongs to NO digest. It records which
+    // instruction sections a scenario is declared to exercise, which changes what
+    // `--affected` selects next time — not what any past run measured. Bucketing it
+    // anywhere would charge a re-run (or at best a re-score) for editing a label,
+    // which is the exact trap the facet split was built to remove.
+    void _coversIsMetadata;
     const { vitest, diff_contains, diff_excludes, post_test, ...restAssert } = assert ?? {};
     const _assertExhaustive = restAssert;
     void _assertExhaustive;
-    const hasGates = diff_contains !== undefined || diff_excludes !== undefined;
+    // `traceAssert` is a GATE, not stimulus: it is evaluated against a trace the run
+    // already saved, so `regate` (free) can re-answer it without re-running the model.
+    // Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
+    // changes what gets executed, the other only what we conclude from it.
+    const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined;
     return {
         // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
         // run executes in the workspace, and neither can be re-evaluated from a saved
         // diff. (`post_test`'s CONTENTS get their own file-path key, hashed separately.)
+        // `extensions` is STIMULUS, not a gate — note the asymmetry with `traceAssert`
+        // below. Changing which extensions load changes what the model can DO, so the
+        // old transcripts describe a different agent and only a re-run can answer.
+        // Changing an assertion only changes what we conclude from evidence already on
+        // disk, which `regate` can redo for free.
+        // APPENDED CONDITIONALLY, never as a fixed slot. This tuple is positional and
+        // its hash is stored in every published results.yaml, so adding an
+        // unconditional element re-hashes every scenario that never used the field —
+        // measured: 62 real lint findings became 261 across the reference corpus, all of
+        // them demanding paid re-runs for scenarios nobody had edited.
         stimulus: JSON.stringify([
             id, mode, turns, workspace, remote, systemPromptFile ?? null,
             fixture ?? null, vitest ?? null, post_test ?? null,
+            ...(extensions ? [extensions] : []),
         ]),
         rubric: JSON.stringify([id, title, checklist]),
         policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
-        gates: hasGates ? JSON.stringify([id, diff_contains ?? null, diff_excludes ?? null]) : null,
+        // Same rule as `stimulus` above: conditional, so a needle-gated scenario that
+        // declares no trace assertions keeps the digest it was published with.
+        gates: hasGates
+            ? JSON.stringify([id, diff_contains ?? null, diff_excludes ?? null, ...(traceAssert ? [traceAssert] : [])])
+            : null,
     };
 }
 function sha(canonical) {
@@ -266,7 +293,12 @@ export function sourceHashes(ctx) {
     const hashes = {};
     // UNREADABLE rather than omission on every branch below: a source we failed to
     // hash must stay visible to lint, not vanish from the record. See UNREADABLE.
-    hashes["SKILL.md"] = fileSha256(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
+    // Two digests of one file, and both are load-bearing. `SKILL.md` is the raw bytes,
+    // kept so an older skill-harness reading this record still has the key it knows and so
+    // `restamp` has a proof to upgrade from; `skill:prompt` is the text the model actually
+    // receives, and is the one lint compares (see `isSupersededKey`).
+    hashes[SKILL_KEY] = fileSha256(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
+    hashes[SKILL_PROMPT_KEY] = promptDocDigestOfFile(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
     hashes[PERSONA_KEY] = personaDigest(ctx.judgePersona);
     for (const s of ctx.scenarios) {
         // Split, not combined: each facet's drift has a different cheapest remedy, and a
@@ -278,8 +310,22 @@ export function sourceHashes(ctx) {
         const gates = gatesDigest(s);
         if (gates !== null)
             hashes[GATES_PREFIX + s.id] = gates;
+        // An agent file is a prompt document too, delivered with `--append-system-prompt`,
+        // and its `tools:` header is read by the harness rather than the model — so it gets
+        // the same pair of digests, for the same reasons, as SKILL.md above.
         if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
-            hashes[s.systemPromptFile] = fileSha256(resolve(ctx.specDir, s.systemPromptFile)) ?? UNREADABLE;
+            const abs = resolve(ctx.specDir, s.systemPromptFile);
+            hashes[s.systemPromptFile] = fileSha256(abs) ?? UNREADABLE;
+            hashes[PROMPT_PREFIX + s.systemPromptFile] = promptDocDigestOfFile(abs) ?? UNREADABLE;
+        }
+        // Extension CONTENTS, not just the paths the stimulus digest already covers.
+        // An orchestration scenario's subagent tool lives in these files: editing one
+        // changes what the model could do without changing a single character of the
+        // spec, which is precisely the drift the staleness gate exists to catch.
+        for (const ext of s.extensions ?? []) {
+            if (ext in hashes)
+                continue;
+            hashes[ext] = fileSha256(resolve(ctx.specDir, ext)) ?? UNREADABLE;
         }
         // The post-test IS the gate on a post_test scenario, and it lives outside the
         // fixture tree by convention (`fixture: fixtures/A1`, `post_test: post/A1.test.ts`),
@@ -309,8 +355,16 @@ export function sourceHashes(ctx) {
  * from drifting: a new key kind is defined once, for both sides.
  */
 export function currentHashFor(key, ctx) {
-    if (key === "SKILL.md")
+    if (key === SKILL_KEY)
         return fileSha256(resolve(ctx.skillDir, "SKILL.md"));
+    if (key === SKILL_PROMPT_KEY)
+        return promptDocDigestOfFile(resolve(ctx.skillDir, "SKILL.md"));
+    // Before the unknown-prefix guard below, which would otherwise swallow these as keys
+    // from a newer skill-harness. A deleted agent file must still resolve to null ("gone"),
+    // not undefined ("not comparable").
+    if (key.startsWith(PROMPT_PREFIX)) {
+        return promptDocDigestOfFile(resolve(ctx.specDir, key.slice(PROMPT_PREFIX.length)));
+    }
     if (key === PERSONA_KEY)
         return personaDigest(ctx.judgePersona);
     // Facet keys, and the legacy combined key, all resolve per scenario id. A removed
@@ -348,6 +402,13 @@ export function currentHashFor(key, ctx) {
 export function describeSourceKey(key) {
     if (key === PERSONA_KEY)
         return "the judge persona";
+    // Deliberately the same label as the raw-bytes key: only one of the pair is ever
+    // compared (see `isSupersededKey`), and a reader of a lint finding cares which FILE
+    // moved, not which of two digests of it was the one consulted.
+    if (key === SKILL_PROMPT_KEY)
+        return SKILL_KEY;
+    if (key.startsWith(PROMPT_PREFIX))
+        return key.slice(PROMPT_PREFIX.length);
     if (key.startsWith(STIMULUS_PREFIX))
         return `the stimulus for \`${key.slice(STIMULUS_PREFIX.length)}\``;
     if (key.startsWith(RUBRIC_PREFIX))
@@ -392,6 +453,149 @@ export function remedyForKey(key) {
 }
 /** The skill-text key. Skill-wide: it belongs to every scenario at once. */
 export const SKILL_KEY = "SKILL.md";
+/**
+ * The model-visible digests of the two prompt documents a run delivers: the skill text
+ * and any `system_prompt_file`. They measure what the model RECEIVES, where the bare
+ * keys above (`SKILL.md`, `<agent path>`) measure the file's raw bytes.
+ *
+ * | key | contents |
+ * |---|---|
+ * | `skill:prompt` | SKILL.md's body + its model-visible frontmatter |
+ * | `prompt:<path>` | the same, for one `system_prompt_file` |
+ *
+ * Both are prefixed, so a skill-harness OLDER than this one reads them through
+ * `currentHashFor`'s unknown-prefix guard and reports "not comparable" rather than a
+ * confident wrong finding about a source that is fine.
+ */
+export const SKILL_PROMPT_KEY = "skill:prompt";
+export const PROMPT_PREFIX = "prompt:";
+/**
+ * Frontmatter keys the model never receives: capability declarations the HARNESS
+ * consumes to build a tool allowlist.
+ *
+ * Why a denylist and not an allowlist of "body + description". An allowlist makes the
+ * safe-looking choice the dangerous one: the day a harness starts putting some new
+ * frontmatter key in context, every published number silently stops being protected and
+ * nothing says so. A denylist fails the other way — a newly-invented inert key charges
+ * one unnecessary re-run until it is named here, which is a one-line patch and a visible
+ * complaint. This module already takes that side everywhere else (see `UNREADABLE`, and
+ * the unknown-prefix guard in `currentHashFor`): a blind spot is the worst outcome, and
+ * a false alarm is merely expensive.
+ *
+ * `tools` and `allowed-tools` are the two spellings of the same ceiling in the reference
+ * corpus. Both are read by pi/pi-daddy to build a `--tools` allowlist; neither is ever
+ * rendered into the model's context.
+ */
+const CAPABILITY_KEYS = new Set(["allowed-tools", "tools"]);
+/** Opening `---` on line 1 only — a `---` further down is a horizontal rule. */
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/;
+/** A prompt document split into its frontmatter text (null when it has none) and its body. */
+function splitPromptDoc(text) {
+    const m = FRONTMATTER_RE.exec(text);
+    // Unterminated frontmatter falls through to "all body", the same call
+    // instruction-coverage.ts makes: treat the whole file as content rather than lose it.
+    return m ? { frontmatter: m[1], body: text.slice(m[0].length) } : { frontmatter: null, body: text };
+}
+/**
+ * Parsed YAML in a stable shape: object keys sorted, string scalars trimmed.
+ *
+ * Trimmed because YAML block style is not stimulus — `description: >` clips a trailing
+ * newline that `description: <text>` does not, so an author refolding a paragraph the
+ * model receives identically would otherwise be charged a re-run.
+ */
+function canonicalValue(v) {
+    if (typeof v === "string")
+        return v.trim();
+    if (Array.isArray(v))
+        return v.map(canonicalValue);
+    if (v && typeof v === "object") {
+        return Object.entries(v)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([k, val]) => [k, canonicalValue(val)]);
+    }
+    return v;
+}
+/**
+ * The frontmatter a run can observe, canonically. Built from the PARSED YAML for the
+ * same reason the scenario digests are built from the parsed scenario: reindenting or
+ * reordering keys changes no stimulus and must not read as a change.
+ *
+ * Unparseable frontmatter is hashed VERBATIM rather than dropped. Dropping it would mean
+ * a file whose frontmatter is malformed quietly loses that half of its protection — and
+ * a malformed header is exactly when an author is most likely to be mid-edit.
+ */
+function modelVisibleFrontmatter(fm) {
+    if (fm === null)
+        return null;
+    let parsed;
+    try {
+        parsed = yaml.load(fm);
+    }
+    catch {
+        return ["unparsed", fm];
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
+        return ["unparsed", fm];
+    return [
+        "parsed",
+        Object.entries(parsed)
+            .filter(([k]) => !CAPABILITY_KEYS.has(k))
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([k, v]) => [k, canonicalValue(v)]),
+    ];
+}
+/**
+ * The digest of a prompt document: its body verbatim, plus every frontmatter field the
+ * model can actually receive.
+ *
+ * The gate exists to protect published claims, so "the current text" has to mean the text
+ * the model was given. Hashing the raw bytes — which is what shipped through 0.7.0 —
+ * charged a full re-run for edits no graded run could observe: measured on the reference
+ * corpus, adding one `allowed-tools:` key to seven SKILL.md files with byte-identical
+ * bodies took lint from 2 findings to 22, and 20 of those demanded paid re-waves.
+ *
+ * Versioned in the hashed tuple (`prompt-doc/1`) because this is a stored-hash format: if
+ * the rule for "model-visible" ever changes, the tag changes with it and old digests stay
+ * legible as what they were.
+ */
+export function promptDocDigest(text) {
+    const { frontmatter, body } = splitPromptDoc(text);
+    return sha(JSON.stringify(["prompt-doc/1", modelVisibleFrontmatter(frontmatter), body]));
+}
+/** `promptDocDigest` of a file, or null when it doesn't exist / isn't readable. */
+export function promptDocDigestOfFile(path) {
+    try {
+        return promptDocDigest(readFileSync(path, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * True when `key` is a raw-bytes key whose model-visible counterpart THIS RUN also
+ * recorded — in which case the counterpart is the honest comparison and the raw-bytes one
+ * must be skipped.
+ *
+ * Both keys keep being written, and this is why: an older skill-harness reading a newer
+ * results.yaml still finds the bare `SKILL.md` key and keeps checking it, and `restamp`
+ * needs the raw-bytes hash as the proof it upgrades from. Superseding at COMPARISON time
+ * rather than dropping the key at record time is what buys both.
+ *
+ * `UNREADABLE` on EITHER side supersedes nothing. If the counterpart is unreadable there
+ * is nothing better to compare against; and if the raw-bytes key itself is unreadable, the
+ * run never verified that document at all — a fact lint must keep reporting, since the
+ * one thing this module refuses to do is let an unhashed source drop out of checking for
+ * the life of a result.
+ */
+export function isSupersededKey(key, recorded) {
+    if (key === SKILL_PROMPT_KEY || key.startsWith(PROMPT_PREFIX))
+        return false;
+    if (recorded[key] === UNREADABLE)
+        return false;
+    const upgraded = key === SKILL_KEY ? SKILL_PROMPT_KEY : PROMPT_PREFIX + key;
+    const v = recorded[upgraded];
+    return v !== undefined && v !== UNREADABLE;
+}
 /**
  * Every recorded key whose drift could change THIS scenario's verdict — excluding the
  * two skill-wide ones (`SKILL.md`, `rubric:__persona`), which callers handle
@@ -423,8 +627,12 @@ export function scenarioSourceKeys(s) {
     ];
     if (gatesDigest(s) !== null)
         keys.push(GATES_PREFIX + s.id);
-    if (s.systemPromptFile)
+    if (s.systemPromptFile) {
         keys.push(s.systemPromptFile); // the agent file IS the stimulus
+        keys.push(PROMPT_PREFIX + s.systemPromptFile); // ...as the model receives it
+    }
+    for (const ext of s.extensions ?? [])
+        keys.push(ext); // an edited extension is new stimulus
     if (s.assert?.post_test)
         keys.push(s.assert.post_test); // its contents are the gate
     const fx = effectiveFixture(s);

package/dist/spec-write.d.ts

@@ -0,0 +1,62 @@
+/**
+ * The single choke point for appending a scenario to an existing
+ * `specification.yaml`.
+ *
+ * Two callers need this — `add-test` and capture promotion — and a second
+ * implementation is how they would drift into disagreeing about what a valid
+ * write is. Everything here is deliberately append-shaped: a spec is
+ * hand-authored and full of comments, and a round trip through
+ * `yaml.load`/`yaml.dump` would silently reformat it and drop every comment the
+ * author wrote. So the existing bytes are never re-serialized — the new block is
+ * concatenated onto them and the *result* is validated before anything is
+ * written.
+ */
+/** Thrown when the spec on disk moved between the caller reading it and writing. */
+export declare class ConcurrentSpecModification extends Error {
+    constructor(specPath: string);
+}
+/** Thrown when the scenario being appended collides with one already in the spec. */
+export declare class DuplicateScenarioId extends Error {
+    constructor(id: string, specPath: string);
+}
+/** SHA-256 of spec text. Callers hold one across a read→confirm→write cycle. */
+export declare function specSha256(text: string): string;
+/**
+ * Render one scenario as a YAML block that can be concatenated onto a spec.
+ *
+ * Dumps `{ scenarios: [scenario] }` and strips the top-level key, leaving the
+ * correctly-indented list item. Going through `yaml.dump` rather than string
+ * templating is what makes arbitrary user text — quotes, colons, newlines,
+ * leading dashes — safe to embed.
+ */
+export declare function renderScenarioBlock(scenario: Record<string, unknown>): string;
+export interface AppendScenarioOptions {
+    specPath: string;
+    /** Plain object in spec field order; serialized by `renderScenarioBlock`. */
+    scenario: Record<string, unknown>;
+    /**
+     * SHA-256 the caller last saw. When supplied and the file no longer matches,
+     * the append is refused rather than layered onto someone else's edit.
+     */
+    baseSha256?: string;
+}
+export interface AppendScenarioResult {
+    id: string;
+    /** SHA-256 of the spec AFTER the append — the caller's new baseline. */
+    sha256: string;
+    /** The block that was appended, for preview/echo. */
+    block: string;
+}
+/**
+ * Validate and atomically append a scenario.
+ *
+ * Order matters and is load-bearing: read → detect concurrent modification →
+ * reject duplicate id → build → **validate the merged text** → write. The
+ * validation is on the merged result, not the block alone, because a block that
+ * parses in isolation can still break the file it lands in.
+ *
+ * The write is temp-file-plus-rename rather than `appendFileSync`. An append
+ * interrupted partway through leaves a syntactically broken spec on disk; a
+ * rename either happened or did not.
+ */
+export declare function appendScenario(opts: AppendScenarioOptions): AppendScenarioResult;

package/dist/spec-write.js

@@ -0,0 +1,106 @@
+import { createHash } from "node:crypto";
+import { readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import yaml from "js-yaml";
+import { parseSpec } from "./spec.js";
+/**
+ * The single choke point for appending a scenario to an existing
+ * `specification.yaml`.
+ *
+ * Two callers need this — `add-test` and capture promotion — and a second
+ * implementation is how they would drift into disagreeing about what a valid
+ * write is. Everything here is deliberately append-shaped: a spec is
+ * hand-authored and full of comments, and a round trip through
+ * `yaml.load`/`yaml.dump` would silently reformat it and drop every comment the
+ * author wrote. So the existing bytes are never re-serialized — the new block is
+ * concatenated onto them and the *result* is validated before anything is
+ * written.
+ */
+/** Thrown when the spec on disk moved between the caller reading it and writing. */
+export class ConcurrentSpecModification extends Error {
+    constructor(specPath) {
+        super(`${specPath} changed on disk since it was read — refusing to append. ` +
+            `Re-read the spec and retry; appending now would validate against a file that no longer exists.`);
+        this.name = "ConcurrentSpecModification";
+    }
+}
+/** Thrown when the scenario being appended collides with one already in the spec. */
+export class DuplicateScenarioId extends Error {
+    constructor(id, specPath) {
+        super(`scenario id \`${id}\` already exists in ${specPath}`);
+        this.name = "DuplicateScenarioId";
+    }
+}
+/** SHA-256 of spec text. Callers hold one across a read→confirm→write cycle. */
+export function specSha256(text) {
+    return createHash("sha256").update(text, "utf8").digest("hex");
+}
+/**
+ * Render one scenario as a YAML block that can be concatenated onto a spec.
+ *
+ * Dumps `{ scenarios: [scenario] }` and strips the top-level key, leaving the
+ * correctly-indented list item. Going through `yaml.dump` rather than string
+ * templating is what makes arbitrary user text — quotes, colons, newlines,
+ * leading dashes — safe to embed.
+ */
+export function renderScenarioBlock(scenario) {
+    const dumped = yaml.dump({ scenarios: [scenario] }, { lineWidth: -1, noRefs: true });
+    return "\n" + dumped.replace(/^scenarios:\n/, "");
+}
+/**
+ * Validate and atomically append a scenario.
+ *
+ * Order matters and is load-bearing: read → detect concurrent modification →
+ * reject duplicate id → build → **validate the merged text** → write. The
+ * validation is on the merged result, not the block alone, because a block that
+ * parses in isolation can still break the file it lands in.
+ *
+ * The write is temp-file-plus-rename rather than `appendFileSync`. An append
+ * interrupted partway through leaves a syntactically broken spec on disk; a
+ * rename either happened or did not.
+ */
+export function appendScenario(opts) {
+    const { specPath, scenario, baseSha256 } = opts;
+    const current = readFileSync(specPath, "utf8");
+    if (baseSha256 !== undefined && specSha256(current) !== baseSha256) {
+        throw new ConcurrentSpecModification(specPath);
+    }
+    const id = scenario.id;
+    if (typeof id !== "string" || id.trim() === "") {
+        throw new Error("scenario needs a non-empty string `id`");
+    }
+    const existing = parseSpec(current, specPath);
+    if (existing.scenarios.some((s) => s.id === id)) {
+        throw new DuplicateScenarioId(id, specPath);
+    }
+    const block = renderScenarioBlock(scenario);
+    const merged = current + block;
+    parseSpec(merged, specPath); // throws if the append broke the spec
+    atomicWrite(specPath, merged);
+    return { id, sha256: specSha256(merged), block };
+}
+/**
+ * Write via a sibling temp file and rename.
+ *
+ * Sibling, not `/tmp`: `rename(2)` is only atomic within a filesystem, and a
+ * cross-device rename would silently degrade to copy-then-delete — exactly the
+ * torn write this exists to prevent.
+ */
+function atomicWrite(path, text) {
+    const tmp = join(dirname(path), `.${Date.now()}-${process.pid}.specwrite.tmp`);
+    try {
+        writeFileSync(tmp, text, "utf8");
+        renameSync(tmp, path);
+    }
+    catch (err) {
+        try {
+            unlinkSync(tmp);
+        }
+        catch {
+            // Best effort: the original file is untouched either way, and masking the
+            // real failure with a cleanup error would hide why the write failed.
+        }
+        throw err;
+    }
+}
+//# sourceMappingURL=spec-write.js.map

package/dist/spec.d.ts

@@ -1,4 +1,5 @@
 import type { WorkspaceKind } from "./workspace.js";
+import { type TraceAssert } from "./trace-gates.js";
 export type ScenarioMode = "inline" | "seeded";
 export interface SeededAssert {
     vitest?: boolean;
@@ -28,9 +29,37 @@ export interface Scenario {
     checklist: string[];
     fixture?: string;
     assert?: SeededAssert;
+    /**
+     * Objective assertions over the execution trace.
+     *
+     * Deliberately NOT part of `SeededAssert`: the other gates read a staged git
+     * diff and are meaningless without a fixture, while a trace exists for any run.
+     * Declaring it opts the scenario into structured (`--mode json`) execution.
+     */
+    traceAssert?: TraceAssert;
     workspace: WorkspaceKind;
     remote: boolean;
     systemPromptFile?: string;
+    /**
+     * `env.extensions`: pi extension files to load, resolved relative to the spec dir.
+     *
+     * Loading is CLOSED, not additive — the adapter passes `--no-extensions` plus one
+     * `--extension` per entry, so exactly these load and nothing discovered does.
+     * (Measured on pi 0.83.0: that flag pair isolates even under `-a` project-local
+     * trust.) Without it, whatever the developer happened to have installed would
+     * silently become part of the test.
+     */
+    extensions?: string[];
+    /**
+     * `covers`: instruction sections this scenario is declared to exercise, e.g.
+     * `SKILL.md#core-principle`.
+     *
+     * METADATA. It stales nothing — see `sources.ts`, where it is deliberately in
+     * no digest. A `covers` edit changes which tests `--affected` selects, not what
+     * any past run measured, so charging a re-run for it would be the exact
+     * "pay tokens to fix a label" trap the facet split exists to remove.
+     */
+    covers?: string[];
     reps?: number;
     passThreshold?: number;
 }

package/dist/spec.js

@@ -1,5 +1,6 @@
 import { readFileSync } from "node:fs";
 import yaml from "js-yaml";
+import { parseTraceAssert } from "./trace-gates.js";
 /** Thrown on any validation failure. Message always carries the spec file path. */
 export class SpecError extends Error {
     constructor(message, file) {
@@ -54,6 +55,34 @@ function resolveWorkspace(env, mode, fixture, id, file) {
     }
     throw new SpecError(`scenario \`${id}\` env.workspace must be none | empty-git | fixture:<path>`, file);
 }
+/**
+ * Resolve `env.extensions` into a list of paths.
+ *
+ * Incompatible with `system_prompt_file` by construction: that flag REPLACES the
+ * system prompt to test a subagent definition in isolation, while an
+ * orchestration scenario tests the PARENT that delegates to one. Allowing both
+ * would silently test neither — the parent's instructions would be gone.
+ */
+function resolveExtensions(env, hasSystemPrompt, id, file) {
+    const raw = env && typeof env === "object" ? env.extensions : undefined;
+    if (raw === undefined)
+        return undefined;
+    if (!Array.isArray(raw) || raw.length === 0) {
+        throw new SpecError(`scenario \`${id}\` env.extensions must be a non-empty list of paths`, file);
+    }
+    const paths = raw.map((p, i) => {
+        if (typeof p !== "string" || p.trim() === "") {
+            throw new SpecError(`scenario \`${id}\` env.extensions[${i}] must be a non-empty path`, file);
+        }
+        return p.trim();
+    });
+    if (hasSystemPrompt) {
+        throw new SpecError(`scenario \`${id}\` sets both env.extensions and system_prompt_file — ` +
+            `system_prompt_file replaces the system prompt to test a subagent in isolation, ` +
+            `while env.extensions tests the parent that delegates to one. Pick one.`, file);
+    }
+    return paths;
+}
 /**
  * Resolve `env.remote`. A remote needs a repo to attach to, so it is only meaningful
  * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
@@ -144,6 +173,12 @@ export function parseSpec(text, file) {
             workspace: "none",
             remote: false,
         };
+        // `assert.trace` is legal for inline AND seeded scenarios — it reads the
+        // execution trace, which every run produces, not a staged diff.
+        const rawAssert = s.assert;
+        if (rawAssert?.trace !== undefined) {
+            scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
+        }
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
                 throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
@@ -208,6 +243,26 @@ export function parseSpec(text, file) {
             }
             scenario.systemPromptFile = s.system_prompt_file.trim();
         }
+        if (s.covers !== undefined) {
+            if (!isStringArray(s.covers) || s.covers.length === 0) {
+                throw new SpecError(`scenario \`${id}\` \`covers\` must be a non-empty list of strings`, file);
+            }
+            const bad = s.covers.find((c) => c.trim() === "");
+            if (bad !== undefined)
+                throw new SpecError(`scenario \`${id}\` \`covers\` has an empty entry`, file);
+            scenario.covers = s.covers.map((c) => c.trim());
+        }
+        // `unchanged_paths` is checked against the workspace's git state, so a scenario
+        // with no repo has nothing to observe. Refused here — free and offline — rather
+        // than at run time, because the alternative shipped for a while: the assertion
+        // silently passed against an empty change list and reported a green safety gate.
+        if (scenario.traceAssert?.unchanged_paths?.length && scenario.workspace === "none") {
+            throw new SpecError(`scenario \`${id}\` declares \`assert.trace.unchanged_paths\` but has no workspace to observe — ` +
+                `set \`env.workspace: empty-git\` or \`fixture:<path>\`, or drop the assertion. ` +
+                `A path policy with nothing to compare against would pass unconditionally.`, file);
+        }
+        // After system_prompt_file, so the incompatibility check sees the resolved value.
+        scenario.extensions = resolveExtensions(s.env, scenario.systemPromptFile !== undefined, id, file);
         if (s.reps !== undefined) {
             if (typeof s.reps !== "number" || !Number.isInteger(s.reps) || s.reps < 1) {
                 throw new SpecError(`scenario \`${id}\` \`reps\` must be a positive integer`, file);

package/dist/stability.js

@@ -2,7 +2,7 @@ import { join } from "node:path";
 import { loadSpec } from "./spec.js";
 import { effectiveVerdicts } from "./results.js";
 import { collectScoredRuns } from "./trends.js";
-import { describeSourceKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY } from "./sources.js";
+import { describeSourceKey, isSupersededKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY, SKILL_PROMPT_KEY } from "./sources.js";
 const DEFAULT_WINDOW = 5;
 /** A verdict that carries evidence about the task rather than about the harness or judge. */
 function conclusive(v) {
@@ -23,6 +23,25 @@ function pointFor(r, s, verdict) {
         unanimous: reps > 1 && s.flakiness === 0,
     };
 }
+/**
+ * Did the skill text the MODEL RECEIVES differ between two runs?
+ *
+ * Prefers `skill:prompt` when both runs recorded it, and falls back to the raw-bytes
+ * `SKILL.md` key for runs that predate it. A frontmatter-only edit moves the raw bytes and
+ * not the prompt, and labelling a flip "SKILL.md changed across that step" on that basis
+ * would point a reader at an edit no model could have seen — the note's whole job is to
+ * say which of two explanations the record supports, so it must not invent a third.
+ */
+function skillTextChanged(prev, cur) {
+    for (const key of [SKILL_PROMPT_KEY, SKILL_KEY]) {
+        const a = prev?.[key];
+        const b = cur?.[key];
+        if (a === undefined || b === undefined)
+            continue;
+        return a !== b;
+    }
+    return false;
+}
 /** reps + threshold, normalised the way lift.ts normalises it (a lone rep has no threshold). */
 function shapeOf(s) {
     const reps = s.reps ?? 1;
@@ -43,6 +62,11 @@ function compareSources(a, b, keys) {
     let shared = 0;
     const changed = [];
     for (const key of keys) {
+        // Same rule as lint's: when both runs recorded the model-visible digest of a prompt
+        // document, THAT is the comparison. Leaving the raw-bytes key in would call two runs
+        // "different questions" over an `allowed-tools:` line neither model could read.
+        if (isSupersededKey(key, a) && isSupersededKey(key, b))
+            continue;
         const va = a[key];
         const vb = b[key];
         if (va === undefined || vb === undefined)
@@ -77,9 +101,7 @@ function stabilityForScenario(group, scenario, window) {
         const cur = raw[i];
         const from = points[i - 1];
         const to = points[i];
-        const skillChanged = prev.r.source_hashes?.[SKILL_KEY] !== undefined &&
-            cur.r.source_hashes?.[SKILL_KEY] !== undefined &&
-            prev.r.source_hashes[SKILL_KEY] !== cur.r.source_hashes[SKILL_KEY];
+        const skillChanged = skillTextChanged(prev.r.source_hashes, cur.r.source_hashes);
         const base = { from, to, flipped: false, skillChanged, changedSources: [], unanimousFlip: false };
         if (!prev.ok || !cur.ok) {
             pairs.push({ ...base, status: "inconclusive" });