@skill-harness/core 0.1.2 → 0.3.0

package/dist/seeded.d.ts

@@ -1,21 +1,101 @@
 import type { Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+import { type ExecResult } from "./util/exec.js";
 interface SeededOpts {
     skillDir: string;
     adapter: HarnessAdapter;
     model: ModelRef;
     mode: RunMode;
     cwd: string;
+    specDir: string;
+    /**
+     * How the vitest gates shell out. Defaults to the real `npx vitest run`.
+     *
+     * A seam, not a mock: a workspace is a bare temp dir, so a test that exercised
+     * the real runner would resolve vitest off the network and be slow and flaky.
+     * Injecting it lets the gate LOGIC — pass, fail, nothing-collected — be tested
+     * deterministically.
+     */
+    runVitest?: (args: string[], cwd: string) => Promise<VitestRun>;
 }
+/**
+ * Result of one vitest invocation — deliberately `ExecResult`, not a narrower
+ * shape of its own.
+ *
+ * An earlier version declared `code: number`. That narrowing was a lie the
+ * compiler happened not to catch (the default's inferred type silently widened
+ * it back), and it is unrepresentable in practice: `exec` SIGKILLs on timeout
+ * and a signal-killed child closes with `code === null`. Declaring non-null
+ * would have made the vitest gate's timeout path — the one an injected double
+ * most needs to reproduce — impossible to express in a test.
+ */
+export type VitestRun = ExecResult;
 export interface SeededOutcome {
     transcript: string;
     gateFailure: string | null;
+    diff: string;
 }
+/**
+ * The added/removed lines of a unified diff — what the model actually *changed*,
+ * with context lines and file headers dropped.
+ *
+ * This is the difference between "the diff mentions `lastIndex`" and "the model
+ * touched `lastIndex`". A unified diff carries three lines of context around every
+ * hunk, so an untouched function sitting near the edit site appears in the diff
+ * verbatim. `build` A2 is exactly that shape — its checklist notes that `lastIndex`
+ * "sits two lines from the edit site" — so a naive substring test against the whole
+ * diff would fail the scenario for every model that fixed the right thing, which is
+ * worse than the prose-dependent item it replaces.
+ *
+ * Classification is HUNK-AWARE rather than prefix-based, because `+++`/`---` are
+ * only headers *outside* a hunk. Filtering on those prefixes anywhere would eat a
+ * changed line whose own source text starts with `++` or `--` — `++counter;` at
+ * column zero, a removed SQL/Lua `-- comment`, a YAML `---` separator. Those
+ * became `+++counter;` and `--- comment` once the diff marker was prepended, were
+ * read as headers, and vanished: `diff_excludes` then reported OK for a diff that
+ * touched the forbidden symbol. A false PASS on an objective gate is worse than
+ * the subjective check it replaced, so the parse follows the format instead of
+ * guessing from prefixes.
+ */
+export declare function changedLines(diff: string): string;
+/**
+ * Cut a diff to a byte budget on a line boundary, appending an explicit marker
+ * naming how much was dropped.
+ *
+ * The marker is not decoration: a silently truncated diff would let the judge
+ * grade "the function is missing" when it was merely cut off, which is the exact
+ * class of false-FAIL this whole change exists to remove. Truncation is reported
+ * as a fact about the transcript, and the untruncated diff is always on disk.
+ */
+export declare function capDiff(diff: string, maxBytes?: number): string;
 /**
  * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
- * the repo, then evaluate objective gates (staged-diff contains + optional vitest
- * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
- * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ * the repo, then evaluate the objective gates it declares — `diff_contains`,
+ * `diff_excludes`, `vitest` and `post_test`. Every gate that is configured runs;
+ * the FIRST failure is what `gateFailure` reports, and a non-null `gateFailure`
+ * makes the scenario an auto-FAIL that never reaches the judge.
+ *
+ * Returns the full staged diff alongside the transcript: the caller persists it
+ * as a run artifact, and a size-capped copy is appended to the transcript under
+ * `=== STAGED DIFF ===` so the judge grades the code rather than the model's
+ * description of it. Workspace creation (fixture copy + git baseline) and
+ * teardown are the caller's responsibility (run.ts).
  */
 export declare function runSeeded(scenario: Scenario, opts: SeededOpts): Promise<SeededOutcome>;
+export interface VitestTally {
+    passed: number;
+    failed: number;
+    skipped: number;
+    todo: number;
+}
+/**
+ * Parse vitest's `Tests  N passed | M skipped (T)` summary line.
+ *
+ * Used to require positive evidence that a hidden `post_test` actually executed
+ * assertions, rather than trusting a zero exit code — which vitest also returns
+ * when every test in the file is skipped. Returns null when no summary line is
+ * present, which the caller treats as "cannot confirm it ran" rather than as a
+ * pass.
+ */
+export declare function vitestTally(out: string): VitestTally | null;
 export {};

package/dist/seeded.js

@@ -1,10 +1,126 @@
+import { copyFileSync, statSync } from "node:fs";
+import { extname, isAbsolute, join, resolve } from "node:path";
 import { exec } from "./util/exec.js";
-const VITEST_TIMEOUT_MS = Number(process.env.SKILL_CHECK_VITEST_TIMEOUT_MS ?? 120_000);
+import { envNum } from "./util/env.js";
+/**
+ * Filename STEM the post-test is copied to at the workspace root; the extension
+ * (`.test.ts`) is appended at the copy site, which is the part that decides
+ * whether vitest collects it at all.
+ *
+ * Harness-owned rather than the author's basename, for two reasons: it cannot
+ * collide with a fixture file by accident, and a model cannot shadow the check by
+ * creating a *file* at the path it guesses we will use — the copy happens after
+ * the model is done and overwrites unconditionally. (A *directory* there makes
+ * the copy fail; that is handled as an infrastructure error, not a model FAIL.)
+ */
+const POST_TEST_BASE = "skill-harness.post";
+const VITEST_TIMEOUT_MS = envNum("VITEST_TIMEOUT_MS", 120_000);
+/**
+ * The added/removed lines of a unified diff — what the model actually *changed*,
+ * with context lines and file headers dropped.
+ *
+ * This is the difference between "the diff mentions `lastIndex`" and "the model
+ * touched `lastIndex`". A unified diff carries three lines of context around every
+ * hunk, so an untouched function sitting near the edit site appears in the diff
+ * verbatim. `build` A2 is exactly that shape — its checklist notes that `lastIndex`
+ * "sits two lines from the edit site" — so a naive substring test against the whole
+ * diff would fail the scenario for every model that fixed the right thing, which is
+ * worse than the prose-dependent item it replaces.
+ *
+ * Classification is HUNK-AWARE rather than prefix-based, because `+++`/`---` are
+ * only headers *outside* a hunk. Filtering on those prefixes anywhere would eat a
+ * changed line whose own source text starts with `++` or `--` — `++counter;` at
+ * column zero, a removed SQL/Lua `-- comment`, a YAML `---` separator. Those
+ * became `+++counter;` and `--- comment` once the diff marker was prepended, were
+ * read as headers, and vanished: `diff_excludes` then reported OK for a diff that
+ * touched the forbidden symbol. A false PASS on an objective gate is worse than
+ * the subjective check it replaced, so the parse follows the format instead of
+ * guessing from prefixes.
+ */
+export function changedLines(diff) {
+    const out = [];
+    let inHunk = false;
+    for (const line of diff.split("\n")) {
+        if (line.startsWith("@@")) {
+            inHunk = true; // a hunk header opens the region where +/- mean "changed"
+            continue;
+        }
+        if (line.startsWith("diff --git ")) {
+            inHunk = false; // a new file section closes it; its ---/+++ are headers again
+            continue;
+        }
+        if (!inHunk)
+            continue; // index/mode/---/+++ preamble lines
+        if (line.startsWith("+") || line.startsWith("-"))
+            out.push(line);
+    }
+    return out.join("\n");
+}
+/**
+ * Byte cap on the diff copy embedded in the judged transcript. The judge prompt
+ * is a single request, so an unbounded diff (a fixture-wide refactor, a
+ * regenerated lockfile) could overflow the context window and turn a gradeable
+ * run into a judge ERROR. The artifact on disk is never capped — only the copy
+ * the judge reads.
+ */
+const DIFF_MAX_BYTES = envNum("DIFF_MAX_BYTES", 64_000);
+/**
+ * Cut a diff to a byte budget on a line boundary, appending an explicit marker
+ * naming how much was dropped.
+ *
+ * The marker is not decoration: a silently truncated diff would let the judge
+ * grade "the function is missing" when it was merely cut off, which is the exact
+ * class of false-FAIL this whole change exists to remove. Truncation is reported
+ * as a fact about the transcript, and the untruncated diff is always on disk.
+ */
+export function capDiff(diff, maxBytes = DIFF_MAX_BYTES) {
+    const total = Buffer.byteLength(diff, "utf8");
+    if (total <= maxBytes)
+        return diff;
+    // Accumulate whole lines until the next one would exceed the budget. `used`
+    // counts the separators actually emitted by join() — n lines carry n-1 of
+    // them — so the omitted figure in the marker is exact rather than one byte
+    // short. This file's whole thesis is telling the judge accurately what it was
+    // not shown, so an off-by-one here is a small lie in the wrong place.
+    const kept = [];
+    let used = 0;
+    for (const line of diff.split("\n")) {
+        const cost = Buffer.byteLength(line, "utf8") + (kept.length > 0 ? 1 : 0);
+        if (used + cost > maxBytes)
+            break;
+        kept.push(line);
+        used += cost;
+    }
+    // A single line longer than the whole budget (a minified bundle, a lockfile,
+    // a generated blob) would otherwise keep nothing at all and hand the judge a
+    // marker with zero code under it. Show a byte-safe prefix instead: some
+    // evidence beats none, and the marker still says what was cut.
+    if (kept.length === 0) {
+        const head = Buffer.from(diff, "utf8").subarray(0, maxBytes).toString("utf8");
+        // toString() on a boundary-split multibyte sequence yields U+FFFD; drop a
+        // trailing one rather than show the judge a corrupted character.
+        const clean = head.endsWith("�") ? head.slice(0, -1) : head;
+        kept.push(clean);
+        used = Buffer.byteLength(clean, "utf8");
+    }
+    const omitted = total - used;
+    return (kept.join("\n") +
+        `\n[… diff truncated: ${omitted} of ${total} bytes omitted (cap ${maxBytes}). ` +
+        `The complete diff is saved beside this transcript as this scenario's .diff.txt artifact. ` +
+        `Do not treat anything below the cut as absent — it was not shown to you. …]`);
+}
 /**
  * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
- * the repo, then evaluate objective gates (staged-diff contains + optional vitest
- * pass). A failed gate short-circuits to an auto-FAIL. Workspace creation (fixture
- * copy + git baseline) and teardown are the caller's responsibility (run.ts).
+ * the repo, then evaluate the objective gates it declares — `diff_contains`,
+ * `diff_excludes`, `vitest` and `post_test`. Every gate that is configured runs;
+ * the FIRST failure is what `gateFailure` reports, and a non-null `gateFailure`
+ * makes the scenario an auto-FAIL that never reaches the judge.
+ *
+ * Returns the full staged diff alongside the transcript: the caller persists it
+ * as a run artifact, and a size-capped copy is appended to the transcript under
+ * `=== STAGED DIFF ===` so the judge grades the code rather than the model's
+ * description of it. Workspace creation (fixture copy + git baseline) and
+ * teardown are the caller's responsibility (run.ts).
  */
 export async function runSeeded(scenario, opts) {
     const repo = opts.cwd;
@@ -15,26 +131,157 @@ export async function runSeeded(scenario, opts) {
         turns: scenario.turns,
         cwd: repo,
     });
-    await git(repo, ["add", "-A"]);
-    const diff = (await git(repo, ["diff", "--cached"])).stdout;
     const parts = [harnessOut, "", "=== SEEDED GATES ==="];
     let gateFailure = null;
-    const wantDiff = scenario.assert?.diff_contains ?? [];
-    for (const needle of wantDiff) {
-        const ok = diff.includes(needle);
+    const runVitest = opts.runVitest ??
+        ((args, cwd) => exec("npx", ["vitest", "run", ...args], { cwd, timeoutMs: VITEST_TIMEOUT_MS }));
+    // `exec` never throws on a non-zero exit and, on timeout, SIGKILLs the child and
+    // resolves with whatever stdout accumulated (code === null). Ignoring that here
+    // let a truncated or failed capture flow into every downstream consumer as
+    // evidence: diff_contains FAILs blamed on the model, diff_excludes silently
+    // passing, the transcript asserting "the model left no staged changes", and a
+    // partial diff persisted as the "complete" artifact — under the cap, so the
+    // truncation marker never fires. A capture we cannot trust is infrastructure,
+    // and must say so instead of being graded.
+    const add = await git(repo, ["add", "-A"]);
+    const show = await git(repo, ["diff", "--cached"]);
+    const diff = show.stdout;
+    const gitFailure = [add, show].find((r) => r.code !== 0);
+    if (gitFailure) {
+        const why = gitFailure.code === null ? "timed out and was killed" : `exited ${gitFailure.code}`;
+        const msg = `staged diff could not be captured — git ${why} — infrastructure, not model behavior` +
+            (gitFailure.stderr.trim() ? `: ${gitFailure.stderr.trim().split("\n")[0]}` : "");
+        parts.push(`  staged diff: ERROR (${msg})`);
+        gateFailure = msg;
+        return finish(parts, gateFailure, diff);
+    }
+    // BOTH needle gates read the changed lines only, never context. A unified diff
+    // carries three lines of context per hunk, so an untouched symbol near the edit
+    // site appears verbatim in the diff text — and matching that means neither gate
+    // is answering the question it was asked.
+    //
+    // The positive gate is the one this was measured on. `build` A4 asserts
+    // diff_contains ["divide", "ok"], and its fixture's baseline already contains
+    // both — `ok` in `{ ok: true; value: T }`, `divide` inside `divideByZero` — so
+    // any edit near those lines satisfied the gate whether or not the model wrote
+    // either token. Every published A4 result recorded that as an objective pass.
+    // Read against changed lines it means what the checklist means: the model
+    // returned a Result.
+    const changed = changedLines(diff);
+    for (const needle of scenario.assert?.diff_contains ?? []) {
+        const ok = changed.includes(needle);
         parts.push(`  diff_contains ${JSON.stringify(needle)}: ${ok ? "OK" : "MISSING"}`);
         if (!ok && !gateFailure)
             gateFailure = `staged diff missing ${JSON.stringify(needle)}`;
     }
+    // Scope discipline, stated as a fact about the diff rather than inferred from
+    // whether the model remembered to say "I left lastIndex alone".
+    for (const needle of scenario.assert?.diff_excludes ?? []) {
+        const ok = !changed.includes(needle);
+        parts.push(`  diff_excludes ${JSON.stringify(needle)}: ${ok ? "OK" : "PRESENT"}`);
+        if (!ok && !gateFailure)
+            gateFailure = `staged diff touches forbidden ${JSON.stringify(needle)}`;
+    }
     if (scenario.assert?.vitest) {
-        const v = await exec("npx", ["vitest", "run"], { cwd: repo, timeoutMs: VITEST_TIMEOUT_MS });
+        const v = await runVitest([], repo);
+        // code === null means exec SIGKILLed it at the timeout. That is infrastructure,
+        // not a failing test, and must not be reported as the model's fault.
+        const killed = v.code === null;
         const passed = v.code === 0;
-        parts.push(`  vitest run: ${passed ? "PASS" : `FAIL (exit ${v.code})`}`);
-        parts.push(indent(v.stdout.trim() || v.stderr.trim()));
-        if (!passed && !gateFailure)
-            gateFailure = `vitest failed (exit ${v.code})`;
+        parts.push(killed
+            ? `  vitest run: ERROR (timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior)`
+            : `  vitest run: ${passed ? "PASS" : `FAIL (exit ${v.code})`}`);
+        parts.push(indent(bothStreams(v)));
+        if (!passed && !gateFailure) {
+            gateFailure = killed
+                ? `vitest timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior`
+                : `vitest failed (exit ${v.code})`;
+        }
     }
-    return { transcript: parts.join("\n"), gateFailure };
+    const postTest = scenario.assert?.post_test;
+    if (postTest) {
+        const src = isAbsolute(postTest) ? postTest : resolve(opts.specDir, postTest);
+        // statSync().isFile(), not existsSync: a DIRECTORY "exists", and copyFileSync
+        // then throws EISDIR. That rejection escapes runSeeded, and runRep guards only
+        // workspace setup — so it reaches runSkillModel, writeResults never runs, and a
+        // paid multi-scenario run loses every scenario already completed. One spec typo
+        // (`post_test: post` for `post/A1.test.ts`) must cost a scenario, not a run.
+        if (!isReadableFile(src)) {
+            // Not model behavior. It still fails the scenario — a silently skipped gate is
+            // worse than a loud one — but the message says whose fault it is so nobody
+            // reads it as "the model broke the test".
+            const msg = `post_test is not a readable file: ${postTest} — spec error, not model behavior`;
+            parts.push(`  post_test: ERROR (${msg})`);
+            if (!gateFailure)
+                gateFailure = msg;
+        }
+        else {
+            // `.test` is part of the name we build, never taken from the source: extname()
+            // of "A2.test.ts" is ".ts", so appending only that produced
+            // "skill-harness.post.ts" — a file vitest does not collect as a test at all,
+            // which would have made the gate silently vacuous.
+            const dest = join(repo, `${POST_TEST_BASE}.test${extname(src) || ".ts"}`);
+            try {
+                copyFileSync(src, dest);
+            }
+            catch (e) {
+                // Permissions, a race, a full disk. Same reasoning as the branch above:
+                // degrade this scenario, never take the whole run down with it.
+                const msg = `post_test could not be copied into the workspace ` +
+                    `(${e instanceof Error ? e.message : String(e)}) — infrastructure, not model behavior`;
+                parts.push(`  post_test: ERROR (${msg})`);
+                if (!gateFailure)
+                    gateFailure = msg;
+                return finish(parts, gateFailure, diff);
+            }
+            const v = await runVitest([POST_TEST_BASE], repo);
+            const out = `${v.stdout}\n${v.stderr}`;
+            // This gate must never pass without positive evidence that assertions ran.
+            // Absence-of-failure is not enough: a `.skip`/`.todo` left in the file exits
+            // ZERO and prints "Tests  1 skipped", so keying off the exit code reported
+            // PASS for a hidden gate that executed nothing — the exact vacuous-gate
+            // shape post_test exists to prevent, and one nobody would ever notice
+            // because a passing gate produces output no one reads.
+            const tally = vitestTally(out);
+            // Anchored to line start: the unanchored form also matched this string
+            // appearing anywhere in test output or a model-authored console.log, which
+            // would flip a genuine pass into a phantom "fixture is broken". The exact
+            // wording is vitest's ("No test files found, exiting with code 1", verified
+            // against 2.1.9) — recheck it when the vitest major changes.
+            const notCollected = /^\s*No test files found/im.test(out);
+            const killed = v.code === null;
+            let problem = null;
+            if (killed) {
+                problem = `post_test ${JSON.stringify(postTest)} timed out after ${VITEST_TIMEOUT_MS}ms — infrastructure, not model behavior`;
+            }
+            else if (notCollected) {
+                problem = `post_test ${JSON.stringify(postTest)} was never collected by vitest — spec/fixture error, not model behavior`;
+            }
+            else if (tally === null) {
+                problem = `post_test ${JSON.stringify(postTest)} produced no parseable vitest summary (exit ${v.code}) — cannot confirm it ran`;
+            }
+            else if (v.code !== 0 || tally.failed > 0) {
+                // A real failure is checked BEFORE the vacuity conditions below: a run
+                // where everything failed also has zero passes, and reporting that as
+                // "ran no assertions" would point the author at their spec instead of at
+                // the model's code, which is the actual news.
+                problem = `post_test ${JSON.stringify(postTest)} failed (exit ${v.code})`;
+            }
+            else if (tally.skipped > 0 || tally.todo > 0) {
+                problem = `post_test ${JSON.stringify(postTest)} has ${tally.skipped + tally.todo} skipped/todo test(s) — a hidden gate must actually run; spec error, not model behavior`;
+            }
+            else if (tally.passed === 0) {
+                problem = `post_test ${JSON.stringify(postTest)} ran no assertions — spec error, not model behavior`;
+            }
+            parts.push(problem === null
+                ? `  post_test ${JSON.stringify(postTest)}: PASS (${tally.passed} assertion-bearing test(s))`
+                : `  post_test ${JSON.stringify(postTest)}: ${v.code === 0 && !killed ? "ERROR" : "FAIL"} (${problem})`);
+            parts.push(indent(bothStreams(v)));
+            if (problem && !gateFailure)
+                gateFailure = problem;
+        }
+    }
+    return finish(parts, gateFailure, diff);
 }
 function git(cwd, args) {
     return exec("git", args, { cwd, timeoutMs: 30_000 });
@@ -42,4 +289,57 @@ function git(cwd, args) {
 function indent(s) {
     return s.split("\n").map((l) => `    ${l}`).join("\n");
 }
+/** True only for a regular file we can stat. Never throws — a directory, a dangling symlink or EACCES all read as "not usable". */
+function isReadableFile(p) {
+    try {
+        return statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Both streams, labelled.
+ *
+ * The previous `stdout.trim() || stderr.trim()` dropped stderr entirely whenever
+ * stdout was non-empty — which it always is once vitest prints its banner. That
+ * discarded exactly the diagnostics worth keeping, including exec's own
+ * `[skill-harness] killed after …ms timeout` notice, leaving a transcript that
+ * showed partial test output and an unexplained failure.
+ */
+function bothStreams(v) {
+    const o = v.stdout.trim();
+    const e = v.stderr.trim();
+    if (o && e)
+        return `${o}\n[stderr]\n${e}`;
+    return o || e;
+}
+/** Append the staged diff and return the outcome. Every exit path goes through here, so the judge always sees the same sections in the same order. */
+function finish(parts, gateFailure, diff) {
+    // The code itself, last — the gates above only prove that keywords appeared.
+    // Without this section a seeded checklist item about what the code *does* is
+    // graded from the model's own description of its work.
+    parts.push("", "=== STAGED DIFF ===");
+    parts.push(diff.trim() === "" ? "  (empty — the model left no staged changes)" : capDiff(diff));
+    return { transcript: parts.join("\n"), gateFailure, diff };
+}
+/**
+ * Parse vitest's `Tests  N passed | M skipped (T)` summary line.
+ *
+ * Used to require positive evidence that a hidden `post_test` actually executed
+ * assertions, rather than trusting a zero exit code — which vitest also returns
+ * when every test in the file is skipped. Returns null when no summary line is
+ * present, which the caller treats as "cannot confirm it ran" rather than as a
+ * pass.
+ */
+export function vitestTally(out) {
+    const line = /^\s*Tests\s+(.+)$/m.exec(out);
+    if (!line)
+        return null;
+    const read = (word) => {
+        const m = new RegExp(`(\\d+)\\s+${word}`).exec(line[1]);
+        return m ? Number(m[1]) : 0;
+    };
+    return { passed: read("passed"), failed: read("failed"), skipped: read("skipped"), todo: read("todo") };
+}
 //# sourceMappingURL=seeded.js.map

package/dist/sources.d.ts

@@ -0,0 +1,115 @@
+import type { Scenario } from "./spec.js";
+/**
+ * What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
+ * lint can prove a published result still describes the current inputs.
+ *
+ * A run measures more than the skill text. It measures the SKILL.md, any agent
+ * file, **the scenario definition itself** (turns, checklist, gates) and **the
+ * fixture** the scenario starts from. Hashing only the first two — which is what
+ * shipped through 0.2.1 — meant editing a checklist or swapping a fixture left
+ * every published result looking current.
+ *
+ * ## Key scheme
+ *
+ * | key | means |
+ * |---|---|
+ * | `SKILL.md` | the skill text, resolved against the skill dir |
+ * | `scenario:<id>` | the semantic content of one scenario in the spec |
+ * | `fixture:<path>` | every file under one fixture dir |
+ * | anything else | a file path resolved against the spec's dir (`system_prompt_file`) |
+ *
+ * Separation from bare-path keys is conventional, not guaranteed: `scenario:A1`
+ * is a legal POSIX filename, so nothing stops someone naming an agent file that.
+ * In practice `system_prompt_file` and `post_test` values are ordinary relative
+ * paths, and a `<name>:` prefix is reserved for this scheme. Old results carrying
+ * only bare-path keys keep resolving exactly as before.
+ *
+ * ## Why per-scenario, not one hash of specification.yaml
+ *
+ * Hashing the whole spec file would mark **every** historical run stale the
+ * moment a spec grows by one scenario — the precise noise `lint`'s scenario-set
+ * check already exists to prevent ("a spec reshape must not consistency-flag
+ * every historical run"). A per-scenario digest says what actually changed:
+ * editing A1's checklist marks A1 stale and leaves A2 alone; appending a new
+ * scenario marks nothing stale, because nothing already measured changed.
+ *
+ * It also ignores formatting: the digest is built from the *parsed* scenario, so
+ * reindenting the YAML or reordering scenarios is correctly a no-op, while
+ * changing a single checklist word is correctly a change.
+ */
+export declare const SCENARIO_PREFIX = "scenario:";
+export declare const FIXTURE_PREFIX = "fixture:";
+/**
+ * Recorded in place of a hash when a source existed but could not be read.
+ *
+ * Omitting it instead — which is what an early version did — is the worst
+ * available option: `lint` only ever iterates the keys a run recorded, so a
+ * source dropped at record time is never compared again for the life of that
+ * result. A fixture briefly unreadable during a run could then be replaced
+ * wholesale and `lint` would still report 0 findings, which is verbatim the miss
+ * this module was written to close.
+ *
+ * Not valid sha256 hex, so it can never equal a real digest and always surfaces.
+ */
+export declare const UNREADABLE = "unreadable";
+/** sha256 of a file, or null when it doesn't exist / isn't readable. */
+export declare function fileSha256(path: string): string | null;
+/**
+ * Stable sha256 over a directory tree: every file's relative path (POSIX-slashed,
+ * sorted) and contents. Null if the directory is missing or unreadable.
+ *
+ * Sorting is what makes it stable — readdir order is filesystem-dependent, so an
+ * unsorted walk would produce different digests for identical trees on different
+ * machines and turn CI into a staleness alarm. Paths are hashed alongside
+ * contents so that renaming a fixture file is a change, and separators are
+ * normalised so a Linux-recorded hash still matches on Windows.
+ */
+export declare function dirSha256(dir: string): string | null;
+/**
+ * A scenario's semantic digest: everything that changes what the scenario
+ * measures, and nothing that doesn't.
+ *
+ * Built from the parsed scenario rather than its YAML text, so formatting is
+ * irrelevant. `critical` is included because it changes whether the scenario can
+ * block a ship; `title` is included because it is what a reader of the scorecard
+ * believes was tested.
+ */
+export declare function scenarioDigest(s: Scenario): string;
+/**
+ * The fixture path a scenario actually runs in, or undefined.
+ *
+ * The EFFECTIVE workspace fixture, which is not always `scenario.fixture`: an
+ * inline scenario with `env.workspace: fixture:PATH` sets `workspace.fixture`
+ * and leaves `scenario.fixture` unset. Exported and shared with lint, which
+ * needs the identical rule — a second copy of this expression is how the hashed
+ * set and the checked set drift apart.
+ */
+export declare function effectiveFixture(s: Scenario): string | undefined;
+export interface SourceContext {
+    skillDir: string;
+    specDir: string;
+    scenarios: Scenario[];
+}
+/**
+ * Hash every source this run measures: SKILL.md, each distinct
+ * `system_prompt_file`, each scenario's definition, and each distinct fixture
+ * tree. Entries that can't be read are omitted — a missing source is lint's
+ * problem to report, not run's to crash on.
+ */
+export declare function sourceHashes(ctx: SourceContext): Record<string, string>;
+/**
+ * The current hash for a recorded key, or null when the source is gone.
+ *
+ * `undefined` is distinct from `null` and means "not comparable": the key names a
+ * scenario the spec no longer has. That is a spec *reshape*, not staleness — the
+ * same stance lint's scenario-set check already takes — so the caller stays quiet
+ * rather than reporting a removed scenario as a stale measurement.
+ *
+ * Sharing this resolver with `sourceHashes` is what keeps recording and checking
+ * from drifting: a new key kind is defined once, for both sides.
+ */
+export declare function currentHashFor(key: string, ctx: SourceContext): string | null | undefined;
+/** Human label for a recorded key, used in lint messages. */
+export declare function describeSourceKey(key: string): string;
+/** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
+export declare function scenarioIdForKey(key: string, scenarios: Scenario[]): string | undefined;