@skill-harness/core 0.3.2 → 0.5.0

package/dist/sources.js

@@ -39,8 +39,44 @@ import { isAbsolute, join, resolve } from "node:path";
  * reindenting the YAML or reordering scenarios is correctly a no-op, while
  * changing a single checklist word is correctly a change.
  */
+/**
+ * The pre-0.4.0 combined key: one digest over a scenario's stimulus, rubric, policy
+ * and gates together. Still read (runs recorded with it must keep comparing), never
+ * written. See `scenarioDigest`.
+ */
 export const SCENARIO_PREFIX = "scenario:";
 export const FIXTURE_PREFIX = "fixture:";
+/**
+ * The split: three (four, with gates) digests per scenario, each mapped to the
+ * cheapest tool that can honestly restore freshness.
+ *
+ * | key | contents | drift means | remedy |
+ * |---|---|---|---|
+ * | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
+ * | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
+ * | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
+ * | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (free; judges only flipped reps) |
+ * | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
+ *
+ * Why this matters more than it looks: with one key, lint had exactly one remedy for
+ * any drift — "re-run" — so **correcting a rubric cost model spend**. Measured on the
+ * reference corpus, two parked branches (one needle, one checklist rewrite) demanded
+ * 135 rep-executions to restore freshness while producing zero new information about
+ * the models. A gate that charges that much to fix a known-bad rubric is pressure to
+ * leave the rubric in place, which inverts the point of having a gate.
+ *
+ * The strictness is unchanged: every edit still marks something stale. Only the price
+ * of getting back to fresh changed.
+ */
+export const STIMULUS_PREFIX = "stimulus:";
+export const RUBRIC_PREFIX = "rubric:";
+export const POLICY_PREFIX = "policy:";
+export const GATES_PREFIX = "gates:";
+/**
+ * The spec-level rubric key. `__persona` cannot collide with a scenario id: ids are
+ * validated as `[A-Za-z][A-Za-z0-9_-]*`, so none can begin with an underscore.
+ */
+export const PERSONA_KEY = `${RUBRIC_PREFIX}__persona`;
 /**
  * Recorded in place of a hash when a source existed but could not be read.
  *
@@ -120,34 +156,87 @@ function walk(dir, prefix = "") {
  * block a ship; `title` is included because it is what a reader of the scorecard
  * believes was tested.
  */
-export function scenarioDigest(s) {
-    // Destructured, with the remainder pinned to `Record<string, never>`, so that
-    // adding a field to `Scenario` or `SeededAssert` FAILS THE BUILD here instead of
-    // silently escaping the digest. A field nobody remembered to add is a permanent
-    // staleness blind spot — edit it and every published result still looks current,
-    // which is precisely the bug this module exists to kill. This PR added two
-    // SeededAssert fields and remembered both; that only proves the discipline works
-    // while someone is looking, so the compiler now does the looking.
+/**
+ * A scenario's fields, sorted into the four buckets, as canonical JSON.
+ *
+ * One function so the exhaustive-destructure trick below covers all four digests at
+ * once: adding a field to `Scenario` or `SeededAssert` fails the build **here** until
+ * someone decides which bucket — and therefore which remedy — it belongs to. A field
+ * nobody assigned is a permanent staleness blind spot, and a field assigned to the
+ * wrong bucket is worse than that: it would tell a user `rescore` is enough when the
+ * transcripts are actually invalid.
+ */
+function facets(s) {
     const { id, title, critical, mode, turns, checklist, fixture, assert, workspace, remote, systemPromptFile, reps, passThreshold, ...restScenario } = s;
     const _scenarioExhaustive = restScenario;
     void _scenarioExhaustive;
     const { vitest, diff_contains, diff_excludes, post_test, ...restAssert } = assert ?? {};
     const _assertExhaustive = restAssert;
     void _assertExhaustive;
+    const hasGates = diff_contains !== undefined || diff_excludes !== undefined;
+    return {
+        // `vitest` and the `post_test` PATH are stimulus, not gates: both change what the
+        // run executes in the workspace, and neither can be re-evaluated from a saved
+        // diff. (`post_test`'s CONTENTS get their own file-path key, hashed separately.)
+        stimulus: JSON.stringify([
+            id, mode, turns, workspace, remote, systemPromptFile ?? null,
+            fixture ?? null, vitest ?? null, post_test ?? null,
+        ]),
+        rubric: JSON.stringify([id, title, checklist]),
+        policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
+        gates: hasGates ? JSON.stringify([id, diff_contains ?? null, diff_excludes ?? null]) : null,
+    };
+}
+function sha(canonical) {
+    return createHash("sha256").update(canonical).digest("hex");
+}
+export function stimulusDigest(s) {
+    return sha(facets(s).stimulus);
+}
+export function rubricDigest(s) {
+    return sha(facets(s).rubric);
+}
+export function policyDigest(s) {
+    return sha(facets(s).policy);
+}
+/** Null when the scenario declares no needle gates — no key is recorded for it. */
+export function gatesDigest(s) {
+    const g = facets(s).gates;
+    return g === null ? null : sha(g);
+}
+/** The spec-level judge persona, which is rubric for every scenario at once. */
+export function personaDigest(persona) {
+    return sha(JSON.stringify(["__persona", persona]));
+}
+/**
+ * The pre-0.4.0 combined digest: everything about a scenario in one hash.
+ *
+ * **Read-only now** — `sourceHashes` writes the four split keys instead. Kept because
+ * `lint` must still compare runs that recorded `scenario:<id>`, and those runs are
+ * every scorecard published before 0.4.0. Deleting it would turn "no findings" into
+ * "no comparison" for the entire existing corpus, silently.
+ *
+ * Its bytes must therefore never change again: this is a stored-hash format, not an
+ * implementation detail. The facet digests take new fields; this one is frozen at the
+ * 0.3.x field set, which is why it does not go through `facets()`.
+ */
+export function scenarioDigest(s) {
     const canonical = JSON.stringify([
-        id,
-        title,
-        critical,
-        mode,
-        turns,
-        checklist,
-        fixture ?? null,
-        assert ? [vitest ?? null, diff_contains ?? null, diff_excludes ?? null, post_test ?? null] : null,
-        workspace,
-        remote,
-        systemPromptFile ?? null,
-        reps ?? null,
-        passThreshold ?? null,
+        s.id,
+        s.title,
+        s.critical,
+        s.mode,
+        s.turns,
+        s.checklist,
+        s.fixture ?? null,
+        s.assert
+            ? [s.assert.vitest ?? null, s.assert.diff_contains ?? null, s.assert.diff_excludes ?? null, s.assert.post_test ?? null]
+            : null,
+        s.workspace,
+        s.remote,
+        s.systemPromptFile ?? null,
+        s.reps ?? null,
+        s.passThreshold ?? null,
     ]);
     return createHash("sha256").update(canonical).digest("hex");
 }
@@ -178,8 +267,17 @@ export function sourceHashes(ctx) {
     // UNREADABLE rather than omission on every branch below: a source we failed to
     // hash must stay visible to lint, not vanish from the record. See UNREADABLE.
     hashes["SKILL.md"] = fileSha256(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
+    hashes[PERSONA_KEY] = personaDigest(ctx.judgePersona);
     for (const s of ctx.scenarios) {
-        hashes[SCENARIO_PREFIX + s.id] = scenarioDigest(s);
+        // Split, not combined: each facet's drift has a different cheapest remedy, and a
+        // single key could only ever name the most expensive one. The legacy
+        // `scenario:<id>` key is deliberately NOT written any more — see scenarioDigest.
+        hashes[STIMULUS_PREFIX + s.id] = stimulusDigest(s);
+        hashes[RUBRIC_PREFIX + s.id] = rubricDigest(s);
+        hashes[POLICY_PREFIX + s.id] = policyDigest(s);
+        const gates = gatesDigest(s);
+        if (gates !== null)
+            hashes[GATES_PREFIX + s.id] = gates;
         if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
             hashes[s.systemPromptFile] = fileSha256(resolve(ctx.specDir, s.systemPromptFile)) ?? UNREADABLE;
         }
@@ -213,10 +311,27 @@ export function sourceHashes(ctx) {
 export function currentHashFor(key, ctx) {
     if (key === "SKILL.md")
         return fileSha256(resolve(ctx.skillDir, "SKILL.md"));
-    if (key.startsWith(SCENARIO_PREFIX)) {
-        const id = key.slice(SCENARIO_PREFIX.length);
-        const s = ctx.scenarios.find((x) => x.id === id);
-        return s ? scenarioDigest(s) : undefined; // removed → reshape, not stale
+    if (key === PERSONA_KEY)
+        return personaDigest(ctx.judgePersona);
+    // Facet keys, and the legacy combined key, all resolve per scenario id. A removed
+    // scenario is a reshape, not staleness, on every kind.
+    const facetResolvers = [
+        [STIMULUS_PREFIX, stimulusDigest],
+        [RUBRIC_PREFIX, rubricDigest],
+        [POLICY_PREFIX, policyDigest],
+        [GATES_PREFIX, gatesDigest],
+        [SCENARIO_PREFIX, scenarioDigest],
+    ];
+    for (const [prefix, digest] of facetResolvers) {
+        if (!key.startsWith(prefix))
+            continue;
+        const s = ctx.scenarios.find((x) => x.id === key.slice(prefix.length));
+        if (!s)
+            return undefined; // removed → reshape, not stale
+        // gatesDigest returns null when the scenario no longer declares needles. That is
+        // a real change (the gate the run recorded is gone), so null — "no longer
+        // exists" — is the honest answer rather than "not comparable".
+        return digest(s);
     }
     if (key.startsWith(FIXTURE_PREFIX)) {
         return dirSha256(fixtureAbs(ctx.specDir, key.slice(FIXTURE_PREFIX.length)));
@@ -231,14 +346,58 @@ export function currentHashFor(key, ctx) {
 }
 /** Human label for a recorded key, used in lint messages. */
 export function describeSourceKey(key) {
+    if (key === PERSONA_KEY)
+        return "the judge persona";
+    if (key.startsWith(STIMULUS_PREFIX))
+        return `the stimulus for \`${key.slice(STIMULUS_PREFIX.length)}\``;
+    if (key.startsWith(RUBRIC_PREFIX))
+        return `the rubric for \`${key.slice(RUBRIC_PREFIX.length)}\``;
+    if (key.startsWith(POLICY_PREFIX))
+        return `the scoring policy for \`${key.slice(POLICY_PREFIX.length)}\``;
+    if (key.startsWith(GATES_PREFIX))
+        return `the gates for \`${key.slice(GATES_PREFIX.length)}\``;
     if (key.startsWith(SCENARIO_PREFIX))
         return `scenario \`${key.slice(SCENARIO_PREFIX.length)}\``;
     if (key.startsWith(FIXTURE_PREFIX))
         return `fixture \`${key.slice(FIXTURE_PREFIX.length)}\``;
     return key;
 }
+/**
+ * The cheapest command that honestly restores freshness for this key kind.
+ *
+ * This string is the feature. Before the split, lint's only remedy was "re-run", so a
+ * one-word checklist fix cost a full model pass — pressure to leave a known-bad rubric
+ * in place. Naming the actual remedy is what converts that into a free command.
+ */
+export function remedyForKey(key) {
+    if (key === PERSONA_KEY) {
+        return "re-grade each model's saved transcripts (`grade <run-dir>`) — judge-only, no model spend";
+    }
+    if (key.startsWith(RUBRIC_PREFIX)) {
+        return "re-grade from the saved transcripts (`grade <run-dir>`) — judge-only, no model spend";
+    }
+    if (key.startsWith(POLICY_PREFIX)) {
+        return "re-score the saved reps (`rescore <run-dir>`) — free, offline";
+    }
+    if (key.startsWith(GATES_PREFIX)) {
+        return "re-evaluate the needles against the saved diffs (`regate <run-dir>`) — free, and it judges only the reps whose gate verdict flipped";
+    }
+    // A pre-split run recorded one hash over stimulus + rubric + policy + gates, so
+    // which of them moved is genuinely unknowable from the record. Naming a cheap
+    // remedy here would be a guess dressed as a fact.
+    if (key.startsWith(SCENARIO_PREFIX)) {
+        return "re-run — this run predates the stimulus/rubric/policy split, so which part changed cannot be told from what it recorded";
+    }
+    return "re-run"; // stimulus:, SKILL.md, fixture:, agent files, post_test contents
+}
 /** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
 export function scenarioIdForKey(key, scenarios) {
+    if (key === PERSONA_KEY)
+        return undefined; // spec-level: belongs to no single scenario
+    for (const p of [STIMULUS_PREFIX, RUBRIC_PREFIX, POLICY_PREFIX, GATES_PREFIX]) {
+        if (key.startsWith(p))
+            return key.slice(p.length);
+    }
     if (key.startsWith(SCENARIO_PREFIX))
         return key.slice(SCENARIO_PREFIX.length);
     if (key.startsWith(FIXTURE_PREFIX)) {

package/dist/trends.d.ts

@@ -14,6 +14,17 @@ export interface TrendRun {
 export interface TrendModel {
     model: string;
     tag: string;
+    /**
+     * The delivery mode every run in this series shares (`green` or `force`).
+     *
+     * A series is per tag AND per mode, never pooled: the two modes are different
+     * deliveries of the same text, and placement moves verdicts in both directions at
+     * once (measured on identical skill text: `build` A1 0/3 → 3/3 with force, `plan`
+     * C2 3/3 → 0/3). A sparkline that ran green then force would draw that epoch
+     * change as skill progress — or regression — which is the one thing a trend line
+     * must not invent.
+     */
+    mode: string;
     runs: TrendRun[];
     truncated: boolean;
     skipped: number;
@@ -35,15 +46,18 @@ export interface TrendData {
  * rule: an override resolves a misfire) + reps flakiness. Read-only; no
  * absolute paths in the result.
  *
- * Only scored (mode === "green") runs are included in the history — a
- * red/force run has no real grade (`effective_grade` is a "not scored"
- * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
- * the sparkline/grid. Non-green runs are deliberately excluded, which is
- * distinct from `skipped`: a run's mode can only be known after reading its
- * results.yaml, so every candidate run-dir in the tag is read (not just the
- * most recent `limit`) before filtering to green and applying the `limit`
- * window — trends is a bounded, on-demand, local view, so this extra read
- * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ * Only scored runs are included in the history — a red baseline has no real grade
+ * (`effective_grade` is a "not scored" placeholder; see run.ts) and would otherwise
+ * plot as a misleading 0% dip in the sparkline/grid. Red runs are deliberately
+ * excluded, which is distinct from `skipped`: a run's mode can only be known after
+ * reading its results.yaml, so every candidate run-dir in the tag is read (not just
+ * the most recent `limit`) before filtering and applying the `limit` window —
+ * trends is a bounded, on-demand, local view, so this extra read cost is
+ * acceptable.
+ *
+ * Green and force runs both count, but never in the same series: a tag with both
+ * yields one TrendModel per mode (see `TrendModel.mode`), each with its own
+ * `limit` window. A tag with no scored run at all is omitted entirely.
  *
  * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
  * write) is logged via `console.warn` and skipped — never surfaced or thrown —

package/dist/trends.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults, effectiveVerdicts } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
 /** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
 function isDir(p) {
     try {
@@ -19,15 +19,18 @@ function isDir(p) {
  * rule: an override resolves a misfire) + reps flakiness. Read-only; no
  * absolute paths in the result.
  *
- * Only scored (mode === "green") runs are included in the history — a
- * red/force run has no real grade (`effective_grade` is a "not scored"
- * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
- * the sparkline/grid. Non-green runs are deliberately excluded, which is
- * distinct from `skipped`: a run's mode can only be known after reading its
- * results.yaml, so every candidate run-dir in the tag is read (not just the
- * most recent `limit`) before filtering to green and applying the `limit`
- * window — trends is a bounded, on-demand, local view, so this extra read
- * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ * Only scored runs are included in the history — a red baseline has no real grade
+ * (`effective_grade` is a "not scored" placeholder; see run.ts) and would otherwise
+ * plot as a misleading 0% dip in the sparkline/grid. Red runs are deliberately
+ * excluded, which is distinct from `skipped`: a run's mode can only be known after
+ * reading its results.yaml, so every candidate run-dir in the tag is read (not just
+ * the most recent `limit`) before filtering and applying the `limit` window —
+ * trends is a bounded, on-demand, local view, so this extra read cost is
+ * acceptable.
+ *
+ * Green and force runs both count, but never in the same series: a tag with both
+ * yields one TrendModel per mode (see `TrendModel.mode`), each with its own
+ * `limit` window. A tag with no scored run at all is omitted entirely.
  *
  * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
  * write) is logged via `console.warn` and skipped — never surfaced or thrown —
@@ -54,10 +57,11 @@ export function collectTrends(skillDir, limit = 20) {
             if (runDirs.length === 0)
                 continue;
             // Read every candidate run (mode isn't knowable from the dir name) and
-            // filter to green (scored) runs before applying the `limit` window —
-            // filtering after the slice would let red/force runs consume window
-            // slots, undercounting the green history even when more exists.
-            const greenRuns = [];
+            // filter to scored runs before applying the `limit` window — filtering
+            // after the slice would let red runs consume window slots, undercounting
+            // the history even when more exists. Bucketed by mode, in first-seen
+            // order, so each delivery epoch gets its own series and its own window.
+            const byMode = new Map();
             let skipped = 0;
             for (const rd of runDirs) {
                 let r;
@@ -71,30 +75,35 @@ export function collectTrends(skillDir, limit = 20) {
                     skipped++;
                     continue;
                 }
-                if (r.mode !== "green")
-                    continue; // not scored — deliberate exclusion, not a skip
-                greenRuns.push(r);
+                if (!isScoredMode(r.mode))
+                    continue; // baseline — deliberate exclusion, not a skip
+                (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
             }
-            if (greenRuns.length === 0)
+            if (byMode.size === 0)
                 continue;
-            const truncated = greenRuns.length > limit;
-            const kept = greenRuns.slice(-limit); // most recent `limit`, newest last
-            const runs = [];
-            let model = "";
-            for (const r of kept) {
-                // effectiveVerdicts is the single source of truth for the
-                // override-aware verdict/suspect rule (suspect = s.suspect &&
-                // s.override == null — an override resolves the misfire); zip in
-                // flakiness from the matching ScenarioResult.
-                const verdicts = effectiveVerdicts(r.scenarios);
-                const cells = {};
-                r.scenarios.forEach((s, i) => {
-                    cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
-                });
-                runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
-                model = r.model; // last successfully-read run (kept is ascending) wins
+            for (const [mode, scoredRuns] of byMode) {
+                const truncated = scoredRuns.length > limit;
+                const kept = scoredRuns.slice(-limit); // most recent `limit`, newest last
+                const runs = [];
+                let model = "";
+                for (const r of kept) {
+                    // effectiveVerdicts is the single source of truth for the
+                    // override-aware verdict/suspect rule (suspect = s.suspect &&
+                    // s.override == null — an override resolves the misfire); zip in
+                    // flakiness from the matching ScenarioResult.
+                    const verdicts = effectiveVerdicts(r.scenarios);
+                    const cells = {};
+                    r.scenarios.forEach((s, i) => {
+                        cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
+                    });
+                    runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
+                    model = r.model; // last successfully-read run (kept is ascending) wins
+                }
+                // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
+                // two series reports the same count on both — the alternative is attributing a
+                // parse failure to a mode nobody could read.
+                models.push({ model, tag, mode, runs, truncated, skipped });
             }
-            models.push({ model, tag, runs, truncated, skipped });
         }
     }
     return { skill: spec.skill, scenarios, models };

package/dist/version.d.ts

@@ -0,0 +1 @@
+export declare const HARNESS_VERSION: string;

package/dist/version.js

@@ -0,0 +1,22 @@
+import { createRequire } from "node:module";
+/**
+ * The version of the harness that is running, read from this package's own
+ * `package.json`.
+ *
+ * Why this exists: `schema` is the wrong sentinel for "can these two numbers be
+ * compared". 0.2.1 → 0.3.0 kept `results.yaml` at `schema: 2` while changing what
+ * a verdict *means* — the judge started seeing the staged diff, and needle gates
+ * started matching changed lines rather than raw diff text. A record that says
+ * only `schema: 2` cannot tell you which of those it was graded under, and a stale
+ * global install produces plausible-looking numbers with no warning.
+ *
+ * `createRequire` rather than a JSON import: an `import ... from
+ * "../package.json"` needs `resolveJsonModule` and changes the emit layout under
+ * `tsc -b` (the JSON is copied into `dist/`, shifting relative depths). A runtime
+ * require resolves `../package.json` against this module's own location, which is
+ * the package root in both the source tree (`src/version.ts`) and the published
+ * build (`dist/version.js`).
+ */
+const require = createRequire(import.meta.url);
+export const HARNESS_VERSION = require("../package.json").version;
+//# sourceMappingURL=version.js.map

package/dist/workspace.js

@@ -1,4 +1,4 @@
-import { cpSync, existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { appendFileSync, cpSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
 import { execFileSync } from "node:child_process";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, resolve } from "node:path";
@@ -77,6 +77,34 @@ function assertKnownMarkers(src) {
             `${MARKERS.map((m) => `\`${m}/\``).join(" and ")}. Rename it, or move it deeper if it is ordinary content.`);
     }
 }
+/**
+ * Paths a *tool* creates during a run, excluded from the workspace repo so they never
+ * reach the captured diff.
+ *
+ * `runSeeded` records what the model did as `git add -A` + `git diff --cached`, which
+ * cannot distinguish the model's edits from vitest's cache. Measured in all four
+ * `post-diff-remeasure-full` runs of the reference corpus: every diff carried
+ * `node_modules/.vite/vitest/<sha>/results.json`, whose contents are test file paths
+ * and `"failed":false` booleans. No gate there was affected, but the channel runs both
+ * ways — a `diff_contains` needle matching a test filename can pass for free, and a
+ * `diff_excludes` needle can false-fail on a path string — and it pads every diff the
+ * judge reads.
+ */
+const TOOL_ARTIFACTS = ["node_modules/", "coverage/", ".vitest/"];
+/**
+ * Exclude tool artifacts via `.git/info/exclude`, deliberately not a `.gitignore`.
+ *
+ * `.git/info/exclude` is not a worktree file, so: it cannot contaminate a scenario
+ * that is *about* `.gitignore`, the model can neither read nor delete it, a fixture's
+ * own `.gitignore` is left byte-identical, and it applies to every git call in the
+ * workspace rather than to the ones someone remembered to add a pathspec to.
+ */
+function excludeToolArtifacts(cwd) {
+    const excludeFile = join(cwd, ".git", "info", "exclude");
+    const existing = existsSync(excludeFile) ? readFileSync(excludeFile, "utf8") : "";
+    const nl = existing.length > 0 && !existing.endsWith("\n") ? "\n" : "";
+    appendFileSync(excludeFile, `${nl}# skill-harness: tool output, never the model's work\n${TOOL_ARTIFACTS.join("\n")}\n`, "utf8");
+}
 /**
  * git init + a baseline commit, so a later `git diff --cached` shows only edits.
  * Pinned to `main`: the host's init.defaultBranch is not ours to depend on, and
@@ -84,6 +112,9 @@ function assertKnownMarkers(src) {
  */
 function gitBaseline(cwd) {
     execFileSync("git", ["init", "-q", "-b", "main"], { cwd, timeout: GIT_TIMEOUT_MS });
+    // Before the baseline `add -A`, so a fixture that ships a stray node_modules/ does
+    // not commit it either.
+    excludeToolArtifacts(cwd);
     execFileSync("git", ["add", "-A"], { cwd, timeout: GIT_TIMEOUT_MS });
     execFileSync("git", ["-c", "user.email=sh@local", "-c", "user.name=skill-harness", "commit", "-q", "--allow-empty", "-m", "baseline"], { cwd, timeout: GIT_TIMEOUT_MS });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.3.2",
+    "version": "0.5.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",