@skill-harness/core 0.5.0 → 0.7.0

package/dist/trends.js

@@ -11,6 +11,67 @@ function isDir(p) {
         return false;
     }
 }
+/**
+ * Walk `<skillDir>/tests/results/` and group every SCORED run by model tag × delivery
+ * mode, chronologically (timestamp-slug dir names sort correctly).
+ *
+ * The single history reader: `collectTrends` renders it, `collectStability` derives
+ * run-over-run flips from it. Two walkers over the same tree is how "which runs count"
+ * drifts — the mistake that had force runs excluded from scoring in seven places at
+ * once (see SCORED_MODES).
+ *
+ * Red runs are excluded: a baseline has no grade, and pairing it with anything would
+ * compare a skill-off run to a skill-on one. Green and force are never pooled into one
+ * group — placement moves verdicts, so a green run and a force run of the same scenario
+ * are two measurements, not two samples.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic write) is
+ * logged via `console.warn`, skipped, and counted in `skipped` — never thrown, because
+ * one torn file must not take down a whole read-only view.
+ */
+export function collectScoredRuns(skillDir) {
+    const resultsRoot = join(skillDir, "tests", "results");
+    if (!existsSync(resultsRoot))
+        return [];
+    const groups = [];
+    const tags = readdirSync(resultsRoot)
+        .filter((n) => isDir(join(resultsRoot, n)))
+        .sort();
+    for (const tag of tags) {
+        const tagDir = join(resultsRoot, tag);
+        const runDirs = readdirSync(tagDir)
+            .map((n) => join(tagDir, n))
+            .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
+            .sort(); // timestamp-slug dir names ⇒ chronological ascending
+        if (runDirs.length === 0)
+            continue;
+        // Every candidate run is read: a run's mode is not knowable from its dir name, so
+        // filtering has to happen after the read. Bucketed by mode in first-seen order.
+        const byMode = new Map();
+        let skipped = 0;
+        for (const rd of runDirs) {
+            let r;
+            try {
+                r = readResults(rd);
+            }
+            catch (e) {
+                console.warn(`skill-harness: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
+                skipped++;
+                continue;
+            }
+            if (!isScoredMode(r.mode))
+                continue; // baseline — deliberate exclusion, not a skip
+            (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
+        }
+        for (const [mode, runs] of byMode) {
+            // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
+            // two series reports the same count on both — the alternative is attributing a
+            // parse failure to a mode nobody could read.
+            groups.push({ tag, mode, model: runs[runs.length - 1].model, runs, skipped });
+        }
+    }
+    return groups;
+}
 /**
  * Per model-tag, read the full run history (not just the latest) from
  * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort
@@ -42,69 +103,23 @@ export function collectTrends(skillDir, limit = 20) {
     const specPath = join(skillDir, "tests", "specification.yaml");
     const spec = loadSpec(specPath);
     const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
-    const resultsRoot = join(skillDir, "tests", "results");
     const models = [];
-    if (existsSync(resultsRoot)) {
-        const tags = readdirSync(resultsRoot)
-            .filter((n) => isDir(join(resultsRoot, n)))
-            .sort();
-        for (const tag of tags) {
-            const tagDir = join(resultsRoot, tag);
-            const runDirs = readdirSync(tagDir)
-                .map((n) => join(tagDir, n))
-                .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
-                .sort(); // timestamp-slug dir names ⇒ chronological ascending
-            if (runDirs.length === 0)
-                continue;
-            // Read every candidate run (mode isn't knowable from the dir name) and
-            // filter to scored runs before applying the `limit` window — filtering
-            // after the slice would let red runs consume window slots, undercounting
-            // the history even when more exists. Bucketed by mode, in first-seen
-            // order, so each delivery epoch gets its own series and its own window.
-            const byMode = new Map();
-            let skipped = 0;
-            for (const rd of runDirs) {
-                let r;
-                try {
-                    r = readResults(rd);
-                }
-                catch (e) {
-                    // A corrupt/truncated results.yaml must not take down the whole
-                    // trends view — skip that run, but surface the failure.
-                    console.warn(`skill-harness trends: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
-                    skipped++;
-                    continue;
-                }
-                if (!isScoredMode(r.mode))
-                    continue; // baseline — deliberate exclusion, not a skip
-                (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
-            }
-            if (byMode.size === 0)
-                continue;
-            for (const [mode, scoredRuns] of byMode) {
-                const truncated = scoredRuns.length > limit;
-                const kept = scoredRuns.slice(-limit); // most recent `limit`, newest last
-                const runs = [];
-                let model = "";
-                for (const r of kept) {
-                    // effectiveVerdicts is the single source of truth for the
-                    // override-aware verdict/suspect rule (suspect = s.suspect &&
-                    // s.override == null — an override resolves the misfire); zip in
-                    // flakiness from the matching ScenarioResult.
-                    const verdicts = effectiveVerdicts(r.scenarios);
-                    const cells = {};
-                    r.scenarios.forEach((s, i) => {
-                        cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
-                    });
-                    runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
-                    model = r.model; // last successfully-read run (kept is ascending) wins
-                }
-                // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
-                // two series reports the same count on both — the alternative is attributing a
-                // parse failure to a mode nobody could read.
-                models.push({ model, tag, mode, runs, truncated, skipped });
-            }
+    for (const group of collectScoredRuns(skillDir)) {
+        const truncated = group.runs.length > limit;
+        const kept = group.runs.slice(-limit); // most recent `limit`, newest last
+        const runs = [];
+        for (const r of kept) {
+            // effectiveVerdicts is the single source of truth for the override-aware
+            // verdict/suspect rule (suspect = s.suspect && s.override == null — an override
+            // resolves the misfire); zip in flakiness from the matching ScenarioResult.
+            const verdicts = effectiveVerdicts(r.scenarios);
+            const cells = {};
+            r.scenarios.forEach((s, i) => {
+                cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
+            });
+            runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
         }
+        models.push({ model: group.model, tag: group.tag, mode: group.mode, runs, truncated, skipped: group.skipped });
     }
     return { skill: spec.skill, scenarios, models };
 }

package/dist/workspace.d.ts

@@ -47,3 +47,39 @@ export declare function createWorkspace(kind: WorkspaceKind, opts: {
     specDir: string;
     remote?: boolean;
 }): Workspace;
+/**
+ * A content snapshot of every file in a workspace: relative path → sha256.
+ *
+ * Taken immediately before the model runs, and compared after. Three reasons it
+ * is a content walk rather than the obvious `git diff`:
+ *
+ * 1. **`git add -A` honours `.gitignore`.** The canonical assertion this feature
+ *    exists for is `unchanged_paths: [".env"]`, and `.env` is the canonical
+ *    gitignored file. Overwriting it produced an empty diff, which read as
+ *    "observed, nothing changed" — a safety gate reporting green on precisely
+ *    the file class that motivated it. It also covers `TOOL_ARTIFACTS`, which
+ *    the harness itself writes into `.git/info/exclude`.
+ * 2. **The baseline commit is not the pre-run state.** `createWorkspace` applies
+ *    a fixture's `_staged/` and `_uncommitted/` trees AFTER `gitBaseline`, so
+ *    those files are already dirty before the model does anything. Diffing
+ *    against the baseline blamed the model for the fixture's own contents.
+ * 3. **The harness writes to the workspace too** — `runSeeded` copies the
+ *    post-test in. A snapshot taken after setup contains it, so it cancels out
+ *    instead of being attributed to the model.
+ */
+export type PathSnapshot = Map<string, string>;
+/** Snapshot a workspace, or null when there is no workspace to look at. */
+export declare function snapshotPaths(cwd: string | undefined, kind: WorkspaceKind): PathSnapshot | null;
+/**
+ * Paths whose content changed between two snapshots — added, removed, modified.
+ *
+ * The ONLY evidence `assert.trace.unchanged_paths` can honestly rest on. A tool
+ * trace proves which tool was called with which arguments; it cannot prove what
+ * that tool then did to the filesystem, so a path policy has to be checked
+ * against the filesystem.
+ *
+ * Returns null when either snapshot is missing — the caller must treat that as
+ * MISSING EVIDENCE, never as "nothing changed". An empty array means
+ * observed-and-nothing-changed; null means we could not look.
+ */
+export declare function diffSnapshots(before: PathSnapshot | null, after: PathSnapshot | null): string[] | null;

package/dist/workspace.js

@@ -1,5 +1,6 @@
 import { appendFileSync, cpSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
 import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, resolve } from "node:path";
 const GIT_TIMEOUT_MS = 30_000;
@@ -189,4 +190,64 @@ export function createWorkspace(kind, opts) {
     }
     return { cwd, cleanup };
 }
+/** Never the model's work, and never worth hashing. */
+const SNAPSHOT_SKIP = new Set([".git", "node_modules", "coverage", ".vitest"]);
+/** Snapshot a workspace, or null when there is no workspace to look at. */
+export function snapshotPaths(cwd, kind) {
+    if (kind === "none" || !cwd || !existsSync(cwd))
+        return null;
+    const out = new Map();
+    const walk = (dir, prefix) => {
+        let entries;
+        try {
+            entries = readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return; // an unreadable subtree is not evidence about the model
+        }
+        for (const e of entries) {
+            if (SNAPSHOT_SKIP.has(e.name))
+                continue;
+            const rel = prefix ? `${prefix}/${e.name}` : e.name;
+            const abs = join(dir, e.name);
+            if (e.isDirectory()) {
+                walk(abs, rel);
+            }
+            else if (e.isFile()) {
+                try {
+                    out.set(rel, createHash("sha256").update(readFileSync(abs)).digest("hex"));
+                }
+                catch {
+                    out.set(rel, "<unreadable>");
+                }
+            }
+        }
+    };
+    walk(cwd, "");
+    return out;
+}
+/**
+ * Paths whose content changed between two snapshots — added, removed, modified.
+ *
+ * The ONLY evidence `assert.trace.unchanged_paths` can honestly rest on. A tool
+ * trace proves which tool was called with which arguments; it cannot prove what
+ * that tool then did to the filesystem, so a path policy has to be checked
+ * against the filesystem.
+ *
+ * Returns null when either snapshot is missing — the caller must treat that as
+ * MISSING EVIDENCE, never as "nothing changed". An empty array means
+ * observed-and-nothing-changed; null means we could not look.
+ */
+export function diffSnapshots(before, after) {
+    if (!before || !after)
+        return null;
+    const changed = new Set();
+    for (const [path, hash] of after)
+        if (before.get(path) !== hash)
+            changed.add(path);
+    for (const path of before.keys())
+        if (!after.has(path))
+            changed.add(path);
+    return [...changed].sort();
+}
 //# sourceMappingURL=workspace.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.5.0",
+    "version": "0.7.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",