@skill-harness/core 0.4.0 → 0.6.0

package/dist/trends.js

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { loadSpec } from "./spec.js";
-import { readResults, effectiveVerdicts } from "./results.js";
+import { readResults, effectiveVerdicts, isScoredMode } from "./results.js";
 /** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
 function isDir(p) {
     try {
@@ -11,6 +11,67 @@ function isDir(p) {
         return false;
     }
 }
+/**
+ * Walk `<skillDir>/tests/results/` and group every SCORED run by model tag × delivery
+ * mode, chronologically (timestamp-slug dir names sort correctly).
+ *
+ * The single history reader: `collectTrends` renders it, `collectStability` derives
+ * run-over-run flips from it. Two walkers over the same tree is how "which runs count"
+ * drifts — the mistake that had force runs excluded from scoring in seven places at
+ * once (see SCORED_MODES).
+ *
+ * Red runs are excluded: a baseline has no grade, and pairing it with anything would
+ * compare a skill-off run to a skill-on one. Green and force are never pooled into one
+ * group — placement moves verdicts, so a green run and a force run of the same scenario
+ * are two measurements, not two samples.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic write) is
+ * logged via `console.warn`, skipped, and counted in `skipped` — never thrown, because
+ * one torn file must not take down a whole read-only view.
+ */
+export function collectScoredRuns(skillDir) {
+    const resultsRoot = join(skillDir, "tests", "results");
+    if (!existsSync(resultsRoot))
+        return [];
+    const groups = [];
+    const tags = readdirSync(resultsRoot)
+        .filter((n) => isDir(join(resultsRoot, n)))
+        .sort();
+    for (const tag of tags) {
+        const tagDir = join(resultsRoot, tag);
+        const runDirs = readdirSync(tagDir)
+            .map((n) => join(tagDir, n))
+            .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
+            .sort(); // timestamp-slug dir names ⇒ chronological ascending
+        if (runDirs.length === 0)
+            continue;
+        // Every candidate run is read: a run's mode is not knowable from its dir name, so
+        // filtering has to happen after the read. Bucketed by mode in first-seen order.
+        const byMode = new Map();
+        let skipped = 0;
+        for (const rd of runDirs) {
+            let r;
+            try {
+                r = readResults(rd);
+            }
+            catch (e) {
+                console.warn(`skill-harness: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
+                skipped++;
+                continue;
+            }
+            if (!isScoredMode(r.mode))
+                continue; // baseline — deliberate exclusion, not a skip
+            (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
+        }
+        for (const [mode, runs] of byMode) {
+            // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
+            // two series reports the same count on both — the alternative is attributing a
+            // parse failure to a mode nobody could read.
+            groups.push({ tag, mode, model: runs[runs.length - 1].model, runs, skipped });
+        }
+    }
+    return groups;
+}
 /**
  * Per model-tag, read the full run history (not just the latest) from
  * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort
@@ -19,15 +80,18 @@ function isDir(p) {
  * rule: an override resolves a misfire) + reps flakiness. Read-only; no
  * absolute paths in the result.
  *
- * Only scored (mode === "green") runs are included in the history — a
- * red/force run has no real grade (`effective_grade` is a "not scored"
- * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
- * the sparkline/grid. Non-green runs are deliberately excluded, which is
- * distinct from `skipped`: a run's mode can only be known after reading its
- * results.yaml, so every candidate run-dir in the tag is read (not just the
- * most recent `limit`) before filtering to green and applying the `limit`
- * window — trends is a bounded, on-demand, local view, so this extra read
- * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ * Only scored runs are included in the history — a red baseline has no real grade
+ * (`effective_grade` is a "not scored" placeholder; see run.ts) and would otherwise
+ * plot as a misleading 0% dip in the sparkline/grid. Red runs are deliberately
+ * excluded, which is distinct from `skipped`: a run's mode can only be known after
+ * reading its results.yaml, so every candidate run-dir in the tag is read (not just
+ * the most recent `limit`) before filtering and applying the `limit` window —
+ * trends is a bounded, on-demand, local view, so this extra read cost is
+ * acceptable.
+ *
+ * Green and force runs both count, but never in the same series: a tag with both
+ * yields one TrendModel per mode (see `TrendModel.mode`), each with its own
+ * `limit` window. A tag with no scored run at all is omitted entirely.
  *
  * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
  * write) is logged via `console.warn` and skipped — never surfaced or thrown —
@@ -39,63 +103,23 @@ export function collectTrends(skillDir, limit = 20) {
     const specPath = join(skillDir, "tests", "specification.yaml");
     const spec = loadSpec(specPath);
     const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
-    const resultsRoot = join(skillDir, "tests", "results");
     const models = [];
-    if (existsSync(resultsRoot)) {
-        const tags = readdirSync(resultsRoot)
-            .filter((n) => isDir(join(resultsRoot, n)))
-            .sort();
-        for (const tag of tags) {
-            const tagDir = join(resultsRoot, tag);
-            const runDirs = readdirSync(tagDir)
-                .map((n) => join(tagDir, n))
-                .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
-                .sort(); // timestamp-slug dir names ⇒ chronological ascending
-            if (runDirs.length === 0)
-                continue;
-            // Read every candidate run (mode isn't knowable from the dir name) and
-            // filter to green (scored) runs before applying the `limit` window —
-            // filtering after the slice would let red/force runs consume window
-            // slots, undercounting the green history even when more exists.
-            const greenRuns = [];
-            let skipped = 0;
-            for (const rd of runDirs) {
-                let r;
-                try {
-                    r = readResults(rd);
-                }
-                catch (e) {
-                    // A corrupt/truncated results.yaml must not take down the whole
-                    // trends view — skip that run, but surface the failure.
-                    console.warn(`skill-harness trends: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
-                    skipped++;
-                    continue;
-                }
-                if (r.mode !== "green")
-                    continue; // not scored — deliberate exclusion, not a skip
-                greenRuns.push(r);
-            }
-            if (greenRuns.length === 0)
-                continue;
-            const truncated = greenRuns.length > limit;
-            const kept = greenRuns.slice(-limit); // most recent `limit`, newest last
-            const runs = [];
-            let model = "";
-            for (const r of kept) {
-                // effectiveVerdicts is the single source of truth for the
-                // override-aware verdict/suspect rule (suspect = s.suspect &&
-                // s.override == null — an override resolves the misfire); zip in
-                // flakiness from the matching ScenarioResult.
-                const verdicts = effectiveVerdicts(r.scenarios);
-                const cells = {};
-                r.scenarios.forEach((s, i) => {
-                    cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
-                });
-                runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
-                model = r.model; // last successfully-read run (kept is ascending) wins
-            }
-            models.push({ model, tag, runs, truncated, skipped });
+    for (const group of collectScoredRuns(skillDir)) {
+        const truncated = group.runs.length > limit;
+        const kept = group.runs.slice(-limit); // most recent `limit`, newest last
+        const runs = [];
+        for (const r of kept) {
+            // effectiveVerdicts is the single source of truth for the override-aware
+            // verdict/suspect rule (suspect = s.suspect && s.override == null — an override
+            // resolves the misfire); zip in flakiness from the matching ScenarioResult.
+            const verdicts = effectiveVerdicts(r.scenarios);
+            const cells = {};
+            r.scenarios.forEach((s, i) => {
+                cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
+            });
+            runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
         }
+        models.push({ model: group.model, tag: group.tag, mode: group.mode, runs, truncated, skipped: group.skipped });
     }
     return { skill: spec.skill, scenarios, models };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.4.0",
+    "version": "0.6.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",