@skill-harness/core 0.5.0 → 0.6.0

package/dist/index.d.ts

@@ -26,3 +26,4 @@ export * from "./judge-policy.js";
 export * from "./regate.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
+export * from "./stability.js";

package/dist/index.js

@@ -26,4 +26,5 @@ export * from "./judge-policy.js";
 export * from "./regate.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
+export * from "./stability.js";
 //# sourceMappingURL=index.js.map

package/dist/lint.d.ts

@@ -1,10 +1,25 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "stability" | "lint-error";
+/**
+ * How much a finding means.
+ *
+ * `error` (the default, and every code that existed through 0.5.0) fails the gate.
+ * `info` reports something a reader should know that is NOT a defect: a boundary cell
+ * is a statement about how much one run of a scenario is worth, not a broken spec, and
+ * turning it red would make "this cell needs more reps" indistinguishable from "your
+ * fixture is missing". Omitted rather than written on every finding so the shape stays
+ * backward-compatible for anything already reading this list.
+ */
+export type LintSeverity = "error" | "info";
 export interface LintFinding {
     readonly skill: string;
     readonly scenario?: string;
     readonly code: LintCode;
     readonly message: string;
+    /** Absent means `error` — only findings that must not fail CI carry this. */
+    readonly severity?: LintSeverity;
 }
+/** True when a finding fails the gate. The single place the exit-code rule lives. */
+export declare function failsGate(f: LintFinding): boolean;
 /**
  * Validate one skill's spec + fixtures statically (and results-consistency when
  * committed results exist — see the consistency block). Never throws: a bad spec

package/dist/lint.js

@@ -5,7 +5,13 @@ import { loadSpec, SpecError } from "./spec.js";
 import { readResults, finalizeResults, findTranscriptFiles, resultsPath, scoreContextFor } from "./results.js";
 import { currentHashFor, describeSourceKey, remedyForKey, scenarioIdForKey, effectiveFixture, SCENARIO_PREFIX, STIMULUS_PREFIX, UNREADABLE } from "./sources.js";
 import { downgradeWarning } from "./downgrade.js";
+import { collectScoredRuns } from "./trends.js";
+import { boundaryCells, stabilityFrom, stabilityNote } from "./stability.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
+/** True when a finding fails the gate. The single place the exit-code rule lives. */
+export function failsGate(f) {
+    return (f.severity ?? "error") === "error";
+}
 /** True if `p` exists and is a directory. Never throws (TOCTOU-safe: a race or dangling
  * symlink between the check and the stat is treated as "not a directory", not an error). */
 function isDir(p) {
@@ -265,6 +271,29 @@ export function lintSkill(skillDir) {
             }
         }
     }
+    // Run-over-run stability — INFO, never a gate failure. Derived from committed
+    // history at zero cost, and it answers a question no single results.yaml can: a
+    // scenario can be internally unanimous in every run and still land on a different
+    // side each time. Measured in the reference corpus: two consecutive full runs, one
+    // 3/3 PASS and the next 0/3 FAIL, each `flakiness 0.00`.
+    //
+    // In lint because that is where a repo already looks, and free because it reads what
+    // is on disk. Wrapped: lintSkill must never throw, and a stability read touches every
+    // run file in the tree.
+    try {
+        for (const cell of boundaryCells(stabilityFrom(collectScoredRuns(skillDir), spec))) {
+            findings.push({
+                skill, scenario: cell.id, code: "stability", severity: "info",
+                message: `${cell.tag} mode=${cell.mode}: ${stabilityNote(cell)}`,
+            });
+        }
+    }
+    catch (e) {
+        findings.push({
+            skill, code: "stability", severity: "info",
+            message: `run-over-run stability could not be derived: ${e instanceof Error ? e.message : String(e)}`,
+        });
+    }
     return findings;
 }
 /** Model-tag dirs under tests/results (each holds timestamped run dirs). */

package/dist/report.d.ts

@@ -20,6 +20,18 @@ export interface RunColumn {
         flakiness?: number;
         override: string | null;
         note: string;
+        /**
+         * Run-over-run history for this cell, derived from the tag's other runs in the
+         * SAME mode. Present only for a cell that flipped (`state: "boundary"`) — a
+         * marker on every cell would bury the one signal it exists to show, and the
+         * per-cell `flakiness` beside it is a within-run number that cannot see this.
+         */
+        stability?: {
+            flips: number;
+            compared: number;
+            volatility: number | null;
+            note: string;
+        };
     }>;
     /**
      * Baseline-vs-skill lift for this model, when the tag has both a red baseline and
@@ -83,6 +95,18 @@ export declare function publicView(data: ReportData): {
             flakiness?: number;
             override: string | null;
             note: string;
+            /**
+             * Run-over-run history for this cell, derived from the tag's other runs in the
+             * SAME mode. Present only for a cell that flipped (`state: "boundary"`) — a
+             * marker on every cell would bury the one signal it exists to show, and the
+             * per-cell `flakiness` beside it is a within-run number that cannot see this.
+             */
+            stability?: {
+                flips: number;
+                compared: number;
+                volatility: number | null;
+                note: string;
+            };
         }>;
     }[];
 };

package/dist/report.js

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import { loadSpec } from "./spec.js";
 import { readResults } from "./results.js";
 import { collectLift, liftHeadline } from "./lift.js";
+import { boundaryCells, collectStability, stabilityNote } from "./stability.js";
 /** Most-recent run dir (by name, which is an ISO-ish slug) under a model-tag dir. */
 function latestRunDir(tagDir) {
     if (!statSync(tagDir).isDirectory())
@@ -24,6 +25,9 @@ export function collectReport(skillDir) {
     const resultsRoot = join(skillDir, "tests", "results");
     // Lift is keyed by model tag, the same key columns are built from.
     const liftByTag = new Map(collectLift(skillDir).map((l) => [l.tag, l]));
+    // Stability is keyed by tag + mode + scenario: a column shows one delivery mode, and
+    // green and force histories are never one series (placement moves verdicts).
+    const boundaryByCell = new Map(boundaryCells(collectStability(skillDir)).map((c) => [`${c.tag}\u0000${c.mode}\u0000${c.id}`, c]));
     const columns = [];
     if (existsSync(resultsRoot)) {
         const tags = readdirSync(resultsRoot)
@@ -35,9 +39,19 @@ export function collectReport(skillDir) {
             if (!runDir)
                 continue;
             const r = readResults(runDir);
+            const tagName = tagDir.split("/").pop();
             const cells = {};
             for (const s of r.scenarios) {
+                const boundary = boundaryByCell.get(`${tagName}\u0000${r.mode}\u0000${s.id}`);
                 cells[s.id] = {
+                    ...(boundary
+                        ? {
+                            stability: {
+                                flips: boundary.flips, compared: boundary.compared,
+                                volatility: boundary.volatility, note: stabilityNote(boundary),
+                            },
+                        }
+                        : {}),
                     judge_verdict: s.judge_verdict,
                     judge_reason: s.judge_reason,
                     suspect: s.suspect ?? false, // suspect defaults false for older results that predate the field
@@ -49,7 +63,7 @@ export function collectReport(skillDir) {
                     note: s.note,
                 };
             }
-            const tag = tagDir.split("/").pop();
+            const tag = tagName;
             // A column is the tag's LATEST run, which is not necessarily the skill-side
             // one — record a red baseline after a green run and the newest run in the tag
             // is red. The review UI recomputes lift from `cells` (so author overrides move

package/dist/run.d.ts

@@ -2,6 +2,7 @@ import type { Spec } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
 import { type ResultsFile } from "./results.js";
 import { type Lift } from "./lift.js";
+import { type ScenarioStability } from "./stability.js";
 export interface RunOptions {
     spec: Spec;
     skillDir: string;
@@ -55,5 +56,12 @@ export declare function hasEmptyAssistantTurn(transcript: string): boolean;
  * exists. Passing it for a green run turns the scorecard from "the skill scored
  * B" into "the skill *did* this much" — without a baseline the grade alone can't
  * distinguish a skill that works from a model that never needed it.
+ *
+ * `stability` is the run-over-run half of the same argument, and it is derived from
+ * history rather than from this run: a cell that flipped its verdict between the last
+ * two runs of this skill × model × mode is worth less than the ✓ beside it suggests,
+ * and this run's own `flaky 0.00` cannot say so — it only ever looked at one run. Pass
+ * the cells for THIS tag and mode; anything else would report another model's history
+ * under this model's scorecard.
  */
-export declare function formatScorecard(summary: RunSummary, lift?: Lift): string;
+export declare function formatScorecard(summary: RunSummary, lift?: Lift, stability?: ScenarioStability[]): string;

package/dist/run.js

@@ -11,6 +11,7 @@ import { runPool } from "./scheduler.js";
 import { outcomesToResult } from "./reps.js";
 import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
+import { boundaryCells, stabilityNote } from "./stability.js";
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -251,8 +252,15 @@ async function runRep(scenario, rep, repCount, ctx) {
  * exists. Passing it for a green run turns the scorecard from "the skill scored
  * B" into "the skill *did* this much" — without a baseline the grade alone can't
  * distinguish a skill that works from a model that never needed it.
+ *
+ * `stability` is the run-over-run half of the same argument, and it is derived from
+ * history rather than from this run: a cell that flipped its verdict between the last
+ * two runs of this skill × model × mode is worth less than the ✓ beside it suggests,
+ * and this run's own `flaky 0.00` cannot say so — it only ever looked at one run. Pass
+ * the cells for THIS tag and mode; anything else would report another model's history
+ * under this model's scorecard.
  */
-export function formatScorecard(summary, lift) {
+export function formatScorecard(summary, lift, stability) {
     const { results } = summary;
     const g = results.effective_grade;
     const lines = [];
@@ -289,6 +297,12 @@ export function formatScorecard(summary, lift) {
             ` — on pi ≥ 0.83.0 \`--skill\` only discloses the description and the body loads on demand.` +
             ` Use --mode force for delivery that cannot silently degrade, or --canary to prove it per run.`);
     }
+    // Boundary cells last, because they qualify the verdicts above: a ✓ on a cell that
+    // flipped between the last two runs is one draw, whatever its rep count said.
+    const ran = new Set(results.scenarios.map((s) => s.id));
+    for (const s of boundaryCells(stability ?? []).filter((c) => ran.has(c.id))) {
+        lines.push(`  ⇄ ${stabilityNote(s)}`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run.js.map

package/dist/sources.d.ts

@@ -172,5 +172,31 @@ export declare function describeSourceKey(key: string): string;
  * in place. Naming the actual remedy is what converts that into a free command.
  */
 export declare function remedyForKey(key: string): string;
+/** The skill-text key. Skill-wide: it belongs to every scenario at once. */
+export declare const SKILL_KEY = "SKILL.md";
+/**
+ * Every recorded key whose drift could change THIS scenario's verdict — excluding the
+ * two skill-wide ones (`SKILL.md`, `rubric:__persona`), which callers handle
+ * separately because they move every scenario at once.
+ *
+ * Written for run-over-run comparison (see stability.ts): "did these two runs ask this
+ * scenario the same question, judged by the same rubric?" is answerable from the
+ * recorded hashes, and only if you know which keys belong to the scenario. Derived
+ * from the spec rather than from the key strings, because the path-shaped keys
+ * (`system_prompt_file`, `post_test`) carry no scenario id at all.
+ *
+ * `policy:<id>` is deliberately NOT here. Its `reps`/`pass_threshold` half is already
+ * compared as an *aggregation* shape (1 draw vs a majority of 3 is the comparison a
+ * hash cannot express), and its `critical` half changes whether a verdict can block a
+ * ship, never what the verdict is. Including it would report a critical-set edit as
+ * "these runs measured different things", which is false.
+ *
+ * Both key generations are returned: 0.4.0+ runs carry the split facet keys, older
+ * ones the combined `scenario:<id>`. A caller comparing two runs must not treat a
+ * combined digest and a split one as comparable — they hash different byte layouts —
+ * so it compares only keys BOTH runs recorded, and treats "no shared key" as
+ * unverifiable rather than unchanged.
+ */
+export declare function scenarioSourceKeys(s: Scenario): string[];
 /** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
 export declare function scenarioIdForKey(key: string, scenarios: Scenario[]): string | undefined;

package/dist/sources.js

@@ -390,6 +390,48 @@ export function remedyForKey(key) {
     }
     return "re-run"; // stimulus:, SKILL.md, fixture:, agent files, post_test contents
 }
+/** The skill-text key. Skill-wide: it belongs to every scenario at once. */
+export const SKILL_KEY = "SKILL.md";
+/**
+ * Every recorded key whose drift could change THIS scenario's verdict — excluding the
+ * two skill-wide ones (`SKILL.md`, `rubric:__persona`), which callers handle
+ * separately because they move every scenario at once.
+ *
+ * Written for run-over-run comparison (see stability.ts): "did these two runs ask this
+ * scenario the same question, judged by the same rubric?" is answerable from the
+ * recorded hashes, and only if you know which keys belong to the scenario. Derived
+ * from the spec rather than from the key strings, because the path-shaped keys
+ * (`system_prompt_file`, `post_test`) carry no scenario id at all.
+ *
+ * `policy:<id>` is deliberately NOT here. Its `reps`/`pass_threshold` half is already
+ * compared as an *aggregation* shape (1 draw vs a majority of 3 is the comparison a
+ * hash cannot express), and its `critical` half changes whether a verdict can block a
+ * ship, never what the verdict is. Including it would report a critical-set edit as
+ * "these runs measured different things", which is false.
+ *
+ * Both key generations are returned: 0.4.0+ runs carry the split facet keys, older
+ * ones the combined `scenario:<id>`. A caller comparing two runs must not treat a
+ * combined digest and a split one as comparable — they hash different byte layouts —
+ * so it compares only keys BOTH runs recorded, and treats "no shared key" as
+ * unverifiable rather than unchanged.
+ */
+export function scenarioSourceKeys(s) {
+    const keys = [
+        STIMULUS_PREFIX + s.id,
+        RUBRIC_PREFIX + s.id,
+        SCENARIO_PREFIX + s.id, // legacy combined (pre-0.4.0 runs)
+    ];
+    if (gatesDigest(s) !== null)
+        keys.push(GATES_PREFIX + s.id);
+    if (s.systemPromptFile)
+        keys.push(s.systemPromptFile); // the agent file IS the stimulus
+    if (s.assert?.post_test)
+        keys.push(s.assert.post_test); // its contents are the gate
+    const fx = effectiveFixture(s);
+    if (fx)
+        keys.push(FIXTURE_PREFIX + fx);
+    return keys;
+}
 /** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
 export function scenarioIdForKey(key, scenarios) {
     if (key === PERSONA_KEY)

package/dist/stability.d.ts

@@ -0,0 +1,144 @@
+import { type Spec } from "./spec.js";
+import { type ScoredRunGroup } from "./trends.js";
+import type { Verdict } from "./score.js";
+/**
+ * Run-over-run verdict stability, per scenario, derived on read from committed
+ * history. No new measurement, nothing persisted.
+ *
+ * ## Why this exists
+ *
+ * Measured in the reference corpus (`plan`, deepseek, two consecutive full force runs,
+ * 2026-08-05 → 2026-08-06): **A5 went 3/3 PASS to 0/3 FAIL and D1 went 1/3 to 3/3, each
+ * run internally `flakiness 0.00`.** Two unanimous runs, opposite verdicts.
+ *
+ * `flakiness` is a within-run number: it measures how much the reps of ONE run
+ * disagreed. A scenario sitting on a behavioural boundary can be unanimous inside every
+ * run and still land on a different side each time — and then `flakiness 0.00` reads as
+ * confidence when it is the opposite. Nothing in a single `results.yaml` can see this,
+ * because the evidence is spread across files.
+ *
+ * ## What makes a flip a *stability* signal
+ *
+ * A verdict that changes because the scenario changed is not instability, it is an
+ * edit — and reporting edits as instability would make this feature noise. So a pair of
+ * adjacent runs is only compared when all of the following hold, and each rejection is
+ * reported with its reason rather than silently dropped:
+ *
+ * | gate | rejected as | why |
+ * |---|---|---|
+ * | both verdicts conclusive | `inconclusive` | an ERROR or unresolved misfire says nothing about behaviour |
+ * | same reps + pass threshold | `aggregation` | 1 draw vs a majority of 3 is not the same measurement (as in lift.ts) |
+ * | the scenario's own recorded sources identical | `sources` | different question, or different rubric |
+ * | those sources comparable at all | `unverified` | one run predates `source_hashes`, or the two use different key generations |
+ *
+ * **`SKILL.md` is deliberately NOT one of those gates.** In the measured case the skill
+ * text HAD changed — an edit aimed at a different scenario — while A5's own stimulus and
+ * rubric were byte-identical. Excluding that pair would have hidden the exact finding
+ * this feature was asked to surface. Such a flip is reported with `skillChanged`, and
+ * the note says what it means: either a side effect of that edit or a boundary cell.
+ * Which of the two it is cannot be told from the record, and pretending otherwise would
+ * be a guess dressed as a fact.
+ *
+ * Modes are never mixed (`collectScoredRuns` groups per mode), because placement moves
+ * verdicts on identical text — a green run and a force run are two deployments.
+ */
+/** One run's contribution to a scenario's history. */
+export interface StabilityPoint {
+    timestamp: string;
+    label: string | null;
+    /** Override-aware, matching what the scorecard claims (an override IS the author's verdict). */
+    verdict: Verdict;
+    overridden: boolean;
+    /** True for an `--only` run: real evidence about this scenario, but not a full run. */
+    partial: boolean;
+    reps: number;
+    /**
+     * Every rep in this run agreed AND there was more than one rep. A single rep is not
+     * unanimous, it is one draw — the distinction the headline finding turns on.
+     */
+    unanimous: boolean;
+}
+/** Why a pair of adjacent runs could, or could not, be compared. */
+export type PairStatus = "compared" | "inconclusive" | "aggregation" | "sources" | "unverified";
+export interface StabilityPair {
+    from: StabilityPoint;
+    to: StabilityPoint;
+    status: PairStatus;
+    /** The verdict changed. Only meaningful when `status === "compared"`. */
+    flipped: boolean;
+    /** The skill text differed between these two runs (not a rejection — see the module doc). */
+    skillChanged: boolean;
+    /** Human labels of the scenario's own sources that differed (`status === "sources"`). */
+    changedSources: string[];
+    /** A flip where BOTH runs were internally unanimous — invisible to within-run flakiness. */
+    unanimousFlip: boolean;
+}
+export type StabilityState = "stable" | "boundary" | "unmeasured";
+export interface ScenarioStability {
+    id: string;
+    title: string;
+    critical: boolean;
+    tag: string;
+    mode: string;
+    model: string;
+    /** The window, chronologically ascending (oldest first). */
+    points: StabilityPoint[];
+    /** Adjacent pairs within the window, oldest first. */
+    pairs: StabilityPair[];
+    compared: number;
+    flips: number;
+    /** Of `flips`, how many happened across a SKILL.md edit. */
+    flipsAcrossSkillEdit: number;
+    /** Of `flips`, how many were between two internally-unanimous runs. */
+    unanimousFlips: number;
+    /**
+     * `flips / compared`, or null when nothing was comparable. 0 = never flipped in the
+     * window; 1 = flipped at every opportunity.
+     *
+     * Same polarity as `flakiness` (0 is the quiet end) on purpose, and named for the
+     * thing being counted rather than for its absence: `stability: 0.0` would have to
+     * mean "perfectly stable", which reads exactly backwards next to `flaky 0.00`.
+     */
+    volatility: number | null;
+    state: StabilityState;
+}
+export interface StabilityOptions {
+    /** How many of the most recent scored runs to look at. Default 5. */
+    window?: number;
+}
+/** Derive stability for every scenario × tag × mode from an already-read history. */
+export declare function stabilityFrom(groups: ScoredRunGroup[], spec: Spec, opts?: StabilityOptions): ScenarioStability[];
+/**
+ * Read `<skillDir>/tests/results/` and derive run-over-run stability per scenario ×
+ * model tag × delivery mode. Free and offline: it reads committed results.yaml files
+ * and computes; it never runs a model, a judge, or a harness.
+ *
+ * Deliberately derived on read rather than stored in results.yaml, for the reason lift
+ * is: stability is a fact about a SET of runs, so a copy inside one run's file would be
+ * wrong the moment the next run lands — the stale-scorecard failure `source_hashes`
+ * exists to prevent. It also means this works retroactively on history recorded by
+ * every earlier version.
+ */
+export declare function collectStability(skillDir: string, opts?: StabilityOptions): ScenarioStability[];
+/** Scenarios that flipped at least once — the cells a single run reads as too certain. */
+export declare function boundaryCells(all: ScenarioStability[]): ScenarioStability[];
+/**
+ * The window as one readable string: `PASS!→FAIL!` or `FAIL⋯PASS→PASS!`.
+ *
+ * `→` is a step this comparison counted; `⋯` is a step it rejected (an edit, a
+ * different aggregation, unverifiable hashes); `!` marks a run whose reps were
+ * internally unanimous. The two arrows have to differ, or a path reading `FAIL→PASS`
+ * would sit next to "held its verdict" and look like a contradiction — the window shows
+ * every run, but only some of the steps between them are evidence.
+ */
+export declare const PATH_LEGEND = "\u2192 comparable step \u00B7 \u22EF step not comparable \u00B7 ! that run's reps were unanimous";
+export declare function verdictPath(s: ScenarioStability): string;
+/**
+ * The one-line human statement. This string is the feature: a number nobody can read
+ * ("volatility 1.00") would leave the reader exactly where a single run left them.
+ *
+ * Every branch says what the record supports and no more — which of "the edit did it"
+ * and "the cell is bimodal" is true cannot be told from committed results, so the
+ * across-an-edit wording names both.
+ */
+export declare function stabilityNote(s: ScenarioStability): string;

package/dist/stability.js

@@ -0,0 +1,232 @@
+import { join } from "node:path";
+import { loadSpec } from "./spec.js";
+import { effectiveVerdicts } from "./results.js";
+import { collectScoredRuns } from "./trends.js";
+import { describeSourceKey, scenarioSourceKeys, PERSONA_KEY, SKILL_KEY } from "./sources.js";
+const DEFAULT_WINDOW = 5;
+/** A verdict that carries evidence about the task rather than about the harness or judge. */
+function conclusive(v) {
+    // Same rule lift.ts applies, for the same reason: ERROR is a harness failure and an
+    // unresolved misfire is a judge failure. Counting either as a side of a flip would
+    // report infrastructure noise as behavioural instability.
+    return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "JUDGE-AMBIGUOUS";
+}
+function pointFor(r, s, verdict) {
+    const reps = s.reps ?? 1;
+    return {
+        timestamp: r.timestamp,
+        label: r.label,
+        verdict,
+        overridden: s.override != null,
+        partial: Boolean(r.partial),
+        reps,
+        unanimous: reps > 1 && s.flakiness === 0,
+    };
+}
+/** reps + threshold, normalised the way lift.ts normalises it (a lone rep has no threshold). */
+function shapeOf(s) {
+    const reps = s.reps ?? 1;
+    return JSON.stringify([reps, reps > 1 ? s.pass_threshold ?? null : null]);
+}
+/**
+ * Compare the recorded hashes of one scenario's own sources across two runs.
+ *
+ * Only keys BOTH runs recorded are compared: a key one side never hashed cannot be
+ * shown to be unchanged. `shared === 0` means unverifiable (a pre-`source_hashes` run,
+ * or a split-key run against a legacy combined-key one), which is reported as
+ * `unverified` rather than assumed identical — the whole value of this feature is that
+ * it does not claim more than the record supports.
+ */
+function compareSources(a, b, keys) {
+    if (!a || !b)
+        return { shared: 0, changed: [] };
+    let shared = 0;
+    const changed = [];
+    for (const key of keys) {
+        const va = a[key];
+        const vb = b[key];
+        if (va === undefined || vb === undefined)
+            continue;
+        shared++;
+        if (va !== vb)
+            changed.push(describeSourceKey(key));
+    }
+    return { shared, changed };
+}
+/** Derive one scenario's stability within one tag × mode group. */
+function stabilityForScenario(group, scenario, window) {
+    // The window is over runs that HOLD this scenario: an `--only` run elsewhere in the
+    // history must not consume a slot and shrink the comparison to nothing.
+    const relevant = group.runs.filter((r) => r.scenarios.some((s) => s.id === scenario.id));
+    const kept = relevant.slice(-window);
+    // Skill-wide rubric: the persona moves every verdict in the skill, so it belongs with
+    // the scenario's own sources rather than with the SKILL.md caveat.
+    const keys = [...scenarioSourceKeys(scenario), PERSONA_KEY];
+    const points = [];
+    const raw = [];
+    for (const r of kept) {
+        const i = r.scenarios.findIndex((s) => s.id === scenario.id);
+        const s = r.scenarios[i];
+        const eff = effectiveVerdicts(r.scenarios)[i];
+        points.push(pointFor(r, s, eff.verdict));
+        raw.push({ r, s, ok: conclusive(eff) });
+    }
+    const pairs = [];
+    for (let i = 1; i < raw.length; i++) {
+        const prev = raw[i - 1];
+        const cur = raw[i];
+        const from = points[i - 1];
+        const to = points[i];
+        const skillChanged = prev.r.source_hashes?.[SKILL_KEY] !== undefined &&
+            cur.r.source_hashes?.[SKILL_KEY] !== undefined &&
+            prev.r.source_hashes[SKILL_KEY] !== cur.r.source_hashes[SKILL_KEY];
+        const base = { from, to, flipped: false, skillChanged, changedSources: [], unanimousFlip: false };
+        if (!prev.ok || !cur.ok) {
+            pairs.push({ ...base, status: "inconclusive" });
+            continue;
+        }
+        if (shapeOf(prev.s) !== shapeOf(cur.s)) {
+            pairs.push({ ...base, status: "aggregation" });
+            continue;
+        }
+        const src = compareSources(prev.r.source_hashes, cur.r.source_hashes, keys);
+        if (src.shared === 0) {
+            pairs.push({ ...base, status: "unverified" });
+            continue;
+        }
+        if (src.changed.length > 0) {
+            pairs.push({ ...base, status: "sources", changedSources: src.changed });
+            continue;
+        }
+        const flipped = from.verdict !== to.verdict;
+        pairs.push({
+            ...base,
+            status: "compared",
+            flipped,
+            unanimousFlip: flipped && from.unanimous && to.unanimous,
+        });
+    }
+    const compared = pairs.filter((p) => p.status === "compared").length;
+    const flipped = pairs.filter((p) => p.status === "compared" && p.flipped);
+    return {
+        id: scenario.id,
+        title: scenario.title,
+        critical: scenario.critical,
+        tag: group.tag,
+        mode: group.mode,
+        model: group.model,
+        points,
+        pairs,
+        compared,
+        flips: flipped.length,
+        flipsAcrossSkillEdit: flipped.filter((p) => p.skillChanged).length,
+        unanimousFlips: flipped.filter((p) => p.unanimousFlip).length,
+        volatility: compared === 0 ? null : flipped.length / compared,
+        // "unmeasured" is a third state on purpose: a scenario with one run, or with no
+        // comparable pair, has NOT been shown to be stable. Collapsing it into "stable"
+        // would turn absence of evidence into evidence — the same conflation lift.ts
+        // refuses when it reports "no red baseline" instead of a zero.
+        state: compared === 0 ? "unmeasured" : flipped.length > 0 ? "boundary" : "stable",
+    };
+}
+/** Derive stability for every scenario × tag × mode from an already-read history. */
+export function stabilityFrom(groups, spec, opts = {}) {
+    const window = Math.max(2, opts.window ?? DEFAULT_WINDOW); // a window of 1 has no pair to compare
+    const out = [];
+    for (const group of groups) {
+        for (const scenario of spec.scenarios) {
+            out.push(stabilityForScenario(group, scenario, window));
+        }
+    }
+    return out;
+}
+/**
+ * Read `<skillDir>/tests/results/` and derive run-over-run stability per scenario ×
+ * model tag × delivery mode. Free and offline: it reads committed results.yaml files
+ * and computes; it never runs a model, a judge, or a harness.
+ *
+ * Deliberately derived on read rather than stored in results.yaml, for the reason lift
+ * is: stability is a fact about a SET of runs, so a copy inside one run's file would be
+ * wrong the moment the next run lands — the stale-scorecard failure `source_hashes`
+ * exists to prevent. It also means this works retroactively on history recorded by
+ * every earlier version.
+ */
+export function collectStability(skillDir, opts = {}) {
+    const spec = loadSpec(join(skillDir, "tests", "specification.yaml"));
+    return stabilityFrom(collectScoredRuns(skillDir), spec, opts);
+}
+/** Scenarios that flipped at least once — the cells a single run reads as too certain. */
+export function boundaryCells(all) {
+    return all.filter((s) => s.state === "boundary");
+}
+/**
+ * The window as one readable string: `PASS!→FAIL!` or `FAIL⋯PASS→PASS!`.
+ *
+ * `→` is a step this comparison counted; `⋯` is a step it rejected (an edit, a
+ * different aggregation, unverifiable hashes); `!` marks a run whose reps were
+ * internally unanimous. The two arrows have to differ, or a path reading `FAIL→PASS`
+ * would sit next to "held its verdict" and look like a contradiction — the window shows
+ * every run, but only some of the steps between them are evidence.
+ */
+export const PATH_LEGEND = "→ comparable step · ⋯ step not comparable · ! that run's reps were unanimous";
+export function verdictPath(s) {
+    const label = (p) => `${p.verdict}${p.unanimous ? "!" : ""}${p.overridden ? "(override)" : ""}`;
+    let out = s.points.length > 0 ? label(s.points[0]) : "";
+    s.pairs.forEach((pair, i) => {
+        out += `${pair.status === "compared" ? "→" : "⋯"}${label(s.points[i + 1])}`;
+    });
+    return out;
+}
+/**
+ * The one-line human statement. This string is the feature: a number nobody can read
+ * ("volatility 1.00") would leave the reader exactly where a single run left them.
+ *
+ * Every branch says what the record supports and no more — which of "the edit did it"
+ * and "the cell is bimodal" is true cannot be told from committed results, so the
+ * across-an-edit wording names both.
+ */
+export function stabilityNote(s) {
+    if (s.state === "boundary") {
+        const parts = [
+            `${s.id} flipped its verdict in ${s.flips} of ${s.compared} comparable run-to-run step(s) (${verdictPath(s)})`,
+        ];
+        if (s.unanimousFlips > 0) {
+            parts.push(`${s.unanimousFlips === s.flips ? "each flip was" : `${s.unanimousFlips} flip(s) were`} between runs that were` +
+                ` INTERNALLY UNANIMOUS (flakiness 0.00) — within-run reps cannot see this`);
+        }
+        if (s.flipsAcrossSkillEdit === s.flips && s.flips > 0) {
+            parts.push(`SKILL.md changed across ${s.flips === 1 ? "that step" : "those steps"}, while this scenario's own stimulus` +
+                ` and rubric did not — so it is either a side effect of that edit or a boundary cell, and the record cannot say which`);
+        }
+        else if (s.flipsAcrossSkillEdit > 0) {
+            parts.push(`${s.flipsAcrossSkillEdit} of them across a SKILL.md edit`);
+        }
+        else {
+            parts.push(`on unchanged skill text — treat a single run of this cell as one draw, not a measurement`);
+        }
+        return parts.join("; ");
+    }
+    if (s.state === "stable") {
+        // "across N comparable step(s)", not "across N runs": the window can hold runs whose
+        // steps were rejected, and claiming those as agreement would overstate the evidence.
+        return `${s.id} held its verdict across ${s.compared} comparable run-to-run step(s) (${verdictPath(s)})`;
+    }
+    const why = new Map();
+    for (const p of s.pairs)
+        if (p.status !== "compared")
+            why.set(p.status, (why.get(p.status) ?? 0) + 1);
+    const reasons = [...why.entries()].map(([status, n]) => `${n} ${REJECTION[status]}`);
+    const changed = [...new Set(s.pairs.flatMap((p) => p.changedSources))];
+    const detail = changed.length > 0 ? ` (${changed.join(", ")} changed — an edit, not a flip)` : "";
+    return s.points.length < 2
+        ? `${s.id} has ${s.points.length} run in this mode — no run-over-run comparison exists yet`
+        : `${s.id} has no comparable run-to-run step: ${reasons.join(", ")}${detail}`;
+}
+const REJECTION = {
+    compared: "compared",
+    inconclusive: "step(s) with an ERROR or unresolved misfire",
+    aggregation: "step(s) aggregated differently (reps or pass threshold)",
+    sources: "step(s) where the scenario's own sources changed",
+    unverified: "step(s) whose recorded hashes cannot be compared",
+};
+//# sourceMappingURL=stability.js.map

package/dist/trends.d.ts

@@ -38,6 +38,34 @@ export interface TrendData {
     }[];
     models: TrendModel[];
 }
+/** One model tag's scored run history in ONE delivery mode, chronologically ascending. */
+export interface ScoredRunGroup {
+    tag: string;
+    mode: string;
+    model: string;
+    runs: ResultsFile[];
+    /** Runs in this tag whose results.yaml could not be parsed (per tag, not per mode). */
+    skipped: number;
+}
+/**
+ * Walk `<skillDir>/tests/results/` and group every SCORED run by model tag × delivery
+ * mode, chronologically (timestamp-slug dir names sort correctly).
+ *
+ * The single history reader: `collectTrends` renders it, `collectStability` derives
+ * run-over-run flips from it. Two walkers over the same tree is how "which runs count"
+ * drifts — the mistake that had force runs excluded from scoring in seven places at
+ * once (see SCORED_MODES).
+ *
+ * Red runs are excluded: a baseline has no grade, and pairing it with anything would
+ * compare a skill-off run to a skill-on one. Green and force are never pooled into one
+ * group — placement moves verdicts, so a green run and a force run of the same scenario
+ * are two measurements, not two samples.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic write) is
+ * logged via `console.warn`, skipped, and counted in `skipped` — never thrown, because
+ * one torn file must not take down a whole read-only view.
+ */
+export declare function collectScoredRuns(skillDir: string): ScoredRunGroup[];
 /**
  * Per model-tag, read the full run history (not just the latest) from
  * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort

package/dist/trends.js

@@ -11,6 +11,67 @@ function isDir(p) {
         return false;
     }
 }
+/**
+ * Walk `<skillDir>/tests/results/` and group every SCORED run by model tag × delivery
+ * mode, chronologically (timestamp-slug dir names sort correctly).
+ *
+ * The single history reader: `collectTrends` renders it, `collectStability` derives
+ * run-over-run flips from it. Two walkers over the same tree is how "which runs count"
+ * drifts — the mistake that had force runs excluded from scoring in seven places at
+ * once (see SCORED_MODES).
+ *
+ * Red runs are excluded: a baseline has no grade, and pairing it with anything would
+ * compare a skill-off run to a skill-on one. Green and force are never pooled into one
+ * group — placement moves verdicts, so a green run and a force run of the same scenario
+ * are two measurements, not two samples.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic write) is
+ * logged via `console.warn`, skipped, and counted in `skipped` — never thrown, because
+ * one torn file must not take down a whole read-only view.
+ */
+export function collectScoredRuns(skillDir) {
+    const resultsRoot = join(skillDir, "tests", "results");
+    if (!existsSync(resultsRoot))
+        return [];
+    const groups = [];
+    const tags = readdirSync(resultsRoot)
+        .filter((n) => isDir(join(resultsRoot, n)))
+        .sort();
+    for (const tag of tags) {
+        const tagDir = join(resultsRoot, tag);
+        const runDirs = readdirSync(tagDir)
+            .map((n) => join(tagDir, n))
+            .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
+            .sort(); // timestamp-slug dir names ⇒ chronological ascending
+        if (runDirs.length === 0)
+            continue;
+        // Every candidate run is read: a run's mode is not knowable from its dir name, so
+        // filtering has to happen after the read. Bucketed by mode in first-seen order.
+        const byMode = new Map();
+        let skipped = 0;
+        for (const rd of runDirs) {
+            let r;
+            try {
+                r = readResults(rd);
+            }
+            catch (e) {
+                console.warn(`skill-harness: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
+                skipped++;
+                continue;
+            }
+            if (!isScoredMode(r.mode))
+                continue; // baseline — deliberate exclusion, not a skip
+            (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
+        }
+        for (const [mode, runs] of byMode) {
+            // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
+            // two series reports the same count on both — the alternative is attributing a
+            // parse failure to a mode nobody could read.
+            groups.push({ tag, mode, model: runs[runs.length - 1].model, runs, skipped });
+        }
+    }
+    return groups;
+}
 /**
  * Per model-tag, read the full run history (not just the latest) from
  * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort
@@ -42,69 +103,23 @@ export function collectTrends(skillDir, limit = 20) {
     const specPath = join(skillDir, "tests", "specification.yaml");
     const spec = loadSpec(specPath);
     const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
-    const resultsRoot = join(skillDir, "tests", "results");
     const models = [];
-    if (existsSync(resultsRoot)) {
-        const tags = readdirSync(resultsRoot)
-            .filter((n) => isDir(join(resultsRoot, n)))
-            .sort();
-        for (const tag of tags) {
-            const tagDir = join(resultsRoot, tag);
-            const runDirs = readdirSync(tagDir)
-                .map((n) => join(tagDir, n))
-                .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
-                .sort(); // timestamp-slug dir names ⇒ chronological ascending
-            if (runDirs.length === 0)
-                continue;
-            // Read every candidate run (mode isn't knowable from the dir name) and
-            // filter to scored runs before applying the `limit` window — filtering
-            // after the slice would let red runs consume window slots, undercounting
-            // the history even when more exists. Bucketed by mode, in first-seen
-            // order, so each delivery epoch gets its own series and its own window.
-            const byMode = new Map();
-            let skipped = 0;
-            for (const rd of runDirs) {
-                let r;
-                try {
-                    r = readResults(rd);
-                }
-                catch (e) {
-                    // A corrupt/truncated results.yaml must not take down the whole
-                    // trends view — skip that run, but surface the failure.
-                    console.warn(`skill-harness trends: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
-                    skipped++;
-                    continue;
-                }
-                if (!isScoredMode(r.mode))
-                    continue; // baseline — deliberate exclusion, not a skip
-                (byMode.get(r.mode) ?? byMode.set(r.mode, []).get(r.mode)).push(r);
-            }
-            if (byMode.size === 0)
-                continue;
-            for (const [mode, scoredRuns] of byMode) {
-                const truncated = scoredRuns.length > limit;
-                const kept = scoredRuns.slice(-limit); // most recent `limit`, newest last
-                const runs = [];
-                let model = "";
-                for (const r of kept) {
-                    // effectiveVerdicts is the single source of truth for the
-                    // override-aware verdict/suspect rule (suspect = s.suspect &&
-                    // s.override == null — an override resolves the misfire); zip in
-                    // flakiness from the matching ScenarioResult.
-                    const verdicts = effectiveVerdicts(r.scenarios);
-                    const cells = {};
-                    r.scenarios.forEach((s, i) => {
-                        cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
-                    });
-                    runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
-                    model = r.model; // last successfully-read run (kept is ascending) wins
-                }
-                // `skipped` is per tag (an unreadable run has no knowable mode), so a tag with
-                // two series reports the same count on both — the alternative is attributing a
-                // parse failure to a mode nobody could read.
-                models.push({ model, tag, mode, runs, truncated, skipped });
-            }
+    for (const group of collectScoredRuns(skillDir)) {
+        const truncated = group.runs.length > limit;
+        const kept = group.runs.slice(-limit); // most recent `limit`, newest last
+        const runs = [];
+        for (const r of kept) {
+            // effectiveVerdicts is the single source of truth for the override-aware
+            // verdict/suspect rule (suspect = s.suspect && s.override == null — an override
+            // resolves the misfire); zip in flakiness from the matching ScenarioResult.
+            const verdicts = effectiveVerdicts(r.scenarios);
+            const cells = {};
+            r.scenarios.forEach((s, i) => {
+                cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
+            });
+            runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
         }
+        models.push({ model: group.model, tag: group.tag, mode: group.mode, runs, truncated, skipped: group.skipped });
     }
     return { skill: spec.skill, scenarios, models };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.5.0",
+    "version": "0.6.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",