@skill-harness/core 0.3.0 → 0.3.2

package/dist/lift.d.ts

@@ -37,10 +37,49 @@ export interface Lift {
     /** Ids the green run covered that the red baseline did not (and vice versa). */
     greenOnly: string[];
     redOnly: string[];
+    /**
+     * Ids both runs covered that a lift cannot speak to, because the harness runs
+     * them identically in red and green. Reported rather than compared: folding them
+     * in would credit the red side with passes the skill itself produced. See
+     * `LiftOptions.modeInsensitive`.
+     */
+    modeInsensitive: string[];
+    /**
+     * Ids both runs covered whose two verdicts were produced by different
+     * aggregations, so the comparison is not like-for-like. Excluded rather than
+     * compared: see `comparableAggregation`.
+     */
+    aggregationMismatch: LiftAggregationMismatch[];
     /** True when either side was an `--only` run, so coverage is a subset by construction. */
     partial: boolean;
     cells: Record<string, LiftCell>;
 }
+/**
+ * How one side produced a scenario's verdict: over how many reps, and under which
+ * majority threshold (null when no aggregation happened).
+ */
+export interface AggregationShape {
+    reps: number;
+    threshold: number | null;
+}
+export interface LiftAggregationMismatch {
+    id: string;
+    red: AggregationShape;
+    green: AggregationShape;
+}
+export interface LiftOptions {
+    /**
+     * Scenario ids whose red and green runs are the same run by construction, so
+     * comparing them measures nothing.
+     *
+     * The case that exists today is `system_prompt_file`: the pi adapter treats an
+     * agent-file scenario's file AS the system prompt and passes `--no-skills`
+     * *whatever the mode*, so the skill is loaded on both sides. Left in, such a
+     * cell lands in `kept` (or `both-fail`) and drags the denominator down —
+     * understating lift with evidence that the skill worked.
+     */
+    modeInsensitive?: Iterable<string>;
+}
 /**
  * Compare a red (baseline, skill off) run against a green (skill active) run of
  * the same model: the "does this skill actually do anything?" measurement.
@@ -48,22 +87,10 @@ export interface Lift {
  * Both sides go through `effectiveVerdicts`, so an author override is what
  * counts and an override resolves a misfire — the same rule scoring uses. Only
  * the intersection of scenario ids is compared; a lift cannot speak to a
- * scenario one side never ran.
+ * scenario one side never ran, nor to one the harness ran identically in both
+ * modes (`opts.modeInsensitive`).
  */
-export declare function computeLift(red: ResultsFile, green: ResultsFile): Lift;
+export declare function computeLift(red: ResultsFile, green: ResultsFile, opts?: LiftOptions): Lift;
 /** One line for a human: what the skill did, and what it cost. */
 export declare function liftHeadline(lift: Lift): string;
-/**
- * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
- * with the most recent green run and compute the lift.
- *
- * Deliberately derived on read rather than persisted into results.yaml: a lift
- * is a fact about a *pair* of runs, so caching it inside one run's file would go
- * stale the moment a new baseline lands — the stale-scorecard failure mode
- * `source_hashes` exists to prevent. Deriving also means lift works
- * retroactively on results already committed by 0.1.x/0.2.0 users.
- *
- * A tag with no red baseline is omitted entirely rather than reported as a zero
- * lift: "not measured" and "measured no effect" are different claims.
- */
 export declare function collectLift(skillDir: string): Lift[];

package/dist/lift.js

@@ -1,6 +1,27 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { readResults, effectiveVerdicts } from "./results.js";
+import { loadSpec } from "./spec.js";
+function aggregationShape(s) {
+    const reps = s.reps ?? 1;
+    // At one rep `outcomesToResult` keeps the single judge verdict and never calls
+    // `aggregateReps`, so a `pass_threshold` sitting beside it was applied to
+    // nothing. Normalizing it away keeps a stray field from faking a mismatch.
+    return { reps, threshold: reps > 1 ? s.pass_threshold ?? null : null };
+}
+/**
+ * Whether two verdicts were produced the same way, and so mean the same thing.
+ *
+ * A one-rep verdict is a single draw; a three-rep verdict is a majority over
+ * three. Across that gap `red FAIL -> green PASS` can be sampling alone, and
+ * `gained` would be reporting the harness's own asymmetry as skill value — the
+ * inverse of the `modeInsensitive` error, and pointing the number the *other*
+ * way. The threshold counts too: 1-of-3 versus 3-of-3 is a different majority
+ * policy at the same N, so the aggregate is not the same measurement.
+ */
+function comparableAggregation(red, green) {
+    return red.reps === green.reps && red.threshold === green.threshold;
+}
 /** A verdict that carries real evidence about the task, rather than about the harness or the judge. */
 function conclusive(verdict, suspect) {
     // ERROR is a harness failure (timeout, empty reply) — it says nothing about
@@ -30,21 +51,40 @@ function classify(red, green) {
  * Both sides go through `effectiveVerdicts`, so an author override is what
  * counts and an override resolves a misfire — the same rule scoring uses. Only
  * the intersection of scenario ids is compared; a lift cannot speak to a
- * scenario one side never ran.
+ * scenario one side never ran, nor to one the harness ran identically in both
+ * modes (`opts.modeInsensitive`).
  */
-export function computeLift(red, green) {
+export function computeLift(red, green, opts = {}) {
+    const insensitive = new Set(opts.modeInsensitive ?? []);
     const redV = new Map(effectiveVerdicts(red.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
     const greenV = new Map(effectiveVerdicts(green.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
+    const redShape = new Map(red.scenarios.map((s) => [s.id, aggregationShape(s)]));
+    const greenShape = new Map(green.scenarios.map((s) => [s.id, aggregationShape(s)]));
     const cells = {};
     const counts = { gained: 0, regressed: 0, kept: 0, "both-fail": 0, inconclusive: 0 };
     let redPassed = 0;
     let greenPassed = 0;
     // Green order drives display order (it is the run the author is looking at),
     // restricted to ids the red baseline also covered.
+    const modeInsensitive = [];
+    const aggregationMismatch = [];
     for (const [id, g] of greenV) {
         const r = redV.get(id);
         if (!r)
             continue;
+        if (insensitive.has(id)) {
+            modeInsensitive.push(id);
+            continue;
+        }
+        // Checked before classification, and reported separately, for the reason
+        // modeInsensitive is: there is no honest bucket for two verdicts that were
+        // not measured the same way.
+        const rShape = redShape.get(id) ?? { reps: 1, threshold: null };
+        const gShape = greenShape.get(id) ?? { reps: 1, threshold: null };
+        if (!comparableAggregation(rShape, gShape)) {
+            aggregationMismatch.push({ id, red: rShape, green: gShape });
+            continue;
+        }
         const cls = classify(r, g);
         cells[id] = { red: r.verdict, redSuspect: r.suspect, green: g.verdict, class: cls };
         counts[cls]++;
@@ -71,14 +111,48 @@ export function computeLift(red, green) {
         delta: greenPassed - redPassed,
         greenOnly: [...greenV.keys()].filter((id) => !redV.has(id)),
         redOnly: [...redV.keys()].filter((id) => !greenV.has(id)),
+        modeInsensitive,
+        aggregationMismatch,
         partial: Boolean(red.partial || green.partial),
         cells,
     };
 }
+function reps(n) {
+    return n === 1 ? "1 rep" : `${n} reps`;
+}
+/** What differs between the two sides, in the words of the flag that caused it. */
+function describeMismatch(ms) {
+    const distinct = new Set(ms.map((m) => m.red.reps !== m.green.reps
+        ? `red ${reps(m.red.reps)} vs ${reps(m.green.reps)}`
+        : `red pass threshold ${m.red.threshold} vs ${m.green.threshold}`));
+    return distinct.size === 1 ? [...distinct][0] : "red and green aggregated differently";
+}
+/** The one command that would make the comparison measurable. */
+function mismatchRemedy(ms) {
+    const greenReps = new Set(ms.map((m) => m.green.reps));
+    if (greenReps.size === 1 && ms.every((m) => m.red.reps !== m.green.reps)) {
+        return `re-run the baseline with --reps ${[...greenReps][0]}`;
+    }
+    return "re-measure both sides the same way";
+}
 /** One line for a human: what the skill did, and what it cost. */
 export function liftHeadline(lift) {
-    if (lift.compared === 0)
+    if (lift.compared === 0) {
+        // Excluded-but-shared is not the same as never-shared. Claiming the runs had
+        // no scenario in common would hide the reason the lift is empty.
+        const mismatched = lift.aggregationMismatch.length;
+        const insensitive = lift.modeInsensitive.length;
+        if (mismatched > 0 && insensitive > 0) {
+            return `nothing comparable (${insensitive} run identically in both modes, ${mismatched} ${describeMismatch(lift.aggregationMismatch)})`;
+        }
+        if (mismatched > 0) {
+            return `nothing comparable (${mismatched} shared, ${describeMismatch(lift.aggregationMismatch)} — ${mismatchRemedy(lift.aggregationMismatch)})`;
+        }
+        if (insensitive > 0) {
+            return `nothing comparable (${insensitive} shared, all run identically in both modes)`;
+        }
         return "no shared scenarios to compare";
+    }
     // Everything inconclusive is NOT "no effect" — it is no measurement. Saying
     // "no measured effect" here would be the same not-measured/measured-no-effect
     // conflation this module refuses to make when a red baseline is missing.
@@ -98,6 +172,12 @@ export function liftHeadline(lift) {
     }
     if (lift.inconclusive > 0)
         segments.push(`${lift.inconclusive} inconclusive`);
+    if (lift.modeInsensitive.length > 0) {
+        segments.push(`${lift.modeInsensitive.length} not comparable (same run in both modes)`);
+    }
+    if (lift.aggregationMismatch.length > 0) {
+        segments.push(`${lift.aggregationMismatch.length} not comparable (${describeMismatch(lift.aggregationMismatch)})`);
+    }
     if (lift.partial)
         segments.push("partial run");
     return segments.join(" · ");
@@ -124,10 +204,31 @@ function isDir(p) {
  * A tag with no red baseline is omitted entirely rather than reported as a zero
  * lift: "not measured" and "measured no effect" are different claims.
  */
+/**
+ * Scenario ids the harness runs identically in red and green, read from the spec
+ * rather than from results.yaml: a lift is derived on read, so this has to work
+ * on runs recorded before the field existed — and `scenario:<id>` source hashes
+ * fold the value in without preserving it.
+ *
+ * Never throws: an unparseable spec must degrade to "nothing excluded" rather
+ * than take down a view that is otherwise readable from the results alone.
+ */
+function modeInsensitiveIds(skillDir) {
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    if (!existsSync(specPath))
+        return [];
+    try {
+        return loadSpec(specPath).scenarios.filter((s) => s.systemPromptFile).map((s) => s.id);
+    }
+    catch {
+        return [];
+    }
+}
 export function collectLift(skillDir) {
     const resultsRoot = join(skillDir, "tests", "results");
     if (!existsSync(resultsRoot))
         return [];
+    const modeInsensitive = modeInsensitiveIds(skillDir);
     const lifts = [];
     for (const tag of readdirSync(resultsRoot).filter((n) => isDir(join(resultsRoot, n))).sort()) {
         const tagDir = join(resultsRoot, tag);
@@ -156,7 +257,7 @@ export function collectLift(skillDir) {
         }
         if (!red || !green)
             continue;
-        lifts.push({ ...computeLift(red, green), tag });
+        lifts.push({ ...computeLift(red, green, { modeInsensitive }), tag });
     }
     return lifts;
 }

package/package.json

@@ -1,19 +1,45 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.3.0",
+    "version": "0.3.2",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",
     "main": "./dist/index.js",
     "types": "./dist/index.d.ts",
-    "exports": { ".": "./dist/index.js" },
-    "keywords": ["agent-skills", "skill", "llm", "testing", "eval", "grading", "pi", "ci", "harness"],
-    "files": ["dist/**/*.js", "dist/**/*.d.ts", "LICENSE", "README.md"],
-    "repository": { "type": "git", "url": "git+https://github.com/mojomanyana/skill-harness.git" },
-    "publishConfig": { "access": "public" },
-    "engines": { "node": ">=20" },
+    "exports": {
+        ".": "./dist/index.js"
+    },
+    "keywords": [
+        "agent-skills",
+        "skill",
+        "llm",
+        "testing",
+        "eval",
+        "grading",
+        "pi",
+        "ci",
+        "harness"
+    ],
+    "files": [
+        "dist/**/*.js",
+        "dist/**/*.d.ts",
+        "LICENSE",
+        "README.md"
+    ],
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/mojomanyana/skill-harness.git"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "engines": {
+        "node": ">=20"
+    },
     "scripts": {
         "prepack": "cp ../../LICENSE ./LICENSE"
     },
-    "dependencies": { "js-yaml": "^4.1.0" }
+    "dependencies": {
+        "js-yaml": "^4.1.0"
+    }
 }