@skill-harness/core 0.3.0 → 0.3.1

package/dist/lift.d.ts

@@ -37,10 +37,30 @@ export interface Lift {
     /** Ids the green run covered that the red baseline did not (and vice versa). */
     greenOnly: string[];
     redOnly: string[];
+    /**
+     * Ids both runs covered that a lift cannot speak to, because the harness runs
+     * them identically in red and green. Reported rather than compared: folding them
+     * in would credit the red side with passes the skill itself produced. See
+     * `LiftOptions.modeInsensitive`.
+     */
+    modeInsensitive: string[];
     /** True when either side was an `--only` run, so coverage is a subset by construction. */
     partial: boolean;
     cells: Record<string, LiftCell>;
 }
+export interface LiftOptions {
+    /**
+     * Scenario ids whose red and green runs are the same run by construction, so
+     * comparing them measures nothing.
+     *
+     * The case that exists today is `system_prompt_file`: the pi adapter treats an
+     * agent-file scenario's file AS the system prompt and passes `--no-skills`
+     * *whatever the mode*, so the skill is loaded on both sides. Left in, such a
+     * cell lands in `kept` (or `both-fail`) and drags the denominator down —
+     * understating lift with evidence that the skill worked.
+     */
+    modeInsensitive?: Iterable<string>;
+}
 /**
  * Compare a red (baseline, skill off) run against a green (skill active) run of
  * the same model: the "does this skill actually do anything?" measurement.
@@ -48,22 +68,10 @@ export interface Lift {
  * Both sides go through `effectiveVerdicts`, so an author override is what
  * counts and an override resolves a misfire — the same rule scoring uses. Only
  * the intersection of scenario ids is compared; a lift cannot speak to a
- * scenario one side never ran.
+ * scenario one side never ran, nor to one the harness ran identically in both
+ * modes (`opts.modeInsensitive`).
  */
-export declare function computeLift(red: ResultsFile, green: ResultsFile): Lift;
+export declare function computeLift(red: ResultsFile, green: ResultsFile, opts?: LiftOptions): Lift;
 /** One line for a human: what the skill did, and what it cost. */
 export declare function liftHeadline(lift: Lift): string;
-/**
- * Per model-tag under <skillDir>/tests/results/, pair the most recent red run
- * with the most recent green run and compute the lift.
- *
- * Deliberately derived on read rather than persisted into results.yaml: a lift
- * is a fact about a *pair* of runs, so caching it inside one run's file would go
- * stale the moment a new baseline lands — the stale-scorecard failure mode
- * `source_hashes` exists to prevent. Deriving also means lift works
- * retroactively on results already committed by 0.1.x/0.2.0 users.
- *
- * A tag with no red baseline is omitted entirely rather than reported as a zero
- * lift: "not measured" and "measured no effect" are different claims.
- */
 export declare function collectLift(skillDir: string): Lift[];

package/dist/lift.js

@@ -1,6 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
 import { readResults, effectiveVerdicts } from "./results.js";
+import { loadSpec } from "./spec.js";
 /** A verdict that carries real evidence about the task, rather than about the harness or the judge. */
 function conclusive(verdict, suspect) {
     // ERROR is a harness failure (timeout, empty reply) — it says nothing about
@@ -30,9 +31,11 @@ function classify(red, green) {
  * Both sides go through `effectiveVerdicts`, so an author override is what
  * counts and an override resolves a misfire — the same rule scoring uses. Only
  * the intersection of scenario ids is compared; a lift cannot speak to a
- * scenario one side never ran.
+ * scenario one side never ran, nor to one the harness ran identically in both
+ * modes (`opts.modeInsensitive`).
  */
-export function computeLift(red, green) {
+export function computeLift(red, green, opts = {}) {
+    const insensitive = new Set(opts.modeInsensitive ?? []);
     const redV = new Map(effectiveVerdicts(red.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
     const greenV = new Map(effectiveVerdicts(green.scenarios).map((v) => [v.id, { verdict: v.verdict, suspect: v.suspect ?? false }]));
     const cells = {};
@@ -41,10 +44,15 @@ export function computeLift(red, green) {
     let greenPassed = 0;
     // Green order drives display order (it is the run the author is looking at),
     // restricted to ids the red baseline also covered.
+    const modeInsensitive = [];
     for (const [id, g] of greenV) {
         const r = redV.get(id);
         if (!r)
             continue;
+        if (insensitive.has(id)) {
+            modeInsensitive.push(id);
+            continue;
+        }
         const cls = classify(r, g);
         cells[id] = { red: r.verdict, redSuspect: r.suspect, green: g.verdict, class: cls };
         counts[cls]++;
@@ -71,14 +79,21 @@ export function computeLift(red, green) {
         delta: greenPassed - redPassed,
         greenOnly: [...greenV.keys()].filter((id) => !redV.has(id)),
         redOnly: [...redV.keys()].filter((id) => !greenV.has(id)),
+        modeInsensitive,
         partial: Boolean(red.partial || green.partial),
         cells,
     };
 }
 /** One line for a human: what the skill did, and what it cost. */
 export function liftHeadline(lift) {
-    if (lift.compared === 0)
+    if (lift.compared === 0) {
+        // Excluded-but-shared is not the same as never-shared. Claiming the runs had
+        // no scenario in common would hide the reason the lift is empty.
+        if (lift.modeInsensitive.length > 0) {
+            return `nothing comparable (${lift.modeInsensitive.length} shared, all run identically in both modes)`;
+        }
         return "no shared scenarios to compare";
+    }
     // Everything inconclusive is NOT "no effect" — it is no measurement. Saying
     // "no measured effect" here would be the same not-measured/measured-no-effect
     // conflation this module refuses to make when a red baseline is missing.
@@ -98,6 +113,9 @@ export function liftHeadline(lift) {
     }
     if (lift.inconclusive > 0)
         segments.push(`${lift.inconclusive} inconclusive`);
+    if (lift.modeInsensitive.length > 0) {
+        segments.push(`${lift.modeInsensitive.length} not comparable (same run in both modes)`);
+    }
     if (lift.partial)
         segments.push("partial run");
     return segments.join(" · ");
@@ -124,10 +142,31 @@ function isDir(p) {
  * A tag with no red baseline is omitted entirely rather than reported as a zero
  * lift: "not measured" and "measured no effect" are different claims.
  */
+/**
+ * Scenario ids the harness runs identically in red and green, read from the spec
+ * rather than from results.yaml: a lift is derived on read, so this has to work
+ * on runs recorded before the field existed — and `scenario:<id>` source hashes
+ * fold the value in without preserving it.
+ *
+ * Never throws: an unparseable spec must degrade to "nothing excluded" rather
+ * than take down a view that is otherwise readable from the results alone.
+ */
+function modeInsensitiveIds(skillDir) {
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    if (!existsSync(specPath))
+        return [];
+    try {
+        return loadSpec(specPath).scenarios.filter((s) => s.systemPromptFile).map((s) => s.id);
+    }
+    catch {
+        return [];
+    }
+}
 export function collectLift(skillDir) {
     const resultsRoot = join(skillDir, "tests", "results");
     if (!existsSync(resultsRoot))
         return [];
+    const modeInsensitive = modeInsensitiveIds(skillDir);
     const lifts = [];
     for (const tag of readdirSync(resultsRoot).filter((n) => isDir(join(resultsRoot, n))).sort()) {
         const tagDir = join(resultsRoot, tag);
@@ -156,7 +195,7 @@ export function collectLift(skillDir) {
         }
         if (!red || !green)
             continue;
-        lifts.push({ ...computeLift(red, green), tag });
+        lifts.push({ ...computeLift(red, green, { modeInsensitive }), tag });
     }
     return lifts;
 }

package/package.json

@@ -1,19 +1,45 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.3.0",
+    "version": "0.3.1",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",
     "main": "./dist/index.js",
     "types": "./dist/index.d.ts",
-    "exports": { ".": "./dist/index.js" },
-    "keywords": ["agent-skills", "skill", "llm", "testing", "eval", "grading", "pi", "ci", "harness"],
-    "files": ["dist/**/*.js", "dist/**/*.d.ts", "LICENSE", "README.md"],
-    "repository": { "type": "git", "url": "git+https://github.com/mojomanyana/skill-harness.git" },
-    "publishConfig": { "access": "public" },
-    "engines": { "node": ">=20" },
+    "exports": {
+        ".": "./dist/index.js"
+    },
+    "keywords": [
+        "agent-skills",
+        "skill",
+        "llm",
+        "testing",
+        "eval",
+        "grading",
+        "pi",
+        "ci",
+        "harness"
+    ],
+    "files": [
+        "dist/**/*.js",
+        "dist/**/*.d.ts",
+        "LICENSE",
+        "README.md"
+    ],
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/mojomanyana/skill-harness.git"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "engines": {
+        "node": ">=20"
+    },
     "scripts": {
         "prepack": "cp ../../LICENSE ./LICENSE"
     },
-    "dependencies": { "js-yaml": "^4.1.0" }
+    "dependencies": {
+        "js-yaml": "^4.1.0"
+    }
 }