@skill-harness/core 0.3.2 → 0.5.0

package/dist/adapters/types.d.ts

@@ -32,4 +32,14 @@ export interface HarnessAdapter {
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     judge(req: JudgeReq): Promise<string>;
+    /**
+     * The harness CLI's own version, recorded in `results.yaml` as
+     * `harness_cli_version`. Null when it cannot be determined — a version this
+     * adapter had to guess at is worse than none, because the whole point of the
+     * field is to identify which CLI produced a transcript.
+     *
+     * Optional so a test double or a future adapter need not implement it; callers
+     * treat a missing method exactly like a null answer.
+     */
+    version?(): Promise<string | null>;
 }

package/dist/canary.d.ts

@@ -0,0 +1,44 @@
+import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+/**
+ * The probe target: the longest `## ` heading in the skill's body.
+ *
+ * Body-only by construction — the frontmatter `description` is always in context
+ * under progressive disclosure, so anything quotable from it would pass against a
+ * model that never read the instructions. Longest rather than first because the
+ * check is "did you see this text", and `## Overview` is guessable while
+ * `## Refuse a metered judge, whatever chose it` is not.
+ *
+ * Null when the body has no `## ` heading: there is then nothing to ask for that a
+ * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
+ * worse than none.
+ */
+export declare function deliveryAnchor(skillMd: string): string | null;
+export interface CanaryResult {
+    /** `pass` = the anchor came back; `fail` = it did not; `skipped` = nothing safe to probe for. */
+    status: "pass" | "fail" | "skipped";
+    /** What the probe looked for (null when skipped). */
+    anchor: string | null;
+    /** Why it was skipped, or what the model said instead (trimmed) — for the log and the journal. */
+    detail: string;
+}
+export declare function canaryPrompt(skillName: string, anchor: string): string;
+export interface CanaryOptions {
+    adapter: HarnessAdapter;
+    model: ModelRef;
+    skillDir: string;
+    skillName: string;
+    /** Neutral cwd, same as a scenario gets — the probe must not see a repo either. */
+    cwd: string;
+}
+/**
+ * Run the probe in green mode and report whether the skill body was reachable.
+ *
+ * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
+ * with the reply in `detail`, because "the harness answered without the skill" and
+ * "the model said something odd" are both reasons not to spend a wave. An adapter
+ * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
+ * setup errors with their own messages.
+ */
+export declare function runDeliveryCanary(opts: CanaryOptions): Promise<CanaryResult>;
+/** The abort message for a failed canary: what was measured, and what to do instead. */
+export declare function canaryFailure(skillName: string, result: CanaryResult, cliVersion: string | null): string;

package/dist/canary.js

@@ -0,0 +1,123 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+/**
+ * The delivery canary: one cheap probe, before any scenario runs, that the skill
+ * under test actually reached the model.
+ *
+ * Why this exists. `--mode green` asks the harness to activate the skill, and the
+ * harness can decline without saying so. Measured on pi: 0.80.x wrapped the prompt
+ * with the skill body; 0.83.0 switched to progressive disclosure (only the
+ * description is in context, the body loads on demand — "models don't always do
+ * this", pi's own docs); and a nonexistent `--skill` path is accepted silently,
+ * exit 0 with a normal answer. The reference corpus ran two full waves in that
+ * state: `architect` came back 7/14, ≈ its no-skill baseline, and looked entirely
+ * plausible — the only tell was a contradictory failure mix (over-ceremony AND
+ * capitulation at once) that no single skill edit produces.
+ *
+ * What it can and cannot prove. A pass means the skill's body is *reachable* in
+ * this exact invocation — which is what kills the whole silent-non-delivery class:
+ * a dropped flag, a wrong path, a harness that stopped honoring the mode. It does
+ * NOT prove the body entered context for every later scenario; under progressive
+ * disclosure that is the model's choice per turn, and no probe can promise it. The
+ * mode whose delivery needs no promise is `force` (SKILL.md as system prompt),
+ * which is why `run` recommends it rather than pretending the canary is equivalent.
+ *
+ * Cost: exactly one subject call per run, and only when asked for (`--canary`).
+ * A run that aborts here has spent one rep instead of a wave.
+ */
+/** Frontmatter-stripped body of a SKILL.md. */
+function skillBody(text) {
+    const m = /^---\r?\n[\s\S]*?\r?\n---\r?\n/.exec(text);
+    return m ? text.slice(m[0].length) : text;
+}
+/**
+ * The probe target: the longest `## ` heading in the skill's body.
+ *
+ * Body-only by construction — the frontmatter `description` is always in context
+ * under progressive disclosure, so anything quotable from it would pass against a
+ * model that never read the instructions. Longest rather than first because the
+ * check is "did you see this text", and `## Overview` is guessable while
+ * `## Refuse a metered judge, whatever chose it` is not.
+ *
+ * Null when the body has no `## ` heading: there is then nothing to ask for that a
+ * plausible-sounding answer couldn't fake, and a canary that can be bluffed is
+ * worse than none.
+ */
+export function deliveryAnchor(skillMd) {
+    const headings = [...skillBody(skillMd).matchAll(/^##[ \t]+(.+?)[ \t]*$/gm)].map((m) => m[1].trim());
+    if (headings.length === 0)
+        return null;
+    return headings.reduce((a, b) => (b.length > a.length ? b : a));
+}
+/** Normalize for comparison: case, whitespace runs, and markdown emphasis/backticks. */
+function normalize(s) {
+    return s.toLowerCase().replace(/[`*_]/g, "").replace(/\s+/g, " ").trim();
+}
+export function canaryPrompt(skillName, anchor) {
+    // Asks for every heading, not just the anchor, because that is a question a model
+    // WITH the instructions answers naturally and a model without them can only
+    // invent. The instruction to say NOT_AVAILABLE keeps an honest miss from looking
+    // like a refusal to follow format.
+    return (`Answer from the instructions you have loaded — do not perform any task.\n\n` +
+        `List every level-2 markdown heading (lines starting with "## ") in the instructions of ` +
+        `the skill named "${skillName}", verbatim, one per line, with no other text.\n` +
+        `If you have no such instructions available, reply exactly: NOT_AVAILABLE\n\n` +
+        `(The heading text is what matters; keep it exact.)`);
+}
+/**
+ * What the model actually said, per the adapters' shared transcript convention
+ * (`>>> USER:` / `<<< ASSISTANT:`). Falls back to the whole text for an adapter
+ * that doesn't use the markers.
+ */
+function assistantReply(transcript) {
+    const parts = transcript.split(/^<<< ASSISTANT:\s*$/m);
+    return (parts.length > 1 ? parts[parts.length - 1] : transcript).trim();
+}
+/**
+ * Run the probe in green mode and report whether the skill body was reachable.
+ *
+ * Never throws for a model-side outcome: an empty or off-format reply is a `fail`
+ * with the reply in `detail`, because "the harness answered without the skill" and
+ * "the model said something odd" are both reasons not to spend a wave. An adapter
+ * that throws (pi missing, the skill-dir tripwire) is left to propagate — those are
+ * setup errors with their own messages.
+ */
+export async function runDeliveryCanary(opts) {
+    const skillMd = readFileSync(join(opts.skillDir, "SKILL.md"), "utf8");
+    const anchor = deliveryAnchor(skillMd);
+    if (!anchor) {
+        return {
+            status: "skipped",
+            anchor: null,
+            detail: `${opts.skillName}/SKILL.md has no \`## \` heading to probe for — nothing a reply could prove`,
+        };
+    }
+    const transcript = await opts.adapter.run({
+        skillDir: opts.skillDir,
+        model: opts.model,
+        mode: "green",
+        turns: [canaryPrompt(opts.skillName, anchor)],
+        cwd: opts.cwd,
+    });
+    const ok = normalize(transcript).includes(normalize(anchor));
+    return {
+        status: ok ? "pass" : "fail",
+        anchor,
+        // The reply, not the transcript: the transcript opens with our own prompt, and a
+        // failure report whose first 400 characters are the question is useless.
+        detail: ok ? "" : assistantReply(transcript).slice(0, 400),
+    };
+}
+/** The abort message for a failed canary: what was measured, and what to do instead. */
+export function canaryFailure(skillName, result, cliVersion) {
+    return (`delivery canary FAILED for ${skillName}: the model could not quote its own skill instructions ` +
+        `(looked for the heading \`${result.anchor}\`).\n` +
+        `  The skill is not reaching the model, so every scenario in this run would measure a naked ` +
+        `model and score like a result. Nothing has been spent beyond this one probe.\n` +
+        `  harness CLI: ${cliVersion ?? "unknown"}. On pi ≥ 0.83.0 \`--skill\` is progressive disclosure ` +
+        `(description in context, body on demand) and a nonexistent path is accepted silently.\n` +
+        `  Fix: re-run with \`--mode force\` (SKILL.md as the system prompt — delivery no version has made ` +
+        `conditional), or check that the skill dir is the one you meant.\n` +
+        `  What the model said instead: ${result.detail || "(nothing)"}`);
+}
+//# sourceMappingURL=canary.js.map

package/dist/defaults.d.ts

@@ -0,0 +1,30 @@
+/**
+ * The judge used when nothing else says otherwise: Opus through the **`claude-code`
+ * provider**, which authenticates with the user's Claude subscription (OAuth via
+ * `claude -p`) rather than a metered API key.
+ *
+ * The model is deliberately the strongest available — judging is the one place
+ * where a weak model silently corrupts every number in a scorecard. The *provider*
+ * is what changed in 0.3.3: the default was `anthropic:claude-opus-4-8`, a metered
+ * API, and it billed a corpus once by accident because nothing in the tool surface
+ * distinguishes "the flag I forgot" from "the flag I meant". A default must not be
+ * able to spend money that was not asked for. The metered path is still one flag
+ * away (`--judge anthropic:claude-opus-4-8`) for anyone who wants it — an API key
+ * scales past a subscription's rate limits, which matters for a large `--reps` run.
+ */
+export declare const BAKED_DEFAULT_JUDGE = "claude-code:claude-opus-4-8";
+/**
+ * Resolve the default judge: `SKILL_HARNESS_JUDGE` if set, else the baked value.
+ * An explicit `--judge` always wins over both — this is only the default.
+ *
+ * The env layer exists because this harness is built for someone who steers a
+ * process rather than typing every flag: judge policy belongs to the repo or the
+ * shell, set once. It is also how you opt *into* the metered API deliberately
+ * (`SKILL_HARNESS_JUDGE=anthropic:claude-opus-4-8`) instead of by forgetting a
+ * flag. Read through `readEnv`, so the pre-rename `SKILL_CHECK_JUDGE` keeps working
+ * with the usual one-time notice.
+ *
+ * Resolved per call, not at module load: tests and long-lived processes (the pi
+ * extension) must see an env change without a reload.
+ */
+export declare function defaultJudge(): string;

package/dist/defaults.js

@@ -0,0 +1,34 @@
+import { readEnv } from "./util/env.js";
+/**
+ * The judge used when nothing else says otherwise: Opus through the **`claude-code`
+ * provider**, which authenticates with the user's Claude subscription (OAuth via
+ * `claude -p`) rather than a metered API key.
+ *
+ * The model is deliberately the strongest available — judging is the one place
+ * where a weak model silently corrupts every number in a scorecard. The *provider*
+ * is what changed in 0.3.3: the default was `anthropic:claude-opus-4-8`, a metered
+ * API, and it billed a corpus once by accident because nothing in the tool surface
+ * distinguishes "the flag I forgot" from "the flag I meant". A default must not be
+ * able to spend money that was not asked for. The metered path is still one flag
+ * away (`--judge anthropic:claude-opus-4-8`) for anyone who wants it — an API key
+ * scales past a subscription's rate limits, which matters for a large `--reps` run.
+ */
+export const BAKED_DEFAULT_JUDGE = "claude-code:claude-opus-4-8";
+/**
+ * Resolve the default judge: `SKILL_HARNESS_JUDGE` if set, else the baked value.
+ * An explicit `--judge` always wins over both — this is only the default.
+ *
+ * The env layer exists because this harness is built for someone who steers a
+ * process rather than typing every flag: judge policy belongs to the repo or the
+ * shell, set once. It is also how you opt *into* the metered API deliberately
+ * (`SKILL_HARNESS_JUDGE=anthropic:claude-opus-4-8`) instead of by forgetting a
+ * flag. Read through `readEnv`, so the pre-rename `SKILL_CHECK_JUDGE` keeps working
+ * with the usual one-time notice.
+ *
+ * Resolved per call, not at module load: tests and long-lived processes (the pi
+ * extension) must see an env change without a reload.
+ */
+export function defaultJudge() {
+    return readEnv("JUDGE") ?? BAKED_DEFAULT_JUDGE;
+}
+//# sourceMappingURL=defaults.js.map

package/dist/discover.d.ts

@@ -8,6 +8,13 @@ export interface DiscoveredSkill {
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
  * SKILL.md. It is testable iff `<skill>/tests/specification.yaml` exists.
  * Returns skills sorted by name (testable or not).
+ *
+ * `dir` and `specPath` are ABSOLUTE, whatever `root` was. They are handed to child
+ * processes that run in a neutral cwd of the harness's choosing (`pi --skill
+ * <dir>`), so a relative `--skills .` used to produce a path that resolved to
+ * nothing over there — and pi accepts a nonexistent `--skill` path silently, exit 0
+ * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
+ * but the honest fix is here, where the path is built.
  */
 export declare function discover(root: string): DiscoveredSkill[];
 /**

package/dist/discover.js

@@ -1,19 +1,27 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 /**
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
  * SKILL.md. It is testable iff `<skill>/tests/specification.yaml` exists.
  * Returns skills sorted by name (testable or not).
+ *
+ * `dir` and `specPath` are ABSOLUTE, whatever `root` was. They are handed to child
+ * processes that run in a neutral cwd of the harness's choosing (`pi --skill
+ * <dir>`), so a relative `--skills .` used to produce a path that resolved to
+ * nothing over there — and pi accepts a nonexistent `--skill` path silently, exit 0
+ * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
+ * but the honest fix is here, where the path is built.
  */
 export function discover(root) {
-    if (!existsSync(root) || !statSync(root).isDirectory()) {
+    const absRoot = resolve(root);
+    if (!existsSync(absRoot) || !statSync(absRoot).isDirectory()) {
         throw new Error(`skills root is not a directory: ${root}`);
     }
     const skills = [];
-    for (const name of readdirSync(root)) {
+    for (const name of readdirSync(absRoot)) {
         if (name.startsWith("."))
             continue;
-        const dir = join(root, name);
+        const dir = join(absRoot, name);
         if (!statSync(dir).isDirectory())
             continue;
         if (!existsSync(join(dir, "SKILL.md")))
@@ -32,7 +40,7 @@ export function discover(root) {
 export function resolveSkill(root, name) {
     const skill = discover(root).find((s) => s.name === name);
     if (!skill) {
-        const dir = join(root, name);
+        const dir = join(resolve(root), name);
         if (existsSync(dir) && statSync(dir).isDirectory() && !existsSync(join(dir, "SKILL.md"))) {
             throw new Error(`skill \`${name}\` has no SKILL.md (looked in ${dir})`);
         }

package/dist/downgrade.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Compare two semver-ish version strings by numeric component.
+ *
+ * String comparison is the trap this exists to avoid: `"0.10.0" < "0.9.0"` is true
+ * lexically and false in fact, which would make the tripwire fire backwards exactly
+ * once — on the release where it mattered most. A prerelease suffix is dropped rather
+ * than ordered: `0.4.0-rc.1` vs `0.4.0` is not a distinction worth a refusal, and
+ * getting it wrong in either direction is worse than treating them as equal.
+ */
+export declare function compareVersions(a: string, b: string): number;
+/**
+ * The newest `harness_version` recorded anywhere under `<skillDir>/tests/results/`, or
+ * null when no run records one.
+ *
+ * Null is the honest answer for a pre-0.3.3 tree and must stay silent: those runs
+ * carry no version, so there is nothing to compare and no basis for a warning. The
+ * tripwire therefore only sharpens as fresh runs land — the same forward-looking
+ * bargain `source_hashes` made.
+ */
+export declare function newestRecordedVersion(skillDir: string): string | null;
+/**
+ * Refuse to write a fresh measurement with an older tool than the one that produced
+ * the records already in the tree.
+ *
+ * The failure this kills, measured on the reference corpus: a stale global **0.1.0**
+ * install would have spent ~102 rep-executions grading *without showing the judge the
+ * staged diff* — the exact defect the run was meant to correct — and every resulting
+ * number would have looked entirely plausible. It also emitted 38 spurious findings
+ * that the current version does not. Nothing announced any of it.
+ *
+ * `schema` cannot serve here: 0.2.1 → 0.3.0 kept `schema: 2` while changing what a
+ * verdict *means*. Only the writing version distinguishes those measurements.
+ *
+ * Refusal is for `run` alone, because only `run` mints a new measurement that would sit
+ * beside newer ones as if comparable. `grade` and `lint` warn (see `downgradeWarning`):
+ * both are how someone diagnoses this in the first place, and blocking diagnosis is a
+ * bad trade.
+ */
+export declare function assertNotDowngraded(skillDir: string, command: "run" | "grade" | "lint"): void;
+/** The loud-but-not-fatal version, for `grade` and `lint`. Null when nothing is newer. */
+export declare function downgradeWarning(skillDir: string): string | null;

package/dist/downgrade.js

@@ -0,0 +1,100 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { readResults } from "./results.js";
+import { HARNESS_VERSION } from "./version.js";
+/**
+ * Compare two semver-ish version strings by numeric component.
+ *
+ * String comparison is the trap this exists to avoid: `"0.10.0" < "0.9.0"` is true
+ * lexically and false in fact, which would make the tripwire fire backwards exactly
+ * once — on the release where it mattered most. A prerelease suffix is dropped rather
+ * than ordered: `0.4.0-rc.1` vs `0.4.0` is not a distinction worth a refusal, and
+ * getting it wrong in either direction is worse than treating them as equal.
+ */
+export function compareVersions(a, b) {
+    const parts = (v) => v.split("-")[0].split(".").map((n) => Number(n) || 0);
+    const [pa, pb] = [parts(a), parts(b)];
+    for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+        const d = (pa[i] ?? 0) - (pb[i] ?? 0);
+        if (d !== 0)
+            return d > 0 ? 1 : -1;
+    }
+    return 0;
+}
+function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * The newest `harness_version` recorded anywhere under `<skillDir>/tests/results/`, or
+ * null when no run records one.
+ *
+ * Null is the honest answer for a pre-0.3.3 tree and must stay silent: those runs
+ * carry no version, so there is nothing to compare and no basis for a warning. The
+ * tripwire therefore only sharpens as fresh runs land — the same forward-looking
+ * bargain `source_hashes` made.
+ */
+export function newestRecordedVersion(skillDir) {
+    const root = join(skillDir, "tests", "results");
+    if (!existsSync(root))
+        return null;
+    let newest = null;
+    for (const tag of readdirSync(root).filter((n) => isDir(join(root, n)))) {
+        const tagDir = join(root, tag);
+        for (const run of readdirSync(tagDir).filter((n) => isDir(join(tagDir, n)))) {
+            try {
+                const v = readResults(join(tagDir, run)).harness_version;
+                if (v && (newest === null || compareVersions(v, newest) > 0))
+                    newest = v;
+            }
+            catch {
+                // Unreadable results are the consistency check's problem, not this one's.
+            }
+        }
+    }
+    return newest;
+}
+function upgradeAdvice(recorded) {
+    return (`You are running skill-harness ${HARNESS_VERSION}; this tree holds results recorded by ${recorded}.\n` +
+        `  A global install goes stale silently — check with \`skill-harness --version\`, and prefer\n` +
+        `  \`npx skill-harness@${recorded}\` (or \`npm i -g skill-harness@latest\`) so the tool matches the records.`);
+}
+/**
+ * Refuse to write a fresh measurement with an older tool than the one that produced
+ * the records already in the tree.
+ *
+ * The failure this kills, measured on the reference corpus: a stale global **0.1.0**
+ * install would have spent ~102 rep-executions grading *without showing the judge the
+ * staged diff* — the exact defect the run was meant to correct — and every resulting
+ * number would have looked entirely plausible. It also emitted 38 spurious findings
+ * that the current version does not. Nothing announced any of it.
+ *
+ * `schema` cannot serve here: 0.2.1 → 0.3.0 kept `schema: 2` while changing what a
+ * verdict *means*. Only the writing version distinguishes those measurements.
+ *
+ * Refusal is for `run` alone, because only `run` mints a new measurement that would sit
+ * beside newer ones as if comparable. `grade` and `lint` warn (see `downgradeWarning`):
+ * both are how someone diagnoses this in the first place, and blocking diagnosis is a
+ * bad trade.
+ */
+export function assertNotDowngraded(skillDir, command) {
+    if (command !== "run")
+        return;
+    const recorded = newestRecordedVersion(skillDir);
+    if (!recorded || compareVersions(recorded, HARNESS_VERSION) <= 0)
+        return;
+    throw new Error(`refusing to run: these results were recorded by a NEWER skill-harness, so a run from this one would not be comparable.\n  ` +
+        upgradeAdvice(recorded));
+}
+/** The loud-but-not-fatal version, for `grade` and `lint`. Null when nothing is newer. */
+export function downgradeWarning(skillDir) {
+    const recorded = newestRecordedVersion(skillDir);
+    if (!recorded || compareVersions(recorded, HARNESS_VERSION) <= 0)
+        return null;
+    return `warning: this skill-harness is older than the tool that recorded these results.\n  ${upgradeAdvice(recorded)}`;
+}
+//# sourceMappingURL=downgrade.js.map

package/dist/index.d.ts

@@ -20,3 +20,9 @@ export * from "./adapters/types.js";
 export * from "./util/exec.js";
 export * from "./util/env.js";
 export * from "./scaffold.js";
+export * from "./version.js";
+export * from "./defaults.js";
+export * from "./judge-policy.js";
+export * from "./regate.js";
+export * from "./downgrade.js";
+export * from "./canary.js";

package/dist/index.js

@@ -20,4 +20,10 @@ export * from "./adapters/types.js";
 export * from "./util/exec.js";
 export * from "./util/env.js";
 export * from "./scaffold.js";
+export * from "./version.js";
+export * from "./defaults.js";
+export * from "./judge-policy.js";
+export * from "./regate.js";
+export * from "./downgrade.js";
+export * from "./canary.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -22,12 +22,26 @@ export type JournalEvent = {
     skill: string;
     harness: string;
     model: string;
+    /** The harness CLI's own version (`pi --version`), or null when it could not be asked. */
+    harness_cli_version?: string | null;
     judge: {
         provider: string;
         model: string;
     };
     mode: string;
     label: string | null;
+}
+/**
+ * The pre-flight delivery probe (green mode, `--canary`): did the model quote a
+ * body-only heading of its own skill back? `fail` aborts the run, so a journal
+ * carrying a failed canary is the record of a wave that was NOT spent.
+ */
+ | {
+    event: "delivery-canary";
+    ts: string;
+    status: "pass" | "fail" | "skipped";
+    anchor: string | null;
+    detail: string;
 } | {
     event: "scenario-started";
     ts: string;
@@ -68,6 +82,20 @@ export type JournalEvent = {
     total: number;
     pct: number;
     ship: boolean;
+}
+/**
+ * A regate: needle gates re-evaluated against the saved staged diffs. `judge_calls`
+ * is on the record because regate is advertised as free apart from the reps whose
+ * gate verdict flipped — a claim the journal should be able to settle. `skipped`
+ * names scenarios it could not regate (vitest/post_test, or missing diff artifacts).
+ */
+ | {
+    event: "regate";
+    ts: string;
+    scenarios: string[];
+    changed: string[];
+    judge_calls: number;
+    skipped?: string[];
 } | {
     event: "score";
     ts: string;

package/dist/judge-policy.d.ts

@@ -0,0 +1,42 @@
+import type { ModelRef } from "./adapters/types.js";
+/** Whether judging with this ref can charge a per-token API. */
+export declare function isMeteredJudge(judge: ModelRef): boolean;
+/** Whether the user has explicitly accepted metered judging for this repo/shell. */
+export declare function allowMeteredJudge(): boolean;
+export interface JudgeAllowOpts {
+    /**
+     * Where the judge came from, in the user's own vocabulary — `--judge`,
+     * `SKILL_HARNESS_JUDGE`, `the run's recorded judge`. A refusal has to say which
+     * knob to turn, and for a regrade the answer is not the one the user expects:
+     * nobody typed anything, the run's own `results.yaml` supplied it.
+     */
+    source: string;
+    /** Per-invocation opt-in (`--allow-metered-judge`); the env var is read here. */
+    allowMetered?: boolean;
+}
+/**
+ * Refuse to judge through a metered API unless the user explicitly asked for it.
+ *
+ * Why a hard refusal and not a warning: judging is the only cost in this tool that
+ * happens *without* a decision. The subject model is chosen per run and is the point
+ * of the exercise; the judge is a default, and a default that bills is a bug — it
+ * already billed a corpus once. A warning scrolls past inside a run's progress
+ * output, and by then the money is spent.
+ *
+ * Three paths could reach a metered API, and only one of them involves typing a
+ * flag: `--judge anthropic:…`; a `SKILL_HARNESS_JUDGE` set (or mistyped) to a
+ * metered provider; and `grade`, which re-judges with the judge the run *recorded* —
+ * so a run whose `results.yaml` names a metered judge bills on every later regrade,
+ * with no flag involved at all.
+ *
+ * That third path is latent rather than live in the corpus this was built against:
+ * checked 2026-08-05, all ~140 committed `results.yaml` in `principal-pi-skills`
+ * record `provider: claude-code`, because its owner always passed the subscription
+ * judge explicitly. The old default was reachable, not taken. Worth stating
+ * precisely — "your whole archive bills on regrade" would have been a scarier claim
+ * than the evidence supports.
+ *
+ * Deliberately not applied to the subject model: paying to run the model under test
+ * is what a run *is*.
+ */
+export declare function assertJudgeAllowed(judge: ModelRef, opts: JudgeAllowOpts): void;

package/dist/judge-policy.js

@@ -0,0 +1,61 @@
+import { envFlag } from "./util/env.js";
+import { BAKED_DEFAULT_JUDGE } from "./defaults.js";
+/**
+ * Judge providers that cannot bill a per-token API.
+ *
+ * `claude-code` shells out to the `claude` CLI, authenticated with the user's
+ * Claude subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local` are local
+ * runtimes. Everything else is assumed to charge.
+ *
+ * An **allow-list**, deliberately, not a deny-list of known-paid providers: a
+ * provider nobody has classified yet should be treated as able to bill, because
+ * being wrong in that direction produces a surprise invoice while being wrong in
+ * this direction produces one extra flag.
+ */
+const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "ollama", "lmstudio", "llamacpp", "local"]);
+/** Whether judging with this ref can charge a per-token API. */
+export function isMeteredJudge(judge) {
+    return !FREE_JUDGE_PROVIDERS.has(judge.provider);
+}
+/** Whether the user has explicitly accepted metered judging for this repo/shell. */
+export function allowMeteredJudge() {
+    return envFlag("ALLOW_METERED_JUDGE");
+}
+/**
+ * Refuse to judge through a metered API unless the user explicitly asked for it.
+ *
+ * Why a hard refusal and not a warning: judging is the only cost in this tool that
+ * happens *without* a decision. The subject model is chosen per run and is the point
+ * of the exercise; the judge is a default, and a default that bills is a bug — it
+ * already billed a corpus once. A warning scrolls past inside a run's progress
+ * output, and by then the money is spent.
+ *
+ * Three paths could reach a metered API, and only one of them involves typing a
+ * flag: `--judge anthropic:…`; a `SKILL_HARNESS_JUDGE` set (or mistyped) to a
+ * metered provider; and `grade`, which re-judges with the judge the run *recorded* —
+ * so a run whose `results.yaml` names a metered judge bills on every later regrade,
+ * with no flag involved at all.
+ *
+ * That third path is latent rather than live in the corpus this was built against:
+ * checked 2026-08-05, all ~140 committed `results.yaml` in `principal-pi-skills`
+ * record `provider: claude-code`, because its owner always passed the subscription
+ * judge explicitly. The old default was reachable, not taken. Worth stating
+ * precisely — "your whole archive bills on regrade" would have been a scarier claim
+ * than the evidence supports.
+ *
+ * Deliberately not applied to the subject model: paying to run the model under test
+ * is what a run *is*.
+ */
+export function assertJudgeAllowed(judge, opts) {
+    if (!isMeteredJudge(judge))
+        return;
+    if (opts.allowMetered || allowMeteredJudge())
+        return;
+    const token = `${judge.provider}:${judge.model}`;
+    throw new Error(`refusing to judge with ${token}: \`${judge.provider}\` bills a per-token API key, and it came from ${opts.source}.\n` +
+        `  Judging is meant to cost nothing you did not ask for.\n` +
+        `  • judge on your Claude subscription instead:  --judge ${BAKED_DEFAULT_JUDGE}\n` +
+        `  • allow the metered API for this command:     --allow-metered-judge\n` +
+        `  • allow it for this repo or shell:            export SKILL_HARNESS_ALLOW_METERED_JUDGE=1`);
+}
+//# sourceMappingURL=judge-policy.js.map

package/dist/lift.d.ts

@@ -24,6 +24,19 @@ export interface Lift {
     model: string;
     redTimestamp: string;
     greenTimestamp: string;
+    /**
+     * How the skill was delivered on the non-baseline side: `green` (the harness
+     * activated it) or `force` (SKILL.md as the system prompt).
+     *
+     * The `green*` field names are kept as the *skill-active side*, not as a claim
+     * about the mode — they are the wire format the committed report assets and the
+     * review UI read, and renaming them would break every published report to say
+     * something the `mode` field already says. A red baseline is mode-independent
+     * (`--no-skills` either way), so red-vs-force is as valid a comparison as
+     * red-vs-green — but which one you are looking at changes what the number means,
+     * so it is recorded rather than implied.
+     */
+    mode: string;
     /** Scenario ids present in both runs — the only ones a lift can speak to. */
     compared: number;
     gained: number;