@skill-harness/core 0.10.0 → 0.12.0

package/README.md

@@ -1,7 +1,8 @@
 # @skill-harness/core
 
 The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
-spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results.
+spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results,
+and the separately versioned durable qualification-runner-v1 primitives.
 
 Internal API — use the `skill-harness` CLI. See the repo for docs.
 

package/dist/adapters/types.d.ts

@@ -11,6 +11,21 @@ export interface ModelRef {
 export declare function parseModelRef(token: string): ModelRef;
 /** Stable, filesystem-safe slug for a model ref (used in run-dir names). */
 export declare function modelSlug(ref: ModelRef): string;
+export type PromptMechanism = "none" | "pi-skill" | "append-system-prompt" | "system-prompt-file";
+export interface PromptProvenance {
+    capture_version: "prompt-provenance-v1";
+    request_index: number;
+    raw_sha256: string;
+    normalized_sha256: string;
+    normalization_rule: "cwd-line-v1";
+    bytes: number;
+    contract_sha256: string;
+    contract_bytes: number;
+    contract_occurrences: number;
+    mechanism: PromptMechanism;
+    status: "PASS" | "NOT-MEASURED" | "ERROR";
+    error?: string;
+}
 export interface RunReq {
     skillDir: string;
     model: ModelRef;
@@ -44,6 +59,14 @@ export interface RunReq {
     extensions?: string[];
     /** Workspace-local native ledgers to normalize after the subject finishes. */
     eventSources?: TrajectoryEventSource[];
+    /**
+     * Extra env for the subject process, from the arm. Merged over `process.env` by
+     * the adapter — an arm must be able to add `PI_GRANTS_*` without the harness
+     * inheriting a caller's copy of them.
+     */
+    armEnv?: Record<string, string>;
+    /** Adapter-produced observation callback. Callers receive evidence; they never supply delivery status. */
+    onPromptObservation?: (observation: PromptProvenance) => void;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {
@@ -60,9 +83,16 @@ export interface StructuredRun {
     events?: TrajectoryEventV1[];
     /** Native sources that were required but missing/malformed. Never treated as an empty success. */
     eventErrors?: string[];
+    /**
+     * Set when pi failed provider-side (auth, transport) rather than the model
+     * answering badly. `run.ts` turns this into ERROR — never a model verdict.
+     */
+    providerFailure?: string;
 }
 export interface HarnessAdapter {
     name: string;
+    /** True only when the adapter computes final provider-payload observations. */
+    observesPrompts?: boolean;
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     /**

package/dist/adjudication.d.ts

@@ -1,5 +1,5 @@
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
-import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile } from "./results.js";
+import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile, CriterionVote } from "./results.js";
 import type { Scenario, ShipBar, Spec } from "./spec.js";
 import type { Verdict } from "./score.js";
 /**
@@ -33,6 +33,11 @@ export interface CellState {
     suspect: boolean;
     /** Per-rep verdicts, when the cell ran with reps. Empty for a single-rep cell. */
     repVerdicts?: Verdict[];
+    criteria?: CriterionVote[];
+    /** Delivery prerequisite for schema-3 cells; non-PASS cells are never judged. */
+    deliveryStatus?: "PASS" | "NOT-MEASURED" | "ERROR";
+    /** The actual selected repetition's primary vote; aggregate fields remain for planning only. */
+    primaryJudgment?: Judgment;
 }
 export interface TriggerDecision {
     id: string;
@@ -105,11 +110,14 @@ export declare function collapseJudgments(judgments: Judgment[], trigger: Trigge
  */
 export declare function boundAdjudicationToRepetitions(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): AdjudicationResult;
 export declare function projectAdjudication(result: ScenarioResult, adj: AdjudicationResult): ScenarioResult;
+/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
+export declare function projectAdjudicationForScenario(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): ScenarioResult;
 /** Ask one judge about one cell's saved transcript. Supplied by the caller. */
 export type RejudgeFn = (id: string, judge: ModelRef) => Promise<{
     verdict: Verdict;
     reason: string;
     suspect: boolean;
+    criteria?: CriterionVote[];
 }>;
 export interface RunAdjudicationOptions {
     plan: AdjudicationPlan;

package/dist/adjudication.js

@@ -1,9 +1,10 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, } from "./results.js";
+import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, completeCriterionVotes, } from "./results.js";
 import { score } from "./score.js";
 import { buildJudgePrompt, judgeInWorkspace, parseVerdict } from "./grade.js";
 import { appendJournal } from "./journal.js";
+import { collapseVotePanel } from "./vote-panel.js";
 /** Maximum judgments per cell, first wave included. Hard cap, not a default. */
 export const MAX_JUDGMENTS = 3;
 /**
@@ -18,6 +19,9 @@ export function planAdjudication(input) {
     const enabled = new Set(input.enabled ?? ["ambiguous", "contradictory", "non_unanimous", "ship_deciding"]);
     const decisions = [];
     for (const cell of input.cells) {
+        // Delivery-gated schema-3 cells carry no behavioral claim to adjudicate.
+        if (cell.deliveryStatus && cell.deliveryStatus !== "PASS")
+            continue;
         const triggers = [];
         // `JUDGE-AMBIGUOUS` is what the parser emits when a judge's verdict blocks
         // disagree. `ERROR` is what it emits when nothing parsed at all — and that
@@ -92,24 +96,13 @@ function flipsShipDecision(cell, input) {
  * cast a deciding vote would launder exactly the unreliability being measured.
  */
 export function collapseJudgments(judgments, trigger) {
-    const clean = judgments.filter((j) => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
-    const base = { trigger, judgments };
-    if (clean.length < 2) {
-        // Nothing to compare: one clean vote (or none) cannot confirm anything.
-        return { ...base, state: "unresolved" };
-    }
-    const passes = clean.filter((j) => j.verdict === "PASS").length;
-    const fails = clean.length - passes;
-    if (passes === 0 || fails === 0) {
-        return { ...base, state: "confirmed", verdict: clean[0].verdict };
-    }
-    // A clean strict majority breaks the tie. With exactly 2 clean votes that
-    // disagree there is no majority, so this correctly falls through.
-    if (passes > fails)
-        return { ...base, state: "tie_broken", verdict: "PASS" };
-    if (fails > passes)
-        return { ...base, state: "tie_broken", verdict: "FAIL" };
-    return { ...base, state: "unresolved" };
+    const collapsed = collapseVotePanel(judgments);
+    return {
+        trigger,
+        judgments,
+        state: collapsed.state,
+        ...(collapsed.verdict ? { verdict: collapsed.verdict } : {}),
+    };
 }
 /**
  * Project an adjudication onto the compatibility fields the scorer reads.
@@ -149,6 +142,15 @@ export function projectAdjudication(result, adj) {
         adjudication: adj,
     };
 }
+/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
+export function projectAdjudicationForScenario(result, scenario, adj) {
+    const bounded = boundAdjudicationToRepetitions(result, scenario, adj);
+    const projected = projectAdjudication(result, bounded);
+    if (bounded !== adj) {
+        projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
+    }
+    return projected;
+}
 function reasonFor(adj) {
     const n = adj.judgments.length;
     const verb = adj.state === "confirmed" ? "confirmed by" : "resolved by majority of";
@@ -179,7 +181,9 @@ export async function runAdjudication(opts) {
         // in the fixed order `planAdjudication` evaluates them.
         const trigger = decision.triggers[0];
         const judgments = [
-            { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect },
+            cell.primaryJudgment
+                ? { ...cell.primaryJudgment, ordinal: 1 }
+                : { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect, criteria: cell.criteria ?? [] },
         ];
         const second = await opts.rejudge(decision.id, opts.secondaryJudge);
         callsMade++;
@@ -242,11 +246,8 @@ export async function adjudicateRun(opts) {
         if (!adj)
             return s;
         const scenario = byIdScenario.get(s.id);
-        const boundedAdj = scenario ? boundAdjudicationToRepetitions(s, scenario, adj) : adj;
-        const projected = projectAdjudication(s, boundedAdj);
-        if (boundedAdj !== adj) {
-            projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
-        }
+        const withRepetition = { ...adj, repetition: s.rep_judgments?.[0]?.repetition ?? 0 };
+        const projected = scenario ? projectAdjudicationForScenario(s, scenario, withRepetition) : projectAdjudication(s, withRepetition);
         const extraCalls = adj.judgments.filter((judgment) => judgment.ordinal > 1).length;
         projected.metrics = mergeScenarioMetrics(s.metrics, {
             wall_time_ms: 0,
@@ -299,7 +300,7 @@ async function judgeCell(opts) {
         event: "judge-verdict", ts: opts.now(), id: opts.scenario.id,
         verdict: g.verdict, reason: g.reason, suspect: g.suspect,
     });
-    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
+    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, opts.scenario.checklist.length) };
 }
 /**
  * Build the plan's input cells from a run.
@@ -311,13 +312,20 @@ async function judgeCell(opts) {
  * about — inverting the one invariant this feature actually promises.
  */
 export function cellsFromResults(runDir, results) {
-    return results.scenarios.map((s) => ({
-        id: s.id,
-        verdict: s.judge_verdict,
-        reason: s.judge_reason,
-        suspect: s.suspect,
-        repVerdicts: repVerdictsOf(runDir, s, results.mode),
-    }));
+    return results.scenarios.map((s) => {
+        const primaryJudgment = s.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0];
+        const deliveryStatus = s.objective?.assertions.find(assertion => assertion.kind === "skill_delivered")?.status;
+        return {
+            id: s.id,
+            verdict: s.judge_verdict,
+            reason: s.judge_reason,
+            suspect: s.suspect,
+            repVerdicts: repVerdictsOf(runDir, s, results.mode),
+            criteria: primaryJudgment?.criteria,
+            ...(deliveryStatus === "PASS" || deliveryStatus === "NOT-MEASURED" || deliveryStatus === "ERROR" ? { deliveryStatus } : {}),
+            ...(primaryJudgment ? { primaryJudgment } : {}),
+        };
+    });
 }
 /** Per-rep verdicts from saved judge artifacts, for the non-unanimous trigger. */
 function repVerdictsOf(runDir, s, mode) {

package/dist/arms.d.ts

@@ -0,0 +1,51 @@
+/**
+ * An arm: a named bundle of harness-side conditions a run is measured under.
+ *
+ * Why arms live HERE and not in `specification.yaml`. An arm is meant to be
+ * A/B'd — the control and the treatment must be the same experiment run twice.
+ * Declaring it in the spec would move the spec digest, which both stales every
+ * committed run in the corpus and makes control and treatment textually
+ * different experiments. So `arms.yaml` is deliberately part of NO digest, and
+ * the arm is carried in the run-dir TAG instead: `lint` and `stability` key on
+ * the tag, so two arms are separate lineages that can never be misread as one
+ * lineage flipping its verdict run-over-run.
+ */
+export interface Arm {
+    name: string;
+    /** Absolute paths, one `--extension` each. */
+    extensions: string[];
+    /** Dirs (relative to the skills root) copied into `<workspace>/.pi/skills/`. */
+    seedSkills: string[];
+    /** Minimum definitions the seeding must produce, else the run ERRORs. */
+    requireDefinitions: number;
+    /** Env for the subject process. `<run-dir>` and `<workspace>` are substituted per rep. */
+    env: Record<string, string>;
+}
+/** The implicit control: today's behaviour, byte-identical, no extensions. */
+export declare const NONE_ARM: Arm;
+/** `<skills-root>/tests/arms.yaml`, or an empty map when the corpus declares none. */
+export declare function loadArms(skillsRoot: string): Map<string, Arm>;
+/** The named arm, or the control when no `--arm` was given. */
+export declare function resolveArm(skillsRoot: string, name: string | null): Arm;
+/**
+ * Copy an arm's definitions into `<workspace>/.pi/skills/` and return the count.
+ *
+ * Three refusals, each with a negative control in the suite, because a
+ * positive-only check measures less than it claims:
+ *
+ *  1. **Fewer definitions than required.** pi-daddy spawns definitions by path.
+ *     Zero definitions means the arm has nothing to spawn, so it would run green
+ *     and measure nothing — a vacuous result shaped exactly like a finding.
+ *  2. **A missing seed directory** — the `--skill <nonexistent>` class: pi accepts
+ *     silently, so the run would look fine and measure a different thing.
+ *  3. **A non-empty ambient skills root.** pi-daddy reads `~/.pi/agent/skills` as
+ *     well as the workspace, so anything there is an uncontrolled variable in the
+ *     measurement. Empty on the reference box today — that is luck, not a
+ *     guarantee, and this check is what turns it into one. It runs for any
+ *     non-control arm, even one that declares no `seed_skills` — pi-daddy
+ *     reads that root independently of whether the workspace was seeded, so
+ *     an arm can't opt out of the guarantee just by not seeding.
+ */
+export declare function seedArmDefinitions(arm: Arm, skillsRoot: string, workspaceCwd: string, opts?: {
+    ambientSkillsDir?: string;
+}): number;

package/dist/arms.js

@@ -0,0 +1,255 @@
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { isAbsolute, join, resolve, sep } from "node:path";
+import yaml from "js-yaml";
+/** The implicit control: today's behaviour, byte-identical, no extensions. */
+export const NONE_ARM = { name: "none", extensions: [], seedSkills: [], requireDefinitions: 0, env: {} };
+/** `none` is the control's name; an arm may not shadow it. */
+const RESERVED = new Set(["none"]);
+/**
+ * Arm names become a path segment, so they are restricted to what survives one
+ * unambiguously — no separators, no dots, nothing needing quoting.
+ */
+const NAME_RE = /^[A-Za-z0-9_-]+$/;
+function expandHome(p) {
+    return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p;
+}
+/**
+ * `realpathSync(p)`, or `p` when it cannot be resolved.
+ *
+ * A path that does not exist has no real path, and refusing here would replace
+ * the two callers' specific messages ("outside the skills root", "cannot be read
+ * — pi would start with nothing to spawn") with a bare ENOENT. The lexical path
+ * is the strictly safer fallback for the containment check: an absent path
+ * cannot be a symlink out of the corpus.
+ */
+function realpathOr(p) {
+    try {
+        return realpathSync(p);
+    }
+    catch {
+        return p;
+    }
+}
+/**
+ * `require_definitions`, refusing anything that is not a non-negative integer.
+ *
+ * A tolerant `Number(x) || 0` reads `require_definitions: six` (and any value
+ * YAML hands over as a string) as 0, which does not disable a typo — it disables
+ * the REFUSAL the key exists to make, so an arm that seeds nothing runs green and
+ * measures nothing. The one failure mode this key defends against is the one a
+ * silent default reintroduces.
+ */
+function parseRequireDefinitions(file, name, raw) {
+    if (raw === undefined || raw === null)
+        return 0;
+    const n = typeof raw === "number" ? raw : Number(String(raw).trim());
+    if (!Number.isInteger(n) || n < 0) {
+        throw new Error(`${file}: arm \`${name}\` require_definitions is ${JSON.stringify(raw)} — it must be a non-negative integer, ` +
+            `and a value that silently read as 0 would disable the refusal it exists to make.`);
+    }
+    return n;
+}
+/** `<skills-root>/tests/arms.yaml`, or an empty map when the corpus declares none. */
+export function loadArms(skillsRoot) {
+    const file = join(skillsRoot, "tests", "arms.yaml");
+    const out = new Map();
+    if (!existsSync(file))
+        return out;
+    const doc = yaml.load(readFileSync(file, "utf8"));
+    const raw = Array.isArray(doc?.arms) ? doc.arms : [];
+    for (const entry of raw) {
+        const name = typeof entry?.name === "string" ? entry.name.trim() : "";
+        if (!NAME_RE.test(name)) {
+            throw new Error(`${file}: arm name \`${name}\` must match ${NAME_RE} — it becomes a run-directory segment`);
+        }
+        if (RESERVED.has(name))
+            throw new Error(`${file}: arm name \`${name}\` is reserved for the implicit control`);
+        if (out.has(name))
+            throw new Error(`${file}: two arms are both named \`${name}\``);
+        // Resolved here, but NOT existence-checked here: `loadArms` reads every
+        // declared arm, so validating all of them would make one arm's
+        // machine-specific path (the corpus's `pi-daddy` points into a checkout that
+        // exists on one box) fail `--arm <some-other-arm>` — and `resolveArm`'s
+        // "unknown arm" message — for an arm the caller never asked for. The refusal
+        // moved to `resolveArm`, where only the SELECTED arm is checked.
+        const extensions = (Array.isArray(entry.extensions) ? entry.extensions : []).map((p) => {
+            const expanded = expandHome(String(p));
+            return isAbsolute(expanded) ? expanded : resolve(skillsRoot, expanded);
+        });
+        const env = {};
+        for (const [k, v] of Object.entries((entry.env ?? {}))) {
+            // A nested object/array value stringifies as the literal text
+            // "[object Object]" (or a comma-joined array) via `String(v)`, which would
+            // reach the subject process as a corrupted env var with no indication
+            // anything went wrong. Refused instead — env values must be scalars.
+            if (v !== null && typeof v === "object") {
+                throw new Error(`${file}: arm \`${name}\` env.${k} is not a scalar value — got ${JSON.stringify(v)}`);
+            }
+            env[k] = String(v);
+        }
+        out.set(name, {
+            name,
+            extensions,
+            seedSkills: (Array.isArray(entry.seed_skills) ? entry.seed_skills : []).map(String),
+            requireDefinitions: parseRequireDefinitions(file, name, entry.require_definitions),
+            env,
+        });
+    }
+    return out;
+}
+/** The named arm, or the control when no `--arm` was given. */
+export function resolveArm(skillsRoot, name) {
+    if (!name || name === NONE_ARM.name)
+        return NONE_ARM;
+    const arms = loadArms(skillsRoot);
+    const arm = arms.get(name);
+    if (!arm) {
+        const known = [...arms.keys()];
+        throw new Error(`unknown arm \`${name}\` — ${known.length ? `declared arms: ${known.join(", ")}` : `no arms declared in ${join(skillsRoot, "tests", "arms.yaml")}`}`);
+    }
+    // Same refusal `extensionFlags` makes, made earlier and for the same reason:
+    // pi accepts a missing `--extension` path by starting without it, so the
+    // scenario would silently test an agent with no delegation tool at all.
+    for (const abs of arm.extensions) {
+        if (!existsSync(abs)) {
+            throw new Error(`arm \`${arm.name}\` names extension ${abs}, which does not exist — pi would start without it`);
+        }
+    }
+    return arm;
+}
+/** pi's second skill root, which pi-daddy reads too. */
+function defaultAmbientSkillsDir() {
+    return join(homedir(), ".pi", "agent", "skills");
+}
+/**
+ * Copy an arm's definitions into `<workspace>/.pi/skills/` and return the count.
+ *
+ * Three refusals, each with a negative control in the suite, because a
+ * positive-only check measures less than it claims:
+ *
+ *  1. **Fewer definitions than required.** pi-daddy spawns definitions by path.
+ *     Zero definitions means the arm has nothing to spawn, so it would run green
+ *     and measure nothing — a vacuous result shaped exactly like a finding.
+ *  2. **A missing seed directory** — the `--skill <nonexistent>` class: pi accepts
+ *     silently, so the run would look fine and measure a different thing.
+ *  3. **A non-empty ambient skills root.** pi-daddy reads `~/.pi/agent/skills` as
+ *     well as the workspace, so anything there is an uncontrolled variable in the
+ *     measurement. Empty on the reference box today — that is luck, not a
+ *     guarantee, and this check is what turns it into one. It runs for any
+ *     non-control arm, even one that declares no `seed_skills` — pi-daddy
+ *     reads that root independently of whether the workspace was seeded, so
+ *     an arm can't opt out of the guarantee just by not seeding.
+ */
+export function seedArmDefinitions(arm, skillsRoot, workspaceCwd, opts = {}) {
+    // The control arm loads no extension at all, so nothing can read the
+    // ambient root on its behalf — it's the only arm exempt from the check
+    // below. Every other arm must pass it even when it declares no
+    // seed_skills: pi-daddy reads ~/.pi/agent/skills independently of seeding,
+    // so an arm that loads pi-daddy via `extensions` alone must not be able to
+    // skip the guarantee that root is empty.
+    if (arm.name === NONE_ARM.name)
+        return 0;
+    const ambient = opts.ambientSkillsDir ?? defaultAmbientSkillsDir();
+    let ambientEntries = [];
+    try {
+        ambientEntries = readdirSync(ambient);
+    }
+    catch (err) {
+        // Absent is as good as empty: nothing can leak from a directory that
+        // isn't there. Anything else (e.g. EACCES) is not equivalent to empty —
+        // it means we couldn't verify the guarantee, so it must surface, not be
+        // silently read as "clean".
+        if (err.code === "ENOENT") {
+            ambientEntries = [];
+        }
+        else {
+            throw new Error(`arm \`${arm.name}\`: could not read the ambient skill root ${ambient}: ${err.message} — ` +
+                `pi-daddy reads it as well as the workspace, so this can't be verified empty.`);
+        }
+    }
+    if (ambientEntries.length > 0) {
+        throw new Error(`arm \`${arm.name}\`: the ambient skill root ${ambient} is not empty (${ambientEntries.slice(0, 5).join(", ")}) — ` +
+            `pi-daddy reads it as well as the workspace, so those definitions would be an uncontrolled variable in the measurement. ` +
+            `Move them aside for the run.`);
+    }
+    // Checked BEFORE the early return below, not after: an empty (or vacuously
+    // typo'd — `seed_skill:` instead of `seed_skills:`) list must hit the same
+    // `require_definitions` refusal as a directory that seeds too few. This is
+    // the same bypass shape the ambient-root check above was already hoisted to
+    // avoid — an arm with `require_definitions: 6` and no `seed_skills` would
+    // otherwise seed 0, return early, and run tagged `+pi-daddy` with nothing to
+    // spawn: the exact vacuous arm this refusal exists to prevent.
+    if (arm.seedSkills.length === 0) {
+        if (arm.requireDefinitions > 0) {
+            throw new Error(`arm \`${arm.name}\`: seeded 0 definition(s) (no \`seed_skills\` declared) but require_definitions is ${arm.requireDefinitions} — ` +
+                `pi-daddy would have nothing to spawn, and the arm would measure nothing while looking green.`);
+        }
+        return 0;
+    }
+    const dest = join(workspaceCwd, ".pi", "skills");
+    mkdirSync(dest, { recursive: true });
+    // `realpathSync` and not `resolve`: `resolve` is pure lexical arithmetic on the
+    // path string, so a `seed_skills` entry naming a SYMLINK inside the corpus that
+    // points outside it passes a `startsWith` check on the lexical path and seeds
+    // from outside the corpus anyway — the containment guarantee held for `../..`
+    // and not for the equivalent written as a link.
+    const resolvedRoot = realpathOr(resolve(skillsRoot));
+    const seen = new Set();
+    for (const rel of arm.seedSkills) {
+        const src = realpathOr(resolve(skillsRoot, rel));
+        // `seed_skills` entries are meant to name directories WITHIN the corpus —
+        // `resolve(skillsRoot, "../..")` is accepted by `resolve()` without complaint,
+        // and would seed the workspace from whatever happens to sit outside the
+        // corpus root. Refused rather than silently followed.
+        if (src !== resolvedRoot && !src.startsWith(resolvedRoot + sep)) {
+            throw new Error(`arm \`${arm.name}\`: seed_skills entry ${JSON.stringify(rel)} resolves to ${src}, which is outside the skills root ${resolvedRoot} — refusing to seed from outside the corpus`);
+        }
+        let names;
+        try {
+            names = readdirSync(src);
+        }
+        catch {
+            throw new Error(`arm \`${arm.name}\`: seed_skills names ${src}, which cannot be read — pi would start with nothing to spawn`);
+        }
+        for (const name of names) {
+            const from = join(src, name);
+            if (!name.endsWith(".md"))
+                continue;
+            let isFile;
+            try {
+                isFile = statSync(from).isFile();
+            }
+            catch (err) {
+                // A dangling symlink (or a file removed between readdir and stat) throws
+                // here uncaught otherwise — an unclassified crash instead of a message
+                // naming the offending file.
+                throw new Error(`arm \`${arm.name}\`: seed_skills entry ${JSON.stringify(rel)} contains ${from}, which cannot be read (${err.message}) — likely a dangling symlink`);
+            }
+            if (!isFile)
+                continue;
+            // `dest` is ONE flat directory, so two seed dirs shipping the same basename
+            // resolve to the same destination path: the second copy silently overwrites
+            // the first. Counting copies rather than files then lets `require_definitions:
+            // 6` be satisfied by 6 copies that left 3 files on disk — the vacuous arm the
+            // refusal exists to prevent, reached through the refusal itself. Refused,
+            // because the alternative is that whichever entry is listed last wins and
+            // nothing says so.
+            if (seen.has(name)) {
+                throw new Error(`arm \`${arm.name}\`: two seed_skills entries both provide \`${name}\` (latest: ${from}) — ` +
+                    `they are copied into the one flat directory ${dest}, so one would silently overwrite the other. ` +
+                    `Rename one, or drop the duplicate entry.`);
+            }
+            seen.add(name);
+            copyFileSync(from, join(dest, name));
+        }
+    }
+    // Distinct definitions ON DISK, which is what pi-daddy can actually spawn.
+    const count = seen.size;
+    if (count < arm.requireDefinitions) {
+        throw new Error(`arm \`${arm.name}\`: seeded ${count} definition(s) into ${dest} but require_definitions is ${arm.requireDefinitions} — ` +
+            `pi-daddy would have nothing (or too little) to spawn, and the arm would measure nothing while looking green.`);
+    }
+    return count;
+}
+//# sourceMappingURL=arms.js.map

package/dist/capture-trace-types.d.ts

@@ -83,12 +83,14 @@ export interface TraceResultMeta {
  * conversation carries in the session dir, not in the event stream). This
  * mirrors the loop the adapter already runs.
  */
+export type CostSource = "provider-reported" | "subscription" | "unreported";
 export interface TraceMetrics {
     input_tokens: number;
     output_tokens: number;
     cache_read_tokens: number;
     cache_write_tokens: number;
     cost_usd: number;
+    cost_source: CostSource;
     tool_calls: number;
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */

package/dist/command-cost.d.ts

@@ -0,0 +1,9 @@
+/**
+ * CLI commands guaranteed to make no subject or judge calls.
+ *
+ * Keep this list narrow: commands that can conditionally spend (notably `regate`)
+ * do not belong here even when their common path only replays saved artifacts.
+ */
+export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "coverage", "init", "judge-agreement", "lint", "list", "mutation-test", "rescore", "restamp", "screen", "stability"];
+export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
+export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;

package/dist/command-cost.js

@@ -0,0 +1,23 @@
+/**
+ * CLI commands guaranteed to make no subject or judge calls.
+ *
+ * Keep this list narrow: commands that can conditionally spend (notably `regate`)
+ * do not belong here even when their common path only replays saved artifacts.
+ */
+export const FREE_OFFLINE_COMMANDS = [
+    "affected",
+    "coverage",
+    "init",
+    "judge-agreement",
+    "lint",
+    "list",
+    "mutation-test",
+    "rescore",
+    "restamp",
+    "screen",
+    "stability",
+];
+export function isFreeOfflineCommand(command) {
+    return FREE_OFFLINE_COMMANDS.some((candidate) => candidate === command);
+}
+//# sourceMappingURL=command-cost.js.map

package/dist/comparison.d.ts

@@ -45,6 +45,7 @@ export interface AggregateMetrics {
     cache_read_tokens: number | null;
     cache_write_tokens: number | null;
     subject_cost_usd: number | null;
+    cost_source: "provider-reported" | "subscription" | "unreported" | null;
     tool_calls: number | null;
     delegated_children: number | null;
     max_concurrency: number | null;

package/dist/comparison.js

@@ -51,8 +51,8 @@ export function buildComparison(input) {
     const behavioral = cells.filter((cell) => cell.change === "regression").map((cell) => cell.id);
     const criticalRegressions = cells.filter((cell) => cell.change === "regression" && cell.critical).map((cell) => cell.id);
     const byReference = new Map(reference.scenarios.map((scenario) => [scenario.id, scenario]));
-    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
-        cell.candidate === "ERROR" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
+    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "NOT-MEASURED" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
+        cell.candidate === "ERROR" || cell.candidate === "NOT-MEASURED" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
     const metrics = { reference: aggregateMetrics(reference.scenarios), candidate: aggregateMetrics(candidate.scenarios) };
     const costRegressions = thresholdFindings(metrics.reference, metrics.candidate, input.thresholds);
     const partial = input.partial === true || reference.partial === true || candidate.partial === true;
@@ -137,6 +137,10 @@ export function aggregateMetrics(scenarios) {
         cache_read_tokens: sumOptional("cache_read_tokens"),
         cache_write_tokens: sumOptional("cache_write_tokens"),
         subject_cost_usd: sumOptional("subject_cost_usd"),
+        cost_source: (() => {
+            const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
+            return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
+        })(),
         tool_calls: sumOptional("tool_calls"),
         delegated_children: sumOptional("delegated_children"),
         max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
@@ -147,11 +151,13 @@ function effective(result) {
         return result.override;
     if (result.objective?.status === "ERROR")
         return "ERROR";
+    if (result.objective?.status === "NOT-MEASURED")
+        return "NOT-MEASURED";
     if (result.objective?.status === "FAIL")
         return "FAIL";
     return result.judge_verdict;
 }
-function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS"; }
+function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "NOT-MEASURED" || verdict === "JUDGE-AMBIGUOUS"; }
 function sorted(value) { return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); }
 function increase(reference, candidate) { return reference === null || candidate === null || reference <= 0 ? null : (candidate - reference) / reference; }
 function thresholdFindings(reference, candidate, thresholds) {