@skill-harness/core 0.9.0 → 0.11.0

package/dist/adapters/types.d.ts

@@ -44,6 +44,12 @@ export interface RunReq {
     extensions?: string[];
     /** Workspace-local native ledgers to normalize after the subject finishes. */
     eventSources?: TrajectoryEventSource[];
+    /**
+     * Extra env for the subject process, from the arm. Merged over `process.env` by
+     * the adapter — an arm must be able to add `PI_GRANTS_*` without the harness
+     * inheriting a caller's copy of them.
+     */
+    armEnv?: Record<string, string>;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {
@@ -60,6 +66,11 @@ export interface StructuredRun {
     events?: TrajectoryEventV1[];
     /** Native sources that were required but missing/malformed. Never treated as an empty success. */
     eventErrors?: string[];
+    /**
+     * Set when pi failed provider-side (auth, transport) rather than the model
+     * answering badly. `run.ts` turns this into ERROR — never a model verdict.
+     */
+    providerFailure?: string;
 }
 export interface HarnessAdapter {
     name: string;

package/dist/arms.d.ts

@@ -0,0 +1,51 @@
+/**
+ * An arm: a named bundle of harness-side conditions a run is measured under.
+ *
+ * Why arms live HERE and not in `specification.yaml`. An arm is meant to be
+ * A/B'd — the control and the treatment must be the same experiment run twice.
+ * Declaring it in the spec would move the spec digest, which both stales every
+ * committed run in the corpus and makes control and treatment textually
+ * different experiments. So `arms.yaml` is deliberately part of NO digest, and
+ * the arm is carried in the run-dir TAG instead: `lint` and `stability` key on
+ * the tag, so two arms are separate lineages that can never be misread as one
+ * lineage flipping its verdict run-over-run.
+ */
+export interface Arm {
+    name: string;
+    /** Absolute paths, one `--extension` each. */
+    extensions: string[];
+    /** Dirs (relative to the skills root) copied into `<workspace>/.pi/skills/`. */
+    seedSkills: string[];
+    /** Minimum definitions the seeding must produce, else the run ERRORs. */
+    requireDefinitions: number;
+    /** Env for the subject process. `<run-dir>` is substituted per run. */
+    env: Record<string, string>;
+}
+/** The implicit control: today's behaviour, byte-identical, no extensions. */
+export declare const NONE_ARM: Arm;
+/** `<skills-root>/tests/arms.yaml`, or an empty map when the corpus declares none. */
+export declare function loadArms(skillsRoot: string): Map<string, Arm>;
+/** The named arm, or the control when no `--arm` was given. */
+export declare function resolveArm(skillsRoot: string, name: string | null): Arm;
+/**
+ * Copy an arm's definitions into `<workspace>/.pi/skills/` and return the count.
+ *
+ * Three refusals, each with a negative control in the suite, because a
+ * positive-only check measures less than it claims:
+ *
+ *  1. **Fewer definitions than required.** pi-daddy spawns definitions by path.
+ *     Zero definitions means the arm has nothing to spawn, so it would run green
+ *     and measure nothing — a vacuous result shaped exactly like a finding.
+ *  2. **A missing seed directory** — the `--skill <nonexistent>` class: pi accepts
+ *     silently, so the run would look fine and measure a different thing.
+ *  3. **A non-empty ambient skills root.** pi-daddy reads `~/.pi/agent/skills` as
+ *     well as the workspace, so anything there is an uncontrolled variable in the
+ *     measurement. Empty on the reference box today — that is luck, not a
+ *     guarantee, and this check is what turns it into one. It runs for any
+ *     non-control arm, even one that declares no `seed_skills` — pi-daddy
+ *     reads that root independently of whether the workspace was seeded, so
+ *     an arm can't opt out of the guarantee just by not seeding.
+ */
+export declare function seedArmDefinitions(arm: Arm, skillsRoot: string, workspaceCwd: string, opts?: {
+    ambientSkillsDir?: string;
+}): number;

package/dist/arms.js

@@ -0,0 +1,255 @@
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { isAbsolute, join, resolve, sep } from "node:path";
+import yaml from "js-yaml";
+/** The implicit control: today's behaviour, byte-identical, no extensions. */
+export const NONE_ARM = { name: "none", extensions: [], seedSkills: [], requireDefinitions: 0, env: {} };
+/** `none` is the control's name; an arm may not shadow it. */
+const RESERVED = new Set(["none"]);
+/**
+ * Arm names become a path segment, so they are restricted to what survives one
+ * unambiguously — no separators, no dots, nothing needing quoting.
+ */
+const NAME_RE = /^[A-Za-z0-9_-]+$/;
+function expandHome(p) {
+    return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p;
+}
+/**
+ * `realpathSync(p)`, or `p` when it cannot be resolved.
+ *
+ * A path that does not exist has no real path, and refusing here would replace
+ * the two callers' specific messages ("outside the skills root", "cannot be read
+ * — pi would start with nothing to spawn") with a bare ENOENT. The lexical path
+ * is the strictly safer fallback for the containment check: an absent path
+ * cannot be a symlink out of the corpus.
+ */
+function realpathOr(p) {
+    try {
+        return realpathSync(p);
+    }
+    catch {
+        return p;
+    }
+}
+/**
+ * `require_definitions`, refusing anything that is not a non-negative integer.
+ *
+ * A tolerant `Number(x) || 0` reads `require_definitions: six` (and any value
+ * YAML hands over as a string) as 0, which does not disable a typo — it disables
+ * the REFUSAL the key exists to make, so an arm that seeds nothing runs green and
+ * measures nothing. The one failure mode this key defends against is the one a
+ * silent default reintroduces.
+ */
+function parseRequireDefinitions(file, name, raw) {
+    if (raw === undefined || raw === null)
+        return 0;
+    const n = typeof raw === "number" ? raw : Number(String(raw).trim());
+    if (!Number.isInteger(n) || n < 0) {
+        throw new Error(`${file}: arm \`${name}\` require_definitions is ${JSON.stringify(raw)} — it must be a non-negative integer, ` +
+            `and a value that silently read as 0 would disable the refusal it exists to make.`);
+    }
+    return n;
+}
+/** `<skills-root>/tests/arms.yaml`, or an empty map when the corpus declares none. */
+export function loadArms(skillsRoot) {
+    const file = join(skillsRoot, "tests", "arms.yaml");
+    const out = new Map();
+    if (!existsSync(file))
+        return out;
+    const doc = yaml.load(readFileSync(file, "utf8"));
+    const raw = Array.isArray(doc?.arms) ? doc.arms : [];
+    for (const entry of raw) {
+        const name = typeof entry?.name === "string" ? entry.name.trim() : "";
+        if (!NAME_RE.test(name)) {
+            throw new Error(`${file}: arm name \`${name}\` must match ${NAME_RE} — it becomes a run-directory segment`);
+        }
+        if (RESERVED.has(name))
+            throw new Error(`${file}: arm name \`${name}\` is reserved for the implicit control`);
+        if (out.has(name))
+            throw new Error(`${file}: two arms are both named \`${name}\``);
+        // Resolved here, but NOT existence-checked here: `loadArms` reads every
+        // declared arm, so validating all of them would make one arm's
+        // machine-specific path (the corpus's `pi-daddy` points into a checkout that
+        // exists on one box) fail `--arm <some-other-arm>` — and `resolveArm`'s
+        // "unknown arm" message — for an arm the caller never asked for. The refusal
+        // moved to `resolveArm`, where only the SELECTED arm is checked.
+        const extensions = (Array.isArray(entry.extensions) ? entry.extensions : []).map((p) => {
+            const expanded = expandHome(String(p));
+            return isAbsolute(expanded) ? expanded : resolve(skillsRoot, expanded);
+        });
+        const env = {};
+        for (const [k, v] of Object.entries((entry.env ?? {}))) {
+            // A nested object/array value stringifies as the literal text
+            // "[object Object]" (or a comma-joined array) via `String(v)`, which would
+            // reach the subject process as a corrupted env var with no indication
+            // anything went wrong. Refused instead — env values must be scalars.
+            if (v !== null && typeof v === "object") {
+                throw new Error(`${file}: arm \`${name}\` env.${k} is not a scalar value — got ${JSON.stringify(v)}`);
+            }
+            env[k] = String(v);
+        }
+        out.set(name, {
+            name,
+            extensions,
+            seedSkills: (Array.isArray(entry.seed_skills) ? entry.seed_skills : []).map(String),
+            requireDefinitions: parseRequireDefinitions(file, name, entry.require_definitions),
+            env,
+        });
+    }
+    return out;
+}
+/** The named arm, or the control when no `--arm` was given. */
+export function resolveArm(skillsRoot, name) {
+    if (!name || name === NONE_ARM.name)
+        return NONE_ARM;
+    const arms = loadArms(skillsRoot);
+    const arm = arms.get(name);
+    if (!arm) {
+        const known = [...arms.keys()];
+        throw new Error(`unknown arm \`${name}\` — ${known.length ? `declared arms: ${known.join(", ")}` : `no arms declared in ${join(skillsRoot, "tests", "arms.yaml")}`}`);
+    }
+    // Same refusal `extensionFlags` makes, made earlier and for the same reason:
+    // pi accepts a missing `--extension` path by starting without it, so the
+    // scenario would silently test an agent with no delegation tool at all.
+    for (const abs of arm.extensions) {
+        if (!existsSync(abs)) {
+            throw new Error(`arm \`${arm.name}\` names extension ${abs}, which does not exist — pi would start without it`);
+        }
+    }
+    return arm;
+}
+/** pi's second skill root, which pi-daddy reads too. */
+function defaultAmbientSkillsDir() {
+    return join(homedir(), ".pi", "agent", "skills");
+}
+/**
+ * Copy an arm's definitions into `<workspace>/.pi/skills/` and return the count.
+ *
+ * Three refusals, each with a negative control in the suite, because a
+ * positive-only check measures less than it claims:
+ *
+ *  1. **Fewer definitions than required.** pi-daddy spawns definitions by path.
+ *     Zero definitions means the arm has nothing to spawn, so it would run green
+ *     and measure nothing — a vacuous result shaped exactly like a finding.
+ *  2. **A missing seed directory** — the `--skill <nonexistent>` class: pi accepts
+ *     silently, so the run would look fine and measure a different thing.
+ *  3. **A non-empty ambient skills root.** pi-daddy reads `~/.pi/agent/skills` as
+ *     well as the workspace, so anything there is an uncontrolled variable in the
+ *     measurement. Empty on the reference box today — that is luck, not a
+ *     guarantee, and this check is what turns it into one. It runs for any
+ *     non-control arm, even one that declares no `seed_skills` — pi-daddy
+ *     reads that root independently of whether the workspace was seeded, so
+ *     an arm can't opt out of the guarantee just by not seeding.
+ */
+export function seedArmDefinitions(arm, skillsRoot, workspaceCwd, opts = {}) {
+    // The control arm loads no extension at all, so nothing can read the
+    // ambient root on its behalf — it's the only arm exempt from the check
+    // below. Every other arm must pass it even when it declares no
+    // seed_skills: pi-daddy reads ~/.pi/agent/skills independently of seeding,
+    // so an arm that loads pi-daddy via `extensions` alone must not be able to
+    // skip the guarantee that root is empty.
+    if (arm.name === NONE_ARM.name)
+        return 0;
+    const ambient = opts.ambientSkillsDir ?? defaultAmbientSkillsDir();
+    let ambientEntries = [];
+    try {
+        ambientEntries = readdirSync(ambient);
+    }
+    catch (err) {
+        // Absent is as good as empty: nothing can leak from a directory that
+        // isn't there. Anything else (e.g. EACCES) is not equivalent to empty —
+        // it means we couldn't verify the guarantee, so it must surface, not be
+        // silently read as "clean".
+        if (err.code === "ENOENT") {
+            ambientEntries = [];
+        }
+        else {
+            throw new Error(`arm \`${arm.name}\`: could not read the ambient skill root ${ambient}: ${err.message} — ` +
+                `pi-daddy reads it as well as the workspace, so this can't be verified empty.`);
+        }
+    }
+    if (ambientEntries.length > 0) {
+        throw new Error(`arm \`${arm.name}\`: the ambient skill root ${ambient} is not empty (${ambientEntries.slice(0, 5).join(", ")}) — ` +
+            `pi-daddy reads it as well as the workspace, so those definitions would be an uncontrolled variable in the measurement. ` +
+            `Move them aside for the run.`);
+    }
+    // Checked BEFORE the early return below, not after: an empty (or vacuously
+    // typo'd — `seed_skill:` instead of `seed_skills:`) list must hit the same
+    // `require_definitions` refusal as a directory that seeds too few. This is
+    // the same bypass shape the ambient-root check above was already hoisted to
+    // avoid — an arm with `require_definitions: 6` and no `seed_skills` would
+    // otherwise seed 0, return early, and run tagged `+pi-daddy` with nothing to
+    // spawn: the exact vacuous arm this refusal exists to prevent.
+    if (arm.seedSkills.length === 0) {
+        if (arm.requireDefinitions > 0) {
+            throw new Error(`arm \`${arm.name}\`: seeded 0 definition(s) (no \`seed_skills\` declared) but require_definitions is ${arm.requireDefinitions} — ` +
+                `pi-daddy would have nothing to spawn, and the arm would measure nothing while looking green.`);
+        }
+        return 0;
+    }
+    const dest = join(workspaceCwd, ".pi", "skills");
+    mkdirSync(dest, { recursive: true });
+    // `realpathSync` and not `resolve`: `resolve` is pure lexical arithmetic on the
+    // path string, so a `seed_skills` entry naming a SYMLINK inside the corpus that
+    // points outside it passes a `startsWith` check on the lexical path and seeds
+    // from outside the corpus anyway — the containment guarantee held for `../..`
+    // and not for the equivalent written as a link.
+    const resolvedRoot = realpathOr(resolve(skillsRoot));
+    const seen = new Set();
+    for (const rel of arm.seedSkills) {
+        const src = realpathOr(resolve(skillsRoot, rel));
+        // `seed_skills` entries are meant to name directories WITHIN the corpus —
+        // `resolve(skillsRoot, "../..")` is accepted by `resolve()` without complaint,
+        // and would seed the workspace from whatever happens to sit outside the
+        // corpus root. Refused rather than silently followed.
+        if (src !== resolvedRoot && !src.startsWith(resolvedRoot + sep)) {
+            throw new Error(`arm \`${arm.name}\`: seed_skills entry ${JSON.stringify(rel)} resolves to ${src}, which is outside the skills root ${resolvedRoot} — refusing to seed from outside the corpus`);
+        }
+        let names;
+        try {
+            names = readdirSync(src);
+        }
+        catch {
+            throw new Error(`arm \`${arm.name}\`: seed_skills names ${src}, which cannot be read — pi would start with nothing to spawn`);
+        }
+        for (const name of names) {
+            const from = join(src, name);
+            if (!name.endsWith(".md"))
+                continue;
+            let isFile;
+            try {
+                isFile = statSync(from).isFile();
+            }
+            catch (err) {
+                // A dangling symlink (or a file removed between readdir and stat) throws
+                // here uncaught otherwise — an unclassified crash instead of a message
+                // naming the offending file.
+                throw new Error(`arm \`${arm.name}\`: seed_skills entry ${JSON.stringify(rel)} contains ${from}, which cannot be read (${err.message}) — likely a dangling symlink`);
+            }
+            if (!isFile)
+                continue;
+            // `dest` is ONE flat directory, so two seed dirs shipping the same basename
+            // resolve to the same destination path: the second copy silently overwrites
+            // the first. Counting copies rather than files then lets `require_definitions:
+            // 6` be satisfied by 6 copies that left 3 files on disk — the vacuous arm the
+            // refusal exists to prevent, reached through the refusal itself. Refused,
+            // because the alternative is that whichever entry is listed last wins and
+            // nothing says so.
+            if (seen.has(name)) {
+                throw new Error(`arm \`${arm.name}\`: two seed_skills entries both provide \`${name}\` (latest: ${from}) — ` +
+                    `they are copied into the one flat directory ${dest}, so one would silently overwrite the other. ` +
+                    `Rename one, or drop the duplicate entry.`);
+            }
+            seen.add(name);
+            copyFileSync(from, join(dest, name));
+        }
+    }
+    // Distinct definitions ON DISK, which is what pi-daddy can actually spawn.
+    const count = seen.size;
+    if (count < arm.requireDefinitions) {
+        throw new Error(`arm \`${arm.name}\`: seeded ${count} definition(s) into ${dest} but require_definitions is ${arm.requireDefinitions} — ` +
+            `pi-daddy would have nothing (or too little) to spawn, and the arm would measure nothing while looking green.`);
+    }
+    return count;
+}
+//# sourceMappingURL=arms.js.map

package/dist/index.d.ts

@@ -39,3 +39,5 @@ export * from "./affected.js";
 export * from "./adjudication.js";
 export * from "./comparison.js";
 export * from "./sandbox.js";
+export * from "./provider-failure.js";
+export * from "./arms.js";

package/dist/index.js

@@ -39,4 +39,6 @@ export * from "./affected.js";
 export * from "./adjudication.js";
 export * from "./comparison.js";
 export * from "./sandbox.js";
+export * from "./provider-failure.js";
+export * from "./arms.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -90,6 +90,7 @@ export type JournalEvent = {
     ts: string;
     id: string;
     attempt: number;
+    reason?: string;
     rep?: number;
 } | {
     event: "rescore";

package/dist/provider-failure.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Provider-side failure detection.
+ *
+ * A provider outage is INFRASTRUCTURE, never a model verdict. Measured 2026-08-22:
+ * an invalidated `openai-codex` OAuth token makes pi report
+ * `stopReason: "error"` with a `provider_transport_failure` diagnostic and **exit 0**
+ * in `--mode json`, and exit 1 with `Encountered invalidated oauth token for user,
+ * failing request` on stderr in text mode. Neither path was classified, so a wave
+ * against a dead token produced model FAILs shaped exactly like findings.
+ *
+ * Two entry points because the two run paths surface it differently, and the
+ * structured one is the WEAKER signal — it hides the real message behind a generic
+ * transport error and exits 0. Do not assume `--structured` improves diagnosis.
+ */
+/**
+ * Written into a transcript by the adapter when pi failed provider-side, so the
+ * text path can carry a machine-readable signal through a `string` return without
+ * widening `HarnessAdapter.run`. Read back by `providerFailureFromTranscript`.
+ *
+ * The marker must not be forgeable by a model that types the same words into its
+ * answer — and a verdict is what hangs on it, since a forged marker converts a
+ * would-be FAIL into ERROR and (via `judgeOneRep`'s short-circuit) permanently
+ * suppresses the judge on every later re-grade. Line-anchoring alone does not do
+ * that: nothing stops a model emitting a newline and then the marker.
+ *
+ * So position, not spelling, is the guarantee. The marker is written ONLY into the
+ * transcript PREAMBLE — everything before the first `>>> ` turn header — which the
+ * adapter composes before any model output exists. There is no ordering by which a
+ * model's text can land there. See `withProviderFailure` (write) and
+ * `providerFailureFromTranscript` (read); both live here so the two halves of the
+ * convention cannot drift apart.
+ */
+export declare const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
+/**
+ * Prefix `transcript` with the provider-failure marker, or return it unchanged.
+ *
+ * The one supported way to write the marker: it must precede the first turn header
+ * to be readable back, and putting the rule next to the reader keeps a future
+ * adapter from re-introducing the forgeable placement.
+ */
+export declare function withProviderFailure(transcript: string, failure: string | null): string;
+/**
+ * A provider failure named by one `pi --mode json` line, or null.
+ *
+ * Fail-closed on the diagnostic, fail-open on the parse: an unreadable line is not
+ * evidence of a failure, and throwing here would abort a wave over one bad line.
+ */
+export declare function providerFailureFromJsonLine(line: string): string | null;
+/**
+ * The failure the adapter recorded in a transcript's preamble, or null.
+ *
+ * Scanning STOPS at the first turn header. Everything from there on is model
+ * output or interleaved with it, so a marker found there is not evidence the
+ * provider failed — it is evidence the model typed the words.
+ */
+export declare function providerFailureFromTranscript(transcript: string): string | null;

package/dist/provider-failure.js

@@ -0,0 +1,98 @@
+/**
+ * Provider-side failure detection.
+ *
+ * A provider outage is INFRASTRUCTURE, never a model verdict. Measured 2026-08-22:
+ * an invalidated `openai-codex` OAuth token makes pi report
+ * `stopReason: "error"` with a `provider_transport_failure` diagnostic and **exit 0**
+ * in `--mode json`, and exit 1 with `Encountered invalidated oauth token for user,
+ * failing request` on stderr in text mode. Neither path was classified, so a wave
+ * against a dead token produced model FAILs shaped exactly like findings.
+ *
+ * Two entry points because the two run paths surface it differently, and the
+ * structured one is the WEAKER signal — it hides the real message behind a generic
+ * transport error and exits 0. Do not assume `--structured` improves diagnosis.
+ */
+/**
+ * Written into a transcript by the adapter when pi failed provider-side, so the
+ * text path can carry a machine-readable signal through a `string` return without
+ * widening `HarnessAdapter.run`. Read back by `providerFailureFromTranscript`.
+ *
+ * The marker must not be forgeable by a model that types the same words into its
+ * answer — and a verdict is what hangs on it, since a forged marker converts a
+ * would-be FAIL into ERROR and (via `judgeOneRep`'s short-circuit) permanently
+ * suppresses the judge on every later re-grade. Line-anchoring alone does not do
+ * that: nothing stops a model emitting a newline and then the marker.
+ *
+ * So position, not spelling, is the guarantee. The marker is written ONLY into the
+ * transcript PREAMBLE — everything before the first `>>> ` turn header — which the
+ * adapter composes before any model output exists. There is no ordering by which a
+ * model's text can land there. See `withProviderFailure` (write) and
+ * `providerFailureFromTranscript` (read); both live here so the two halves of the
+ * convention cannot drift apart.
+ */
+export const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
+/**
+ * The adapters' shared turn-header prefix (`>>> USER:` / `>>> USER (turn n/m):`),
+ * which is where the preamble ends and model-influenced content begins.
+ */
+const TURN_HEADER_PREFIX = ">>> ";
+/**
+ * Prefix `transcript` with the provider-failure marker, or return it unchanged.
+ *
+ * The one supported way to write the marker: it must precede the first turn header
+ * to be readable back, and putting the rule next to the reader keeps a future
+ * adapter from re-introducing the forgeable placement.
+ */
+export function withProviderFailure(transcript, failure) {
+    return failure ? `${PROVIDER_FAILURE_MARKER} ${failure}\n\n${transcript}` : transcript;
+}
+/** Diagnostic types that mean the provider never ran the request. */
+const FAILURE_DIAGNOSTICS = new Set(["provider_transport_failure"]);
+/**
+ * A provider failure named by one `pi --mode json` line, or null.
+ *
+ * Fail-closed on the diagnostic, fail-open on the parse: an unreadable line is not
+ * evidence of a failure, and throwing here would abort a wave over one bad line.
+ */
+export function providerFailureFromJsonLine(line) {
+    let parsed;
+    try {
+        parsed = JSON.parse(line);
+    }
+    catch {
+        return null;
+    }
+    const message = parsed?.message;
+    if (!message || typeof message !== "object")
+        return null;
+    const diagnostics = message.diagnostics;
+    if (!Array.isArray(diagnostics))
+        return null;
+    for (const raw of diagnostics) {
+        if (typeof raw?.type !== "string" || !FAILURE_DIAGNOSTICS.has(raw.type))
+            continue;
+        const provider = typeof message.provider === "string"
+            ? message.provider
+            : "unknown provider";
+        const detail = typeof raw.error?.message === "string" ? raw.error.message : raw.type;
+        return `${provider}: ${detail}`;
+    }
+    return null;
+}
+/**
+ * The failure the adapter recorded in a transcript's preamble, or null.
+ *
+ * Scanning STOPS at the first turn header. Everything from there on is model
+ * output or interleaved with it, so a marker found there is not evidence the
+ * provider failed — it is evidence the model typed the words.
+ */
+export function providerFailureFromTranscript(transcript) {
+    for (const line of transcript.split("\n")) {
+        if (line.startsWith(TURN_HEADER_PREFIX))
+            return null;
+        if (line.startsWith(PROVIDER_FAILURE_MARKER))
+            return line.slice(PROVIDER_FAILURE_MARKER.length).trim();
+    }
+    return null;
+}
+//# sourceMappingURL=provider-failure.js.map

package/dist/regate.js

@@ -305,6 +305,13 @@ export async function regateRun(opts) {
         skill: prev.skill, harness: prev.harness, model: prev.model,
         // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
         harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
+        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+        // only record that a `+<arm>` run actually delegated: rebuilding the draft
+        // field-by-field without it silently deleted `definitions`/`ledger_events`
+        // from any arm run that was ever re-graded, leaving a record
+        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+        // `delivery_canary` and `source_hashes` are carried here.
+        arm: prev.arm,
         judge: { provider: opts.judge.provider, model: opts.judge.model },
         timestamp: prev.timestamp, label: prev.label, mode: prev.mode, partial: prev.partial,
         // Only the `gates:` keys of the scenarios actually re-evaluated. Stimulus, rubric

package/dist/regrade.js

@@ -5,6 +5,7 @@ import { findTranscriptFiles, judgeRawPath, repIndexOf, readResults, writeResult
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
 import { rubricDigest, personaDigest, RUBRIC_PREFIX, PERSONA_KEY } from "./sources.js";
+import { providerFailureFromTranscript } from "./provider-failure.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -37,10 +38,26 @@ export function refreshRubricHashes(recorded, spec, judgedIds) {
 export async function judgeOneRep(opts) {
     const { runDir, spec, scenario, transcript, adapter, judge, specDir, mode, rep, now } = opts;
     const startedAt = performance.now();
+    const repField = rep === undefined ? {} : { rep };
+    // A provider outage recorded in the transcript (either by `run()`'s marker or,
+    // since the fix for I2, by `runStructured` too) must stay ERROR through a
+    // re-judge. Without this check, `grade`/`regrade` reads the saved transcript
+    // with no infrastructure-failure guard at all — the one `run.ts` applies BEFORE
+    // ever calling this function — and re-judges "provider never ran the request"
+    // into a model verdict, spending a judge call to turn infrastructure noise into
+    // a false FAIL.
+    const providerFailure = providerFailureFromTranscript(transcript);
+    if (providerFailure) {
+        const reason = `provider failure — ${providerFailure}`;
+        appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: "ERROR", reason, suspect: false, ...repField });
+        return {
+            verdict: "ERROR", reason, suspect: false,
+            metrics: { wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)), judge_calls: 0, judge_rejudge_calls: 0 },
+        };
+    }
     const prompt = buildJudgePrompt({ skill: spec.skill, persona: spec.judge_persona, scenario, transcript });
     const g = await judgeInWorkspace(adapter, judge, prompt, specDir);
     writeFileSync(judgeRawPath(runDir, scenario.id, mode, rep), g.raw, "utf8");
-    const repField = rep === undefined ? {} : { rep };
     appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict: g.verdict, reason: g.reason, suspect: g.suspect, ...repField });
     if (g.suspect)
         appendJournal(runDir, { event: "misfire-flag", ts: now(), id: scenario.id, reason: g.reason, ...repField });
@@ -180,6 +197,13 @@ export async function regradeRun(opts) {
         // here would credit the old transcripts to a version that never ran them.
         harness_cli_version: prev?.harness_cli_version,
         delivery_canary: prev?.delivery_canary,
+        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+        // only record that a `+<arm>` run actually delegated: rebuilding the draft
+        // field-by-field without it silently deleted `definitions`/`ledger_events`
+        // from any arm run that was ever re-graded, leaving a record
+        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+        // `delivery_canary` and `source_hashes` are carried here.
+        arm: prev?.arm,
         model: prev?.model ?? "unknown",
         judge: { provider: judge.provider, model: judge.model },
         timestamp: prev?.timestamp ?? now(),

package/dist/rescore.js

@@ -85,6 +85,13 @@ export function rescoreRun(opts) {
         // threshold to reps the recorded harness already produced.
         harness_cli_version: prev.harness_cli_version,
         delivery_canary: prev.delivery_canary,
+        // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
+        // only record that a `+<arm>` run actually delegated: rebuilding the draft
+        // field-by-field without it silently deleted `definitions`/`ledger_events`
+        // from any arm run that was ever re-graded, leaving a record
+        // indistinguishable from a vacuous arm. Same reason `harness_cli_version`,
+        // `delivery_canary` and `source_hashes` are carried here.
+        arm: prev.arm,
         // A rescore re-applies the CURRENT policy (thresholds, critical set) to the
         // recorded reps, so `policy:` drift is genuinely resolved by having run this —
         // that is what makes `rescore` the honest remedy lint names for it. Stimulus,

package/dist/results.d.ts

@@ -157,6 +157,39 @@ export interface ResultsFile {
     timestamp: string;
     label: string | null;
     mode: string;
+    /**
+     * The arm this run was measured under, absent for the control.
+     *
+     * Recorded rather than inferred from the tag: the tag says which arm, this says
+     * what the arm actually loaded — a run whose extension list or definition count
+     * differed from today's `arms.yaml` is not comparable, and only the record can
+     * show that.
+     *
+     * `ledger_events` is the arm's delivery proof, `--canary`'s equivalent for this
+     * axis: the pi-daddy ledger itself is gitignored (`*.jsonl`), so without a
+     * count here a vacuous arm run (extension loaded, nothing ever delegated)
+     * commits a record indistinguishable from one that actually exercised
+     * delegation. 0 is a reportable outcome, not an absent field.
+     *
+     * `env` is the arm's declared env for the subject process (`PI_GRANTS_GRANT`,
+     * `PI_GRANTS_MAX_DEPTH`, …) — recorded because it IS the experimental
+     * condition, not decoration around it. Without it, changing `PI_GRANTS_MAX_DEPTH`
+     * from 1 to 3 and re-running lands both runs in the same `+<arm>` tag with
+     * byte-identical `arm` records, and `stability` reads a verdict flip between two
+     * different conditions as one lineage being bimodal — the exact misreading the
+     * tag placement exists to prevent. Recorded as DECLARED, with `<run-dir>`
+     * unsubstituted: the substituted form differs every run for the same condition,
+     * which would make two identical conditions look like two.
+     *
+     * Optional only for records written before it was added; every writer emits it.
+     */
+    arm?: {
+        name: string;
+        extensions: string[];
+        definitions: number;
+        ledger_events: number;
+        env?: Record<string, string>;
+    };
     /** True for an `--only`-filtered run: a scenario subset, never ship-graded, never a release run. */
     partial?: boolean;
     /**
@@ -227,8 +260,21 @@ export interface ScoreContext {
     shipBar: ShipBar;
     critical: string[];
 }
-/** <skillDir>/tests/results/<harness>-<model-slug>/<timestamp-slug>/ */
-export declare function runDirFor(skillDir: string, harness: string, model: ModelRef, timestamp: string): string;
+/**
+ * `<skillDir>/tests/results/<harness>-<model-slug>[+<arm>]/<timestamp-slug>/`
+ *
+ * The arm is part of the run's IDENTITY, not of the spec's digest. `lint` and
+ * `stability` both key on this tag, so two arms become separate lineages that can
+ * never be misread as one lineage flipping run-over-run — and no
+ * `specification.yaml` byte moves, so every committed run stays valid.
+ *
+ * `+` is deliberate and is appended OUTSIDE `modelSlug`, whose character class
+ * (`[^A-Za-z0-9._-]`) cannot emit one. So the separator can never occur inside a
+ * slug and the tag stays unambiguously splittable back into model and arm. The
+ * control arm appends nothing, so its paths are byte-identical to before arms
+ * existed.
+ */
+export declare function runDirFor(skillDir: string, harness: string, model: ModelRef, timestamp: string, armName?: string): string;
 /** Path of a transcript file within a run dir. A rep index (for --reps N>1) is suffixed. */
 export declare function transcriptPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 export declare function reportPath(runDir: string): string;

package/dist/results.js

@@ -87,9 +87,23 @@ export function effectiveThreshold(prevScenario, scenario) {
 function timestampSlug(iso) {
     return iso.replace(/[:.]/g, "-");
 }
-/** <skillDir>/tests/results/<harness>-<model-slug>/<timestamp-slug>/ */
-export function runDirFor(skillDir, harness, model, timestamp) {
-    return join(skillDir, "tests", "results", `${harness}-${modelSlug(model)}`, timestampSlug(timestamp));
+/**
+ * `<skillDir>/tests/results/<harness>-<model-slug>[+<arm>]/<timestamp-slug>/`
+ *
+ * The arm is part of the run's IDENTITY, not of the spec's digest. `lint` and
+ * `stability` both key on this tag, so two arms become separate lineages that can
+ * never be misread as one lineage flipping run-over-run — and no
+ * `specification.yaml` byte moves, so every committed run stays valid.
+ *
+ * `+` is deliberate and is appended OUTSIDE `modelSlug`, whose character class
+ * (`[^A-Za-z0-9._-]`) cannot emit one. So the separator can never occur inside a
+ * slug and the tag stays unambiguously splittable back into model and arm. The
+ * control arm appends nothing, so its paths are byte-identical to before arms
+ * existed.
+ */
+export function runDirFor(skillDir, harness, model, timestamp, armName) {
+    const arm = armName && armName !== "none" ? `+${armName}` : "";
+    return join(skillDir, "tests", "results", `${harness}-${modelSlug(model)}${arm}`, timestampSlug(timestamp));
 }
 /** Path of a transcript file within a run dir. A rep index (for --reps N>1) is suffixed. */
 export function transcriptPath(runDir, scenarioId, mode, rep) {
@@ -182,6 +196,7 @@ export function finalizeResults(draft, ctx) {
         timestamp: draft.timestamp,
         label: draft.label,
         mode: draft.mode,
+        ...(draft.arm ? { arm: draft.arm } : {}),
         ...(draft.partial ? { partial: true } : {}),
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
         effective_grade,

package/dist/run.d.ts

@@ -1,5 +1,6 @@
 import type { Spec } from "./spec.js";
 import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
+import { type Arm } from "./arms.js";
 import { type ResultsFile } from "./results.js";
 import { type Lift } from "./lift.js";
 import { type ScenarioStability } from "./stability.js";
@@ -32,6 +33,19 @@ export interface RunOptions {
      * there) is already refused by the adapter for free.
      */
     canary?: boolean;
+    /**
+     * Take the structured path even when no scenario declares a trace or trajectory
+     * assertion. The subject half of `ScenarioMetrics` — tokens, `subject_cost_usd` —
+     * is only ever populated from traces, so without this a run records no cost or
+     * token data at all, which is why the reference corpus has none.
+     */
+    structured?: boolean;
+    /** The arm this run is measured under. Absent means the control (`none`). */
+    arm?: Arm;
+    /** Skills root — `seed_skills` paths resolve against it. Defaults to the skill's parent. */
+    skillsRoot?: string;
+    /** Injectable ambient skill root, for tests. Production reads `~/.pi/agent/skills`. */
+    ambientSkillsDir?: string;
 }
 export interface RunSummary {
     runDir: string;

package/dist/run.js

@@ -1,7 +1,8 @@
-import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
 import { sourceHashes } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
+import { NONE_ARM, seedArmDefinitions } from "./arms.js";
 import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
@@ -16,6 +17,36 @@ import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./comparison.js";
+import { providerFailureFromTranscript } from "./provider-failure.js";
+/**
+ * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
+ * `PI_GRANTS_LEDGER` at exactly this filename inside the run dir (see
+ * `../principal-pi-skills/tests/arms.yaml` and CODEX-ARMS-RUNBOOK.md §4.3), so
+ * a fixed name here is what makes the count readable without parsing the arm's
+ * own `env` map back out.
+ */
+const LEDGER_FILENAME = "pi-daddy.ledger.jsonl";
+/**
+ * Count the arm's ledger events after the run, or 0 when the file was never
+ * written.
+ *
+ * This is `--canary`'s lesson applied to the arm: the ledger is the arm's
+ * delivery proof, and a delivery claim has to survive a commit. A missing file
+ * records `0`, not an absent field — "the extension never wrote a ledger" and
+ * "the extension wrote one with nothing in it" are both real, reportable
+ * outcomes, and a vacuous arm run must not commit a record indistinguishable
+ * from one that actually delegated.
+ */
+function countLedgerEvents(runDir) {
+    let text;
+    try {
+        text = readFileSync(join(runDir, LEDGER_FILENAME), "utf8");
+    }
+    catch {
+        return 0;
+    }
+    return text.split("\n").filter((line) => line.trim().length > 0).length;
+}
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -39,7 +70,8 @@ export async function runSkillModel(opts) {
         log(`  ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
             `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
     }
-    const runDir = runDirFor(skillDir, adapter.name, model, timestamp);
+    const arm = opts.arm ?? NONE_ARM;
+    const runDir = runDirFor(skillDir, adapter.name, model, timestamp, arm.name);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
     // Which harness CLI delivered the skill, asked once per run and recorded with the
@@ -97,12 +129,13 @@ export async function runSkillModel(opts) {
     const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
     const owners = [];
     const tasks = [];
+    const armDefinitions = { count: 0 };
     scenarios.forEach((scenario, si) => {
         for (let k = 0; k < repCounts[si]; k++) {
             const rep = k;
             const total = repCounts[si];
             owners.push(si);
-            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log }));
+            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log, armDefinitions }));
         }
     });
     const flat = await runPool(tasks, opts.concurrency ?? 1);
@@ -130,6 +163,22 @@ export async function runSkillModel(opts) {
         // coverage of scenarios it skipped.
         source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
         scenarios: scenarioResults,
+        ...(arm.name === NONE_ARM.name ? {} : {
+            arm: {
+                name: arm.name,
+                extensions: arm.extensions,
+                definitions: armDefinitions.count,
+                ledger_events: countLedgerEvents(runDir),
+                // The DECLARED env, `<run-dir>` left unsubstituted. It is the condition
+                // being measured (grant, max depth), so leaving it out made two runs at
+                // different settings byte-identical here and inside the same `+<arm>`
+                // tag — `stability` then reads the verdict difference between two
+                // conditions as one lineage flipping. The substituted form would be the
+                // opposite error: it embeds this run's temp path, so re-running the SAME
+                // condition would record two different-looking arms.
+                env: arm.env,
+            },
+        }),
     }, ctx);
     if (ctx) {
         const g = results.effective_grade;
@@ -159,6 +208,14 @@ async function runRep(scenario, rep, repCount, ctx) {
     const startedAt = performance.now();
     const { spec, judge, mode, runDir, now, log } = ctx;
     const repField = repCount > 1 ? { rep } : {};
+    const arm = ctx.arm ?? NONE_ARM;
+    const skillsRoot = ctx.skillsRoot ?? dirname(ctx.skillDir);
+    // `<run-dir>` in an arm's env values is a real path (temp dirs on this box
+    // routinely contain characters a regex would treat specially), so this is a
+    // plain split/join, not a `.replace(/<run-dir>/g, …)`.
+    const armEnv = Object.keys(arm.env).length
+        ? Object.fromEntries(Object.entries(arm.env).map(([k, v]) => [k, v.split("<run-dir>").join(runDir)]))
+        : undefined;
     if (rep === 0) {
         log(`  ${scenario.id} (${scenario.title})${repCount > 1 ? ` ×${repCount}` : ""} …`);
         appendJournal(runDir, { event: "scenario-started", ts: now(), id: scenario.id, title: scenario.title });
@@ -194,66 +251,158 @@ async function runRep(scenario, rep, repCount, ctx) {
         // a committed results.yaml, naming files the model never touched.
         let before = ws ? snapshotPaths(ws.cwd, scenario.workspace) : null;
         if (ws) {
+            // Seed the arm's definitions into THIS workspace before the subject runs.
+            // Control (`none`) short-circuits inside `seedArmDefinitions` with no
+            // filesystem side effects at all.
+            ctx.armDefinitions.count = seedArmDefinitions(arm, skillsRoot, ws.cwd, { ambientSkillsDir: ctx.ambientSkillsDir });
+        }
+        // Set when the adapter itself threw. Rep-scoped rather than loop-scoped: the
+        // evidence blocks below have to know this rep never ran.
+        let adapterFailure = null;
+        if (ws) {
+            // Decided ONCE, before any attempt, and for both the seeded and non-seeded
+            // branches — these are spec/adapter facts, not per-attempt outcomes, and
+            // deciding them here is what lets the execution below be wrapped in a catch
+            // that must not swallow a configuration error.
+            //
+            // `needsStructuredEvidence` is a gate with nothing to read if the adapter
+            // cannot produce traces: ERROR, never a silent fallback to the unstructured
+            // path, because a gate that could not be evaluated must not look like one
+            // that passed. A bare `--structured` request with no gate depending on it
+            // degrades to the plain path instead — on BOTH branches. The seeded branch
+            // used to pass `trace: {...}` unconditionally and let `runSeeded` throw
+            // ``scenario `X` declares `assert.trace`…`` at a scenario that declares no
+            // such thing.
+            const needsStructuredEvidence = Boolean(scenario.traceAssert || scenario.trajectoryAssert);
+            if (needsStructuredEvidence && !ctx.adapter.runStructured) {
+                throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
+                    ` cannot produce execution traces/events — the gate would have no evidence to read.`);
+            }
+            const useStructured = (Boolean(ctx.structured) || needsStructuredEvidence) && Boolean(ctx.adapter.runStructured);
             // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
             // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
             // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
+            // An adapter that THREW is the same class of event and takes the same retry.
             for (let attempt = 0; attempt < 2; attempt++) {
                 if (attempt > 0) {
-                    appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, ...repField });
-                    log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
+                    const why = adapterFailure ? `adapter failed (${adapterFailure})` : "empty response";
+                    appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, reason: why, ...repField });
+                    log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${why} — retrying once`);
                     ws.cleanup();
                     ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
                     // A fresh workspace needs a fresh baseline, or the retry's diff would
                     // be taken against a directory that no longer exists.
                     before = snapshotPaths(ws.cwd, scenario.workspace);
+                    // And a fresh workspace has nothing to spawn until it, too, is seeded —
+                    // a retry that skipped this would silently run the arm with no
+                    // definitions at all.
+                    ctx.armDefinitions.count = seedArmDefinitions(arm, skillsRoot, ws.cwd, { ambientSkillsDir: ctx.ambientSkillsDir });
                 }
-                if (scenario.mode === "seeded") {
-                    const r = await runSeeded(scenario, {
-                        skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
-                        specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
-                        trace: scenario.traceAssert || scenario.trajectoryAssert ? { scenarioId: scenario.id, rep } : undefined,
-                    });
-                    transcript = r.transcript;
-                    gatePrefix = r.gateFailure;
-                    infrastructureFailure = r.gateError;
-                    stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
-                    traces = r.traces;
-                    events = r.events;
-                    eventErrors = r.eventErrors;
-                }
-                else {
-                    const req = {
-                        skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
-                        // resolved like fixtures: relative to the spec's dir
-                        systemPromptFile: scenario.systemPromptFile
-                            ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
-                            : undefined,
-                        // Absolute before it reaches a child process running in a neutral cwd.
-                        extensions: scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)),
-                        eventSources: scenario.eventSources,
-                    };
-                    if (scenario.traceAssert || scenario.trajectoryAssert) {
-                        // Missing required evidence is ERROR, never a silent fallback to the
-                        // unstructured path: a gate with nothing to read must not look like a
-                        // gate that passed.
-                        if (!ctx.adapter.runStructured) {
-                            throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
-                                ` cannot produce execution traces/events — the gate would have no evidence to read.`);
-                        }
-                        const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
-                        transcript = structured.transcript;
-                        traces = structured.traces;
-                        events = structured.events ?? [];
-                        eventErrors = structured.eventErrors ?? [];
+                // Cleared per attempt, never carried over. A provider failure always
+                // leaves a blank assistant turn, so attempt 0's reason used to survive a
+                // retry that then succeeded: the rep was recorded ERROR citing an outage
+                // while the persisted transcript was attempt 1's clean one — the record
+                // and the artifact disagreeing, and a recovered measurement thrown away.
+                infrastructureFailure = null;
+                adapterFailure = null;
+                try {
+                    if (scenario.mode === "seeded") {
+                        const r = await runSeeded(scenario, {
+                            skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
+                            specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
+                            // `ctx.structured` (a bare `--structured` request, with no gate depending
+                            // on it) must route through the structured path here too, exactly as it
+                            // does for the non-seeded branch below — without it, `--structured` on a
+                            // `mode: seeded` scenario silently called the adapter's plain `run()` and
+                            // recorded zero subject tokens/cost, which is the one thing the flag exists
+                            // to capture. `useStructured` (not the raw request) so an adapter with no
+                            // `runStructured` degrades here exactly as it does below.
+                            trace: useStructured ? { scenarioId: scenario.id, rep } : undefined,
+                            // `runSeeded` merges these with `scenario.extensions` itself when it
+                            // builds the RunReq — the arm's extensions and env (with `<run-dir>`
+                            // already substituted) both must reach pi.
+                            armExtensions: arm.extensions,
+                            ...(armEnv ? { armEnv } : {}),
+                        });
+                        transcript = r.transcript;
+                        gatePrefix = r.gateFailure;
+                        infrastructureFailure = r.gateError;
+                        stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
+                        traces = r.traces;
+                        events = r.events;
+                        eventErrors = r.eventErrors;
                     }
                     else {
-                        transcript = await ctx.adapter.run(req);
+                        const req = {
+                            skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
+                            // resolved like fixtures: relative to the spec's dir
+                            systemPromptFile: scenario.systemPromptFile
+                                ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
+                                : undefined,
+                            // Absolute before it reaches a child process running in a neutral cwd.
+                            // The arm's extensions are added alongside whatever the scenario
+                            // itself declares — both must reach pi.
+                            extensions: [
+                                ...(scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)) ?? []),
+                                ...arm.extensions,
+                            ],
+                            eventSources: scenario.eventSources,
+                            ...(armEnv ? { armEnv } : {}),
+                        };
+                        if (useStructured) {
+                            const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
+                            transcript = structured.transcript;
+                            traces = structured.traces;
+                            events = structured.events ?? [];
+                            eventErrors = structured.eventErrors ?? [];
+                            if (structured.providerFailure)
+                                infrastructureFailure = `provider failure — ${structured.providerFailure}`;
+                        }
+                        else {
+                            transcript = await ctx.adapter.run(req);
+                        }
                     }
                 }
+                catch (e) {
+                    // An adapter that THREW is one rep's infrastructure failure, not the
+                    // wave's. `runPool` is fail-fast and `runRep` had only a `finally`, so
+                    // a single pi crash or `PI_TIMEOUT_MS` timeout unwound the whole run:
+                    // no results.yaml written and every already-graded scenario's verdict
+                    // lost. `--structured` made that reachable for EVERY scenario, because
+                    // `runStructured` throws on a stream with no terminal events where the
+                    // text path merely recorded `[pi exited N]` and carried on.
+                    //
+                    // Configuration errors do not arrive here: the two that used to be
+                    // thrown from inside this block (adapter cannot produce traces for a
+                    // gated scenario) are decided above, before the loop.
+                    adapterFailure = e instanceof Error ? e.message : String(e);
+                    transcript = `[adapter failure] ${adapterFailure}`;
+                    // Nothing partial from the failed attempt may be read as evidence: an
+                    // empty trace list satisfies a `forbid_calls` gate, and a stale diff
+                    // from a previous attempt would be attributed to this one.
+                    gatePrefix = null;
+                    stagedDiff = null;
+                    traces = [];
+                    events = [];
+                    eventErrors = [];
+                }
+                // A provider outage is infrastructure, never a model verdict. Checked on
+                // every path: the text path carries the marker in the transcript, the
+                // structured path sets `providerFailure` above and exits 0 while doing it.
+                if (!infrastructureFailure) {
+                    const provider = providerFailureFromTranscript(transcript);
+                    if (provider)
+                        infrastructureFailure = `provider failure — ${provider}`;
+                }
                 noResponse = hasEmptyAssistantTurn(transcript);
-                if (!noResponse)
+                if (!noResponse && !adapterFailure)
                     break;
             }
+            // Survived the retry: ERROR for this rep, inside a run that still completes
+            // and still writes every other scenario's verdict.
+            if (adapterFailure && !infrastructureFailure) {
+                infrastructureFailure = `adapter failure — ${adapterFailure}`;
+            }
         }
         const repSuffix = repCount > 1 ? rep : undefined;
         writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
@@ -294,8 +443,13 @@ async function runRep(scenario, rep, repCount, ctx) {
         }
         // Objective evidence is persisted for every rep, pass or fail — a failing gate
         // is exactly when someone wants to read what the model actually did.
+        // `!adapterFailure`: a rep whose adapter threw never ran, so there is nothing
+        // to evaluate. Evaluating anyway reports "no execution trace was produced",
+        // which is the symptom — it names the missing artifact instead of the crash or
+        // timeout that caused it, and it is that text, not the real reason, that would
+        // reach the results record. The verdict is ERROR either way.
         let objective;
-        if (scenario.traceAssert || scenario.trajectoryAssert) {
+        if ((scenario.traceAssert || scenario.trajectoryAssert) && !adapterFailure) {
             const assertionResults = [];
             let status = "PASS";
             let traceMeta = {};

package/dist/seeded.d.ts

@@ -27,6 +27,16 @@ interface SeededOpts {
         scenarioId: string;
         rep: number;
     };
+    /**
+     * Absolute paths of an arm's pi extensions, merged with `scenario.extensions`
+     * when the `RunReq` is built below — the same shape the non-seeded path in
+     * `run.ts` produces, so a trace-gated seeded scenario measures the same
+     * delivery as an ungated one (the reason `runStructured` shares `skillFlags`
+     * with `run()` instead of inventing a second convention applies here too).
+     */
+    armExtensions?: string[];
+    /** The arm's env for the subject process; `<run-dir>` already substituted by the caller. */
+    armEnv?: Record<string, string>;
 }
 /**
  * Result of one vitest invocation — deliberately `ExecResult`, not a narrower

package/dist/seeded.js

@@ -166,9 +166,15 @@ export async function runSeeded(scenario, opts) {
         mode: opts.mode,
         turns: scenario.turns,
         cwd: repo,
-        // Resolved against the spec dir, exactly like fixtures and post-tests.
-        extensions: scenario.extensions?.map((e) => resolve(opts.specDir, e)),
+        // Resolved against the spec dir, exactly like fixtures and post-tests —
+        // the arm's extensions (already absolute) join alongside, same as the
+        // non-seeded path in run.ts.
+        extensions: [
+            ...(scenario.extensions?.map((e) => resolve(opts.specDir, e)) ?? []),
+            ...(opts.armExtensions ?? []),
+        ],
         eventSources: scenario.eventSources,
+        ...(opts.armEnv ? { armEnv: opts.armEnv } : {}),
     };
     // A trace-gated seeded scenario runs through the structured path so the tool
     // calls are recorded; everything downstream (gates, diff, transcript) is

package/dist/workspace.js

@@ -90,8 +90,22 @@ function assertKnownMarkers(src) {
  * ways — a `diff_contains` needle matching a test filename can pass for free, and a
  * `diff_excludes` needle can false-fail on a path string — and it pads every diff the
  * judge reads.
+ *
+ * `.pi/skills/` is the same class from a different writer: `seedArmDefinitions` copies
+ * an arm's definitions there for pi-daddy to spawn, and that happens in the same
+ * workspace `runSeeded` later runs `git add -A` + `git diff --cached` against.
+ * Without this exclusion those seeded `.md` files land in the diff as
+ * harness-injected `+` lines — a `diff_contains` needle can match text the model
+ * never wrote (a false objective PASS), the judge is shown the seeded definitions
+ * as if they were the model's edit, and `.pi/skills/*` reads as a model change for
+ * `unchanged_paths`.
+ *
+ * `.pi/skills/` and NOT `.pi/`: only that one directory is harness-written. `.pi/`
+ * at large is content the model can legitimately be asked to author, and excluding
+ * all of it would make that work invisible to every gate — including for the
+ * control arm, which seeds nothing.
  */
-const TOOL_ARTIFACTS = ["node_modules/", "coverage/", ".vitest/"];
+const TOOL_ARTIFACTS = ["node_modules/", "coverage/", ".vitest/", ".pi/skills/"];
 /**
  * Exclude tool artifacts via `.git/info/exclude`, deliberately not a `.gitignore`.
  *
@@ -190,8 +204,20 @@ export function createWorkspace(kind, opts) {
     }
     return { cwd, cleanup };
 }
-/** Never the model's work, and never worth hashing. */
+/** Never the model's work, and never worth hashing — matched on the entry NAME, at any depth. */
 const SNAPSHOT_SKIP = new Set([".git", "node_modules", "coverage", ".vitest"]);
+/**
+ * Skipped by RELATIVE PATH, not by name: `.pi/skills/` is where an arm's seeded
+ * definitions land, and only that directory is the harness's writing.
+ *
+ * Scoped to `skills/` rather than all of `.pi/` on purpose. `.pi/` is also a
+ * perfectly ordinary thing for the model to be asked to write — a skill that
+ * authors pi agent definitions does exactly that, and the corpus keeps a `.pi/`
+ * at its own root — so skipping the whole directory would hide that work from
+ * `unchanged_paths` (a vacuous pass) as well as from `diff_contains` (a false
+ * FAIL), for the control arm too, which seeds nothing at all.
+ */
+const SNAPSHOT_SKIP_RELS = new Set([".pi/skills"]);
 /** Snapshot a workspace, or null when there is no workspace to look at. */
 export function snapshotPaths(cwd, kind) {
     if (kind === "none" || !cwd || !existsSync(cwd))
@@ -209,6 +235,8 @@ export function snapshotPaths(cwd, kind) {
             if (SNAPSHOT_SKIP.has(e.name))
                 continue;
             const rel = prefix ? `${prefix}/${e.name}` : e.name;
+            if (SNAPSHOT_SKIP_RELS.has(rel))
+                continue;
             const abs = join(dir, e.name);
             if (e.isDirectory()) {
                 walk(abs, rel);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.9.0",
+    "version": "0.11.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",