@skill-harness/core 0.10.0 → 0.12.0

package/dist/run.js

@@ -1,8 +1,9 @@
-import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
-import { sourceHashes } from "./sources.js";
+import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { sourceHashes, PROMPT_NORMALIZATION_SOURCE_KEY, PROMPT_NORMALIZATION_SOURCE_DIGEST } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
-import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
+import { NONE_ARM, seedArmDefinitions } from "./arms.js";
+import { runDirFor, transcriptPath, diffPath, tracePath, trajectoryPath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, deliveryStatusForObservations, } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { liftHeadline } from "./lift.js";
 import { runSeeded } from "./seeded.js";
@@ -16,6 +17,36 @@ import { judgeOneRep } from "./regrade.js";
 import { runDeliveryCanary, canaryFailure } from "./canary.js";
 import { boundaryCells, stabilityNote } from "./stability.js";
 import { aggregateMetrics } from "./comparison.js";
+import { providerFailureFromTranscript } from "./provider-failure.js";
+/**
+ * The pi-daddy arm's delivery proof, by convention: `arms.yaml` points
+ * `PI_GRANTS_LEDGER` at exactly this filename inside the run dir (see
+ * `../principal-pi-skills/tests/arms.yaml` and CODEX-ARMS-RUNBOOK.md §4.3), so
+ * a fixed name here is what makes the count readable without parsing the arm's
+ * own `env` map back out.
+ */
+const LEDGER_FILENAME = "pi-daddy.ledger.jsonl";
+/**
+ * Count the arm's ledger events after the run, or 0 when the file was never
+ * written.
+ *
+ * This is `--canary`'s lesson applied to the arm: the ledger is the arm's
+ * delivery proof, and a delivery claim has to survive a commit. A missing file
+ * records `0`, not an absent field — "the extension never wrote a ledger" and
+ * "the extension wrote one with nothing in it" are both real, reportable
+ * outcomes, and a vacuous arm run must not commit a record indistinguishable
+ * from one that actually delegated.
+ */
+function countLedgerEvents(runDir) {
+    let text;
+    try {
+        text = readFileSync(join(runDir, LEDGER_FILENAME), "utf8");
+    }
+    catch {
+        return 0;
+    }
+    return text.split("\n").filter((line) => line.trim().length > 0).length;
+}
 /** Run one skill against one model: run scenarios, grade, score, persist. */
 export async function runSkillModel(opts) {
     const { spec, skillDir, adapter, model, judge, mode, timestamp } = opts;
@@ -39,7 +70,8 @@ export async function runSkillModel(opts) {
         log(`  ⚠ judge (${judge.provider}:${judge.model}) resembles the model under test ` +
             `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
     }
-    const runDir = runDirFor(skillDir, adapter.name, model, timestamp);
+    const arm = opts.arm ?? NONE_ARM;
+    const runDir = runDirFor(skillDir, adapter.name, model, timestamp, arm.name);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
     // Which harness CLI delivered the skill, asked once per run and recorded with the
@@ -97,22 +129,30 @@ export async function runSkillModel(opts) {
     const repCounts = scenarios.map((s) => s.reps ?? opts.reps ?? 1);
     const owners = [];
     const tasks = [];
+    const armDefinitions = { count: 0 };
     scenarios.forEach((scenario, si) => {
         for (let k = 0; k < repCounts[si]; k++) {
             const rep = k;
             const total = repCounts[si];
             owners.push(si);
-            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log }));
+            tasks.push(() => runRep(scenario, rep, total, { ...opts, runDir, now, log, armDefinitions }));
         }
     });
     const flat = await runPool(tasks, opts.concurrency ?? 1);
+    const subjectInvocations = flat.flatMap(outcome => outcome.subject_invocations ?? []);
     const grouped = scenarios.map(() => []);
     flat.forEach((outcome, i) => grouped[owners[i]].push(outcome));
     const scenarioResults = scenarios.map((scenario, si) => {
         const threshold = scenario.critical
             ? effectiveThreshold(undefined, scenario)
             : scenario.passThreshold ?? opts.passThreshold ?? 0.5;
-        return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
+        const result = outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
+        if (adapter.observesPrompts)
+            result.criterion_count = scenario.checklist.length;
+        if (adapter.observesPrompts && !result.rep_judgments) {
+            result.rep_judgments = grouped[si].map((outcome, repetition) => ({ repetition, judgments: [], recorded_verdict: outcome.verdict, ...(outcome.objective ? { objective: outcome.objective } : {}) }));
+        }
+        return result;
     });
     const ctx = scoreContextFor({ mode, partial }, spec);
     const results = writeResults(runDir, {
@@ -128,8 +168,28 @@ export async function runSkillModel(opts) {
         ...(partial ? { partial: true } : {}),
         // Only the scenarios this run actually measured: a --only run must not claim
         // coverage of scenarios it skipped.
-        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
+        source_hashes: {
+            ...sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
+            ...(adapter.observesPrompts ? { [PROMPT_NORMALIZATION_SOURCE_KEY]: PROMPT_NORMALIZATION_SOURCE_DIGEST } : {}),
+        },
         scenarios: scenarioResults,
+        ...(adapter.observesPrompts ? { schema: 3, subject_invocations: subjectInvocations } : {}),
+        ...(arm.name === NONE_ARM.name ? {} : {
+            arm: {
+                name: arm.name,
+                extensions: arm.extensions,
+                definitions: armDefinitions.count,
+                ledger_events: countLedgerEvents(runDir),
+                // The DECLARED env, `<run-dir>` left unsubstituted. It is the condition
+                // being measured (grant, max depth), so leaving it out made two runs at
+                // different settings byte-identical here and inside the same `+<arm>`
+                // tag — `stability` then reads the verdict difference between two
+                // conditions as one lineage flipping. The substituted form would be the
+                // opposite error: it embeds this run's temp path, so re-running the SAME
+                // condition would record two different-looking arms.
+                env: arm.env,
+            },
+        }),
     }, ctx);
     if (ctx) {
         const g = results.effective_grade;
@@ -159,6 +219,15 @@ async function runRep(scenario, rep, repCount, ctx) {
     const startedAt = performance.now();
     const { spec, judge, mode, runDir, now, log } = ctx;
     const repField = repCount > 1 ? { rep } : {};
+    const arm = ctx.arm ?? NONE_ARM;
+    const skillsRoot = ctx.skillsRoot ?? dirname(ctx.skillDir);
+    // Arm paths are per-run/per-workspace values. Plain split/join avoids treating
+    // temp-path characters as regex syntax.
+    const armEnvFor = (workspace) => Object.keys(arm.env).length
+        ? Object.fromEntries(Object.entries(arm.env).map(([k, v]) => [k, v
+                .split("<run-dir>").join(runDir)
+                .split("<workspace>").join(workspace)]))
+        : undefined;
     if (rep === 0) {
         log(`  ${scenario.id} (${scenario.title})${repCount > 1 ? ` ×${repCount}` : ""} …`);
         appendJournal(runDir, { event: "scenario-started", ts: now(), id: scenario.id, title: scenario.title });
@@ -171,6 +240,7 @@ async function runRep(scenario, rep, repCount, ctx) {
     // produces no diff, and writing an empty artifact there would misreport "the
     // model changed nothing" for a rep that never ran.
     let stagedDiff = null;
+    const subjectInvocations = [];
     try {
         try {
             ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
@@ -194,66 +264,161 @@ async function runRep(scenario, rep, repCount, ctx) {
         // a committed results.yaml, naming files the model never touched.
         let before = ws ? snapshotPaths(ws.cwd, scenario.workspace) : null;
         if (ws) {
+            // Seed the arm's definitions into THIS workspace before the subject runs.
+            // Control (`none`) short-circuits inside `seedArmDefinitions` with no
+            // filesystem side effects at all.
+            ctx.armDefinitions.count = seedArmDefinitions(arm, skillsRoot, ws.cwd, { ambientSkillsDir: ctx.ambientSkillsDir });
+        }
+        // Set when the adapter itself threw. Rep-scoped rather than loop-scoped: the
+        // evidence blocks below have to know this rep never ran.
+        let adapterFailure = null;
+        if (ws) {
+            // Decided ONCE, before any attempt, and for both the seeded and non-seeded
+            // branches — these are spec/adapter facts, not per-attempt outcomes, and
+            // deciding them here is what lets the execution below be wrapped in a catch
+            // that must not swallow a configuration error.
+            //
+            // `needsStructuredEvidence` is a gate with nothing to read if the adapter
+            // cannot produce traces: ERROR, never a silent fallback to the unstructured
+            // path, because a gate that could not be evaluated must not look like one
+            // that passed. A bare `--structured` request with no gate depending on it
+            // degrades to the plain path instead — on BOTH branches. The seeded branch
+            // used to pass `trace: {...}` unconditionally and let `runSeeded` throw
+            // ``scenario `X` declares `assert.trace`…`` at a scenario that declares no
+            // such thing.
+            const needsStructuredEvidence = Boolean(scenario.traceAssert || scenario.trajectoryAssert);
+            if (needsStructuredEvidence && !ctx.adapter.runStructured) {
+                throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
+                    ` cannot produce execution traces/events — the gate would have no evidence to read.`);
+            }
+            const useStructured = (Boolean(ctx.structured) || needsStructuredEvidence) && Boolean(ctx.adapter.runStructured);
             // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
             // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
             // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
+            // An adapter that THREW is the same class of event and takes the same retry.
             for (let attempt = 0; attempt < 2; attempt++) {
+                const observe = (prompt) => subjectInvocations.push({ scenario_id: scenario.id, repetition: rep, attempt, prompt });
                 if (attempt > 0) {
-                    appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, ...repField });
-                    log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} empty response — retrying once`);
+                    const why = adapterFailure ? `adapter failed (${adapterFailure})` : "empty response";
+                    appendJournal(runDir, { event: "empty-response-retry", ts: now(), id: scenario.id, attempt, reason: why, ...repField });
+                    log(`  ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${why} — retrying once`);
                     ws.cleanup();
                     ws = createWorkspace(scenario.workspace, { specDir: dirname(ctx.specPath), remote: scenario.remote });
                     // A fresh workspace needs a fresh baseline, or the retry's diff would
                     // be taken against a directory that no longer exists.
                     before = snapshotPaths(ws.cwd, scenario.workspace);
+                    // And a fresh workspace has nothing to spawn until it, too, is seeded —
+                    // a retry that skipped this would silently run the arm with no
+                    // definitions at all.
+                    ctx.armDefinitions.count = seedArmDefinitions(arm, skillsRoot, ws.cwd, { ambientSkillsDir: ctx.ambientSkillsDir });
                 }
-                if (scenario.mode === "seeded") {
-                    const r = await runSeeded(scenario, {
-                        skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
-                        specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
-                        trace: scenario.traceAssert || scenario.trajectoryAssert ? { scenarioId: scenario.id, rep } : undefined,
-                    });
-                    transcript = r.transcript;
-                    gatePrefix = r.gateFailure;
-                    infrastructureFailure = r.gateError;
-                    stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
-                    traces = r.traces;
-                    events = r.events;
-                    eventErrors = r.eventErrors;
-                }
-                else {
-                    const req = {
-                        skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
-                        // resolved like fixtures: relative to the spec's dir
-                        systemPromptFile: scenario.systemPromptFile
-                            ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
-                            : undefined,
-                        // Absolute before it reaches a child process running in a neutral cwd.
-                        extensions: scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)),
-                        eventSources: scenario.eventSources,
-                    };
-                    if (scenario.traceAssert || scenario.trajectoryAssert) {
-                        // Missing required evidence is ERROR, never a silent fallback to the
-                        // unstructured path: a gate with nothing to read must not look like a
-                        // gate that passed.
-                        if (!ctx.adapter.runStructured) {
-                            throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
-                                ` cannot produce execution traces/events — the gate would have no evidence to read.`);
-                        }
-                        const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
-                        transcript = structured.transcript;
-                        traces = structured.traces;
-                        events = structured.events ?? [];
-                        eventErrors = structured.eventErrors ?? [];
+                // Cleared per attempt, never carried over. A provider failure always
+                // leaves a blank assistant turn, so attempt 0's reason used to survive a
+                // retry that then succeeded: the rep was recorded ERROR citing an outage
+                // while the persisted transcript was attempt 1's clean one — the record
+                // and the artifact disagreeing, and a recovered measurement thrown away.
+                infrastructureFailure = null;
+                adapterFailure = null;
+                try {
+                    if (scenario.mode === "seeded") {
+                        const r = await runSeeded(scenario, {
+                            skillDir: ctx.skillDir, adapter: ctx.adapter, model: ctx.model, mode, cwd: ws.cwd,
+                            specDir: dirname(ctx.specPath), // assert.post_test resolves like a fixture
+                            // `ctx.structured` (a bare `--structured` request, with no gate depending
+                            // on it) must route through the structured path here too, exactly as it
+                            // does for the non-seeded branch below — without it, `--structured` on a
+                            // `mode: seeded` scenario silently called the adapter's plain `run()` and
+                            // recorded zero subject tokens/cost, which is the one thing the flag exists
+                            // to capture. `useStructured` (not the raw request) so an adapter with no
+                            // `runStructured` degrades here exactly as it does below.
+                            trace: useStructured ? { scenarioId: scenario.id, rep } : undefined,
+                            // `runSeeded` merges these with `scenario.extensions` itself when it
+                            // builds the RunReq — the arm's extensions and env (with `<run-dir>`
+                            // already substituted) both must reach pi.
+                            armExtensions: arm.extensions,
+                            ...(armEnvFor(ws.cwd) ? { armEnv: armEnvFor(ws.cwd) } : {}),
+                            ...(ctx.adapter.observesPrompts ? { onPromptObservation: observe } : {}),
+                        });
+                        transcript = r.transcript;
+                        gatePrefix = r.gateFailure;
+                        infrastructureFailure = r.gateError;
+                        stagedDiff = r.diff; // a retry replaces the aborted attempt's diff, as it should
+                        traces = r.traces;
+                        events = r.events;
+                        eventErrors = r.eventErrors;
                     }
                     else {
-                        transcript = await ctx.adapter.run(req);
+                        const req = {
+                            skillDir: ctx.skillDir, model: ctx.model, mode, turns: scenario.turns, cwd: ws.cwd,
+                            // resolved like fixtures: relative to the spec's dir
+                            systemPromptFile: scenario.systemPromptFile
+                                ? resolve(dirname(ctx.specPath), scenario.systemPromptFile)
+                                : undefined,
+                            // Absolute before it reaches a child process running in a neutral cwd.
+                            // The arm's extensions are added alongside whatever the scenario
+                            // itself declares — both must reach pi.
+                            extensions: [
+                                ...(scenario.extensions?.map((e) => resolve(dirname(ctx.specPath), e)) ?? []),
+                                ...arm.extensions,
+                            ],
+                            eventSources: scenario.eventSources,
+                            ...(armEnvFor(ws.cwd) ? { armEnv: armEnvFor(ws.cwd) } : {}),
+                            ...(ctx.adapter.observesPrompts ? { onPromptObservation: observe } : {}),
+                        };
+                        if (useStructured) {
+                            const structured = await ctx.adapter.runStructured({ ...req, scenarioId: scenario.id, rep });
+                            transcript = structured.transcript;
+                            traces = structured.traces;
+                            events = structured.events ?? [];
+                            eventErrors = structured.eventErrors ?? [];
+                            if (structured.providerFailure)
+                                infrastructureFailure = `provider failure — ${structured.providerFailure}`;
+                        }
+                        else {
+                            transcript = await ctx.adapter.run(req);
+                        }
                     }
                 }
+                catch (e) {
+                    // An adapter that THREW is one rep's infrastructure failure, not the
+                    // wave's. `runPool` is fail-fast and `runRep` had only a `finally`, so
+                    // a single pi crash or `PI_TIMEOUT_MS` timeout unwound the whole run:
+                    // no results.yaml written and every already-graded scenario's verdict
+                    // lost. `--structured` made that reachable for EVERY scenario, because
+                    // `runStructured` throws on a stream with no terminal events where the
+                    // text path merely recorded `[pi exited N]` and carried on.
+                    //
+                    // Configuration errors do not arrive here: the two that used to be
+                    // thrown from inside this block (adapter cannot produce traces for a
+                    // gated scenario) are decided above, before the loop.
+                    adapterFailure = e instanceof Error ? e.message : String(e);
+                    transcript = `[adapter failure] ${adapterFailure}`;
+                    // Nothing partial from the failed attempt may be read as evidence: an
+                    // empty trace list satisfies a `forbid_calls` gate, and a stale diff
+                    // from a previous attempt would be attributed to this one.
+                    gatePrefix = null;
+                    stagedDiff = null;
+                    traces = [];
+                    events = [];
+                    eventErrors = [];
+                }
+                // A provider outage is infrastructure, never a model verdict. Checked on
+                // every path: the text path carries the marker in the transcript, the
+                // structured path sets `providerFailure` above and exits 0 while doing it.
+                if (!infrastructureFailure) {
+                    const provider = providerFailureFromTranscript(transcript);
+                    if (provider)
+                        infrastructureFailure = `provider failure — ${provider}`;
+                }
                 noResponse = hasEmptyAssistantTurn(transcript);
-                if (!noResponse)
+                if (!noResponse && !adapterFailure)
                     break;
             }
+            // Survived the retry: ERROR for this rep, inside a run that still completes
+            // and still writes every other scenario's verdict.
+            if (adapterFailure && !infrastructureFailure) {
+                infrastructureFailure = `adapter failure — ${adapterFailure}`;
+            }
         }
         const repSuffix = repCount > 1 ? rep : undefined;
         writeFileSync(transcriptPath(runDir, scenario.id, mode, repSuffix), transcript, "utf8");
@@ -292,10 +457,25 @@ async function runRep(scenario, rep, repCount, ctx) {
                 });
             }
         }
+        // Delivery is an objective gate on every Pi subject invocation. It is derived
+        // from captured provider payload bytes, never from mode/argv supplied by a caller.
+        let deliveryObjective;
+        if (ctx.adapter.observesPrompts) {
+            const statuses = subjectInvocations.map(observation => observation.prompt.status);
+            const status = deliveryStatusForObservations(subjectInvocations);
+            deliveryObjective = { status, assertions: [{ kind: "skill_delivered", status, detail: statuses.length === 0 ? "no model-visible prompt observation was retained" : `${statuses.length} provider request(s): ${statuses.join(", ")}` }] };
+            if (status !== "PASS")
+                gatePrefix = `objective: skill_delivered ${status.toLowerCase()} — ${deliveryObjective.assertions[0].detail}`;
+        }
         // Objective evidence is persisted for every rep, pass or fail — a failing gate
         // is exactly when someone wants to read what the model actually did.
-        let objective;
-        if (scenario.traceAssert || scenario.trajectoryAssert) {
+        // `!adapterFailure`: a rep whose adapter threw never ran, so there is nothing
+        // to evaluate. Evaluating anyway reports "no execution trace was produced",
+        // which is the symptom — it names the missing artifact instead of the crash or
+        // timeout that caused it, and it is that text, not the real reason, that would
+        // reach the results record. The verdict is ERROR either way.
+        let objective = deliveryObjective;
+        if ((scenario.traceAssert || scenario.trajectoryAssert) && !adapterFailure) {
             const assertionResults = [];
             let status = "PASS";
             let traceMeta = {};
@@ -340,6 +520,11 @@ async function runRep(scenario, rep, repCount, ctx) {
                     trajectoryMeta = { trajectory_version: gate.event_version, events_sha256: gate.events_sha256 };
                 }
             }
+            if (deliveryObjective) {
+                if (deliveryObjective.status === "ERROR" || (deliveryObjective.status === "NOT-MEASURED" && status !== "ERROR") || (deliveryObjective.status === "FAIL" && status === "PASS"))
+                    status = deliveryObjective.status;
+                assertionResults.unshift(...deliveryObjective.assertions);
+            }
             objective = { status, ...traceMeta, ...trajectoryMeta, assertions: assertionResults };
             if (status !== "PASS") {
                 const details = assertionResults.filter((result) => result.status === status).map((result) => result.detail);
@@ -353,12 +538,18 @@ async function runRep(scenario, rep, repCount, ctx) {
         let verdict;
         let reason;
         let suspect = false;
+        let judgment;
         let judgeCalls = 0;
         if (objective?.status === "ERROR") {
             verdict = "ERROR";
             reason = gatePrefix ?? "objective evidence missing";
             appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
         }
+        else if (objective?.status === "NOT-MEASURED") {
+            verdict = "NOT-MEASURED";
+            reason = gatePrefix ?? "skill delivery was not established";
+            appendJournal(runDir, { event: "judge-verdict", ts: now(), id: scenario.id, verdict, reason, suspect, ...repField });
+        }
         else if (infrastructureFailure) {
             verdict = "ERROR";
             reason = infrastructureFailure;
@@ -382,13 +573,14 @@ async function runRep(scenario, rep, repCount, ctx) {
             });
             verdict = o.verdict;
             reason = o.reason;
-            suspect = o.suspect; // judgeOneRep already journaled (verdict + misfire)
+            suspect = o.suspect;
+            judgment = o.judgment; // judgeOneRep already journaled (verdict + misfire)
             judgeCalls = 1;
         }
         log(`  → ${scenario.id}${repCount > 1 ? `#${rep}` : ""} ${verdict}${reason ? `: ${reason}` : ""}${suspect ? "  ⚠ suspect" : ""}`);
         const subject = mergeTraces(traces)?.metrics;
         return {
-            verdict, reason, suspect, objective,
+            verdict, reason, suspect, objective, judgment, subject_invocations: subjectInvocations,
             metrics: {
                 wall_time_ms: Math.max(0, Math.round(performance.now() - startedAt)),
                 judge_calls: judgeCalls,
@@ -423,8 +615,8 @@ export function formatScorecard(summary, lift, stability) {
     const lines = [];
     lines.push(`── ${results.skill} · ${results.harness} · ${results.model} ──`);
     for (const s of results.scenarios) {
-        const v = s.override ?? (s.objective?.status === "ERROR" ? "ERROR" : s.objective?.status === "FAIL" ? "FAIL" : s.judge_verdict);
-        const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : "?";
+        const v = s.override ?? (s.objective?.status === "ERROR" ? "ERROR" : s.objective?.status === "NOT-MEASURED" ? "NOT-MEASURED" : s.objective?.status === "FAIL" ? "FAIL" : s.judge_verdict);
+        const mark = v === "PASS" ? "✓" : v === "FAIL" ? "✗" : v === "NOT-MEASURED" ? "∅" : "?";
         const ov = s.override ? " (override)" : "";
         const susp = s.suspect ? " ⚠suspect" : "";
         const misfired = s.clean !== undefined && s.reps !== undefined && s.clean < s.reps ? ` · ${s.reps - s.clean} misfired` : "";
@@ -440,7 +632,18 @@ export function formatScorecard(summary, lift, stability) {
             ? "subject tokens unavailable"
             : `subject tokens ${metrics.input_tokens} in / ${metrics.output_tokens ?? 0} out / ${metrics.cache_read_tokens ?? 0} cache-read`;
         const tools = metrics.tool_calls === null ? "tool calls unavailable" : `${metrics.tool_calls} tool call(s), ${metrics.delegated_children ?? 0} delegated, max concurrency ${metrics.max_concurrency ?? 0}`;
-        lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported) · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
+        const cost = metrics.cost_source === "subscription"
+            ? " · subscription ($0 marginal cost recorded)"
+            : metrics.cost_source === "unreported"
+                ? metrics.input_tokens !== null && metrics.input_tokens > 0
+                    ? " · WARNING: subject tokens were used but subject cost was not reported"
+                    : " · subject cost unavailable"
+                : metrics.input_tokens !== null && metrics.input_tokens > 0 && metrics.subject_cost_usd === 0
+                    ? " · WARNING: subject tokens were used but the provider reported $0 cost"
+                    : metrics.subject_cost_usd === null
+                        ? " · subject cost unavailable"
+                        : ` · $${metrics.subject_cost_usd.toFixed(6)} provider-reported`;
+        lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported)${cost} · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
     }
     // Lift is a statement about a skill-delivered run (green or force). On a red run
     // the caller may still have a lift in hand (a scored run exists in the same tag),

package/dist/score.d.ts

@@ -3,7 +3,7 @@ import type { ShipBar } from "./spec.js";
  * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
  * never a pass and never silently resolved — it marks the run for a rejudge.
  */
-export type Verdict = "PASS" | "FAIL" | "ERROR" | "JUDGE-AMBIGUOUS";
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED" | "JUDGE-AMBIGUOUS";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;
@@ -23,13 +23,14 @@ export interface ScoreResult {
     bSeriesFails: number;
     suspectCount: number;
     errorCount: number;
+    notMeasuredCount: number;
     note: string;
 }
 export declare function letterFor(pct: number): string;
 /**
  * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
- * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS are excluded as
- * infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
+ * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS and
+ * NOT-MEASURED are excluded as infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
  * unresolved judge misfire) is excluded from both `passed` and `total` — it is
  * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
  * until an author override resolves it. SHIP otherwise requires: enough total

package/dist/score.js

@@ -11,8 +11,8 @@ export function letterFor(pct) {
 }
 /**
  * Score a set of green-mode verdicts against the ship bar. A scenario PASSes only
- * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS are excluded as
- * infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
+ * on verdict PASS; FAIL is behavioral, while ERROR/JUDGE-AMBIGUOUS and
+ * NOT-MEASURED are excluded as infrastructure/indeterminate and independently block SHIP. A `suspect` verdict (an
  * unresolved judge misfire) is excluded from both `passed` and `total` — it is
  * untrustworthy, neither a pass nor a fail — and any suspect count blocks SHIP
  * until an author override resolves it. SHIP otherwise requires: enough total
@@ -27,11 +27,16 @@ export function score(verdicts, input) {
     let bSeriesFails = 0;
     let suspectCount = 0;
     let errorCount = 0;
+    let notMeasuredCount = 0;
     for (const v of verdicts) {
         if (v.suspect) {
             suspectCount++;
             continue; // untrustworthy: neither pass nor fail
         }
+        if (v.verdict === "NOT-MEASURED") {
+            notMeasuredCount++;
+            continue; // known-undelivered experiment: neither behavioral pass nor fail
+        }
         if (v.verdict === "ERROR" || v.verdict === "JUDGE-AMBIGUOUS") {
             errorCount++;
             continue; // infrastructure/indeterminate: neither behavioral pass nor fail
@@ -55,7 +60,8 @@ export function score(verdicts, input) {
         (!shipBar.no_critical_fail || criticalFails === 0) &&
         bSeriesFails === 0 &&
         suspectCount === 0 &&
-        errorCount === 0;
+        errorCount === 0 &&
+        notMeasuredCount === 0;
     let note = "";
     if (!validBar) {
         note = "invalid ship bar: total/min_pass must be positive integers with min_pass <= total";
@@ -66,12 +72,15 @@ export function score(verdicts, input) {
     else if (errorCount > 0) {
         note = `${errorCount} infrastructure error${errorCount === 1 ? "" : "s"}: retry/repair evidence`;
     }
+    else if (notMeasuredCount > 0) {
+        note = `${notMeasuredCount} not measured: skill delivery was not established`;
+    }
     else if (criticalFails > 0) {
         note = `gated: ${criticalFails} critical fail${criticalFails === 1 ? "" : "s"}`;
     }
     else if (bSeriesFails > 0) {
         note = `gated: ${bSeriesFails} B-series fail${bSeriesFails === 1 ? "" : "s"}`;
     }
-    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, note };
+    return { passed, total, pct, letter, ship, criticalFails, bSeriesFails, suspectCount, errorCount, notMeasuredCount, note };
 }
 //# sourceMappingURL=score.js.map

package/dist/screen.d.ts

@@ -0,0 +1,31 @@
+import { type ResultsFile } from "./results.js";
+export type ScreenClassification = "CEILING" | "FLOOR" | "INFORMATIVE" | "UNKNOWN";
+export interface ScreenRate {
+    passes: number;
+    n: number;
+}
+export interface ScreenScenario {
+    skill: string;
+    model: string;
+    id: string;
+    control: ScreenRate;
+    treatment: ScreenRate;
+    not_measured: number;
+    classification: ScreenClassification;
+}
+export interface ScreenCriterion {
+    skill: string;
+    model: string;
+    scenario_id: string;
+    criterion: number;
+    failures: number;
+    n: number;
+    fail_rate: number;
+}
+export interface ScreenReport {
+    scenarios: ScreenScenario[];
+    criteria: ScreenCriterion[];
+}
+/** Pure/offline: derive only from v3 retained invocation and vote evidence. */
+export declare function screenResults(results: ResultsFile[]): ScreenReport;
+export declare function formatScreen(report: ScreenReport): string;