@skill-harness/core 0.11.0 → 0.13.0

package/README.md

@@ -1,7 +1,8 @@
 # @skill-harness/core
 
 The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
-spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results.
+spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results,
+and the separately versioned durable qualification-runner-v1 primitives.
 
 Internal API — use the `skill-harness` CLI. See the repo for docs.
 

package/dist/adapters/types.d.ts

@@ -11,6 +11,21 @@ export interface ModelRef {
 export declare function parseModelRef(token: string): ModelRef;
 /** Stable, filesystem-safe slug for a model ref (used in run-dir names). */
 export declare function modelSlug(ref: ModelRef): string;
+export type PromptMechanism = "none" | "pi-skill" | "append-system-prompt" | "system-prompt-file";
+export interface PromptProvenance {
+    capture_version: "prompt-provenance-v1";
+    request_index: number;
+    raw_sha256: string;
+    normalized_sha256: string;
+    normalization_rule: "cwd-line-v1";
+    bytes: number;
+    contract_sha256: string;
+    contract_bytes: number;
+    contract_occurrences: number;
+    mechanism: PromptMechanism;
+    status: "PASS" | "NOT-MEASURED" | "ERROR";
+    error?: string;
+}
 export interface RunReq {
     skillDir: string;
     model: ModelRef;
@@ -50,6 +65,8 @@ export interface RunReq {
      * inheriting a caller's copy of them.
      */
     armEnv?: Record<string, string>;
+    /** Adapter-produced observation callback. Callers receive evidence; they never supply delivery status. */
+    onPromptObservation?: (observation: PromptProvenance) => void;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {
@@ -74,6 +91,8 @@ export interface StructuredRun {
 }
 export interface HarnessAdapter {
     name: string;
+    /** True only when the adapter computes final provider-payload observations. */
+    observesPrompts?: boolean;
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     /**

package/dist/adjudication.d.ts

@@ -1,5 +1,5 @@
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
-import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile } from "./results.js";
+import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile, CriterionVote } from "./results.js";
 import type { Scenario, ShipBar, Spec } from "./spec.js";
 import type { Verdict } from "./score.js";
 /**
@@ -33,6 +33,11 @@ export interface CellState {
     suspect: boolean;
     /** Per-rep verdicts, when the cell ran with reps. Empty for a single-rep cell. */
     repVerdicts?: Verdict[];
+    criteria?: CriterionVote[];
+    /** Delivery prerequisite for schema-3 cells; non-PASS cells are never judged. */
+    deliveryStatus?: "PASS" | "NOT-MEASURED" | "ERROR";
+    /** The actual selected repetition's primary vote; aggregate fields remain for planning only. */
+    primaryJudgment?: Judgment;
 }
 export interface TriggerDecision {
     id: string;
@@ -105,11 +110,14 @@ export declare function collapseJudgments(judgments: Judgment[], trigger: Trigge
  */
 export declare function boundAdjudicationToRepetitions(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): AdjudicationResult;
 export declare function projectAdjudication(result: ScenarioResult, adj: AdjudicationResult): ScenarioResult;
+/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
+export declare function projectAdjudicationForScenario(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): ScenarioResult;
 /** Ask one judge about one cell's saved transcript. Supplied by the caller. */
 export type RejudgeFn = (id: string, judge: ModelRef) => Promise<{
     verdict: Verdict;
     reason: string;
     suspect: boolean;
+    criteria?: CriterionVote[];
 }>;
 export interface RunAdjudicationOptions {
     plan: AdjudicationPlan;

package/dist/adjudication.js

@@ -1,9 +1,10 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, } from "./results.js";
+import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, completeCriterionVotes, } from "./results.js";
 import { score } from "./score.js";
 import { buildJudgePrompt, judgeInWorkspace, parseVerdict } from "./grade.js";
 import { appendJournal } from "./journal.js";
+import { collapseVotePanel } from "./vote-panel.js";
 /** Maximum judgments per cell, first wave included. Hard cap, not a default. */
 export const MAX_JUDGMENTS = 3;
 /**
@@ -18,6 +19,9 @@ export function planAdjudication(input) {
     const enabled = new Set(input.enabled ?? ["ambiguous", "contradictory", "non_unanimous", "ship_deciding"]);
     const decisions = [];
     for (const cell of input.cells) {
+        // Delivery-gated schema-3 cells carry no behavioral claim to adjudicate.
+        if (cell.deliveryStatus && cell.deliveryStatus !== "PASS")
+            continue;
         const triggers = [];
         // `JUDGE-AMBIGUOUS` is what the parser emits when a judge's verdict blocks
         // disagree. `ERROR` is what it emits when nothing parsed at all — and that
@@ -92,24 +96,13 @@ function flipsShipDecision(cell, input) {
  * cast a deciding vote would launder exactly the unreliability being measured.
  */
 export function collapseJudgments(judgments, trigger) {
-    const clean = judgments.filter((j) => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
-    const base = { trigger, judgments };
-    if (clean.length < 2) {
-        // Nothing to compare: one clean vote (or none) cannot confirm anything.
-        return { ...base, state: "unresolved" };
-    }
-    const passes = clean.filter((j) => j.verdict === "PASS").length;
-    const fails = clean.length - passes;
-    if (passes === 0 || fails === 0) {
-        return { ...base, state: "confirmed", verdict: clean[0].verdict };
-    }
-    // A clean strict majority breaks the tie. With exactly 2 clean votes that
-    // disagree there is no majority, so this correctly falls through.
-    if (passes > fails)
-        return { ...base, state: "tie_broken", verdict: "PASS" };
-    if (fails > passes)
-        return { ...base, state: "tie_broken", verdict: "FAIL" };
-    return { ...base, state: "unresolved" };
+    const collapsed = collapseVotePanel(judgments);
+    return {
+        trigger,
+        judgments,
+        state: collapsed.state,
+        ...(collapsed.verdict ? { verdict: collapsed.verdict } : {}),
+    };
 }
 /**
  * Project an adjudication onto the compatibility fields the scorer reads.
@@ -149,6 +142,15 @@ export function projectAdjudication(result, adj) {
         adjudication: adj,
     };
 }
+/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
+export function projectAdjudicationForScenario(result, scenario, adj) {
+    const bounded = boundAdjudicationToRepetitions(result, scenario, adj);
+    const projected = projectAdjudication(result, bounded);
+    if (bounded !== adj) {
+        projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
+    }
+    return projected;
+}
 function reasonFor(adj) {
     const n = adj.judgments.length;
     const verb = adj.state === "confirmed" ? "confirmed by" : "resolved by majority of";
@@ -179,7 +181,9 @@ export async function runAdjudication(opts) {
         // in the fixed order `planAdjudication` evaluates them.
         const trigger = decision.triggers[0];
         const judgments = [
-            { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect },
+            cell.primaryJudgment
+                ? { ...cell.primaryJudgment, ordinal: 1 }
+                : { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect, criteria: cell.criteria ?? [] },
         ];
         const second = await opts.rejudge(decision.id, opts.secondaryJudge);
         callsMade++;
@@ -242,11 +246,8 @@ export async function adjudicateRun(opts) {
         if (!adj)
             return s;
         const scenario = byIdScenario.get(s.id);
-        const boundedAdj = scenario ? boundAdjudicationToRepetitions(s, scenario, adj) : adj;
-        const projected = projectAdjudication(s, boundedAdj);
-        if (boundedAdj !== adj) {
-            projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
-        }
+        const withRepetition = { ...adj, repetition: s.rep_judgments?.[0]?.repetition ?? 0 };
+        const projected = scenario ? projectAdjudicationForScenario(s, scenario, withRepetition) : projectAdjudication(s, withRepetition);
         const extraCalls = adj.judgments.filter((judgment) => judgment.ordinal > 1).length;
         projected.metrics = mergeScenarioMetrics(s.metrics, {
             wall_time_ms: 0,
@@ -299,7 +300,7 @@ async function judgeCell(opts) {
         event: "judge-verdict", ts: opts.now(), id: opts.scenario.id,
         verdict: g.verdict, reason: g.reason, suspect: g.suspect,
     });
-    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
+    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, opts.scenario.checklist.length) };
 }
 /**
  * Build the plan's input cells from a run.
@@ -311,13 +312,20 @@ async function judgeCell(opts) {
  * about — inverting the one invariant this feature actually promises.
  */
 export function cellsFromResults(runDir, results) {
-    return results.scenarios.map((s) => ({
-        id: s.id,
-        verdict: s.judge_verdict,
-        reason: s.judge_reason,
-        suspect: s.suspect,
-        repVerdicts: repVerdictsOf(runDir, s, results.mode),
-    }));
+    return results.scenarios.map((s) => {
+        const primaryJudgment = s.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0];
+        const deliveryStatus = s.objective?.assertions.find(assertion => assertion.kind === "skill_delivered")?.status;
+        return {
+            id: s.id,
+            verdict: s.judge_verdict,
+            reason: s.judge_reason,
+            suspect: s.suspect,
+            repVerdicts: repVerdictsOf(runDir, s, results.mode),
+            criteria: primaryJudgment?.criteria,
+            ...(deliveryStatus === "PASS" || deliveryStatus === "NOT-MEASURED" || deliveryStatus === "ERROR" ? { deliveryStatus } : {}),
+            ...(primaryJudgment ? { primaryJudgment } : {}),
+        };
+    });
 }
 /** Per-rep verdicts from saved judge artifacts, for the non-unanimous trigger. */
 function repVerdictsOf(runDir, s, mode) {

package/dist/adoption.d.ts

@@ -0,0 +1,128 @@
+export interface AdoptionInput {
+    hypothesisDigest: string;
+    experimentDigest: string;
+    candidateDigest: string;
+    scopeDigest: string;
+    assessmentPolicyDigest: string;
+    rollbackCandidateDigest: string;
+    activationBoundary: "next-orders";
+    expiresAt: number;
+}
+export interface AdoptionBinding extends AdoptionInput {
+    version: "adoption-binding-v1";
+    id: string;
+}
+export interface AdoptionAuthority {
+    id: string;
+    adoptions: readonly string[];
+    rollbacks: readonly string[];
+}
+export interface AdoptionFacts {
+    experimentDigest: string;
+    candidateDigest: string;
+    scopeDigest: string;
+    assessmentPolicyDigest: string;
+    eligible: boolean;
+}
+export declare function buildAdoptionBinding(input: AdoptionInput): AdoptionBinding;
+/** Independent host declarations and independently verified eligible facts are required again at use. */
+export declare function authorizeAdoption(binding: AdoptionBinding, authority: AdoptionAuthority | null, facts: AdoptionFacts, now: number): Readonly<{
+    id: string;
+    version: "adoption-receipt-v1";
+    bindingId: string;
+    authorityId: string;
+    hypothesisDigest: string;
+    experimentDigest: string;
+    candidateDigest: string;
+    scopeDigest: string;
+    assessmentPolicyDigest: string;
+    rollbackCandidateDigest: string;
+    activationBoundary: "next-orders";
+    grantExpansion: false;
+    expiresAt: number;
+}>;
+export type AdoptionReceipt = ReturnType<typeof authorizeAdoption>;
+export declare function validateAdoptionReceipt(receipt: AdoptionReceipt, binding: AdoptionBinding, authority: AdoptionAuthority | null, facts: AdoptionFacts, now: number): void;
+export interface ProductionObservation {
+    id: string;
+    adoptionId: string;
+    candidateDigest: string;
+    scopeDigest: string;
+    originalRequirementDigest: string;
+    currentRequirementDigest: string;
+    /** Only an independent host may supply confirmed outcomes; observer/model claims are not confirmation. */
+    outcome: "confirmed-defect" | "success" | "unknown";
+    acceptanceDigest: string | null;
+    acceptedArtifactDigest: string | null;
+    observedArtifactDigest: string;
+    evidence: string[];
+}
+export declare function classifyProductionObservation(adoption: AdoptionReceipt, input: ProductionObservation): Readonly<{
+    state: "unknown" | "success" | "out-of-scope" | "changed-requirements" | "changed-artifact" | "caught-defect" | "escape";
+    adoptionId: string;
+    hypothesisDigest: string;
+    experimentDigest: string;
+    observationId: string;
+    acceptanceDigest: string | null;
+    acceptedArtifactDigest: string | null;
+    observedArtifactDigest: string;
+    automaticRollback: false;
+}>;
+export interface RollbackRequest {
+    version: "rollback-request-v1";
+    id: string;
+    adoptionId: string;
+    scopeDigest: string;
+    restoreCandidateDigest: string;
+    reason: "operator-request" | "confirmed-escape";
+    evidence: readonly string[];
+    expiresAt: number;
+}
+export declare function buildRollbackRequest(adoption: AdoptionReceipt, reason: RollbackRequest["reason"], evidence: string[], expiresAt?: number): RollbackRequest;
+/** This validates a recorded choice, not an automatic rollback action or a caller-created authority flag. */
+export declare function authorizeRollback(adoption: AdoptionReceipt, request: RollbackRequest, authority: AdoptionAuthority | null, now: number, current: {
+    adoptionId: string;
+    candidateDigest: string;
+    scopeDigest: string;
+}): Readonly<{
+    requestId: string;
+    adoptionId: string;
+    restoreCandidateDigest: string;
+    scopeDigest: string;
+    grantExpansion: false;
+    application: "not-performed";
+}>;
+export interface LeanOutcome {
+    population: string;
+    complete: boolean;
+    accepted: boolean | null;
+    repairAttempts: number | null;
+    escape: "confirmed" | "none-observed" | "unknown";
+    requirementsChanged: boolean;
+    decisionWaitMs: number | null;
+    questions: number | null;
+}
+export declare function summarizeLeanOutcomes(population: string, rows: readonly LeanOutcome[]): {
+    population: string;
+    firstPassAccepted: number;
+    firstPassDenominator: number;
+    incomplete: number;
+    confirmedEscapes: number;
+    changedRequirements: number;
+    recovery: {
+        known: number;
+        observed: number;
+        total: number;
+    };
+    decisionWait: {
+        known: number;
+        observed: number;
+        total: number;
+    };
+    questions: {
+        known: number;
+        observed: number;
+        total: number;
+    };
+    universalScore: null;
+};

package/dist/adoption.js

@@ -0,0 +1,102 @@
+import { createHash } from "node:crypto";
+const SHA = /^[a-f0-9]{64}$/;
+const text = (x) => typeof x === "string" && x.length > 0 && x.length <= 512 && !/[\u0000-\u001f\u007f]/.test(x);
+const closed = (value, names) => {
+    if (!value || ![Object.prototype, null].includes(Object.getPrototypeOf(value)))
+        return false;
+    const fields = Object.getOwnPropertyDescriptors(value);
+    return Reflect.ownKeys(fields).length === names.length && Reflect.ownKeys(fields).every(key => typeof key === "string" && names.includes(key) && fields[key].enumerable && Object.hasOwn(fields[key], "value"));
+};
+const hash = (value) => createHash("sha256").update(JSON.stringify(Object.keys(value).sort().map(k => [k, value[k]]))).digest("hex");
+export function buildAdoptionBinding(input) {
+    if (!input || !closed(input, ["hypothesisDigest", "experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "rollbackCandidateDigest", "activationBoundary", "expiresAt"])
+        || ![input.hypothesisDigest, input.experimentDigest, input.candidateDigest, input.scopeDigest, input.assessmentPolicyDigest, input.rollbackCandidateDigest].every(s => typeof s === "string" && SHA.test(s))
+        || input.activationBoundary !== "next-orders" || !Number.isSafeInteger(input.expiresAt) || input.expiresAt < 0)
+        throw new Error("invalid scoped adoption binding");
+    const body = { version: "adoption-binding-v1", ...input };
+    return Object.freeze({ ...body, id: hash(body) });
+}
+function bindingValid(binding) {
+    const { version, id, ...input } = binding;
+    if (version !== "adoption-binding-v1" || buildAdoptionBinding(input).id !== id)
+        throw new Error("adoption binding changed");
+}
+function authorityValid(authority) {
+    return !!authority && text(authority.id) && Array.isArray(authority.adoptions) && Array.isArray(authority.rollbacks)
+        && authority.adoptions.length <= 256 && authority.rollbacks.length <= 256 && [...authority.adoptions, ...authority.rollbacks].every(id => typeof id === "string" && SHA.test(id));
+}
+/** Independent host declarations and independently verified eligible facts are required again at use. */
+export function authorizeAdoption(binding, authority, facts, now) {
+    bindingValid(binding);
+    if (!authorityValid(authority) || !authority.adoptions.includes(binding.id))
+        throw new Error("independent adoption authority required");
+    if (!Number.isSafeInteger(now) || now < 0 || now >= binding.expiresAt || !facts || !closed(facts, ["experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "eligible"]) || facts.eligible !== true
+        || facts.experimentDigest !== binding.experimentDigest || facts.candidateDigest !== binding.candidateDigest
+        || facts.scopeDigest !== binding.scopeDigest || facts.assessmentPolicyDigest !== binding.assessmentPolicyDigest)
+        throw new Error("adoption expired, ineligible or outside verified scope");
+    const body = { version: "adoption-receipt-v1", bindingId: binding.id, authorityId: authority.id, hypothesisDigest: binding.hypothesisDigest,
+        experimentDigest: binding.experimentDigest, candidateDigest: binding.candidateDigest, scopeDigest: binding.scopeDigest,
+        assessmentPolicyDigest: binding.assessmentPolicyDigest, rollbackCandidateDigest: binding.rollbackCandidateDigest,
+        activationBoundary: "next-orders", grantExpansion: false, expiresAt: binding.expiresAt };
+    return Object.freeze({ ...body, id: hash(body) });
+}
+export function validateAdoptionReceipt(receipt, binding, authority, facts, now) {
+    if (authorizeAdoption(binding, authority, facts, now).id !== receipt.id)
+        throw new Error("adoption receipt does not match current independent authority/facts");
+    receiptValid(receipt);
+}
+function receiptValid(receipt) {
+    const { id, ...body } = receipt;
+    if (!closed(receipt, ["version", "bindingId", "authorityId", "hypothesisDigest", "experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "rollbackCandidateDigest", "activationBoundary", "grantExpansion", "expiresAt", "id"])
+        || body.version !== "adoption-receipt-v1" || !text(body.authorityId) || body.activationBoundary !== "next-orders" || body.grantExpansion !== false || hash(body) !== id)
+        throw new Error("adoption receipt changed");
+    const binding = buildAdoptionBinding({ hypothesisDigest: body.hypothesisDigest, experimentDigest: body.experimentDigest, candidateDigest: body.candidateDigest,
+        scopeDigest: body.scopeDigest, assessmentPolicyDigest: body.assessmentPolicyDigest, rollbackCandidateDigest: body.rollbackCandidateDigest, activationBoundary: body.activationBoundary, expiresAt: body.expiresAt });
+    if (binding.id !== body.bindingId)
+        throw new Error("adoption receipt binding mismatch");
+}
+export function classifyProductionObservation(adoption, input) {
+    receiptValid(adoption);
+    if (!input || !text(input.id) || input.evidence.length > 256 || input.evidence.some(h => !SHA.test(h))
+        || ![input.originalRequirementDigest, input.currentRequirementDigest, input.candidateDigest, input.scopeDigest, input.observedArtifactDigest].every(h => typeof h === "string" && SHA.test(h))
+        || ![input.acceptanceDigest, input.acceptedArtifactDigest].every(h => h === null || (typeof h === "string" && SHA.test(h)))
+        || (input.acceptanceDigest === null) !== (input.acceptedArtifactDigest === null)
+        || !["confirmed-defect", "success", "unknown"].includes(input.outcome))
+        throw new Error("invalid production observation");
+    const state = input.adoptionId !== adoption.id || input.candidateDigest !== adoption.candidateDigest || input.scopeDigest !== adoption.scopeDigest ? "out-of-scope"
+        : input.originalRequirementDigest !== input.currentRequirementDigest ? "changed-requirements"
+            : input.acceptedArtifactDigest !== null && input.acceptedArtifactDigest !== input.observedArtifactDigest ? "changed-artifact"
+                : input.outcome === "confirmed-defect" && input.evidence.length ? input.acceptanceDigest === null ? "caught-defect" : "escape" : input.outcome === "success" ? "success" : "unknown";
+    return Object.freeze({ state, adoptionId: adoption.id, hypothesisDigest: adoption.hypothesisDigest, experimentDigest: adoption.experimentDigest, observationId: input.id, acceptanceDigest: input.acceptanceDigest, acceptedArtifactDigest: input.acceptedArtifactDigest, observedArtifactDigest: input.observedArtifactDigest, automaticRollback: false });
+}
+export function buildRollbackRequest(adoption, reason, evidence, expiresAt = adoption.expiresAt) {
+    receiptValid(adoption);
+    if (!Number.isSafeInteger(expiresAt) || expiresAt < 0 || !["operator-request", "confirmed-escape"].includes(reason) || !evidence.length || evidence.length > 256 || evidence.some(h => !SHA.test(h)))
+        throw new Error("invalid rollback request");
+    const body = { version: "rollback-request-v1", adoptionId: adoption.id, scopeDigest: adoption.scopeDigest, restoreCandidateDigest: adoption.rollbackCandidateDigest, reason, expiresAt, evidence: Object.freeze([...new Set(evidence)].sort()) };
+    return Object.freeze({ ...body, id: hash(body) });
+}
+/** This validates a recorded choice, not an automatic rollback action or a caller-created authority flag. */
+export function authorizeRollback(adoption, request, authority, now, current) {
+    receiptValid(adoption);
+    const { id, ...body } = request;
+    if (!authorityValid(authority) || authority.id !== adoption.authorityId || !authority.rollbacks.includes(id))
+        throw new Error("independent rollback authority required");
+    if (hash(body) !== id || request.version !== "rollback-request-v1" || request.adoptionId !== adoption.id || request.scopeDigest !== adoption.scopeDigest
+        || request.restoreCandidateDigest !== adoption.rollbackCandidateDigest || !Number.isSafeInteger(now) || now < 0 || !Number.isSafeInteger(request.expiresAt) || now >= request.expiresAt
+        || !current || current.adoptionId !== adoption.id || current.candidateDigest !== adoption.candidateDigest || current.scopeDigest !== adoption.scopeDigest)
+        throw new Error("rollback binding changed or expired");
+    return Object.freeze({ requestId: id, adoptionId: adoption.id, restoreCandidateDigest: request.restoreCandidateDigest, scopeDigest: adoption.scopeDigest, grantExpansion: false, application: "not-performed" });
+}
+export function summarizeLeanOutcomes(population, rows) {
+    if (!text(population) || rows.length > 10000 || rows.some(r => r.population !== population || typeof r.complete !== "boolean" || typeof r.requirementsChanged !== "boolean"
+        || ![true, false, null].includes(r.accepted) || !["confirmed", "none-observed", "unknown"].includes(r.escape)
+        || [r.repairAttempts, r.decisionWaitMs, r.questions].some(n => n !== null && (!Number.isSafeInteger(n) || n < 0))))
+        throw new Error("invalid or incomparable lean population");
+    const complete = rows.filter(r => r.complete && r.accepted !== null && r.repairAttempts !== null);
+    const observed = (field) => ({ known: rows.reduce((n, r) => n + (r[field] ?? 0), 0), observed: rows.filter(r => r[field] !== null).length, total: rows.length });
+    return { population, firstPassAccepted: complete.filter(r => r.accepted && r.repairAttempts === 0).length, firstPassDenominator: complete.length,
+        incomplete: rows.length - complete.length, confirmedEscapes: rows.filter(r => r.escape === "confirmed" && !r.requirementsChanged).length,
+        changedRequirements: rows.filter(r => r.requirementsChanged).length, recovery: observed("repairAttempts"), decisionWait: observed("decisionWaitMs"), questions: observed("questions"), universalScore: null };
+}
+//# sourceMappingURL=adoption.js.map

package/dist/arms.d.ts

@@ -18,7 +18,7 @@ export interface Arm {
     seedSkills: string[];
     /** Minimum definitions the seeding must produce, else the run ERRORs. */
     requireDefinitions: number;
-    /** Env for the subject process. `<run-dir>` is substituted per run. */
+    /** Env for the subject process. `<run-dir>` and `<workspace>` are substituted per rep. */
     env: Record<string, string>;
 }
 /** The implicit control: today's behaviour, byte-identical, no extensions. */

package/dist/capture-trace-types.d.ts

@@ -83,12 +83,14 @@ export interface TraceResultMeta {
  * conversation carries in the session dir, not in the event stream). This
  * mirrors the loop the adapter already runs.
  */
+export type CostSource = "provider-reported" | "subscription" | "unreported";
 export interface TraceMetrics {
     input_tokens: number;
     output_tokens: number;
     cache_read_tokens: number;
     cache_write_tokens: number;
     cost_usd: number;
+    cost_source: CostSource;
     tool_calls: number;
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */

package/dist/command-cost.d.ts

@@ -0,0 +1,9 @@
+/**
+ * CLI commands guaranteed to make no subject or judge calls.
+ *
+ * Keep this list narrow: commands that can conditionally spend (notably `regate`)
+ * do not belong here even when their common path only replays saved artifacts.
+ */
+export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "archive", "coverage", "init", "judge-agreement", "lint", "list", "rescore", "restamp", "screen", "stability"];
+export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
+export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;

package/dist/command-cost.js

@@ -0,0 +1,23 @@
+/**
+ * CLI commands guaranteed to make no subject or judge calls.
+ *
+ * Keep this list narrow: commands that can conditionally spend (notably `regate`)
+ * do not belong here even when their common path only replays saved artifacts.
+ */
+export const FREE_OFFLINE_COMMANDS = [
+    "affected",
+    "archive",
+    "coverage",
+    "init",
+    "judge-agreement",
+    "lint",
+    "list",
+    "rescore",
+    "restamp",
+    "screen",
+    "stability",
+];
+export function isFreeOfflineCommand(command) {
+    return FREE_OFFLINE_COMMANDS.some((candidate) => candidate === command);
+}
+//# sourceMappingURL=command-cost.js.map

package/dist/comparison.d.ts

@@ -45,6 +45,7 @@ export interface AggregateMetrics {
     cache_read_tokens: number | null;
     cache_write_tokens: number | null;
     subject_cost_usd: number | null;
+    cost_source: "provider-reported" | "subscription" | "unreported" | null;
     tool_calls: number | null;
     delegated_children: number | null;
     max_concurrency: number | null;

package/dist/comparison.js

@@ -51,8 +51,8 @@ export function buildComparison(input) {
     const behavioral = cells.filter((cell) => cell.change === "regression").map((cell) => cell.id);
     const criticalRegressions = cells.filter((cell) => cell.change === "regression" && cell.critical).map((cell) => cell.id);
     const byReference = new Map(reference.scenarios.map((scenario) => [scenario.id, scenario]));
-    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
-        cell.candidate === "ERROR" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
+    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "NOT-MEASURED" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
+        cell.candidate === "ERROR" || cell.candidate === "NOT-MEASURED" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
     const metrics = { reference: aggregateMetrics(reference.scenarios), candidate: aggregateMetrics(candidate.scenarios) };
     const costRegressions = thresholdFindings(metrics.reference, metrics.candidate, input.thresholds);
     const partial = input.partial === true || reference.partial === true || candidate.partial === true;
@@ -137,6 +137,10 @@ export function aggregateMetrics(scenarios) {
         cache_read_tokens: sumOptional("cache_read_tokens"),
         cache_write_tokens: sumOptional("cache_write_tokens"),
         subject_cost_usd: sumOptional("subject_cost_usd"),
+        cost_source: (() => {
+            const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
+            return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
+        })(),
         tool_calls: sumOptional("tool_calls"),
         delegated_children: sumOptional("delegated_children"),
         max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
@@ -147,11 +151,13 @@ function effective(result) {
         return result.override;
     if (result.objective?.status === "ERROR")
         return "ERROR";
+    if (result.objective?.status === "NOT-MEASURED")
+        return "NOT-MEASURED";
     if (result.objective?.status === "FAIL")
         return "FAIL";
     return result.judge_verdict;
 }
-function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS"; }
+function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "NOT-MEASURED" || verdict === "JUDGE-AMBIGUOUS"; }
 function sorted(value) { return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); }
 function increase(reference, candidate) { return reference === null || candidate === null || reference <= 0 ? null : (candidate - reference) / reference; }
 function thresholdFindings(reference, candidate, thresholds) {

package/dist/execution-trace.js

@@ -157,12 +157,19 @@ export function parseTrace(lines, meta) {
         }
     }
     const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
+    const subscription = meta.subject.provider === "openai-codex" || meta.subject.provider === "claude-code";
+    const costSource = subscription
+        ? "subscription"
+        : cost !== null
+            ? "provider-reported"
+            : "unreported";
     const metrics = {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
         cache_read_tokens: cacheReadTokens,
         cache_write_tokens: cacheWriteTokens,
         cost_usd: cost ?? 0,
+        cost_source: costSource,
         tool_calls: toolCalls.length,
         delegated_children: toolCalls
             .filter((call) => call.name === "Agent")
@@ -348,12 +355,13 @@ export function mergeTraces(traces) {
             cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
             cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
             cost_usd: sum.cost_usd + trace.metrics.cost_usd,
+            cost_source: sum.cost_source === trace.metrics.cost_source ? sum.cost_source : "unreported",
             tool_calls: sum.tool_calls + trace.metrics.tool_calls,
             delegated_children: sum.delegated_children + trace.metrics.delegated_children,
             max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
         }), {
             input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
-            cost_usd: 0, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
+            cost_usd: 0, cost_source: traces[0].metrics.cost_source, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
         })
         : undefined;
     const last = traces[traces.length - 1];