@skill-harness/core 0.11.0 → 0.12.0

package/README.md

@@ -1,7 +1,8 @@
 # @skill-harness/core
 
 The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
-spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results.
+spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results,
+and the separately versioned durable qualification-runner-v1 primitives.
 
 Internal API — use the `skill-harness` CLI. See the repo for docs.
 

package/dist/adapters/types.d.ts

@@ -11,6 +11,21 @@ export interface ModelRef {
 export declare function parseModelRef(token: string): ModelRef;
 /** Stable, filesystem-safe slug for a model ref (used in run-dir names). */
 export declare function modelSlug(ref: ModelRef): string;
+export type PromptMechanism = "none" | "pi-skill" | "append-system-prompt" | "system-prompt-file";
+export interface PromptProvenance {
+    capture_version: "prompt-provenance-v1";
+    request_index: number;
+    raw_sha256: string;
+    normalized_sha256: string;
+    normalization_rule: "cwd-line-v1";
+    bytes: number;
+    contract_sha256: string;
+    contract_bytes: number;
+    contract_occurrences: number;
+    mechanism: PromptMechanism;
+    status: "PASS" | "NOT-MEASURED" | "ERROR";
+    error?: string;
+}
 export interface RunReq {
     skillDir: string;
     model: ModelRef;
@@ -50,6 +65,8 @@ export interface RunReq {
      * inheriting a caller's copy of them.
      */
     armEnv?: Record<string, string>;
+    /** Adapter-produced observation callback. Callers receive evidence; they never supply delivery status. */
+    onPromptObservation?: (observation: PromptProvenance) => void;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {
@@ -74,6 +91,8 @@ export interface StructuredRun {
 }
 export interface HarnessAdapter {
     name: string;
+    /** True only when the adapter computes final provider-payload observations. */
+    observesPrompts?: boolean;
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     /**

package/dist/adjudication.d.ts

@@ -1,5 +1,5 @@
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
-import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile } from "./results.js";
+import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile, CriterionVote } from "./results.js";
 import type { Scenario, ShipBar, Spec } from "./spec.js";
 import type { Verdict } from "./score.js";
 /**
@@ -33,6 +33,11 @@ export interface CellState {
     suspect: boolean;
     /** Per-rep verdicts, when the cell ran with reps. Empty for a single-rep cell. */
     repVerdicts?: Verdict[];
+    criteria?: CriterionVote[];
+    /** Delivery prerequisite for schema-3 cells; non-PASS cells are never judged. */
+    deliveryStatus?: "PASS" | "NOT-MEASURED" | "ERROR";
+    /** The actual selected repetition's primary vote; aggregate fields remain for planning only. */
+    primaryJudgment?: Judgment;
 }
 export interface TriggerDecision {
     id: string;
@@ -105,11 +110,14 @@ export declare function collapseJudgments(judgments: Judgment[], trigger: Trigge
  */
 export declare function boundAdjudicationToRepetitions(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): AdjudicationResult;
 export declare function projectAdjudication(result: ScenarioResult, adj: AdjudicationResult): ScenarioResult;
+/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
+export declare function projectAdjudicationForScenario(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): ScenarioResult;
 /** Ask one judge about one cell's saved transcript. Supplied by the caller. */
 export type RejudgeFn = (id: string, judge: ModelRef) => Promise<{
     verdict: Verdict;
     reason: string;
     suspect: boolean;
+    criteria?: CriterionVote[];
 }>;
 export interface RunAdjudicationOptions {
     plan: AdjudicationPlan;

package/dist/adjudication.js

@@ -1,9 +1,10 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, } from "./results.js";
+import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, completeCriterionVotes, } from "./results.js";
 import { score } from "./score.js";
 import { buildJudgePrompt, judgeInWorkspace, parseVerdict } from "./grade.js";
 import { appendJournal } from "./journal.js";
+import { collapseVotePanel } from "./vote-panel.js";
 /** Maximum judgments per cell, first wave included. Hard cap, not a default. */
 export const MAX_JUDGMENTS = 3;
 /**
@@ -18,6 +19,9 @@ export function planAdjudication(input) {
     const enabled = new Set(input.enabled ?? ["ambiguous", "contradictory", "non_unanimous", "ship_deciding"]);
     const decisions = [];
     for (const cell of input.cells) {
+        // Delivery-gated schema-3 cells carry no behavioral claim to adjudicate.
+        if (cell.deliveryStatus && cell.deliveryStatus !== "PASS")
+            continue;
         const triggers = [];
         // `JUDGE-AMBIGUOUS` is what the parser emits when a judge's verdict blocks
         // disagree. `ERROR` is what it emits when nothing parsed at all — and that
@@ -92,24 +96,13 @@ function flipsShipDecision(cell, input) {
  * cast a deciding vote would launder exactly the unreliability being measured.
  */
 export function collapseJudgments(judgments, trigger) {
-    const clean = judgments.filter((j) => !j.suspect && (j.verdict === "PASS" || j.verdict === "FAIL"));
-    const base = { trigger, judgments };
-    if (clean.length < 2) {
-        // Nothing to compare: one clean vote (or none) cannot confirm anything.
-        return { ...base, state: "unresolved" };
-    }
-    const passes = clean.filter((j) => j.verdict === "PASS").length;
-    const fails = clean.length - passes;
-    if (passes === 0 || fails === 0) {
-        return { ...base, state: "confirmed", verdict: clean[0].verdict };
-    }
-    // A clean strict majority breaks the tie. With exactly 2 clean votes that
-    // disagree there is no majority, so this correctly falls through.
-    if (passes > fails)
-        return { ...base, state: "tie_broken", verdict: "PASS" };
-    if (fails > passes)
-        return { ...base, state: "tie_broken", verdict: "FAIL" };
-    return { ...base, state: "unresolved" };
+    const collapsed = collapseVotePanel(judgments);
+    return {
+        trigger,
+        judgments,
+        state: collapsed.state,
+        ...(collapsed.verdict ? { verdict: collapsed.verdict } : {}),
+    };
 }
 /**
  * Project an adjudication onto the compatibility fields the scorer reads.
@@ -149,6 +142,15 @@ export function projectAdjudication(result, adj) {
         adjudication: adj,
     };
 }
+/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
+export function projectAdjudicationForScenario(result, scenario, adj) {
+    const bounded = boundAdjudicationToRepetitions(result, scenario, adj);
+    const projected = projectAdjudication(result, bounded);
+    if (bounded !== adj) {
+        projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
+    }
+    return projected;
+}
 function reasonFor(adj) {
     const n = adj.judgments.length;
     const verb = adj.state === "confirmed" ? "confirmed by" : "resolved by majority of";
@@ -179,7 +181,9 @@ export async function runAdjudication(opts) {
         // in the fixed order `planAdjudication` evaluates them.
         const trigger = decision.triggers[0];
         const judgments = [
-            { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect },
+            cell.primaryJudgment
+                ? { ...cell.primaryJudgment, ordinal: 1 }
+                : { ordinal: 1, judge: { ...opts.primaryJudge }, verdict: cell.verdict, reason: cell.reason, suspect: cell.suspect, criteria: cell.criteria ?? [] },
         ];
         const second = await opts.rejudge(decision.id, opts.secondaryJudge);
         callsMade++;
@@ -242,11 +246,8 @@ export async function adjudicateRun(opts) {
         if (!adj)
             return s;
         const scenario = byIdScenario.get(s.id);
-        const boundedAdj = scenario ? boundAdjudicationToRepetitions(s, scenario, adj) : adj;
-        const projected = projectAdjudication(s, boundedAdj);
-        if (boundedAdj !== adj) {
-            projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
-        }
+        const withRepetition = { ...adj, repetition: s.rep_judgments?.[0]?.repetition ?? 0 };
+        const projected = scenario ? projectAdjudicationForScenario(s, scenario, withRepetition) : projectAdjudication(s, withRepetition);
         const extraCalls = adj.judgments.filter((judgment) => judgment.ordinal > 1).length;
         projected.metrics = mergeScenarioMetrics(s.metrics, {
             wall_time_ms: 0,
@@ -299,7 +300,7 @@ async function judgeCell(opts) {
         event: "judge-verdict", ts: opts.now(), id: opts.scenario.id,
         verdict: g.verdict, reason: g.reason, suspect: g.suspect,
     });
-    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect };
+    return { verdict: g.verdict, reason: g.reason, suspect: g.suspect, criteria: completeCriterionVotes(g.criteria, opts.scenario.checklist.length) };
 }
 /**
  * Build the plan's input cells from a run.
@@ -311,13 +312,20 @@ async function judgeCell(opts) {
  * about — inverting the one invariant this feature actually promises.
  */
 export function cellsFromResults(runDir, results) {
-    return results.scenarios.map((s) => ({
-        id: s.id,
-        verdict: s.judge_verdict,
-        reason: s.judge_reason,
-        suspect: s.suspect,
-        repVerdicts: repVerdictsOf(runDir, s, results.mode),
-    }));
+    return results.scenarios.map((s) => {
+        const primaryJudgment = s.rep_judgments?.find(panel => panel.repetition === 0)?.judgments[0];
+        const deliveryStatus = s.objective?.assertions.find(assertion => assertion.kind === "skill_delivered")?.status;
+        return {
+            id: s.id,
+            verdict: s.judge_verdict,
+            reason: s.judge_reason,
+            suspect: s.suspect,
+            repVerdicts: repVerdictsOf(runDir, s, results.mode),
+            criteria: primaryJudgment?.criteria,
+            ...(deliveryStatus === "PASS" || deliveryStatus === "NOT-MEASURED" || deliveryStatus === "ERROR" ? { deliveryStatus } : {}),
+            ...(primaryJudgment ? { primaryJudgment } : {}),
+        };
+    });
 }
 /** Per-rep verdicts from saved judge artifacts, for the non-unanimous trigger. */
 function repVerdictsOf(runDir, s, mode) {

package/dist/arms.d.ts

@@ -18,7 +18,7 @@ export interface Arm {
     seedSkills: string[];
     /** Minimum definitions the seeding must produce, else the run ERRORs. */
     requireDefinitions: number;
-    /** Env for the subject process. `<run-dir>` is substituted per run. */
+    /** Env for the subject process. `<run-dir>` and `<workspace>` are substituted per rep. */
     env: Record<string, string>;
 }
 /** The implicit control: today's behaviour, byte-identical, no extensions. */

package/dist/capture-trace-types.d.ts

@@ -83,12 +83,14 @@ export interface TraceResultMeta {
  * conversation carries in the session dir, not in the event stream). This
  * mirrors the loop the adapter already runs.
  */
+export type CostSource = "provider-reported" | "subscription" | "unreported";
 export interface TraceMetrics {
     input_tokens: number;
     output_tokens: number;
     cache_read_tokens: number;
     cache_write_tokens: number;
     cost_usd: number;
+    cost_source: CostSource;
     tool_calls: number;
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */

package/dist/command-cost.d.ts

@@ -0,0 +1,9 @@
+/**
+ * CLI commands guaranteed to make no subject or judge calls.
+ *
+ * Keep this list narrow: commands that can conditionally spend (notably `regate`)
+ * do not belong here even when their common path only replays saved artifacts.
+ */
+export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "coverage", "init", "judge-agreement", "lint", "list", "mutation-test", "rescore", "restamp", "screen", "stability"];
+export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
+export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;

package/dist/command-cost.js

@@ -0,0 +1,23 @@
+/**
+ * CLI commands guaranteed to make no subject or judge calls.
+ *
+ * Keep this list narrow: commands that can conditionally spend (notably `regate`)
+ * do not belong here even when their common path only replays saved artifacts.
+ */
+export const FREE_OFFLINE_COMMANDS = [
+    "affected",
+    "coverage",
+    "init",
+    "judge-agreement",
+    "lint",
+    "list",
+    "mutation-test",
+    "rescore",
+    "restamp",
+    "screen",
+    "stability",
+];
+export function isFreeOfflineCommand(command) {
+    return FREE_OFFLINE_COMMANDS.some((candidate) => candidate === command);
+}
+//# sourceMappingURL=command-cost.js.map

package/dist/comparison.d.ts

@@ -45,6 +45,7 @@ export interface AggregateMetrics {
     cache_read_tokens: number | null;
     cache_write_tokens: number | null;
     subject_cost_usd: number | null;
+    cost_source: "provider-reported" | "subscription" | "unreported" | null;
     tool_calls: number | null;
     delegated_children: number | null;
     max_concurrency: number | null;

package/dist/comparison.js

@@ -51,8 +51,8 @@ export function buildComparison(input) {
     const behavioral = cells.filter((cell) => cell.change === "regression").map((cell) => cell.id);
     const criticalRegressions = cells.filter((cell) => cell.change === "regression" && cell.critical).map((cell) => cell.id);
     const byReference = new Map(reference.scenarios.map((scenario) => [scenario.id, scenario]));
-    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
-        cell.candidate === "ERROR" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
+    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "NOT-MEASURED" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
+        cell.candidate === "ERROR" || cell.candidate === "NOT-MEASURED" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
     const metrics = { reference: aggregateMetrics(reference.scenarios), candidate: aggregateMetrics(candidate.scenarios) };
     const costRegressions = thresholdFindings(metrics.reference, metrics.candidate, input.thresholds);
     const partial = input.partial === true || reference.partial === true || candidate.partial === true;
@@ -137,6 +137,10 @@ export function aggregateMetrics(scenarios) {
         cache_read_tokens: sumOptional("cache_read_tokens"),
         cache_write_tokens: sumOptional("cache_write_tokens"),
         subject_cost_usd: sumOptional("subject_cost_usd"),
+        cost_source: (() => {
+            const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
+            return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
+        })(),
         tool_calls: sumOptional("tool_calls"),
         delegated_children: sumOptional("delegated_children"),
         max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
@@ -147,11 +151,13 @@ function effective(result) {
         return result.override;
     if (result.objective?.status === "ERROR")
         return "ERROR";
+    if (result.objective?.status === "NOT-MEASURED")
+        return "NOT-MEASURED";
     if (result.objective?.status === "FAIL")
         return "FAIL";
     return result.judge_verdict;
 }
-function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS"; }
+function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "NOT-MEASURED" || verdict === "JUDGE-AMBIGUOUS"; }
 function sorted(value) { return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); }
 function increase(reference, candidate) { return reference === null || candidate === null || reference <= 0 ? null : (candidate - reference) / reference; }
 function thresholdFindings(reference, candidate, thresholds) {

package/dist/execution-trace.js

@@ -157,12 +157,19 @@ export function parseTrace(lines, meta) {
         }
     }
     const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
+    const subscription = meta.subject.provider === "openai-codex" || meta.subject.provider === "claude-code";
+    const costSource = subscription
+        ? "subscription"
+        : cost !== null
+            ? "provider-reported"
+            : "unreported";
     const metrics = {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
         cache_read_tokens: cacheReadTokens,
         cache_write_tokens: cacheWriteTokens,
         cost_usd: cost ?? 0,
+        cost_source: costSource,
         tool_calls: toolCalls.length,
         delegated_children: toolCalls
             .filter((call) => call.name === "Agent")
@@ -348,12 +355,13 @@ export function mergeTraces(traces) {
             cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
             cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
             cost_usd: sum.cost_usd + trace.metrics.cost_usd,
+            cost_source: sum.cost_source === trace.metrics.cost_source ? sum.cost_source : "unreported",
             tool_calls: sum.tool_calls + trace.metrics.tool_calls,
             delegated_children: sum.delegated_children + trace.metrics.delegated_children,
             max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
         }), {
             input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
-            cost_usd: 0, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
+            cost_usd: 0, cost_source: traces[0].metrics.cost_source, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
         })
         : undefined;
     const last = traces[traces.length - 1];

package/dist/grade.d.ts

@@ -1,6 +1,7 @@
 import type { Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
 import type { Verdict } from "./score.js";
+import { type CriterionVote } from "./results.js";
 export interface JudgePromptInput {
     skill: string;
     persona: string;
@@ -36,6 +37,7 @@ export declare function parseVerdict(out: string): ParsedVerdict;
 export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
 export interface GradeResult extends ParsedVerdict {
     raw: string;
+    criteria: CriterionVote[];
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }

package/dist/grade.js

@@ -1,4 +1,5 @@
 import { createWorkspace } from "./workspace.js";
+import { parseCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -133,7 +134,7 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
             parsed.reason = `judge unparseable: ${snippet}`;
     }
     const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect };
+    return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the

package/dist/index.d.ts

@@ -23,8 +23,12 @@ export * from "./util/env.js";
 export * from "./scaffold.js";
 export * from "./version.js";
 export * from "./defaults.js";
+export * from "./command-cost.js";
 export * from "./judge-policy.js";
+export * from "./judge-agreement.js";
 export * from "./regate.js";
+export * from "./screen.js";
+export * from "./prompt-normalization.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
 export * from "./stability.js";
@@ -37,7 +41,15 @@ export * from "./trajectory-gates.js";
 export * from "./instruction-coverage.js";
 export * from "./affected.js";
 export * from "./adjudication.js";
+export * from "./vote-panel.js";
+export * from "./qualification-panels.js";
+export * from "./qualification-panel-store.js";
+export * from "./qualification-validation.js";
 export * from "./comparison.js";
 export * from "./sandbox.js";
 export * from "./provider-failure.js";
 export * from "./arms.js";
+export * from "./qualification-config.js";
+export * from "./qualification-oauth-directory.js";
+export * from "./qualification-store.js";
+export * from "./qualification-runner.js";

package/dist/index.js

@@ -23,8 +23,12 @@ export * from "./util/env.js";
 export * from "./scaffold.js";
 export * from "./version.js";
 export * from "./defaults.js";
+export * from "./command-cost.js";
 export * from "./judge-policy.js";
+export * from "./judge-agreement.js";
 export * from "./regate.js";
+export * from "./screen.js";
+export * from "./prompt-normalization.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
 export * from "./stability.js";
@@ -37,8 +41,16 @@ export * from "./trajectory-gates.js";
 export * from "./instruction-coverage.js";
 export * from "./affected.js";
 export * from "./adjudication.js";
+export * from "./vote-panel.js";
+export * from "./qualification-panels.js";
+export * from "./qualification-panel-store.js";
+export * from "./qualification-validation.js";
 export * from "./comparison.js";
 export * from "./sandbox.js";
 export * from "./provider-failure.js";
 export * from "./arms.js";
+export * from "./qualification-config.js";
+export * from "./qualification-oauth-directory.js";
+export * from "./qualification-store.js";
+export * from "./qualification-runner.js";
 //# sourceMappingURL=index.js.map

package/dist/journal.d.ts

@@ -103,7 +103,7 @@ export type JournalEvent = {
 }
 /**
  * A regate: needle gates re-evaluated against the saved staged diffs. `judge_calls`
- * is on the record because regate is advertised as free apart from the reps whose
+ * is on the record because regate may spend only on the reps whose
  * gate verdict flipped — a claim the journal should be able to settle. `skipped`
  * names scenarios it could not regate (vitest/post_test, or missing diff artifacts).
  */

package/dist/judge-agreement.d.ts

@@ -0,0 +1,19 @@
+import type { ResultsFile } from "./results.js";
+import type { Verdict } from "./score.js";
+export interface JudgeAgreementCell {
+    scenario: string;
+    status: "agree" | "disagree" | "error";
+    judges: string[];
+    verdicts: Verdict[];
+    detail: string;
+}
+export interface JudgeAgreementReport {
+    cells: JudgeAgreementCell[];
+    agree: number;
+    disagree: number;
+    error: number;
+    comparable: number;
+    rate: number | null;
+}
+/** Pure/offline projection of judgments already persisted in results.yaml. */
+export declare function judgeAgreement(results: ResultsFile): JudgeAgreementReport;

package/dist/judge-agreement.js

@@ -0,0 +1,31 @@
+/** Pure/offline projection of judgments already persisted in results.yaml. */
+export function judgeAgreement(results) {
+    const cells = results.scenarios.map((scenario) => {
+        const judgments = scenario.judge_history ?? scenario.adjudication?.judgments ?? [];
+        const distinct = new Map();
+        for (const judgment of judgments) {
+            const key = `${judgment.judge.provider}:${judgment.judge.model}`;
+            distinct.delete(key); // insertion order now reflects each judge's latest persisted vote
+            distinct.set(key, judgment);
+        }
+        const pair = [...distinct.entries()].slice(-2);
+        if (pair.length < 2 || pair.some(([, judgment]) => judgment.suspect || judgment.verdict === "ERROR" || judgment.verdict === "NOT-MEASURED" || judgment.verdict === "JUDGE-AMBIGUOUS")) {
+            return {
+                scenario: scenario.id, status: "error", judges: pair.map(([judge]) => judge),
+                verdicts: pair.map(([, judgment]) => judgment.verdict),
+                detail: pair.length < 2 ? "fewer than two distinct persisted judges" : "one or more judgments is not a clean vote",
+            };
+        }
+        const status = pair[0][1].verdict === pair[1][1].verdict ? "agree" : "disagree";
+        return {
+            scenario: scenario.id, status, judges: pair.map(([judge]) => judge),
+            verdicts: pair.map(([, judgment]) => judgment.verdict), detail: `${pair[0][1].verdict} vs ${pair[1][1].verdict}`,
+        };
+    });
+    const agree = cells.filter((cell) => cell.status === "agree").length;
+    const disagree = cells.filter((cell) => cell.status === "disagree").length;
+    const error = cells.filter((cell) => cell.status === "error").length;
+    const comparable = agree + disagree;
+    return { cells, agree, disagree, error, comparable, rate: comparable ? agree / comparable : null };
+}
+//# sourceMappingURL=judge-agreement.js.map

package/dist/judge-policy.js

@@ -4,15 +4,17 @@ import { BAKED_DEFAULT_JUDGE } from "./defaults.js";
  * Judge providers that cannot bill a per-token API.
  *
  * `claude-code` shells out to the `claude` CLI, authenticated with the user's
- * Claude subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local` are local
- * runtimes. Everything else is assumed to charge.
+ * Claude subscription (OAuth). `openai-codex` stays in Pi and authenticates with
+ * the user's ChatGPT subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local`
+ * are local runtimes. Everything else — including direct `openai` — is assumed
+ * to charge.
  *
  * An **allow-list**, deliberately, not a deny-list of known-paid providers: a
  * provider nobody has classified yet should be treated as able to bill, because
  * being wrong in that direction produces a surprise invoice while being wrong in
  * this direction produces one extra flag.
  */
-const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "ollama", "lmstudio", "llamacpp", "local"]);
+const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "openai-codex", "ollama", "lmstudio", "llamacpp", "local"]);
 /** Whether judging with this ref can charge a per-token API. */
 export function isMeteredJudge(judge) {
     return !FREE_JUDGE_PROVIDERS.has(judge.provider);

package/dist/lift.js

@@ -29,7 +29,7 @@ function conclusive(verdict, suspect) {
     // misfire are judge failures. Treating any of them as a FAIL would make
     // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
     // measure flakiness while looking like skill value.
-    return !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
+    return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
 }
 function classify(red, green) {
     if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))

package/dist/lint.js

@@ -279,7 +279,7 @@ export function lintSkill(skillDir) {
                 else if (current !== recorded) {
                     // The remedy is per key kind, and it is the whole point of the split: a
                     // rubric edit is re-gradeable from saved transcripts, a policy edit is a
-                    // free rescore, a needle edit is a free regate. Only stimulus drift costs
+                    // free rescore, while a gate edit avoids a subject re-run via regate. Only stimulus drift costs
                     // model spend. One message saying "re-run" for all four is what made
                     // correcting a known-bad rubric expensive enough to skip.
                     findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; ${remedyForKey(key)}` });

package/dist/prompt-normalization.d.ts

@@ -0,0 +1,6 @@
+export declare const PROMPT_NORMALIZATION_RULE: "cwd-line-v1";
+export declare const PROMPT_NORMALIZATION_PATTERN = "^(Current working directory:)[^\\r\\n]*(\\r?)$";
+export declare const PROMPT_NORMALIZATION_FLAGS = "gm";
+export declare const PROMPT_NORMALIZATION_REPLACEMENT = "$1<normalized>$2";
+export declare const PROMPT_NORMALIZATION_SOURCE_KEY = "observation:prompt-normalization";
+export declare const PROMPT_NORMALIZATION_SOURCE_DIGEST: string;

package/dist/prompt-normalization.js

@@ -0,0 +1,14 @@
+import { createHash } from "node:crypto";
+export const PROMPT_NORMALIZATION_RULE = "cwd-line-v1";
+export const PROMPT_NORMALIZATION_PATTERN = "^(Current working directory:)[^\\r\\n]*(\\r?)$";
+export const PROMPT_NORMALIZATION_FLAGS = "gm";
+export const PROMPT_NORMALIZATION_REPLACEMENT = "$1<normalized>$2";
+export const PROMPT_NORMALIZATION_SOURCE_KEY = "observation:prompt-normalization";
+export const PROMPT_NORMALIZATION_SOURCE_DIGEST = createHash("sha256").update(JSON.stringify([
+    "prompt-normalization-registry",
+    PROMPT_NORMALIZATION_RULE,
+    PROMPT_NORMALIZATION_PATTERN,
+    PROMPT_NORMALIZATION_FLAGS,
+    PROMPT_NORMALIZATION_REPLACEMENT,
+])).digest("hex");
+//# sourceMappingURL=prompt-normalization.js.map

package/dist/qualification-capture.d.ts

@@ -0,0 +1,44 @@
+export interface QualificationOutputReceipt {
+    path: string;
+    total_bytes: number;
+    captured_bytes: number;
+    truncated: boolean;
+    sha256: string;
+}
+export interface QualificationBoundedCapture {
+    path: string;
+    total_bytes: number;
+    captured_bytes: number;
+    truncated: boolean;
+    write(chunk: Buffer): void;
+    close(): void;
+}
+export interface QualificationArtifactAttestation {
+    ok: boolean;
+    actual: {
+        provider: string;
+        model: string;
+    } | null;
+    fallback: boolean;
+    refused: boolean;
+    error: string | null;
+}
+export declare function redactQualificationOutput(text: string, secretValues?: readonly string[]): string;
+export declare function openQualificationBoundedCapture(path: string, limit: number, redact: (text: string) => string): QualificationBoundedCapture;
+export declare function qualificationOutputReceipt(spoolRoot: string, path: string, capture: QualificationBoundedCapture): QualificationOutputReceipt;
+export declare function verifyQualificationOutputReceipt(spoolRoot: string, receipt: QualificationOutputReceipt): void;
+export declare function attestQualificationPiJsonl(text: string, requested: {
+    provider: string;
+    model: string;
+}): QualificationArtifactAttestation;
+export declare function spawnQualificationCapture(command: string, argv: string[], options: {
+    cwd: string;
+    env: NodeJS.ProcessEnv;
+    timeout_ms: number;
+    output_limit_bytes: number;
+}): Promise<{
+    stdout: string;
+    stderr: string;
+    code: number | null;
+    timed_out: boolean;
+}>;