@jinn-network/client 0.1.7 → 0.1.8-canary.09a3b2f6

package/dist/eval/eval-harness-run.js

@@ -0,0 +1,123 @@
+/**
+ * Daemon-faithful harness run for `jinn eval` (issue #818) and the #822 e2e.
+ *
+ * Live testing proved the eval's earlier hand-rolled harness invocation did NOT
+ * reproduce the daemon's agent runtime: it ran the agent with NO SolverNet
+ * runtime plugins, so the claude-code adapter passed no `--plugin-dir`, the
+ * bundled MCP server never loaded, and the agent had no `submit_typed_payload`
+ * tool — producing no gradeable patch and rendering every task unscorable. The
+ * daemon scores the same tasks because it builds the full HarnessContext with
+ * `solverPluginRoots` from the SolverNet's runtime plugins (engine.ts §runImpl).
+ *
+ * This module is the single source of truth for that daemon-equivalent context.
+ * It rebuilds the engine's HarnessContext MINUS persistence/on-chain wiring and
+ * runs the harness through the SAME freeze-fence the daemon uses. The CLI uses
+ * it now; the #822 e2e delegates to it next.
+ */
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { provisionWorkingDir } from '../harnesses/engine/packaging.js';
+import { runHarnessWithFreezeFence } from '../daemon/freeze-fence.js';
+import { TrajectoryCollector } from '../trajectory/index.js';
+import { loadSolverNets, } from '../solver-nets/registry.js';
+/** Wall-clock budget for a single eval harness run (1h) when the task carries no window. */
+const EVAL_RUN_BUDGET_MS = 3_600_000;
+/**
+ * Resolve the SolverNet runtime plugins for `solverType`, exactly as the daemon
+ * does (`loadSolverNets(...).forSolverType(...)`). These plugins carry the
+ * bundled MCP server that provides `submit_typed_payload`; without them the
+ * agent cannot emit a gradeable patch.
+ *
+ * Fails LOUD with an operator-actionable message when there is no joined
+ * SolverNet for the solverType, or the matched net has no runtime plugins —
+ * mirroring the evaluator's `jinn harnesses enable` precondition.
+ */
+export async function resolveRuntimePluginsForSolverType(solverType, joinedSolverNets, role = 'restoration') {
+    const registry = await loadSolverNets({ joinedSolverNets });
+    const net = registry.forSolverType(solverType, role);
+    if (!net) {
+        throw new Error(`no SolverNet for solverType ${solverType} (role ${role}): the eval runs the agent with the ` +
+            `SAME runtime plugins the daemon would, but this operator has not joined a SolverNet for it. ` +
+            `Join the SolverNet and install its plugins (e.g. set joinedSolverNets[<manifestCid>] in your ` +
+            `config and run \`jinn harnesses enable\`), then re-run eval.`);
+    }
+    if (net.runtimePlugins.length === 0) {
+        throw new Error(`SolverNet ${net.name} for solverType ${solverType} has no runtime plugins: the agent would run ` +
+            `without the bundled MCP server (no submit_typed_payload) and produce no gradeable patch. ` +
+            `Install the SolverNet's plugins (\`jinn harnesses enable\`) before running eval.`);
+    }
+    return net.runtimePlugins;
+}
+/**
+ * Run a harness once for eval, building the FULL daemon-equivalent
+ * HarnessContext (plugins + solverPluginRoots) but with no persistence /
+ * on-chain wiring. Mirrors `HarnessEngine`'s `runImpl` context assembly
+ * (engine.ts §runImpl) and routes through the same freeze-fence.
+ *
+ * Owns the ephemeral working-dir lifecycle: the returned result references only
+ * the patch string + codeDigest (nothing under workingDir), so the dir is safe
+ * to reclaim in `finally` — preventing leaked repo clones across a full slate.
+ */
+export async function runHarnessForEval(args) {
+    const { harness, task, solverType, runtimePlugins, implStateDir, mode } = args;
+    const defaultEndTs = Date.now() + EVAL_RUN_BUDGET_MS;
+    const windowEndTs = task.window?.endTs ?? defaultEndTs;
+    // Fresh ephemeral scratch dir — SEPARATE from implStateDir so the harness's
+    // repo clone + diff harvest never trips the freeze-fence (which hashes
+    // implStateDir only). Provisioned the same way the engine does.
+    const workingDir = mkdtempSync(join(tmpdir(), 'jinn-eval-work-'));
+    const abort = new AbortController();
+    const msUntilEndTs = () => Math.max(0, windowEndTs - Date.now());
+    const endTimer = setTimeout(() => abort.abort(), msUntilEndTs());
+    // Don't keep the event loop alive on this timer (it's cleared in finally).
+    endTimer.unref?.();
+    try {
+        provisionWorkingDir(workingDir, task);
+        // Full daemon-equivalent HarnessContext — mirrors engine.ts §runImpl MINUS
+        // persistence/on-chain. The load-bearing fields are runtimePlugins +
+        // solverPluginRoots: they carry the bundled MCP server (submit_typed_payload).
+        const ctx = {
+            task,
+            requestId: task.id ?? `eval-${solverType}`,
+            taskCid: '',
+            solverNet: {
+                name: args.solverNetName ?? solverType,
+                solverType,
+                ...(args.model ? { model: args.model } : {}),
+            },
+            runtimePlugins,
+            solverPluginRoots: runtimePlugins.map((plugin) => plugin.root),
+            implStateDir,
+            workingDir,
+            log: () => {
+                /* eval run: quiet harness logs */
+            },
+            abort: abort.signal,
+            msUntilEndTs,
+            trajectory: new TrajectoryCollector({ taskCid: '', runId: `eval-${task.id ?? solverType}` }),
+            mode,
+        };
+        const fence = await runHarnessWithFreezeFence(harness, ctx);
+        if (!fence.ok) {
+            return { violation: { taskId: task.id ?? `eval-${solverType}` } };
+        }
+        const patch = fence.output.solutionPayload?.patch;
+        if (typeof patch !== 'string') {
+            throw new Error(`eval harness produced no swe-rebench-v2 patch for task ${task.id ?? solverType} ` +
+                `(solutionPayload.patch missing) — the agent ran but emitted no gradeable diff`);
+        }
+        return {
+            envelope: { executor: { mode, codeDigest: `sha256:${fence.codeDigest}` } },
+            solution: { patch },
+        };
+    }
+    finally {
+        clearTimeout(endTimer);
+        // Always reclaim the per-task scratch dir — swe-rebench-v2 clones a real
+        // upstream repo here; leaking it across a full slate piles onto the
+        // disk-pressure this stack already guards (JINN_EVAL_DISK_FLOOR_GB).
+        rmSync(workingDir, { recursive: true, force: true });
+    }
+}
+//# sourceMappingURL=eval-harness-run.js.map

package/dist/eval/eval-harness-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-harness-run.js","sourceRoot":"","sources":["../../src/eval/eval-harness-run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EACL,cAAc,GAGf,MAAM,4BAA4B,CAAC;AAEpC,4FAA4F;AAC5F,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAErC;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,kCAAkC,CACtD,UAAkB,EAClB,gBAAmE,EACnE,OAA0B,aAAa;IAEvC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,+BAA+B,UAAU,UAAU,IAAI,sCAAsC;YAC3F,8FAA8F;YAC9F,+FAA+F;YAC/F,8DAA8D,CACjE,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CACb,aAAa,GAAG,CAAC,IAAI,mBAAmB,UAAU,+CAA+C;YAC/F,2FAA2F;YAC3F,kFAAkF,CACrF,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC,cAAc,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IASvC;IAKC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;IAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;IACrD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,YAAY,CAAC;IAEvD,4EAA4E;IAC5E,uEAAuE;IACvE,gEAAgE;IAChE,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAElE,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;IACjE,2EAA2E;IAC3E,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;IAEnB,IAAI,CAAC;QACH,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAEtC,2EAA2E;QAC3E,qEAAqE;QACrE,+EAA+E;QAC/E,MAAM,GAAG,GAAmB;YAC1B,IAAI;YACJ,SAAS,EAAE,IAAI,CAAC,EAAE,IAAI,QAAQ,UAAU,EAAE;YAC1C,OAAO,EAAE,EAAE;YACX,SAAS,EAAE;gBACT,IAAI,EAAE,IAAI,CAAC,aAAa,IAAI,UAAU;gBACtC,UAAU;gBACV,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7C;YACD,cAAc;YACd,iBAAiB,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;YAC9D,YAAY;YACZ,UAAU;YACV,GAAG,EAAE,GAAG,EAAE;gBACR,kCAAkC;YACpC,CAAC;YACD,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,YAAY;YACZ,UAAU,EAAE,IAAI,mBAAmB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,EAAE,CAAC;YAC5F,IAAI;SACL,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,IAAI,QAAQ,UAAU,EAAE,EAAE,EAAE,CAAC;QACpE,CAAC;QAED,MAAM,KAAK,GAAI,KAAK,CAAC,MAAM,CAAC,eAAmD,EAAE,KAAK,CAAC;QACvF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,0DAA0D,IAAI,CAAC,EAAE,IAAI,UAAU,GAAG;gBAChF,+EAA+E,CAClF,CAAC;QACJ,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,KAAK,CAAC,UAAU,EAAE,EAAE,EAAE;YAC1E,QAAQ,EAAE,EAAE,KAAK,EAAE;SACpB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvB,yEAAyE;QACzE,oEAAoE;QACpE,qEAAqE;QACrE,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;AACH,CAAC"}

package/dist/eval/orchestrator.d.ts

@@ -0,0 +1,163 @@
+/**
+ * `jinn eval` held-out checkpoint orchestrator (issue #818).
+ *
+ * Runs a held-out slate against a checkpoint in FROZEN mode, persists per-task
+ * pass/fail, and emits a Wilson-CI resolved-rate comparison vs the parent
+ * checkpoint. Every external boundary is constructor-injected — this is the
+ * thin slice #819 drives a deterministic slate against with NO Docker/IPFS.
+ *
+ * Acceptance criteria:
+ *   AC#1 — run the slate frozen, write per-task pass/fail.
+ *   AC#2 — emit a resolved-rate comparison vs the parent with a CI.
+ *   AC#3 — the freeze-fence holds: a `runHarnessOnce` `{ violation }` throws
+ *          LOUD and the instance is NOT recorded (no implStateDir mutation
+ *          slips through; enforcement lives in `runHarnessWithFreezeFence`).
+ *
+ * Per log/decisions/2026-05-28-rl-eval-measurement.md §4: v1-simple. Only large
+ * deltas are trustworthy (disjoint Wilson intervals). No seed control, no
+ * multi-run averaging.
+ */
+import type { Harness, HarnessContext } from '../harnesses/types.js';
+import type { SweRebenchV2Evaluator } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import type { HarnessCheckpointManifest } from '../vendor/@jinn-network/sdk/dist/checkpoint.js';
+import type { LoadedHeldOutSlate } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+import type { ResolvedSlateTask } from './resolve-slate-tasks.js';
+import type { EvalAggregate, EvalResultRecord } from '../store/store.js';
+import { type RateComparison } from './wilson.js';
+import { type PairedComparison, type PairedInput } from './paired.js';
+/** Thrown when `runHarnessOnce` reports a freeze-fence violation (AC#3). */
+export declare class FreezeFenceViolationError extends Error {
+    readonly instanceId: string;
+    constructor(instanceId: string);
+}
+/**
+ * Thrown when the locally-evaluated impl-state's codeDigest does not match the
+ * named checkpoint's manifest (C1). Guards against persisting + comparing
+ * results under a checkpoint name while the operator's local impl-state has
+ * drifted from that checkpoint.
+ */
+export declare class CheckpointStateMismatchError extends Error {
+    readonly checkpointCid: string;
+    readonly manifestCodeDigest: string;
+    readonly evaluatedCodeDigest: string;
+    constructor(checkpointCid: string, manifestCodeDigest: string, evaluatedCodeDigest: string);
+}
+/** Thrown when the parent checkpoint has no aggregate for this slate version (AC#2). */
+export declare class ParentNotEvaluatedError extends Error {
+    readonly parentCheckpointCid: string;
+    readonly slateVersion: string;
+    constructor(parentCheckpointCid: string, slateVersion: string);
+}
+/** A checkpoint whose recorded results carry slate hashes other than the current one. */
+export interface SlateHashDrift {
+    checkpointCid: string;
+    /** The drifted hashes (each != the current slate hash). */
+    hashes: string[];
+}
+/**
+ * Thrown when EITHER compared checkpoint was scored on a DIFFERENT slate
+ * *content* under the same version label (DR-2026-05-28 §2 — confounder control).
+ * A slate is content-addressed and scores are comparable ONLY within identical
+ * content; the store keys aggregates by version, so a content edit that skipped
+ * the version bump (or re-derived the declared hash — which the slate loader
+ * admits passes), or stale rows surviving `recordEvalResult`'s by-instance
+ * upsert, would let the comparison silently compare two checkpoints scored on
+ * different task sets. That is confounder #1 (task-selection) — exactly what the
+ * held-out exam exists to defeat — so we refuse the comparison loudly instead.
+ */
+export declare class SlateHashMismatchError extends Error {
+    readonly slateVersion: string;
+    readonly currentSlateHash: string;
+    readonly drifted: ReadonlyArray<SlateHashDrift>;
+    constructor(slateVersion: string, currentSlateHash: string, drifted: ReadonlyArray<SlateHashDrift>);
+}
+/**
+ * Injected `runHarnessOnce`-shaped boundary. Mirrors the production
+ * `runHarnessOnce` (freeze-fence + mode propagation) but also surfaces the
+ * harness `solution` so the orchestrator can extract the patch to grade. On a
+ * freeze violation it returns `{ violation }` and no solution.
+ */
+export type RunHarnessOnceForEval = (params: {
+    harness: Harness;
+    implStateDir: string;
+    mode: 'train' | 'frozen';
+    task?: HarnessContext['task'];
+}) => Promise<{
+    envelope?: {
+        executor: {
+            mode: 'train' | 'frozen';
+            codeDigest: string;
+        };
+    };
+    violation?: {
+        taskId: string;
+    };
+    solution?: {
+        patch: string;
+    };
+}>;
+export interface EvalOrchestratorDeps {
+    harness: Harness;
+    fetchImplStateDirToLocal(implStateDirCid: string, targetDir: string): Promise<string>;
+    evaluator: SweRebenchV2Evaluator;
+    runHarnessOnce: RunHarnessOnceForEval;
+    store: {
+        recordEvalResult(args: EvalResultRecord): void;
+        getEvalAggregate(checkpointCid: string, slateVersion: string): EvalAggregate;
+        /** Distinct slate_hash values the parent's results were recorded under (drift guard). */
+        getEvalSlateHashes(checkpointCid: string, slateVersion: string): string[];
+        /**
+         * Optional: per-instance results for a (checkpoint, slate version). When
+         * present, the orchestrator also emits the paired (matched-design) McNemar
+         * verdict (DR-2026-06-02-b §2a) — the same slate is scored before & after, so
+         * the matched test is the statistically correct one and is far more powerful
+         * than the marginal disjoint-interval test. Reported ALONGSIDE `comparison`,
+         * never replacing it (the exam is strengthened, not weakened).
+         */
+        getEvalResults?(checkpointCid: string, slateVersion: string): PairedInput[];
+    };
+}
+export interface PerTaskResult {
+    instance_id: string;
+    /** null when unscorable. */
+    passed: boolean | null;
+    unscorable: boolean;
+}
+/**
+ * Provenance of the graded artifact (Legibility). The CLI grades the operator's
+ * LOCAL frozen impl-state — verified == the named checkpoint via the C1
+ * codeDigest guard (CheckpointStateMismatchError) — NOT a state re-fetched from
+ * the checkpoint CID. Surfacing this stops a reader from assuming the tool
+ * fetched and graded the published checkpoint's state.
+ */
+export interface EvalProvenance {
+    /** Local impl-state directory that was actually run. */
+    implStateDir: string;
+    /** Real codeDigest of the evaluated impl-state (verified == manifest.codeDigest). */
+    codeDigest: string;
+    /** Always true on a returned result — a mismatch throws CheckpointStateMismatchError. */
+    matchedCheckpoint: true;
+}
+export interface EvalRunResult {
+    perTask: PerTaskResult[];
+    comparison: RateComparison;
+    /**
+     * Paired (matched-design) McNemar verdict vs the parent — present only when
+     * the store exposes `getEvalResults`. The marginal `comparison` above is
+     * conservative; this is the statistically correct test for the same-slate
+     * before/after design and is reported alongside it (DR-2026-06-02-b §2a).
+     */
+    paired?: PairedComparison;
+    /** What was actually graded (local impl-state), not the checkpoint identity. */
+    evaluated: EvalProvenance;
+}
+export declare function runEval(args: {
+    checkpointManifest: HarnessCheckpointManifest;
+    checkpointCid: string;
+    slate: LoadedHeldOutSlate;
+    tasksWithRows: ResolvedSlateTask[];
+    parentCheckpointCid: string;
+    deps: EvalOrchestratorDeps;
+    /** Working dir for the fetched impl-state-dir (defaults to a checkpoint-scoped tmp path). */
+    implStateDir?: string;
+}): Promise<EvalRunResult>;

package/dist/eval/orchestrator.js

@@ -0,0 +1,232 @@
+/**
+ * `jinn eval` held-out checkpoint orchestrator (issue #818).
+ *
+ * Runs a held-out slate against a checkpoint in FROZEN mode, persists per-task
+ * pass/fail, and emits a Wilson-CI resolved-rate comparison vs the parent
+ * checkpoint. Every external boundary is constructor-injected — this is the
+ * thin slice #819 drives a deterministic slate against with NO Docker/IPFS.
+ *
+ * Acceptance criteria:
+ *   AC#1 — run the slate frozen, write per-task pass/fail.
+ *   AC#2 — emit a resolved-rate comparison vs the parent with a CI.
+ *   AC#3 — the freeze-fence holds: a `runHarnessOnce` `{ violation }` throws
+ *          LOUD and the instance is NOT recorded (no implStateDir mutation
+ *          slips through; enforcement lives in `runHarnessWithFreezeFence`).
+ *
+ * Per log/decisions/2026-05-28-rl-eval-measurement.md §4: v1-simple. Only large
+ * deltas are trustworthy (disjoint Wilson intervals). No seed control, no
+ * multi-run averaging.
+ */
+import { compareRates } from './wilson.js';
+import { comparePaired } from './paired.js';
+/** Thrown when `runHarnessOnce` reports a freeze-fence violation (AC#3). */
+export class FreezeFenceViolationError extends Error {
+    instanceId;
+    constructor(instanceId) {
+        super(`freeze-fence violation on instance ${instanceId}: the harness mutated implStateDir ` +
+            `during a frozen-mode eval run — refusing to record a tainted result`);
+        this.instanceId = instanceId;
+        this.name = 'FreezeFenceViolationError';
+    }
+}
+/**
+ * Thrown when the locally-evaluated impl-state's codeDigest does not match the
+ * named checkpoint's manifest (C1). Guards against persisting + comparing
+ * results under a checkpoint name while the operator's local impl-state has
+ * drifted from that checkpoint.
+ */
+export class CheckpointStateMismatchError extends Error {
+    checkpointCid;
+    manifestCodeDigest;
+    evaluatedCodeDigest;
+    constructor(checkpointCid, manifestCodeDigest, evaluatedCodeDigest) {
+        super(`local impl-state does not match checkpoint ${checkpointCid}: ` +
+            `manifest codeDigest=${manifestCodeDigest} but the evaluated impl-state hashes to ` +
+            `${evaluatedCodeDigest}. Check out the impl-state for this checkpoint or pass the ` +
+            `correct --impl-state-dir; refusing to record results under a codeDigest that is a lie`);
+        this.checkpointCid = checkpointCid;
+        this.manifestCodeDigest = manifestCodeDigest;
+        this.evaluatedCodeDigest = evaluatedCodeDigest;
+        this.name = 'CheckpointStateMismatchError';
+    }
+}
+/** Thrown when the parent checkpoint has no aggregate for this slate version (AC#2). */
+export class ParentNotEvaluatedError extends Error {
+    parentCheckpointCid;
+    slateVersion;
+    constructor(parentCheckpointCid, slateVersion) {
+        super(`parent checkpoint ${parentCheckpointCid} has no eval results for slate ${slateVersion} — ` +
+            `eval the parent checkpoint first (scores are only comparable within a slate version)`);
+        this.parentCheckpointCid = parentCheckpointCid;
+        this.slateVersion = slateVersion;
+        this.name = 'ParentNotEvaluatedError';
+    }
+}
+/**
+ * Thrown when EITHER compared checkpoint was scored on a DIFFERENT slate
+ * *content* under the same version label (DR-2026-05-28 §2 — confounder control).
+ * A slate is content-addressed and scores are comparable ONLY within identical
+ * content; the store keys aggregates by version, so a content edit that skipped
+ * the version bump (or re-derived the declared hash — which the slate loader
+ * admits passes), or stale rows surviving `recordEvalResult`'s by-instance
+ * upsert, would let the comparison silently compare two checkpoints scored on
+ * different task sets. That is confounder #1 (task-selection) — exactly what the
+ * held-out exam exists to defeat — so we refuse the comparison loudly instead.
+ */
+export class SlateHashMismatchError extends Error {
+    slateVersion;
+    currentSlateHash;
+    drifted;
+    constructor(slateVersion, currentSlateHash, drifted) {
+        const detail = drifted
+            .map((d) => `${d.checkpointCid} carries [${d.hashes.join(', ')}]`)
+            .join('; ');
+        super(`a checkpoint was evaluated against a different slate content under the same version label ` +
+            `${slateVersion}: current slate hash is ${currentSlateHash} but ${detail}. Same version must ` +
+            `mean the same exam — a slate content change is a measurement discontinuity that must bump the ` +
+            `version. Bump the slate version and re-evaluate; refusing to report a delta across two ` +
+            `different task sets (confounder #1, task-selection)`);
+        this.slateVersion = slateVersion;
+        this.currentSlateHash = currentSlateHash;
+        this.drifted = drifted;
+        this.name = 'SlateHashMismatchError';
+    }
+}
+export async function runEval(args) {
+    const { deps, slate, checkpointCid, parentCheckpointCid } = args;
+    // Confounder guard (DR-2026-05-28 §2). The store keys aggregates by slate
+    // VERSION; if EITHER checkpoint has results recorded under a different slate
+    // CONTENT for that version — a content edit that skipped the version bump, or
+    // stale rows surviving recordEvalResult's by-instance upsert — comparing
+    // child-vs-parent reintroduces confounder #1 (task-selection: two scores over
+    // different task sets). Both arms' hashes are known up front, so detect drift
+    // in EITHER before the loop and refuse — never burn N× real spend producing a
+    // number we'd have to throw away.
+    const drifted = [parentCheckpointCid, checkpointCid]
+        .map((cid) => ({
+        checkpointCid: cid,
+        hashes: deps.store.getEvalSlateHashes(cid, slate.version).filter((hash) => hash !== slate.hash),
+    }))
+        .filter((arm) => arm.hashes.length > 0);
+    if (drifted.length > 0) {
+        throw new SlateHashMismatchError(slate.version, slate.hash, drifted);
+    }
+    // Hoist the impl-state-dir fetch outside the loop: every slate instance runs
+    // against the SAME checkpoint state.
+    const implStateDir = args.implStateDir ?? `/tmp/jinn-eval-${checkpointCid}`;
+    const localImplStateDir = await deps.fetchImplStateDirToLocal(args.checkpointManifest.implStateDirCid, implStateDir);
+    const perTask = [];
+    const runAtMs = Date.now();
+    let digestChecked = false;
+    // Real digest of the evaluated impl-state (from the freeze-fence), captured on
+    // the first run for provenance. Falls back to the manifest digest, which the
+    // C1 guard proves equal.
+    let evaluatedDigest = args.checkpointManifest.codeDigest;
+    for (const { task, row } of args.tasksWithRows) {
+        let passed;
+        let unscorable = false;
+        let logExcerpt = '';
+        try {
+            const run = await deps.runHarnessOnce({
+                harness: deps.harness,
+                implStateDir: localImplStateDir,
+                mode: 'frozen',
+                // Carry the full SweRebenchV2Task in `spec` + the `solverType` dispatch
+                // key so the harness can restore the repo (clone/checkout from
+                // spec.repo/base_commit). The harness emits a swe-rebench-v2 patch from
+                // spec; id/description/role alone are insufficient for a real run.
+                task: {
+                    id: task.instance_id,
+                    description: task.problem_statement,
+                    role: 'restoration',
+                    solverType: 'swe-rebench-v2.v1',
+                    spec: task,
+                    window: { startTs: 0, endTs: Date.now() + 3_600_000 },
+                },
+            });
+            // AC#3: a freeze violation is loud and terminal — do NOT record this
+            // instance (its implStateDir mutation taints the run). Thrown OUTSIDE the
+            // unscorable catch below so it propagates and aborts the whole eval.
+            if (run.violation) {
+                throw new FreezeFenceViolationError(task.instance_id);
+            }
+            if (!run.solution) {
+                throw new Error(`harness produced no solution for instance ${task.instance_id}`);
+            }
+            // C1: on the FIRST successful run, verify the real digest of the evaluated
+            // impl-state (computed by the freeze-fence) matches the named checkpoint's
+            // manifest. Fail fast — before grading any instance — to avoid Docker spend
+            // and to never persist a result under a codeDigest that is a lie.
+            if (!digestChecked) {
+                digestChecked = true;
+                const runDigest = run.envelope?.executor.codeDigest;
+                if (runDigest && runDigest !== args.checkpointManifest.codeDigest) {
+                    throw new CheckpointStateMismatchError(checkpointCid, args.checkpointManifest.codeDigest, runDigest);
+                }
+                if (runDigest)
+                    evaluatedDigest = runDigest;
+            }
+            const verdict = await deps.evaluator.grade({
+                task,
+                solutionPayload: { schemaVersion: 'swe-rebench-v2-solution.v1', patch: run.solution.patch },
+                row,
+            });
+            passed = verdict.passed_match;
+            logExcerpt = verdict.test_log.slice(0, 1000);
+        }
+        catch (err) {
+            // A freeze-fence violation taints the run and must stay a LOUD terminal
+            // abort (AC#3). Likewise the C1 digest mismatch — never record results
+            // under a codeDigest that is a lie. Both re-throw and abort the eval.
+            if (err instanceof FreezeFenceViolationError || err instanceof CheckpointStateMismatchError) {
+                throw err;
+            }
+            // Everything else is unscorable: excluded from the denominator, NEVER
+            // coerced to a fail (#476). This covers both grade-side failures
+            // (EvalCouldNotGradeError / InsufficientDiskError) and harness-run
+            // failures (Defect A — harvest missing-artifact, "produced no patch",
+            // clone/timeout, etc.); a harness/infra failure to produce a gradeable
+            // solution is environment-side, not an agent capability fail. The slate
+            // continues to the next instance.
+            passed = null;
+            unscorable = true;
+            logExcerpt = (err instanceof Error ? err.message : String(err)).slice(0, 1000);
+        }
+        deps.store.recordEvalResult({
+            checkpoint_cid: checkpointCid,
+            slate_hash: slate.hash,
+            slate_version: slate.version,
+            instance_id: task.instance_id,
+            passed,
+            unscorable,
+            code_digest: args.checkpointManifest.codeDigest,
+            run_at_ms: runAtMs,
+            test_log_excerpt: logExcerpt,
+        });
+        perTask.push({ instance_id: task.instance_id, passed, unscorable });
+    }
+    // AC#2: compare child vs parent at the SAME slate version. A parent with no
+    // rows is a hard error — no cross-version compare, eval the parent first.
+    const parentAgg = deps.store.getEvalAggregate(parentCheckpointCid, slate.version);
+    if (parentAgg.scorable === 0 && parentAgg.unscorable === 0) {
+        throw new ParentNotEvaluatedError(parentCheckpointCid, slate.version);
+    }
+    const childAgg = deps.store.getEvalAggregate(checkpointCid, slate.version);
+    const comparison = compareRates({ passed: childAgg.passed, scorable: childAgg.scorable }, { passed: parentAgg.passed, scorable: parentAgg.scorable });
+    // Paired (matched-design) verdict — same slate scored before & after, so the
+    // matched McNemar test is the correct, higher-power one. Additive: only when
+    // the store exposes per-instance results (DR-2026-06-02-b §2a).
+    let paired;
+    if (deps.store.getEvalResults) {
+        const parentResults = deps.store.getEvalResults(parentCheckpointCid, slate.version);
+        const childResults = deps.store.getEvalResults(checkpointCid, slate.version);
+        paired = comparePaired(parentResults, childResults);
+    }
+    return {
+        perTask,
+        comparison,
+        ...(paired ? { paired } : {}),
+        evaluated: { implStateDir: localImplStateDir, codeDigest: evaluatedDigest, matchedCheckpoint: true },
+    };
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/eval/orchestrator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/eval/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAQH,OAAO,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,aAAa,EAA2C,MAAM,aAAa,CAAC;AAErF,4EAA4E;AAC5E,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IACtB;IAA5B,YAA4B,UAAkB;QAC5C,KAAK,CACH,sCAAsC,UAAU,qCAAqC;YACnF,qEAAqE,CACxE,CAAC;QAJwB,eAAU,GAAV,UAAU,CAAQ;QAK5C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAEnC;IACA;IACA;IAHlB,YACkB,aAAqB,EACrB,kBAA0B,EAC1B,mBAA2B;QAE3C,KAAK,CACH,8CAA8C,aAAa,IAAI;YAC7D,uBAAuB,kBAAkB,0CAA0C;YACnF,GAAG,mBAAmB,6DAA6D;YACnF,uFAAuF,CAC1F,CAAC;QATc,kBAAa,GAAb,aAAa,CAAQ;QACrB,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAQ3C,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;IAC7C,CAAC;CACF;AAED,wFAAwF;AACxF,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACpB;IAA6C;IAAzE,YAA4B,mBAA2B,EAAkB,YAAoB;QAC3F,KAAK,CACH,qBAAqB,mBAAmB,kCAAkC,YAAY,KAAK;YACzF,sFAAsF,CACzF,CAAC;QAJwB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAAkB,iBAAY,GAAZ,YAAY,CAAQ;QAK3F,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AASD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAE7B;IACA;IACA;IAHlB,YACkB,YAAoB,EACpB,gBAAwB,EACxB,OAAsC;QAEtD,MAAM,MAAM,GAAG,OAAO;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;aACjE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CACH,4FAA4F;YAC1F,GAAG,YAAY,2BAA2B,gBAAgB,QAAQ,MAAM,sBAAsB;YAC9F,gGAAgG;YAChG,yFAAyF;YACzF,qDAAqD,CACxD,CAAC;QAbc,iBAAY,GAAZ,YAAY,CAAQ;QACpB,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,YAAO,GAAP,OAAO,CAA+B;QAYtD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AA8ED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAS7B;IACC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,GAAG,IAAI,CAAC;IAEjE,0EAA0E;IAC1E,6EAA6E;IAC7E,8EAA8E;IAC9E,yEAAyE;IACzE,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,OAAO,GAAG,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACjD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACb,aAAa,EAAE,GAAG;QAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;KAChG,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAED,6EAA6E;IAC7E,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,kBAAkB,aAAa,EAAE,CAAC;IAC5E,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC3D,IAAI,CAAC,kBAAkB,CAAC,eAAe,EACvC,YAAY,CACb,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,+EAA+E;IAC/E,6EAA6E;IAC7E,yBAAyB;IACzB,IAAI,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC;IAEzD,KAAK,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/C,IAAI,MAAsB,CAAC;QAC3B,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,iBAAiB;gBAC/B,IAAI,EAAE,QAAQ;gBACd,wEAAwE;gBACxE,+DAA+D;gBAC/D,wEAAwE;gBACxE,mEAAmE;gBACnE,IAAI,EAAE;oBACJ,EAAE,EAAE,IAAI,CAAC,WAAW;oBACpB,WAAW,EAAE,IAAI,CAAC,iBAAiB;oBACnC,IAAI,EAAE,aAAa;oBACnB,UAAU,EAAE,mBAAmB;oBAC/B,IAAI,EAAE,IAA0C;oBAChD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE;iBACtD;aACF,CAAC,CAAC;YAEH,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,2EAA2E;YAC3E,2EAA2E;YAC3E,4EAA4E;YAC5E,kEAAkE;YAClE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC;gBACpD,IAAI,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC;oBAClE,MAAM,IAAI,4BAA4B,CACpC,aAAa,EACb,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAClC,SAAS,CACV,CAAC;gBACJ,CAAC;gBACD,IAAI,SAAS;oBAAE,eAAe,GAAG,SAAS,CAAC;YAC7C,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBACzC,IAAI;gBACJ,eAAe,EAAE,EAAE,aAAa,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAC3F,GAAG;aACJ,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAC9B,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,IAAI,GAAG,YAAY,yBAAyB,IAAI,GAAG,YAAY,4BAA4B,EAAE,CAAC;gBAC5F,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,sEAAsE;YACtE,iEAAiE;YACjE,mEAAmE;YACnE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,MAAM,GAAG,IAAI,CAAC;YACd,UAAU,GAAG,IAAI,CAAC;YAClB,UAAU,GAAG,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;YAC1B,cAAc,EAAE,aAAa;YAC7B,UAAU,EAAE,KAAK,CAAC,IAAI;YACtB,aAAa,EAAE,KAAK,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;YACN,UAAU;YACV,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,UAAU;YAC/C,SAAS,EAAE,OAAO;YAClB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAClF,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,YAAY,CAC7B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EACxD,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAC3D,CAAC;IAEF,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,IAAI,MAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7E,MAAM,GAAG,aAAa,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,OAAO;QACP,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,eAAe,EAAE,iBAAiB,EAAE,IAAI,EAAE;KACrG,CAAC;AACJ,CAAC"}

package/dist/eval/paired.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/** Minimal per-instance result (subset of PerTaskResult / EvalResultRow). */
+export interface PairedInput {
+    instance_id: string;
+    /** null when unscorable. */
+    passed: boolean | null;
+    unscorable: boolean;
+}
+export type PairedVerdict = 'trustworthy' | 'within-noise';
+export interface PairedComparison {
+    /** Instances scorable (passed !== null) in BOTH arms — the matched pairs. */
+    pairs: number;
+    /** b: before fail → after pass (improvements). */
+    improved: number;
+    /** c: before pass → after fail (regressions). */
+    regressed: number;
+    concordantPass: number;
+    concordantFail: number;
+    /** Instances unscorable/absent in either arm — excluded from the paired test. */
+    excluded: number;
+    /** Exact two-sided McNemar p-value over the discordant pairs. */
+    pValue: number;
+    /** 'trustworthy' iff pValue < alpha AND improved > regressed; else 'within-noise'. */
+    verdict: PairedVerdict;
+}
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export declare function mcnemarExact(b: number, c: number): number;
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export declare function comparePaired(before: readonly PairedInput[], after: readonly PairedInput[], opts?: {
+    alpha?: number;
+}): PairedComparison;