@jinn-network/client 0.1.8 → 0.1.9-canary.144d87d2

package/dist/eval/orchestrator.js

@@ -0,0 +1,250 @@
+/**
+ * `jinn eval` held-out checkpoint orchestrator (issue #818).
+ *
+ * Runs a held-out slate against a checkpoint in FROZEN mode, persists per-task
+ * pass/fail, and emits a Wilson-CI resolved-rate comparison vs the parent
+ * checkpoint. Every external boundary is constructor-injected — this is the
+ * thin slice #819 drives a deterministic slate against with NO Docker/IPFS.
+ *
+ * Acceptance criteria:
+ *   AC#1 — run the slate frozen, write per-task pass/fail.
+ *   AC#2 — emit a resolved-rate comparison vs the parent with a CI.
+ *   AC#3 — the freeze-fence holds: a `runHarnessOnce` `{ violation }` throws
+ *          LOUD and the instance is NOT recorded (no implStateDir mutation
+ *          slips through; enforcement lives in `runHarnessWithFreezeFence`).
+ *
+ * Per log/decisions/2026-05-28-rl-eval-measurement.md §4: v1-simple. Only large
+ * deltas are trustworthy (disjoint Wilson intervals). No seed control, no
+ * multi-run averaging.
+ */
+import { compareRates } from './wilson.js';
+import { comparePaired } from './paired.js';
+/** Thrown when `runHarnessOnce` reports a freeze-fence violation (AC#3). */
+export class FreezeFenceViolationError extends Error {
+    instanceId;
+    constructor(instanceId) {
+        super(`freeze-fence violation on instance ${instanceId}: the harness mutated implStateDir ` +
+            `during a frozen-mode eval run — refusing to record a tainted result`);
+        this.instanceId = instanceId;
+        this.name = 'FreezeFenceViolationError';
+    }
+}
+/**
+ * Thrown when the locally-evaluated impl-state's codeDigest does not match the
+ * named checkpoint's manifest (C1). Guards against persisting + comparing
+ * results under a checkpoint name while the operator's local impl-state has
+ * drifted from that checkpoint.
+ */
+export class CheckpointStateMismatchError extends Error {
+    checkpointCid;
+    manifestCodeDigest;
+    evaluatedCodeDigest;
+    constructor(checkpointCid, manifestCodeDigest, evaluatedCodeDigest) {
+        super(`local impl-state does not match checkpoint ${checkpointCid}: ` +
+            `manifest codeDigest=${manifestCodeDigest} but the evaluated impl-state hashes to ` +
+            `${evaluatedCodeDigest}. Check out the impl-state for this checkpoint or pass the ` +
+            `correct --impl-state-dir; refusing to record results under a codeDigest that is a lie`);
+        this.checkpointCid = checkpointCid;
+        this.manifestCodeDigest = manifestCodeDigest;
+        this.evaluatedCodeDigest = evaluatedCodeDigest;
+        this.name = 'CheckpointStateMismatchError';
+    }
+}
+/** Thrown when the parent checkpoint has no aggregate for this slate version (AC#2). */
+export class ParentNotEvaluatedError extends Error {
+    parentCheckpointCid;
+    slateVersion;
+    constructor(parentCheckpointCid, slateVersion) {
+        super(`parent checkpoint ${parentCheckpointCid} has no eval results for slate ${slateVersion} — ` +
+            `eval the parent checkpoint first (scores are only comparable within a slate version)`);
+        this.parentCheckpointCid = parentCheckpointCid;
+        this.slateVersion = slateVersion;
+        this.name = 'ParentNotEvaluatedError';
+    }
+}
+/**
+ * Thrown when EITHER compared checkpoint was scored on a DIFFERENT slate
+ * *content* under the same version label (DR-2026-05-28 §2 — confounder control).
+ * A slate is content-addressed and scores are comparable ONLY within identical
+ * content; the store keys aggregates by version, so a content edit that skipped
+ * the version bump (or re-derived the declared hash — which the slate loader
+ * admits passes), or stale rows surviving `recordEvalResult`'s by-instance
+ * upsert, would let the comparison silently compare two checkpoints scored on
+ * different task sets. That is confounder #1 (task-selection) — exactly what the
+ * held-out exam exists to defeat — so we refuse the comparison loudly instead.
+ */
+export class SlateHashMismatchError extends Error {
+    slateVersion;
+    currentSlateHash;
+    drifted;
+    constructor(slateVersion, currentSlateHash, drifted) {
+        const detail = drifted
+            .map((d) => `${d.checkpointCid} carries [${d.hashes.join(', ')}]`)
+            .join('; ');
+        super(`a checkpoint was evaluated against a different slate content under the same version label ` +
+            `${slateVersion}: current slate hash is ${currentSlateHash} but ${detail}. Same version must ` +
+            `mean the same exam — a slate content change is a measurement discontinuity that must bump the ` +
+            `version. Bump the slate version and re-evaluate; refusing to report a delta across two ` +
+            `different task sets (confounder #1, task-selection)`);
+        this.slateVersion = slateVersion;
+        this.currentSlateHash = currentSlateHash;
+        this.drifted = drifted;
+        this.name = 'SlateHashMismatchError';
+    }
+}
+/**
+ * Default swe-rebench-v2 instance mapping — preserves the exact harness `task`
+ * envelope and grade call the orchestrator used before the backend seam existed.
+ */
+function sweRebenchSlateInstance(rt) {
+    return {
+        instance_id: rt.task.instance_id,
+        // Carry the full SweRebenchV2Task in `spec` + the `solverType` dispatch key
+        // so the harness can restore the repo (clone/checkout from
+        // spec.repo/base_commit). id/description/role alone are insufficient.
+        harnessTask: {
+            id: rt.task.instance_id,
+            description: rt.task.problem_statement,
+            role: 'restoration',
+            solverType: 'swe-rebench-v2.v1',
+            spec: rt.task,
+            window: { startTs: 0, endTs: Date.now() + 3_600_000 },
+        },
+        gradeTask: rt.task,
+        solutionSchemaVersion: 'swe-rebench-v2-solution.v1',
+        row: rt.row,
+    };
+}
+export async function runEval(args) {
+    const { deps, slate, checkpointCid, parentCheckpointCid } = args;
+    const slateInstances = args.slateInstances ?? (args.tasksWithRows ?? []).map(sweRebenchSlateInstance);
+    // Confounder guard (DR-2026-05-28 §2). The store keys aggregates by slate
+    // VERSION; if EITHER checkpoint has results recorded under a different slate
+    // CONTENT for that version — a content edit that skipped the version bump, or
+    // stale rows surviving recordEvalResult's by-instance upsert — comparing
+    // child-vs-parent reintroduces confounder #1 (task-selection: two scores over
+    // different task sets). Both arms' hashes are known up front, so detect drift
+    // in EITHER before the loop and refuse — never burn N× real spend producing a
+    // number we'd have to throw away.
+    const drifted = [parentCheckpointCid, checkpointCid]
+        .map((cid) => ({
+        checkpointCid: cid,
+        hashes: deps.store.getEvalSlateHashes(cid, slate.version).filter((hash) => hash !== slate.hash),
+    }))
+        .filter((arm) => arm.hashes.length > 0);
+    if (drifted.length > 0) {
+        throw new SlateHashMismatchError(slate.version, slate.hash, drifted);
+    }
+    // Hoist the impl-state-dir fetch outside the loop: every slate instance runs
+    // against the SAME checkpoint state.
+    const implStateDir = args.implStateDir ?? `/tmp/jinn-eval-${checkpointCid}`;
+    const localImplStateDir = await deps.fetchImplStateDirToLocal(args.checkpointManifest.implStateDirCid, implStateDir);
+    const perTask = [];
+    const runAtMs = Date.now();
+    let digestChecked = false;
+    // Real digest of the evaluated impl-state (from the freeze-fence), captured on
+    // the first run for provenance. Falls back to the manifest digest, which the
+    // C1 guard proves equal.
+    let evaluatedDigest = args.checkpointManifest.codeDigest;
+    for (const inst of slateInstances) {
+        const task = inst.gradeTask;
+        let passed;
+        let unscorable = false;
+        let logExcerpt = '';
+        try {
+            const run = await deps.runHarnessOnce({
+                harness: deps.harness,
+                implStateDir: localImplStateDir,
+                mode: 'frozen',
+                // The harness task is built by the BACKEND (eval.ts) and is already
+                // leak-controlled (jinn-repo hands the solver `solverView(item)` only —
+                // no gold tests / reference solution). swe-rebench-v2 carries the full
+                // task in `spec` so the harness can restore the repo.
+                task: inst.harnessTask,
+            });
+            // AC#3: a freeze violation is loud and terminal — do NOT record this
+            // instance (its implStateDir mutation taints the run). Thrown OUTSIDE the
+            // unscorable catch below so it propagates and aborts the whole eval.
+            if (run.violation) {
+                throw new FreezeFenceViolationError(inst.instance_id);
+            }
+            if (!run.solution) {
+                throw new Error(`harness produced no solution for instance ${inst.instance_id}`);
+            }
+            // C1: on the FIRST successful run, verify the real digest of the evaluated
+            // impl-state (computed by the freeze-fence) matches the named checkpoint's
+            // manifest. Fail fast — before grading any instance — to avoid Docker spend
+            // and to never persist a result under a codeDigest that is a lie.
+            if (!digestChecked) {
+                digestChecked = true;
+                const runDigest = run.envelope?.executor.codeDigest;
+                if (runDigest && runDigest !== args.checkpointManifest.codeDigest) {
+                    throw new CheckpointStateMismatchError(checkpointCid, args.checkpointManifest.codeDigest, runDigest);
+                }
+                if (runDigest)
+                    evaluatedDigest = runDigest;
+            }
+            const verdict = await deps.evaluator.grade({
+                task,
+                solutionPayload: { schemaVersion: inst.solutionSchemaVersion, patch: run.solution.patch },
+                ...(inst.row !== undefined ? { row: inst.row } : {}),
+            });
+            passed = verdict.passed_match;
+            logExcerpt = verdict.test_log.slice(0, 1000);
+        }
+        catch (err) {
+            // A freeze-fence violation taints the run and must stay a LOUD terminal
+            // abort (AC#3). Likewise the C1 digest mismatch — never record results
+            // under a codeDigest that is a lie. Both re-throw and abort the eval.
+            if (err instanceof FreezeFenceViolationError || err instanceof CheckpointStateMismatchError) {
+                throw err;
+            }
+            // Everything else is unscorable: excluded from the denominator, NEVER
+            // coerced to a fail (#476). This covers both grade-side failures
+            // (EvalCouldNotGradeError / InsufficientDiskError) and harness-run
+            // failures (Defect A — harvest missing-artifact, "produced no patch",
+            // clone/timeout, etc.); a harness/infra failure to produce a gradeable
+            // solution is environment-side, not an agent capability fail. The slate
+            // continues to the next instance.
+            passed = null;
+            unscorable = true;
+            logExcerpt = (err instanceof Error ? err.message : String(err)).slice(0, 1000);
+        }
+        deps.store.recordEvalResult({
+            checkpoint_cid: checkpointCid,
+            slate_hash: slate.hash,
+            slate_version: slate.version,
+            instance_id: inst.instance_id,
+            passed,
+            unscorable,
+            code_digest: args.checkpointManifest.codeDigest,
+            run_at_ms: runAtMs,
+            test_log_excerpt: logExcerpt,
+        });
+        perTask.push({ instance_id: inst.instance_id, passed, unscorable });
+    }
+    // AC#2: compare child vs parent at the SAME slate version. A parent with no
+    // rows is a hard error — no cross-version compare, eval the parent first.
+    const parentAgg = deps.store.getEvalAggregate(parentCheckpointCid, slate.version);
+    if (parentAgg.scorable === 0 && parentAgg.unscorable === 0) {
+        throw new ParentNotEvaluatedError(parentCheckpointCid, slate.version);
+    }
+    const childAgg = deps.store.getEvalAggregate(checkpointCid, slate.version);
+    const comparison = compareRates({ passed: childAgg.passed, scorable: childAgg.scorable }, { passed: parentAgg.passed, scorable: parentAgg.scorable });
+    // Paired (matched-design) verdict — same slate scored before & after, so the
+    // matched McNemar test is the correct, higher-power one. Additive: only when
+    // the store exposes per-instance results (DR-2026-06-02-b §2a).
+    let paired;
+    if (deps.store.getEvalResults) {
+        const parentResults = deps.store.getEvalResults(parentCheckpointCid, slate.version);
+        const childResults = deps.store.getEvalResults(checkpointCid, slate.version);
+        paired = comparePaired(parentResults, childResults);
+    }
+    return {
+        perTask,
+        comparison,
+        ...(paired ? { paired } : {}),
+        evaluated: { implStateDir: localImplStateDir, codeDigest: evaluatedDigest, matchedCheckpoint: true },
+    };
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/eval/orchestrator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/eval/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAOH,OAAO,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,aAAa,EAA2C,MAAM,aAAa,CAAC;AAErF,4EAA4E;AAC5E,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IACtB;IAA5B,YAA4B,UAAkB;QAC5C,KAAK,CACH,sCAAsC,UAAU,qCAAqC;YACnF,qEAAqE,CACxE,CAAC;QAJwB,eAAU,GAAV,UAAU,CAAQ;QAK5C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAEnC;IACA;IACA;IAHlB,YACkB,aAAqB,EACrB,kBAA0B,EAC1B,mBAA2B;QAE3C,KAAK,CACH,8CAA8C,aAAa,IAAI;YAC7D,uBAAuB,kBAAkB,0CAA0C;YACnF,GAAG,mBAAmB,6DAA6D;YACnF,uFAAuF,CAC1F,CAAC;QATc,kBAAa,GAAb,aAAa,CAAQ;QACrB,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAQ3C,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;IAC7C,CAAC;CACF;AAED,wFAAwF;AACxF,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACpB;IAA6C;IAAzE,YAA4B,mBAA2B,EAAkB,YAAoB;QAC3F,KAAK,CACH,qBAAqB,mBAAmB,kCAAkC,YAAY,KAAK;YACzF,sFAAsF,CACzF,CAAC;QAJwB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAAkB,iBAAY,GAAZ,YAAY,CAAQ;QAK3F,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AASD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAE7B;IACA;IACA;IAHlB,YACkB,YAAoB,EACpB,gBAAwB,EACxB,OAAsC;QAEtD,MAAM,MAAM,GAAG,OAAO;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;aACjE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CACH,4FAA4F;YAC1F,GAAG,YAAY,2BAA2B,gBAAgB,QAAQ,MAAM,sBAAsB;YAC9F,gGAAgG;YAChG,yFAAyF;YACzF,qDAAqD,CACxD,CAAC;QAbc,iBAAY,GAAZ,YAAY,CAAQ;QACpB,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,YAAO,GAAP,OAAO,CAA+B;QAYtD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAgID;;;GAGG;AACH,SAAS,uBAAuB,CAAC,EAAqB;IACpD,OAAO;QACL,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW;QAChC,4EAA4E;QAC5E,2DAA2D;QAC3D,sEAAsE;QACtE,WAAW,EAAE;YACX,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW;YACvB,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB;YACtC,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,mBAAmB;YAC/B,IAAI,EAAE,EAAE,CAAC,IAA0C;YACnD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE;SACtD;QACD,SAAS,EAAE,EAAE,CAAC,IAAI;QAClB,qBAAqB,EAAE,4BAA4B;QACnD,GAAG,EAAE,EAAE,CAAC,GAAG;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAiB7B;IACC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,GAAG,IAAI,CAAC;IACjE,MAAM,cAAc,GAClB,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IAEjF,0EAA0E;IAC1E,6EAA6E;IAC7E,8EAA8E;IAC9E,yEAAyE;IACzE,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,OAAO,GAAG,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACjD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACb,aAAa,EAAE,GAAG;QAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;KAChG,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAED,6EAA6E;IAC7E,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,kBAAkB,aAAa,EAAE,CAAC;IAC5E,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC3D,IAAI,CAAC,kBAAkB,CAAC,eAAe,EACvC,YAAY,CACb,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,+EAA+E;IAC/E,6EAA6E;IAC7E,yBAAyB;IACzB,IAAI,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC;IAEzD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC5B,IAAI,MAAsB,CAAC;QAC3B,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,iBAAiB;gBAC/B,IAAI,EAAE,QAAQ;gBACd,oEAAoE;gBACpE,wEAAwE;gBACxE,uEAAuE;gBACvE,sDAAsD;gBACtD,IAAI,EAAE,IAAI,CAAC,WAAW;aACvB,CAAC,CAAC;YAEH,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,2EAA2E;YAC3E,2EAA2E;YAC3E,4EAA4E;YAC5E,kEAAkE;YAClE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC;gBACpD,IAAI,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC;oBAClE,MAAM,IAAI,4BAA4B,CACpC,aAAa,EACb,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAClC,SAAS,CACV,CAAC;gBACJ,CAAC;gBACD,IAAI,SAAS;oBAAE,eAAe,GAAG,SAAS,CAAC;YAC7C,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBACzC,IAAI;gBACJ,eAAe,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBACzF,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAC9B,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,IAAI,GAAG,YAAY,yBAAyB,IAAI,GAAG,YAAY,4BAA4B,EAAE,CAAC;gBAC5F,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,sEAAsE;YACtE,iEAAiE;YACjE,mEAAmE;YACnE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,MAAM,GAAG,IAAI,CAAC;YACd,UAAU,GAAG,IAAI,CAAC;YAClB,UAAU,GAAG,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;YAC1B,cAAc,EAAE,aAAa;YAC7B,UAAU,EAAE,KAAK,CAAC,IAAI;YACtB,aAAa,EAAE,KAAK,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;YACN,UAAU;YACV,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,UAAU;YAC/C,SAAS,EAAE,OAAO;YAClB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAClF,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,YAAY,CAC7B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EACxD,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAC3D,CAAC;IAEF,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,IAAI,MAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7E,MAAM,GAAG,aAAa,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,OAAO;QACP,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,eAAe,EAAE,iBAAiB,EAAE,IAAI,EAAE;KACrG,CAAC;AACJ,CAAC"}

package/dist/eval/paired.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/** Minimal per-instance result (subset of PerTaskResult / EvalResultRow). */
+export interface PairedInput {
+    instance_id: string;
+    /** null when unscorable. */
+    passed: boolean | null;
+    unscorable: boolean;
+}
+export type PairedVerdict = 'trustworthy' | 'within-noise';
+export interface PairedComparison {
+    /** Instances scorable (passed !== null) in BOTH arms — the matched pairs. */
+    pairs: number;
+    /** b: before fail → after pass (improvements). */
+    improved: number;
+    /** c: before pass → after fail (regressions). */
+    regressed: number;
+    concordantPass: number;
+    concordantFail: number;
+    /** Instances unscorable/absent in either arm — excluded from the paired test. */
+    excluded: number;
+    /** Exact two-sided McNemar p-value over the discordant pairs. */
+    pValue: number;
+    /** 'trustworthy' iff pValue < alpha AND improved > regressed; else 'within-noise'. */
+    verdict: PairedVerdict;
+}
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export declare function mcnemarExact(b: number, c: number): number;
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export declare function comparePaired(before: readonly PairedInput[], after: readonly PairedInput[], opts?: {
+    alpha?: number;
+}): PairedComparison;

package/dist/eval/paired.js

@@ -0,0 +1,93 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export function mcnemarExact(b, c) {
+    const n = b + c;
+    if (n === 0)
+        return 1;
+    const k = Math.min(b, c);
+    // sum_{i=0}^{k} C(n,i) * 0.5^n, via prob_0 = 0.5^n, prob_i = prob_{i-1}*(n-i+1)/i.
+    let term = Math.pow(0.5, n); // i = 0
+    let cdf = term;
+    for (let i = 1; i <= k; i++) {
+        term = (term * (n - i + 1)) / i;
+        cdf += term;
+    }
+    return Math.min(1, 2 * cdf);
+}
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export function comparePaired(before, after, opts = {}) {
+    const alpha = opts.alpha ?? 0.05;
+    const scorable = (r) => !r.unscorable && r.passed !== null;
+    const beforeById = new Map(before.map((r) => [r.instance_id, r]));
+    let improved = 0;
+    let regressed = 0;
+    let concordantPass = 0;
+    let concordantFail = 0;
+    let pairs = 0;
+    let excluded = 0;
+    const seen = new Set();
+    for (const a of after) {
+        seen.add(a.instance_id);
+        const b = beforeById.get(a.instance_id);
+        if (!b || !scorable(b) || !scorable(a)) {
+            excluded++;
+            continue;
+        }
+        pairs++;
+        const wasPass = b.passed === true;
+        const nowPass = a.passed === true;
+        if (!wasPass && nowPass)
+            improved++;
+        else if (wasPass && !nowPass)
+            regressed++;
+        else if (wasPass && nowPass)
+            concordantPass++;
+        else
+            concordantFail++;
+    }
+    // before-only instances (absent from after) are excluded too.
+    for (const b of before)
+        if (!seen.has(b.instance_id))
+            excluded++;
+    const pValue = mcnemarExact(improved, regressed);
+    const verdict = pValue < alpha && improved > regressed ? 'trustworthy' : 'within-noise';
+    return { pairs, improved, regressed, concordantPass, concordantFail, excluded, pValue, verdict };
+}
+//# sourceMappingURL=paired.js.map

package/dist/eval/paired.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paired.js","sourceRoot":"","sources":["../../src/eval/paired.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AA6BH;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,mFAAmF;IACnF,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ;IACrC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChC,GAAG,IAAI,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,MAA8B,EAC9B,KAA6B,EAC7B,OAA2B,EAAE;IAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAc,EAAW,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;IACjF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAElE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,KAAK,EAAE,CAAC;QACR,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QAClC,IAAI,CAAC,OAAO,IAAI,OAAO;YAAE,QAAQ,EAAE,CAAC;aAC/B,IAAI,OAAO,IAAI,CAAC,OAAO;YAAE,SAAS,EAAE,CAAC;aACrC,IAAI,OAAO,IAAI,OAAO;YAAE,cAAc,EAAE,CAAC;;YACzC,cAAc,EAAE,CAAC;IACxB,CAAC;IACD,8DAA8D;IAC9D,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;YAAE,QAAQ,EAAE,CAAC;IAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACjD,MAAM,OAAO,GACX,MAAM,GAAG,KAAK,IAAI,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC;IAE1E,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACnG,CAAC"}

package/dist/eval/resolve-slate-tasks.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Resolve a held-out slate's `instance_id`s into `SweRebenchV2Task` objects
+ * for the `jinn eval` orchestrator (issue #818, AC#1).
+ *
+ * A slate stores only `instance_id`s (`_swe-rebench-v2-held-out-slate.ts`). The
+ * evaluator grades against the full HuggingFace row, fetched by
+ * `(hf_dataset, hf_split, instance_id)`. The `HfRow` does NOT echo
+ * `hf_dataset`/`hf_split` (see `swe-rebench-v2-evaluator/index.ts`), so the
+ * caller supplies the slate-level dataset+split as args; the resolver verifies
+ * each id exists by fetching its row and returns the `{ task, row }` pair so the
+ * orchestrator reuses the row at grade time (avoids a second fetch).
+ *
+ * No retry logic here — `HttpHfFetcher` owns retries. A fetcher throw for any id
+ * propagates loudly (a missing slate instance is a hard error, never a silent
+ * drop).
+ */
+import { type SweRebenchV2Task } from '../vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js';
+import type { HfFetcher, HfRow } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import type { PoolTask } from '../solver-types/_swe-rebench-v2-pool.js';
+export interface ResolvedSlateTask {
+    task: SweRebenchV2Task;
+    row: HfRow;
+}
+export declare function resolveSlateTasks(args: {
+    /**
+     * The pool tasks for this slate group, each carrying the real
+     * `problem_statement` / `base_commit` / `language` the agent run needs to
+     * solve the instance (the slate stores only ids; the pool is the source of
+     * these fields).
+     */
+    poolTasks: PoolTask[];
+    hf_dataset: string;
+    hf_split: string;
+    fetcher: HfFetcher;
+}): Promise<ResolvedSlateTask[]>;

package/dist/eval/resolve-slate-tasks.js

@@ -0,0 +1,56 @@
+/**
+ * Resolve a held-out slate's `instance_id`s into `SweRebenchV2Task` objects
+ * for the `jinn eval` orchestrator (issue #818, AC#1).
+ *
+ * A slate stores only `instance_id`s (`_swe-rebench-v2-held-out-slate.ts`). The
+ * evaluator grades against the full HuggingFace row, fetched by
+ * `(hf_dataset, hf_split, instance_id)`. The `HfRow` does NOT echo
+ * `hf_dataset`/`hf_split` (see `swe-rebench-v2-evaluator/index.ts`), so the
+ * caller supplies the slate-level dataset+split as args; the resolver verifies
+ * each id exists by fetching its row and returns the `{ task, row }` pair so the
+ * orchestrator reuses the row at grade time (avoids a second fetch).
+ *
+ * No retry logic here — `HttpHfFetcher` owns retries. A fetcher throw for any id
+ * propagates loudly (a missing slate instance is a hard error, never a silent
+ * drop).
+ */
+import { SweRebenchV2TaskSchema } from '../vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js';
+export async function resolveSlateTasks(args) {
+    const tasks = [...args.poolTasks].sort((a, b) => a.instance_id.localeCompare(b.instance_id));
+    const out = [];
+    for (const poolTask of tasks) {
+        const row = await args.fetcher.fetchTaskRow({
+            hf_dataset: args.hf_dataset,
+            hf_split: args.hf_split,
+            instance_id: poolTask.instance_id,
+        });
+        out.push({ task: buildTask(poolTask, args.hf_dataset, args.hf_split, row), row });
+    }
+    return out;
+}
+/**
+ * Construct the `SweRebenchV2Task` the orchestrator hands to the harness +
+ * evaluator. The agent run needs the real `problem_statement` (what to solve)
+ * and `base_commit` (the repo state to check out) — these are threaded through
+ * from the pool task (the generator's mapping in `swe-rebench-v2.ts`).
+ * `hf_dataset`/`hf_split` are the evaluator's row-fetch key; `repo` comes from
+ * the fetched row. A pool task missing an optional field falls back the same
+ * way the generator does, so the object stays schema-valid.
+ */
+function buildTask(poolTask, hf_dataset, hf_split, row) {
+    const language = SweRebenchV2TaskSchema.shape.language.safeParse(poolTask.language);
+    return {
+        schemaVersion: 'swe-rebench-v2.v1',
+        instance_id: poolTask.instance_id,
+        repo: row.repo,
+        base_commit: poolTask.base_commit ?? '0'.repeat(40),
+        language: language.success ? language.data : 'python',
+        problem_statement: poolTask.problem_statement ?? '',
+        interface: poolTask.interface ?? '',
+        hf_dataset,
+        hf_split,
+        deadline_unix: 1,
+        round_month: hf_split.replace('_', '-'),
+    };
+}
+//# sourceMappingURL=resolve-slate-tasks.js.map

package/dist/eval/resolve-slate-tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolve-slate-tasks.js","sourceRoot":"","sources":["../../src/eval/resolve-slate-tasks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,sBAAsB,EAAyB,MAAM,6CAA6C,CAAC;AAS5G,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAWvC;IACC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7F,MAAM,GAAG,GAAwB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;YAC1C,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,SAAS,CAChB,QAAkB,EAClB,UAAkB,EAClB,QAAgB,EAChB,GAAU;IAEV,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpF,OAAO;QACL,aAAa,EAAE,mBAAmB;QAClC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;QACrD,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB,IAAI,EAAE;QACnD,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;QACnC,UAAU;QACV,QAAQ;QACR,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;KACxC,CAAC;AACJ,CAAC"}

package/dist/eval/screen-discovery.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Held-out screening's authoritative "already trained-on" exclusion source (#986).
+ *
+ * Returns the set of swe-rebench-v2 `instance_id`s that have been ATTEMPTED on
+ * the network for a SolverNet — any verdict envelope, passed OR failed,
+ * cross-operator. An attempted instance was executed by an operator, so the
+ * learner trained on it; holding it out later would make a trained-checkpoint
+ * pass count as memorization, not generalization.
+ *
+ * Why the indexer and not the local generator-state: the local ledger only
+ * reflects THIS box's posting and can be stale (a different active generator —
+ * e.g. a hosted operator — posts to its own ledger). The indexer's
+ * `verdictEnvelopeMeta` is the cross-operator, current record. (The on-chain
+ * task/attempt tables carry no instance_id; only the indexer's IPFS enrichment
+ * resolves it — same backing as `DiscoveryAPI.getInstanceSuccessCounts`, minus
+ * the `actualPassed: true` filter so failed attempts count too.)
+ *
+ * Throws on indexer failure — callers MUST abort rather than screen against an
+ * unknown attempted set, because a missing exclusion can silently contaminate
+ * the exam (the whole point of held-out discipline).
+ */
+export declare function fetchAttemptedInstanceIds(discoveryUrl: string, manifestCid: string, fetchImpl?: typeof fetch): Promise<Set<string>>;

package/dist/eval/screen-discovery.js

@@ -0,0 +1,71 @@
+/**
+ * Held-out screening's authoritative "already trained-on" exclusion source (#986).
+ *
+ * Returns the set of swe-rebench-v2 `instance_id`s that have been ATTEMPTED on
+ * the network for a SolverNet — any verdict envelope, passed OR failed,
+ * cross-operator. An attempted instance was executed by an operator, so the
+ * learner trained on it; holding it out later would make a trained-checkpoint
+ * pass count as memorization, not generalization.
+ *
+ * Why the indexer and not the local generator-state: the local ledger only
+ * reflects THIS box's posting and can be stale (a different active generator —
+ * e.g. a hosted operator — posts to its own ledger). The indexer's
+ * `verdictEnvelopeMeta` is the cross-operator, current record. (The on-chain
+ * task/attempt tables carry no instance_id; only the indexer's IPFS enrichment
+ * resolves it — same backing as `DiscoveryAPI.getInstanceSuccessCounts`, minus
+ * the `actualPassed: true` filter so failed attempts count too.)
+ *
+ * Throws on indexer failure — callers MUST abort rather than screen against an
+ * unknown attempted set, because a missing exclusion can silently contaminate
+ * the exam (the whole point of held-out discipline).
+ */
+const ATTEMPTED_QUERY = `
+query InstanceAttempted($cid: String!, $limit: Int!, $after: String) {
+  verdictEnvelopeMetas(
+    where: {
+      solverNetManifestCid: $cid,
+      solverType_starts_with: "swe-rebench-v2",
+      enrichmentStatus: "ok",
+      instanceId_not: ""
+    },
+    limit: $limit,
+    after: $after,
+    orderBy: "enrichedAtBlock",
+    orderDirection: "asc"
+  ) {
+    items { instanceId }
+    pageInfo { hasNextPage endCursor }
+  }
+}`;
+export async function fetchAttemptedInstanceIds(discoveryUrl, manifestCid, fetchImpl = fetch) {
+    const gqlUrl = discoveryUrl.endsWith('/graphql') ? discoveryUrl : `${discoveryUrl.replace(/\/$/, '')}/graphql`;
+    const ids = new Set();
+    let cursor = null;
+    const MAX_PAGES = 20;
+    const PAGE_LIMIT = 1000;
+    for (let page = 0; page < MAX_PAGES; page++) {
+        const res = await fetchImpl(gqlUrl, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ query: ATTEMPTED_QUERY, variables: { cid: manifestCid, limit: PAGE_LIMIT, after: cursor } }),
+        });
+        if (!res.ok) {
+            throw new Error(`held-out screening: indexer attempted-ids query failed (HTTP ${res.status}) at ${gqlUrl}`);
+        }
+        const json = (await res.json());
+        if (json.errors) {
+            throw new Error(`held-out screening: indexer attempted-ids query errors: ${JSON.stringify(json.errors).slice(0, 300)}`);
+        }
+        const conn = json.data?.verdictEnvelopeMetas;
+        for (const item of conn?.items ?? []) {
+            if (item.instanceId)
+                ids.add(item.instanceId);
+        }
+        const pageInfo = conn?.pageInfo;
+        if (!pageInfo?.hasNextPage || !pageInfo.endCursor)
+            break;
+        cursor = pageInfo.endCursor;
+    }
+    return ids;
+}
+//# sourceMappingURL=screen-discovery.js.map

package/dist/eval/screen-discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-discovery.js","sourceRoot":"","sources":["../../src/eval/screen-discovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;;EAiBtB,CAAC;AAYH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,YAAoB,EACpB,WAAmB,EACnB,YAA0B,KAAK;IAE/B,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC;IAC/G,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,UAAU,GAAG,IAAI,CAAC;IAExB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE;YAClC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;SACpH,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gEAAgE,GAAG,CAAC,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC;QAC9G,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAkB,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2DAA2D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1H,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC;QAC7C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,UAAU;gBAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,EAAE,QAAQ,CAAC;QAChC,IAAI,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAE,MAAM;QACzD,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}