@jinn-network/client 0.1.8 → 0.1.9-canary.050a41b1

package/dist/eval/paired.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/** Minimal per-instance result (subset of PerTaskResult / EvalResultRow). */
+export interface PairedInput {
+    instance_id: string;
+    /** null when unscorable. */
+    passed: boolean | null;
+    unscorable: boolean;
+}
+export type PairedVerdict = 'trustworthy' | 'within-noise';
+export interface PairedComparison {
+    /** Instances scorable (passed !== null) in BOTH arms — the matched pairs. */
+    pairs: number;
+    /** b: before fail → after pass (improvements). */
+    improved: number;
+    /** c: before pass → after fail (regressions). */
+    regressed: number;
+    concordantPass: number;
+    concordantFail: number;
+    /** Instances unscorable/absent in either arm — excluded from the paired test. */
+    excluded: number;
+    /** Exact two-sided McNemar p-value over the discordant pairs. */
+    pValue: number;
+    /** 'trustworthy' iff pValue < alpha AND improved > regressed; else 'within-noise'. */
+    verdict: PairedVerdict;
+}
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export declare function mcnemarExact(b: number, c: number): number;
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export declare function comparePaired(before: readonly PairedInput[], after: readonly PairedInput[], opts?: {
+    alpha?: number;
+}): PairedComparison;

package/dist/eval/paired.js

@@ -0,0 +1,93 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export function mcnemarExact(b, c) {
+    const n = b + c;
+    if (n === 0)
+        return 1;
+    const k = Math.min(b, c);
+    // sum_{i=0}^{k} C(n,i) * 0.5^n, via prob_0 = 0.5^n, prob_i = prob_{i-1}*(n-i+1)/i.
+    let term = Math.pow(0.5, n); // i = 0
+    let cdf = term;
+    for (let i = 1; i <= k; i++) {
+        term = (term * (n - i + 1)) / i;
+        cdf += term;
+    }
+    return Math.min(1, 2 * cdf);
+}
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export function comparePaired(before, after, opts = {}) {
+    const alpha = opts.alpha ?? 0.05;
+    const scorable = (r) => !r.unscorable && r.passed !== null;
+    const beforeById = new Map(before.map((r) => [r.instance_id, r]));
+    let improved = 0;
+    let regressed = 0;
+    let concordantPass = 0;
+    let concordantFail = 0;
+    let pairs = 0;
+    let excluded = 0;
+    const seen = new Set();
+    for (const a of after) {
+        seen.add(a.instance_id);
+        const b = beforeById.get(a.instance_id);
+        if (!b || !scorable(b) || !scorable(a)) {
+            excluded++;
+            continue;
+        }
+        pairs++;
+        const wasPass = b.passed === true;
+        const nowPass = a.passed === true;
+        if (!wasPass && nowPass)
+            improved++;
+        else if (wasPass && !nowPass)
+            regressed++;
+        else if (wasPass && nowPass)
+            concordantPass++;
+        else
+            concordantFail++;
+    }
+    // before-only instances (absent from after) are excluded too.
+    for (const b of before)
+        if (!seen.has(b.instance_id))
+            excluded++;
+    const pValue = mcnemarExact(improved, regressed);
+    const verdict = pValue < alpha && improved > regressed ? 'trustworthy' : 'within-noise';
+    return { pairs, improved, regressed, concordantPass, concordantFail, excluded, pValue, verdict };
+}
+//# sourceMappingURL=paired.js.map

package/dist/eval/paired.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paired.js","sourceRoot":"","sources":["../../src/eval/paired.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AA6BH;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,mFAAmF;IACnF,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ;IACrC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChC,GAAG,IAAI,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,MAA8B,EAC9B,KAA6B,EAC7B,OAA2B,EAAE;IAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAc,EAAW,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;IACjF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAElE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,KAAK,EAAE,CAAC;QACR,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QAClC,IAAI,CAAC,OAAO,IAAI,OAAO;YAAE,QAAQ,EAAE,CAAC;aAC/B,IAAI,OAAO,IAAI,CAAC,OAAO;YAAE,SAAS,EAAE,CAAC;aACrC,IAAI,OAAO,IAAI,OAAO;YAAE,cAAc,EAAE,CAAC;;YACzC,cAAc,EAAE,CAAC;IACxB,CAAC;IACD,8DAA8D;IAC9D,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;YAAE,QAAQ,EAAE,CAAC;IAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACjD,MAAM,OAAO,GACX,MAAM,GAAG,KAAK,IAAI,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC;IAE1E,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACnG,CAAC"}

package/dist/eval/resolve-slate-tasks.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Resolve a held-out slate's `instance_id`s into `SweRebenchV2Task` objects
+ * for the `jinn eval` orchestrator (issue #818, AC#1).
+ *
+ * A slate stores only `instance_id`s (`_swe-rebench-v2-held-out-slate.ts`). The
+ * evaluator grades against the full HuggingFace row, fetched by
+ * `(hf_dataset, hf_split, instance_id)`. The `HfRow` does NOT echo
+ * `hf_dataset`/`hf_split` (see `swe-rebench-v2-evaluator/index.ts`), so the
+ * caller supplies the slate-level dataset+split as args; the resolver verifies
+ * each id exists by fetching its row and returns the `{ task, row }` pair so the
+ * orchestrator reuses the row at grade time (avoids a second fetch).
+ *
+ * No retry logic here — `HttpHfFetcher` owns retries. A fetcher throw for any id
+ * propagates loudly (a missing slate instance is a hard error, never a silent
+ * drop).
+ */
+import { type SweRebenchV2Task } from '../vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js';
+import type { HfFetcher, HfRow } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import type { PoolTask } from '../solver-types/_swe-rebench-v2-pool.js';
+export interface ResolvedSlateTask {
+    task: SweRebenchV2Task;
+    row: HfRow;
+}
+export declare function resolveSlateTasks(args: {
+    /**
+     * The pool tasks for this slate group, each carrying the real
+     * `problem_statement` / `base_commit` / `language` the agent run needs to
+     * solve the instance (the slate stores only ids; the pool is the source of
+     * these fields).
+     */
+    poolTasks: PoolTask[];
+    hf_dataset: string;
+    hf_split: string;
+    fetcher: HfFetcher;
+}): Promise<ResolvedSlateTask[]>;

package/dist/eval/resolve-slate-tasks.js

@@ -0,0 +1,56 @@
+/**
+ * Resolve a held-out slate's `instance_id`s into `SweRebenchV2Task` objects
+ * for the `jinn eval` orchestrator (issue #818, AC#1).
+ *
+ * A slate stores only `instance_id`s (`_swe-rebench-v2-held-out-slate.ts`). The
+ * evaluator grades against the full HuggingFace row, fetched by
+ * `(hf_dataset, hf_split, instance_id)`. The `HfRow` does NOT echo
+ * `hf_dataset`/`hf_split` (see `swe-rebench-v2-evaluator/index.ts`), so the
+ * caller supplies the slate-level dataset+split as args; the resolver verifies
+ * each id exists by fetching its row and returns the `{ task, row }` pair so the
+ * orchestrator reuses the row at grade time (avoids a second fetch).
+ *
+ * No retry logic here — `HttpHfFetcher` owns retries. A fetcher throw for any id
+ * propagates loudly (a missing slate instance is a hard error, never a silent
+ * drop).
+ */
+import { SweRebenchV2TaskSchema } from '../vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js';
+export async function resolveSlateTasks(args) {
+    const tasks = [...args.poolTasks].sort((a, b) => a.instance_id.localeCompare(b.instance_id));
+    const out = [];
+    for (const poolTask of tasks) {
+        const row = await args.fetcher.fetchTaskRow({
+            hf_dataset: args.hf_dataset,
+            hf_split: args.hf_split,
+            instance_id: poolTask.instance_id,
+        });
+        out.push({ task: buildTask(poolTask, args.hf_dataset, args.hf_split, row), row });
+    }
+    return out;
+}
+/**
+ * Construct the `SweRebenchV2Task` the orchestrator hands to the harness +
+ * evaluator. The agent run needs the real `problem_statement` (what to solve)
+ * and `base_commit` (the repo state to check out) — these are threaded through
+ * from the pool task (the generator's mapping in `swe-rebench-v2.ts`).
+ * `hf_dataset`/`hf_split` are the evaluator's row-fetch key; `repo` comes from
+ * the fetched row. A pool task missing an optional field falls back the same
+ * way the generator does, so the object stays schema-valid.
+ */
+function buildTask(poolTask, hf_dataset, hf_split, row) {
+    const language = SweRebenchV2TaskSchema.shape.language.safeParse(poolTask.language);
+    return {
+        schemaVersion: 'swe-rebench-v2.v1',
+        instance_id: poolTask.instance_id,
+        repo: row.repo,
+        base_commit: poolTask.base_commit ?? '0'.repeat(40),
+        language: language.success ? language.data : 'python',
+        problem_statement: poolTask.problem_statement ?? '',
+        interface: poolTask.interface ?? '',
+        hf_dataset,
+        hf_split,
+        deadline_unix: 1,
+        round_month: hf_split.replace('_', '-'),
+    };
+}
+//# sourceMappingURL=resolve-slate-tasks.js.map

package/dist/eval/resolve-slate-tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolve-slate-tasks.js","sourceRoot":"","sources":["../../src/eval/resolve-slate-tasks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,sBAAsB,EAAyB,MAAM,6CAA6C,CAAC;AAS5G,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAWvC;IACC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7F,MAAM,GAAG,GAAwB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;YAC1C,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,SAAS,CAChB,QAAkB,EAClB,UAAkB,EAClB,QAAgB,EAChB,GAAU;IAEV,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpF,OAAO;QACL,aAAa,EAAE,mBAAmB;QAClC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;QACrD,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB,IAAI,EAAE;QACnD,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;QACnC,UAAU;QACV,QAAQ;QACR,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;KACxC,CAAC;AACJ,CAAC"}

package/dist/eval/screen-discovery.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Held-out screening's authoritative "already trained-on" exclusion source (#986).
+ *
+ * Returns the set of swe-rebench-v2 `instance_id`s that have been ATTEMPTED on
+ * the network for a SolverNet — any verdict envelope, passed OR failed,
+ * cross-operator. An attempted instance was executed by an operator, so the
+ * learner trained on it; holding it out later would make a trained-checkpoint
+ * pass count as memorization, not generalization.
+ *
+ * Why the indexer and not the local generator-state: the local ledger only
+ * reflects THIS box's posting and can be stale (a different active generator —
+ * e.g. a hosted operator — posts to its own ledger). The indexer's
+ * `verdictEnvelopeMeta` is the cross-operator, current record. (The on-chain
+ * task/attempt tables carry no instance_id; only the indexer's IPFS enrichment
+ * resolves it — same backing as `DiscoveryAPI.getInstanceSuccessCounts`, minus
+ * the `actualPassed: true` filter so failed attempts count too.)
+ *
+ * Throws on indexer failure — callers MUST abort rather than screen against an
+ * unknown attempted set, because a missing exclusion can silently contaminate
+ * the exam (the whole point of held-out discipline).
+ */
+export declare function fetchAttemptedInstanceIds(discoveryUrl: string, manifestCid: string, fetchImpl?: typeof fetch): Promise<Set<string>>;

package/dist/eval/screen-discovery.js

@@ -0,0 +1,71 @@
+/**
+ * Held-out screening's authoritative "already trained-on" exclusion source (#986).
+ *
+ * Returns the set of swe-rebench-v2 `instance_id`s that have been ATTEMPTED on
+ * the network for a SolverNet — any verdict envelope, passed OR failed,
+ * cross-operator. An attempted instance was executed by an operator, so the
+ * learner trained on it; holding it out later would make a trained-checkpoint
+ * pass count as memorization, not generalization.
+ *
+ * Why the indexer and not the local generator-state: the local ledger only
+ * reflects THIS box's posting and can be stale (a different active generator —
+ * e.g. a hosted operator — posts to its own ledger). The indexer's
+ * `verdictEnvelopeMeta` is the cross-operator, current record. (The on-chain
+ * task/attempt tables carry no instance_id; only the indexer's IPFS enrichment
+ * resolves it — same backing as `DiscoveryAPI.getInstanceSuccessCounts`, minus
+ * the `actualPassed: true` filter so failed attempts count too.)
+ *
+ * Throws on indexer failure — callers MUST abort rather than screen against an
+ * unknown attempted set, because a missing exclusion can silently contaminate
+ * the exam (the whole point of held-out discipline).
+ */
+const ATTEMPTED_QUERY = `
+query InstanceAttempted($cid: String!, $limit: Int!, $after: String) {
+  verdictEnvelopeMetas(
+    where: {
+      solverNetManifestCid: $cid,
+      solverType_starts_with: "swe-rebench-v2",
+      enrichmentStatus: "ok",
+      instanceId_not: ""
+    },
+    limit: $limit,
+    after: $after,
+    orderBy: "enrichedAtBlock",
+    orderDirection: "asc"
+  ) {
+    items { instanceId }
+    pageInfo { hasNextPage endCursor }
+  }
+}`;
+export async function fetchAttemptedInstanceIds(discoveryUrl, manifestCid, fetchImpl = fetch) {
+    const gqlUrl = discoveryUrl.endsWith('/graphql') ? discoveryUrl : `${discoveryUrl.replace(/\/$/, '')}/graphql`;
+    const ids = new Set();
+    let cursor = null;
+    const MAX_PAGES = 20;
+    const PAGE_LIMIT = 1000;
+    for (let page = 0; page < MAX_PAGES; page++) {
+        const res = await fetchImpl(gqlUrl, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ query: ATTEMPTED_QUERY, variables: { cid: manifestCid, limit: PAGE_LIMIT, after: cursor } }),
+        });
+        if (!res.ok) {
+            throw new Error(`held-out screening: indexer attempted-ids query failed (HTTP ${res.status}) at ${gqlUrl}`);
+        }
+        const json = (await res.json());
+        if (json.errors) {
+            throw new Error(`held-out screening: indexer attempted-ids query errors: ${JSON.stringify(json.errors).slice(0, 300)}`);
+        }
+        const conn = json.data?.verdictEnvelopeMetas;
+        for (const item of conn?.items ?? []) {
+            if (item.instanceId)
+                ids.add(item.instanceId);
+        }
+        const pageInfo = conn?.pageInfo;
+        if (!pageInfo?.hasNextPage || !pageInfo.endCursor)
+            break;
+        cursor = pageInfo.endCursor;
+    }
+    return ids;
+}
+//# sourceMappingURL=screen-discovery.js.map

package/dist/eval/screen-discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-discovery.js","sourceRoot":"","sources":["../../src/eval/screen-discovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;;EAiBtB,CAAC;AAYH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,YAAoB,EACpB,WAAmB,EACnB,YAA0B,KAAK;IAE/B,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC;IAC/G,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,UAAU,GAAG,IAAI,CAAC;IAExB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE;YAClC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;SACpH,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gEAAgE,GAAG,CAAC,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC;QAC9G,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAkB,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2DAA2D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1H,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC;QAC7C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,UAAU;gBAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,EAAE,QAAQ,CAAC;QAChC,IAAI,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAE,MAAM;QACzD,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/eval/screen-progress.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Resumability store for held-out screening (#986).
+ *
+ * A real exam cut runs base R-runs + a prover over dozens of candidates — many
+ * hours of inference. This store persists each candidate's {@link ScreenMeasurement}
+ * so an interrupted run (rate limit, crash, disk) resumes instead of restarting:
+ * re-running the same command replays cached candidates for free and the
+ * `maxCandidates` budget bounds only the NEW work, so a long screen proceeds in
+ * budget-sized chunks.
+ *
+ * Keyed by a `signature` of the measurement-determining config (base model,
+ * prover, R, eval-semantics version). A signature mismatch on load → fresh start:
+ * cached measurements are only valid for the exact config that produced them
+ * (e.g. changing R changes the 0/R determination; a stronger base model changes
+ * pass/fail). Stored at `<stateDir>/held-out-screen-progress.json`.
+ */
+import type { ScreenMeasurement } from './screen.js';
+export declare class ScreenProgressStore {
+    private readonly file;
+    private readonly signature;
+    private data;
+    constructor(opts: {
+        stateDir: string;
+        signature: string;
+    });
+    private load;
+    /** Cached measurement for this instance under the current signature, or undefined. */
+    get(instance_id: string): ScreenMeasurement | undefined;
+    /** Persist a freshly-measured candidate (atomic-enough: whole-file rewrite). */
+    record(instance_id: string, m: ScreenMeasurement): void;
+    /** Number of candidates already measured under the current signature. */
+    get size(): number;
+}
+/** Build the cache signature from the measurement-determining config. */
+export declare function screenSignature(args: {
+    baseModel: string;
+    proverHarness: string;
+    proverModel: string;
+    R: number;
+    evalSemanticsVersion: string;
+}): string;

package/dist/eval/screen-progress.js

@@ -0,0 +1,60 @@
+/**
+ * Resumability store for held-out screening (#986).
+ *
+ * A real exam cut runs base R-runs + a prover over dozens of candidates — many
+ * hours of inference. This store persists each candidate's {@link ScreenMeasurement}
+ * so an interrupted run (rate limit, crash, disk) resumes instead of restarting:
+ * re-running the same command replays cached candidates for free and the
+ * `maxCandidates` budget bounds only the NEW work, so a long screen proceeds in
+ * budget-sized chunks.
+ *
+ * Keyed by a `signature` of the measurement-determining config (base model,
+ * prover, R, eval-semantics version). A signature mismatch on load → fresh start:
+ * cached measurements are only valid for the exact config that produced them
+ * (e.g. changing R changes the 0/R determination; a stronger base model changes
+ * pass/fail). Stored at `<stateDir>/held-out-screen-progress.json`.
+ */
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+const SCHEMA_VERSION = 'held-out-screen-progress.v1';
+export class ScreenProgressStore {
+    file;
+    signature;
+    data;
+    constructor(opts) {
+        this.file = join(opts.stateDir, 'held-out-screen-progress.json');
+        this.signature = opts.signature;
+        this.data = this.load();
+    }
+    load() {
+        try {
+            const raw = JSON.parse(readFileSync(this.file, 'utf8'));
+            if (raw?.schemaVersion === SCHEMA_VERSION && raw.signature === this.signature && raw.measurements) {
+                return raw;
+            }
+        }
+        catch {
+            /* absent or corrupt → fresh */
+        }
+        return { schemaVersion: SCHEMA_VERSION, signature: this.signature, measurements: {} };
+    }
+    /** Cached measurement for this instance under the current signature, or undefined. */
+    get(instance_id) {
+        return this.data.measurements[instance_id];
+    }
+    /** Persist a freshly-measured candidate (atomic-enough: whole-file rewrite). */
+    record(instance_id, m) {
+        this.data.measurements[instance_id] = m;
+        mkdirSync(dirname(this.file), { recursive: true });
+        writeFileSync(this.file, `${JSON.stringify(this.data, null, 2)}\n`);
+    }
+    /** Number of candidates already measured under the current signature. */
+    get size() {
+        return Object.keys(this.data.measurements).length;
+    }
+}
+/** Build the cache signature from the measurement-determining config. */
+export function screenSignature(args) {
+    return `base=${args.baseModel}|prover=${args.proverHarness}:${args.proverModel}|R=${args.R}|sem=${args.evalSemanticsVersion}`;
+}
+//# sourceMappingURL=screen-progress.js.map

package/dist/eval/screen-progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-progress.js","sourceRoot":"","sources":["../../src/eval/screen-progress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,cAAc,GAAG,6BAAsC,CAAC;AAS9D,MAAM,OAAO,mBAAmB;IACb,IAAI,CAAS;IACb,SAAS,CAAS;IAC3B,IAAI,CAAe;IAE3B,YAAY,IAA6C;QACvD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAiB,CAAC;YACxE,IAAI,GAAG,EAAE,aAAa,KAAK,cAAc,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBAClG,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;QACD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IACxF,CAAC;IAED,sFAAsF;IACtF,GAAG,CAAC,WAAmB;QACrB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAED,gFAAgF;IAChF,MAAM,CAAC,WAAmB,EAAE,CAAoB;QAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,yEAAyE;IACzE,IAAI,IAAI;QACN,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;IACpD,CAAC;CACF;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,IAM/B;IACC,OAAO,QAAQ,IAAI,CAAC,SAAS,WAAW,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,MAAM,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,oBAAoB,EAAE,CAAC;AAChI,CAAC"}

package/dist/eval/screen-runner.d.ts

@@ -0,0 +1,30 @@
+import { type ScreenResult } from './screen.js';
+export interface ScreenRunOptions {
+    R: number;
+    heldOutCount: number;
+    maxCandidates: number;
+    perRepoCap: number;
+    /** Prover agent harness: `codex` (default) or `claude-code` (e.g. an Opus prover
+     *  via the working Claude auth — useful when codex is rate-limited). */
+    proverHarness?: 'codex' | 'claude-code';
+    /** Prover model: `codexModel` for the codex harness, `claudeModel` for the
+     *  claude-code harness (defaults to `opus` there). */
+    proverModel?: string;
+    /** Restrict candidates to these instance ids (else whole gradeable pool). */
+    instanceIds?: string[];
+    /** Restrict candidates to one repo (org prefix), e.g. `tobymao`. */
+    repo?: string;
+    configPath?: string;
+    log?: (msg: string) => void;
+}
+export interface ScreenRunSummary {
+    result: ScreenResult;
+    baseCodeDigest: string;
+    slatePath: string;
+    reportPath: string;
+    heldOutCount: number;
+    /** Base-failing candidates whose prover run returned no gradeable result
+     *  (excluded as no-headroom) — a signal the prover may be unavailable. */
+    proverUnscorable: number;
+}
+export declare function runScreenHeldOut(opts: ScreenRunOptions): Promise<ScreenRunSummary>;