@jinn-network/client 0.1.8 → 0.1.9-canary.144d87d2

package/dist/eval/screen-progress.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Resumability store for held-out screening (#986).
+ *
+ * A real exam cut runs base R-runs + a prover over dozens of candidates — many
+ * hours of inference. This store persists each candidate's {@link ScreenMeasurement}
+ * so an interrupted run (rate limit, crash, disk) resumes instead of restarting:
+ * re-running the same command replays cached candidates for free and the
+ * `maxCandidates` budget bounds only the NEW work, so a long screen proceeds in
+ * budget-sized chunks.
+ *
+ * Keyed by a `signature` of the measurement-determining config (base model,
+ * prover, R, eval-semantics version). A signature mismatch on load → fresh start:
+ * cached measurements are only valid for the exact config that produced them
+ * (e.g. changing R changes the 0/R determination; a stronger base model changes
+ * pass/fail). Stored at `<stateDir>/held-out-screen-progress.json`.
+ */
+import type { ScreenMeasurement } from './screen.js';
+export declare class ScreenProgressStore {
+    private readonly file;
+    private readonly signature;
+    private data;
+    constructor(opts: {
+        stateDir: string;
+        signature: string;
+    });
+    private load;
+    /** Cached measurement for this instance under the current signature, or undefined. */
+    get(instance_id: string): ScreenMeasurement | undefined;
+    /** Persist a freshly-measured candidate (atomic-enough: whole-file rewrite). */
+    record(instance_id: string, m: ScreenMeasurement): void;
+    /** Number of candidates already measured under the current signature. */
+    get size(): number;
+}
+/** Build the cache signature from the measurement-determining config. */
+export declare function screenSignature(args: {
+    baseModel: string;
+    proverHarness: string;
+    proverModel: string;
+    R: number;
+    evalSemanticsVersion: string;
+}): string;

package/dist/eval/screen-progress.js

@@ -0,0 +1,60 @@
+/**
+ * Resumability store for held-out screening (#986).
+ *
+ * A real exam cut runs base R-runs + a prover over dozens of candidates — many
+ * hours of inference. This store persists each candidate's {@link ScreenMeasurement}
+ * so an interrupted run (rate limit, crash, disk) resumes instead of restarting:
+ * re-running the same command replays cached candidates for free and the
+ * `maxCandidates` budget bounds only the NEW work, so a long screen proceeds in
+ * budget-sized chunks.
+ *
+ * Keyed by a `signature` of the measurement-determining config (base model,
+ * prover, R, eval-semantics version). A signature mismatch on load → fresh start:
+ * cached measurements are only valid for the exact config that produced them
+ * (e.g. changing R changes the 0/R determination; a stronger base model changes
+ * pass/fail). Stored at `<stateDir>/held-out-screen-progress.json`.
+ */
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+const SCHEMA_VERSION = 'held-out-screen-progress.v1';
+export class ScreenProgressStore {
+    file;
+    signature;
+    data;
+    constructor(opts) {
+        this.file = join(opts.stateDir, 'held-out-screen-progress.json');
+        this.signature = opts.signature;
+        this.data = this.load();
+    }
+    load() {
+        try {
+            const raw = JSON.parse(readFileSync(this.file, 'utf8'));
+            if (raw?.schemaVersion === SCHEMA_VERSION && raw.signature === this.signature && raw.measurements) {
+                return raw;
+            }
+        }
+        catch {
+            /* absent or corrupt → fresh */
+        }
+        return { schemaVersion: SCHEMA_VERSION, signature: this.signature, measurements: {} };
+    }
+    /** Cached measurement for this instance under the current signature, or undefined. */
+    get(instance_id) {
+        return this.data.measurements[instance_id];
+    }
+    /** Persist a freshly-measured candidate (atomic-enough: whole-file rewrite). */
+    record(instance_id, m) {
+        this.data.measurements[instance_id] = m;
+        mkdirSync(dirname(this.file), { recursive: true });
+        writeFileSync(this.file, `${JSON.stringify(this.data, null, 2)}\n`);
+    }
+    /** Number of candidates already measured under the current signature. */
+    get size() {
+        return Object.keys(this.data.measurements).length;
+    }
+}
+/** Build the cache signature from the measurement-determining config. */
+export function screenSignature(args) {
+    return `base=${args.baseModel}|prover=${args.proverHarness}:${args.proverModel}|R=${args.R}|sem=${args.evalSemanticsVersion}`;
+}
+//# sourceMappingURL=screen-progress.js.map

package/dist/eval/screen-progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-progress.js","sourceRoot":"","sources":["../../src/eval/screen-progress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,cAAc,GAAG,6BAAsC,CAAC;AAS9D,MAAM,OAAO,mBAAmB;IACb,IAAI,CAAS;IACb,SAAS,CAAS;IAC3B,IAAI,CAAe;IAE3B,YAAY,IAA6C;QACvD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAiB,CAAC;YACxE,IAAI,GAAG,EAAE,aAAa,KAAK,cAAc,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBAClG,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;QACD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IACxF,CAAC;IAED,sFAAsF;IACtF,GAAG,CAAC,WAAmB;QACrB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAED,gFAAgF;IAChF,MAAM,CAAC,WAAmB,EAAE,CAAoB;QAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,yEAAyE;IACzE,IAAI,IAAI;QACN,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;IACpD,CAAC;CACF;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,IAM/B;IACC,OAAO,QAAQ,IAAI,CAAC,SAAS,WAAW,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,MAAM,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,oBAAoB,EAAE,CAAC;AAChI,CAAC"}

package/dist/eval/screen-runner.d.ts

@@ -0,0 +1,30 @@
+import { type ScreenResult } from './screen.js';
+export interface ScreenRunOptions {
+    R: number;
+    heldOutCount: number;
+    maxCandidates: number;
+    perRepoCap: number;
+    /** Prover agent harness: `codex` (default) or `claude-code` (e.g. an Opus prover
+     *  via the working Claude auth — useful when codex is rate-limited). */
+    proverHarness?: 'codex' | 'claude-code';
+    /** Prover model: `codexModel` for the codex harness, `claudeModel` for the
+     *  claude-code harness (defaults to `opus` there). */
+    proverModel?: string;
+    /** Restrict candidates to these instance ids (else whole gradeable pool). */
+    instanceIds?: string[];
+    /** Restrict candidates to one repo (org prefix), e.g. `tobymao`. */
+    repo?: string;
+    configPath?: string;
+    log?: (msg: string) => void;
+}
+export interface ScreenRunSummary {
+    result: ScreenResult;
+    baseCodeDigest: string;
+    slatePath: string;
+    reportPath: string;
+    heldOutCount: number;
+    /** Base-failing candidates whose prover run returned no gradeable result
+     *  (excluded as no-headroom) — a signal the prover may be unavailable. */
+    proverUnscorable: number;
+}
+export declare function runScreenHeldOut(opts: ScreenRunOptions): Promise<ScreenRunSummary>;

package/dist/eval/screen-runner.js

@@ -0,0 +1,289 @@
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadConfig } from '../config.js';
+import { Store } from '../store/store.js';
+import { hashImplStateDir } from '../harnesses/freeze.js';
+import { LearnerHarness } from '../harnesses/impls/learner/harness.js';
+import { ClaudeCodeHarnessAdapter } from '../harnesses/impls/learner/adapters/claude-code.js';
+import { CodexCodeHarnessAdapter } from '../harnesses/impls/learner/adapters/codex-code.js';
+import { CODEX_HARNESS } from '../harnesses/names.js';
+import { runHarnessForEval, resolveRuntimePluginsForSolverType } from './eval-harness-run.js';
+import { corpusEnvFromConfig } from '../cli/commands/eval.js';
+import { loadSweRebenchV2Pool, defaultStateDir, getSweRebenchV2ValidatedPoolStore, } from '../solver-types/swe-rebench-v2.js';
+import { PoolCacheStore, loadPoolWithCacheFallback } from '../solver-types/_swe-rebench-v2-pool-cache.js';
+import { validatePoolInstances, EVAL_SEMANTICS_VERSION, } from '../solver-types/_swe-rebench-v2-validated-pool.js';
+import { resolveSlateTasks } from './resolve-slate-tasks.js';
+import { loadActiveHeldOutSlateIds, ACTIVE_HELD_OUT_SLATE_VERSIONS, loadHeldOutSlate, } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+import { GeneratorStateStore } from '../solver-types/_swe-rebench-v2-state.js';
+import { fetchAttemptedInstanceIds } from './screen-discovery.js';
+import { DEFAULT_TESTNET_DISCOVERY_URL } from '../config.js';
+import { solverTypeFromJoinedContract } from '../solver-nets/registry.js';
+import { SweRebenchV2Evaluator } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import { HttpHfFetcher } from '../harnesses/impls/swe-rebench-v2-evaluator/hf-fetcher.js';
+import { PythonEvalRunner } from '../harnesses/impls/swe-rebench-v2-evaluator/eval-runner.js';
+import { readEnabledState, defaultSweRebenchV2EvaluatorImplStateDir } from '../harnesses/impls/swe-rebench-v2-evaluator/harness.js';
+import { stratifyByRepo, screenBaseFailures, buildV2SlateFile, } from './screen.js';
+import { ScreenProgressStore, screenSignature } from './screen-progress.js';
+const DISPATCH_SOLVER_TYPE = 'swe-rebench-v2.v1';
+const SLATE_VERSION = 'v2';
+const RUN_BUDGET_MS = 3_600_000;
+/** dist/src parity: the shipped slate JSON lives next to the compiled module. */
+function slatesDir() {
+    return join(dirname(fileURLToPath(import.meta.url)), '..', 'solver-types', 'slates');
+}
+export async function runScreenHeldOut(opts) {
+    const log = opts.log ?? (() => { });
+    const config = loadConfig(opts.configPath);
+    // Precondition: evaluator enabled (upstream repo cloned).
+    const enabled = readEnabledState(defaultSweRebenchV2EvaluatorImplStateDir());
+    if (!enabled) {
+        throw new Error('swe-rebench-v2 evaluator not enabled — run `jinn harnesses enable swe-rebench-v2-evaluator` first');
+    }
+    const upstreamRepoDir = enabled.upstreamRepoDir;
+    const stateDir = process.env['JINN_SWE_REBENCH_V2_STATE_DIR'] ?? defaultStateDir();
+    const fetcher = new HttpHfFetcher();
+    const evaluator = new SweRebenchV2Evaluator({ fetcher, runner: new PythonEvalRunner({ upstreamRepoDir }) });
+    const validatedStore = getSweRebenchV2ValidatedPoolStore();
+    const runtimePlugins = await resolveRuntimePluginsForSolverType(DISPATCH_SOLVER_TYPE, config.joinedSolverNets);
+    // Common adapter wiring (mirrors buildEvalHarness in cli/commands/eval.ts).
+    const daemonApiToken = process.env['DAEMON_API_TOKEN']?.trim();
+    const corpusEnv = corpusEnvFromConfig(config);
+    const common = {
+        claudePath: config.claudePath ?? 'claude',
+        storePath: config.dbPath,
+        daemonApiUrl: `http://127.0.0.1:${config.apiPort}`,
+        ...(daemonApiToken ? { daemonApiToken } : {}),
+        ...(corpusEnv ? { corpusEnv } : {}),
+    };
+    const baseHarness = new LearnerHarness({
+        adapter: new ClaudeCodeHarnessAdapter({ ...common, claudeModel: config.claudeModel }),
+        claudePath: common.claudePath,
+    });
+    // Prover harness: codex (default) or claude-code (e.g. an Opus prover via the
+    // working Claude auth, sidestepping a codex rate limit; same-family Haiku→Opus
+    // is a clean capability ladder for "proven headroom").
+    const proverKind = opts.proverHarness ?? 'codex';
+    const proverHarness = proverKind === 'claude-code'
+        ? new LearnerHarness({
+            adapter: new ClaudeCodeHarnessAdapter({ ...common, claudeModel: opts.proverModel ?? 'opus' }),
+            claudePath: common.claudePath,
+        })
+        : new LearnerHarness({
+            name: CODEX_HARNESS,
+            adapter: new CodexCodeHarnessAdapter({ ...common, ...(opts.proverModel ? { codexModel: opts.proverModel } : {}) }),
+            claudePath: common.claudePath,
+            ...(config.codexPath !== undefined ? { codexPath: config.codexPath } : {}),
+        });
+    // Candidate pool (whole gradeable pool by default; scopeable).
+    const cacheResult = await loadPoolWithCacheFallback({
+        loadPool: loadSweRebenchV2Pool, cache: new PoolCacheStore({ stateDir }), currentPool: [],
+    });
+    let pool = cacheResult.pool;
+    if (pool.length === 0)
+        throw new Error(`SWE-rebench v2 pool empty${cacheResult.error ? ` (${cacheResult.error.message})` : ''}`);
+    // Held-out discipline (#986): draw the exam from the never-trained, never-held-out
+    // remainder. Union three exclusion sources:
+    //  - active held-out slate (would overlap an existing exam);
+    //  - already-ATTEMPTED on-network (indexer verdictEnvelopeMeta, any verdict,
+    //    cross-operator, CURRENT) — the authoritative "the learner trained on it"
+    //    signal; and
+    //  - this box's posted ledger (belt — may be STALE when another generator is
+    //    the active poster, e.g. a hosted operator; hence the indexer is the truth).
+    // A trained instance held out later would make a trained-checkpoint pass count
+    // as memorization, not generalization.
+    const heldOutIds = loadActiveHeldOutSlateIds(DISPATCH_SOLVER_TYPE, ACTIVE_HELD_OUT_SLATE_VERSIONS);
+    const postedIds = await new GeneratorStateStore({ stateDir }).postedInstanceIds();
+    let attemptedIds = new Set();
+    const discoveryUrl = config.discovery?.url?.trim()
+        || (config.network === 'testnet' ? DEFAULT_TESTNET_DISCOVERY_URL : undefined);
+    const joinedNet = Object.values(config.joinedSolverNets ?? {}).find((n) => solverTypeFromJoinedContract(n) === DISPATCH_SOLVER_TYPE);
+    if (discoveryUrl && joinedNet?.manifestCid) {
+        attemptedIds = await fetchAttemptedInstanceIds(discoveryUrl, joinedNet.manifestCid);
+        log(`[screen] indexer: ${attemptedIds.size} instance(s) already attempted on-network (authoritative, cross-operator) → excluded`);
+    }
+    else {
+        log('[screen] WARNING: no discovery URL / manifestCid — cannot exclude already-attempted instances; relying on the local posted ledger, which may be stale');
+    }
+    const excludeIds = new Set([...heldOutIds, ...postedIds, ...attemptedIds]);
+    if (opts.instanceIds?.length) {
+        // Explicit operator override — screen exactly these, but warn if any are
+        // already contaminated (attempted/posted/held-out) so an intentional pick is informed.
+        const want = new Set(opts.instanceIds);
+        pool = pool.filter((t) => want.has(t.instance_id));
+        const tainted = pool.filter((t) => excludeIds.has(t.instance_id)).map((t) => t.instance_id);
+        if (tainted.length > 0) {
+            log(`[screen] WARNING: ${tainted.length} explicitly-named instance(s) are already attempted/posted/held-out (NOT clean held-out candidates): ${tainted.join(', ')}`);
+        }
+    }
+    else {
+        if (opts.repo)
+            pool = pool.filter((t) => t.instance_id.startsWith(`${opts.repo}__`));
+        const before = pool.length;
+        pool = pool.filter((t) => !excludeIds.has(t.instance_id));
+        // Restrict to ALREADY-VALIDATED-SCORABLE candidates (#986): the never-validated
+        // tail is mostly not-gradeable (~90% deeper in the pool), so base-screening it
+        // wastes inference. Discovery of gradeability is `validate-pool`'s job; the
+        // screen selects held-out FROM the scorable set. Falls back to the full
+        // remainder only when no validation data exists yet (and warns).
+        const scorableIds = await validatedStore.getScorableIds(EVAL_SEMANTICS_VERSION);
+        if (scorableIds) {
+            const beforeScorable = pool.length;
+            pool = pool.filter((t) => scorableIds.has(t.instance_id));
+            log(`[screen] candidate pool ${before} → ${beforeScorable} (excluded ${excludeIds.size}: ${heldOutIds.size} held-out ∪ ${postedIds.size} posted ∪ ${attemptedIds.size} attempted) → ${pool.length} validated-scorable (run validate-pool to grow this)`);
+        }
+        else {
+            log(`[screen] WARNING: no validated-scorable data — screening the full ${pool.length}-task remainder (mostly not-gradeable; run validate-pool first for efficiency)`);
+        }
+    }
+    const candidates = stratifyByRepo(pool);
+    log(`[screen] ${candidates.length} candidate(s) after stratification`);
+    // Resolve a single instance to the {task,row} the harness + grader need.
+    const byId = new Map(pool.map((t) => [t.instance_id, t]));
+    async function runOnce(harness, poolTask) {
+        const implStateDir = mkdtempSync(join(tmpdir(), 'jinn-screen-state-'));
+        // Track which stage we're in so an unscorable result names its cause (#476
+        // excludes infra failures from the denominator — but it must be diagnosable,
+        // not an opaque black box).
+        let stage = 'resolve';
+        try {
+            const [resolved] = await resolveSlateTasks({
+                poolTasks: [poolTask], hf_dataset: poolTask.hf_dataset, hf_split: poolTask.hf_split, fetcher,
+            });
+            if (!resolved)
+                return { passed: null, unscorableReason: 'resolve: instance not in pool' };
+            const task = {
+                id: poolTask.instance_id,
+                description: resolved.task.problem_statement,
+                role: 'restoration',
+                solverType: DISPATCH_SOLVER_TYPE,
+                spec: resolved.task,
+                window: { startTs: 0, endTs: Date.now() + RUN_BUDGET_MS },
+            };
+            stage = 'harness';
+            const run = await runHarnessForEval({
+                harness, task, solverType: DISPATCH_SOLVER_TYPE, runtimePlugins, implStateDir, mode: 'frozen',
+            });
+            if (run.violation)
+                return { passed: null, unscorableReason: 'harness: freeze-fence violation' };
+            if (!run.solution)
+                return { passed: null, unscorableReason: 'harness: no solution produced' };
+            stage = 'grade';
+            const verdict = await evaluator.grade({
+                task: resolved.task,
+                solutionPayload: { schemaVersion: 'swe-rebench-v2-solution.v1', patch: run.solution.patch },
+                row: resolved.row,
+            });
+            return { passed: verdict.passed_match };
+        }
+        catch (err) {
+            // Any harness/grader/infra failure ⇒ unscorable, never a fail (#476). Name
+            // the stage + error so the exclusion is diagnosable. An "agent produced no
+            // patch" throw is flagged distinctly from a true infra/grader error.
+            const msg = err instanceof Error ? err.message : String(err);
+            const reason = /produced no\b|no .*patch/i.test(msg)
+                ? `${stage}: agent produced no patch`
+                : `${stage}-error: ${msg.slice(0, 200)}`;
+            return { passed: null, unscorableReason: reason };
+        }
+        finally {
+            rmSync(implStateDir, { recursive: true, force: true });
+        }
+    }
+    const emptyBaseDir = mkdtempSync(join(tmpdir(), 'jinn-screen-base-'));
+    const hashOpts = baseHarness.freezeStateHashIgnore?.length
+        ? { ignoreRelPaths: [...baseHarness.freezeStateHashIgnore] } : undefined;
+    const baseCodeDigest = `sha256:${await hashImplStateDir(emptyBaseDir, hashOpts)}`;
+    rmSync(emptyBaseDir, { recursive: true, force: true });
+    // Resumability: cache each candidate's measurement under a config signature so
+    // an interrupted run resumes (re-run the same command). The base policy is fixed
+    // (empty impl-state) so its measurement is stable; the signature invalidates the
+    // cache if the base model / prover / R / semantics change.
+    const proverModelLabel = opts.proverModel ?? (proverKind === 'claude-code' ? 'opus' : 'codex-default');
+    const progress = new ScreenProgressStore({
+        stateDir,
+        signature: screenSignature({
+            baseModel: config.claudeModel, proverHarness: proverKind, proverModel: proverModelLabel,
+            R: opts.R, evalSemanticsVersion: EVAL_SEMANTICS_VERSION,
+        }),
+    });
+    if (progress.size > 0)
+        log(`[screen] resuming: ${progress.size} candidate(s) already measured (cached) for this config`);
+    const deps = {
+        log,
+        getCachedMeasurement: (id) => progress.get(id),
+        recordMeasurement: (id, m) => progress.record(id, m),
+        ensureGradeable: async (task) => {
+            await validatePoolInstances([task], {
+                fetcher, runner: new PythonEvalRunner({ upstreamRepoDir }), store: validatedStore,
+                semanticsVersion: EVAL_SEMANTICS_VERSION, upstreamRepoDir,
+            }, {});
+            return (await validatedStore.getEntry(task.instance_id, EVAL_SEMANTICS_VERSION))?.scorable === true;
+        },
+        runBaseFrozen: (task) => runOnce(baseHarness, byId.get(task.instance_id)),
+        runProverFrozen: (task) => runOnce(proverHarness, byId.get(task.instance_id)),
+    };
+    const result = await screenBaseFailures(candidates, deps, {
+        R: opts.R, heldOutCount: opts.heldOutCount, maxCandidates: opts.maxCandidates, perRepoCap: opts.perRepoCap,
+    });
+    // Diagnosability: a base-failing candidate routed to the prover that comes
+    // back `proverPassed: null` means the prover produced NO gradeable result
+    // (errored / no patch), not a clean "the prover can't solve it". That silently
+    // routes to `no-headroom` and can yield a misleadingly empty slate when the
+    // prover is simply unavailable (e.g. codex CLI < 0.133.0, or auth missing).
+    // Surface it loudly rather than swallow it.
+    const proverUnscorable = result.screened.filter((s) => s.reason === 'no-headroom' && s.proverPassed === null).length;
+    if (proverUnscorable > 0) {
+        log(`[screen] WARNING: the prover returned no gradeable result on ${proverUnscorable} base-failing ` +
+            `candidate(s) — excluded as no-headroom, but this likely means the prover is UNAVAILABLE rather than ` +
+            `unable. Verify the codex CLI (>=0.133.0) + auth, then re-run. See proverPassed=null rows in the report.`);
+    }
+    // Cumulative v2: union the EXISTING slate with this run's NEW held-out, so
+    // re-running to GROW the exam never drops already-reserved instances. (The
+    // active-slate exclusion keeps existing held-out OUT of candidates, so
+    // result.heldOut is only the new admits.) Growing changes the slate's content
+    // hash; re-recording the base arm for the FULL set below upserts the prior rows
+    // to the new hash, so the orchestrator's slate-hash-drift guard stays satisfied.
+    let existingHeldOut = [];
+    try {
+        existingHeldOut = [...loadHeldOutSlate(DISPATCH_SOLVER_TYPE, SLATE_VERSION).instanceIds];
+    }
+    catch {
+        /* no v2 slate yet — this is the first cut */
+    }
+    const newHeldOut = result.heldOut.map((h) => h.instance_id);
+    const allHeldOutIds = [...new Set([...existingHeldOut, ...newHeldOut])];
+    const generatedAt = new Date().toISOString();
+    const slateFile = buildV2SlateFile(allHeldOutIds, generatedAt);
+    mkdirSync(slatesDir(), { recursive: true });
+    const slatePath = join(slatesDir(), 'held-out-slate.swe-rebench-v2.v2.json');
+    writeFileSync(slatePath, `${JSON.stringify(slateFile, null, 2)}\n`);
+    const reportPath = join(slatesDir(), 'held-out-slate.swe-rebench-v2.v2.screening-report.json');
+    writeFileSync(reportPath, `${JSON.stringify({
+        generatedAt, evalSemanticsVersion: EVAL_SEMANTICS_VERSION, baseCodeDigest,
+        R: opts.R, proverHarness: proverKind,
+        proverModel: opts.proverModel ?? (proverKind === 'claude-code' ? 'opus' : 'codex-default'),
+        heldOutTotal: allHeldOutIds.length, newThisRun: newHeldOut, carriedOver: existingHeldOut,
+        screened: result.screened,
+    }, null, 2)}\n`);
+    // Persist the base arm (all-fail) for the FULL held-out set under the (possibly
+    // new) slate hash. The upsert is keyed by (checkpoint, slate_version, instance_id),
+    // so prior rows get their slate_hash refreshed to match the grown slate — no drift.
+    const store = new Store(config.dbPath);
+    try {
+        const runAtMs = Date.now();
+        for (const id of allHeldOutIds) {
+            store.recordEvalResult({
+                checkpoint_cid: baseCodeDigest, slate_hash: slateFile.hash, slate_version: SLATE_VERSION,
+                instance_id: id, passed: false, unscorable: false, code_digest: baseCodeDigest,
+                run_at_ms: runAtMs, test_log_excerpt: 'base arm (screening): consistent fail 0/R',
+            });
+        }
+    }
+    finally {
+        store.close?.();
+    }
+    return { result, baseCodeDigest, slatePath, reportPath, heldOutCount: allHeldOutIds.length, proverUnscorable };
+}
+//# sourceMappingURL=screen-runner.js.map

package/dist/eval/screen-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-runner.js","sourceRoot":"","sources":["../../src/eval/screen-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,oDAAoD,CAAC;AAC9F,OAAO,EAAE,uBAAuB,EAAE,MAAM,mDAAmD,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,kCAAkC,EAAE,MAAM,uBAAuB,CAAC;AAC9F,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EACL,oBAAoB,EAAE,eAAe,EAAE,iCAAiC,GACzE,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,+CAA+C,CAAC;AAC1G,OAAO,EACL,qBAAqB,EAAE,sBAAsB,GAC9C,MAAM,mDAAmD,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EACL,yBAAyB,EAAE,8BAA8B,EAAE,gBAAgB,GAC5E,MAAM,mDAAmD,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,qBAAqB,EAAE,MAAM,sDAAsD,CAAC;AAC7F,OAAO,EAAE,aAAa,EAAE,MAAM,2DAA2D,CAAC;AAC1F,OAAO,EAAE,gBAAgB,EAAE,MAAM,4DAA4D,CAAC;AAC9F,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,MAAM,wDAAwD,CAAC;AACpI,OAAO,EACL,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,GAErD,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5E,MAAM,oBAAoB,GAAG,mBAAmB,CAAC;AACjD,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,aAAa,GAAG,SAAS,CAAC;AAgChC,iFAAiF;AACjF,SAAS,SAAS;IAChB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;AACvF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAsB;IAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE3C,0DAA0D;IAC1D,MAAM,OAAO,GAAG,gBAAgB,CAAC,wCAAwC,EAAE,CAAC,CAAC;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IACD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAEhD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,IAAI,eAAe,EAAE,CAAC;IACnF,MAAM,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,IAAI,qBAAqB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5G,MAAM,cAAc,GAAG,iCAAiC,EAAE,CAAC;IAC3D,MAAM,cAAc,GAAoB,MAAM,kCAAkC,CAC9E,oBAAoB,EAAE,MAAM,CAAC,gBAAgB,CAC9C,CAAC;IAEF,4EAA4E;IAC5E,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,CAAC;IAC/D,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG;QACb,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,QAAQ;QACzC,SAAS,EAAE,MAAM,CAAC,MAAM;QACxB,YAAY,EAAE,oBAAoB,MAAM,CAAC,OAAO,EAAE;QAClD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpC,CAAC;IACF,MAAM,WAAW,GAAY,IAAI,cAAc,CAAC;QAC9C,OAAO,EAAE,IAAI,wBAAwB,CAAC,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;QACrF,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IACH,8EAA8E;IAC9E,+EAA+E;IAC/E,uDAAuD;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC;IACjD,MAAM,aAAa,GAAY,UAAU,KAAK,aAAa;QACzD,CAAC,CAAC,IAAI,cAAc,CAAC;YACjB,OAAO,EAAE,IAAI,wBAAwB,CAAC,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,MAAM,EAAE,CAAC;YAC7F,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;QACJ,CAAC,CAAC,IAAI,cAAc,CAAC;YACjB,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,IAAI,uBAAuB,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAClH,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3E,CAAC,CAAC;IAEP,+DAA+D;IAC/D,MAAM,WAAW,GAAG,MAAM,yBAAyB,CAAC;QAClD,QAAQ,EAAE,oBAAoB,EAAE,KAAK,EAAE,IAAI,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE;KACzF,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;IAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjI,mFAAmF;IACnF,4CAA4C;IAC5C,6DAA6D;IAC7D,6EAA6E;IAC7E,8EAA8E;IAC9E,iBAAiB;IACjB,6EAA6E;IAC7E,iFAAiF;IACjF,+EAA+E;IAC/E,uCAAuC;IACvC,MAAM,UAAU,GAAG,yBAAyB,CAAC,oBAAoB,EAAE,8BAA8B,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,MAAM,IAAI,mBAAmB,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAClF,IAAI,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE;WAC7C,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,CACjE,CAAC,CAAC,EAAE,EAAE,CAAC,4BAA4B,CAAC,CAAC,CAAC,KAAK,oBAAoB,CAChE,CAAC;IACF,IAAI,YAAY,IAAI,SAAS,EAAE,WAAW,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,YAAY,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;QACpF,GAAG,CAAC,qBAAqB,YAAY,CAAC,IAAI,sFAAsF,CAAC,CAAC;IACpI,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,uJAAuJ,CAAC,CAAC;IAC/J,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,UAAU,EAAE,GAAG,SAAS,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;IACnF,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QAC7B,yEAAyE;QACzE,uFAAuF;QACvF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,qBAAqB,OAAO,CAAC,MAAM,wGAAwG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvK,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,IAAI,CAAC,IAAI;YAAE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACrF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1D,gFAAgF;QAChF,+EAA+E;QAC/E,4EAA4E;QAC5E,wEAAwE;QACxE,iEAAiE;QACjE,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;QAChF,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;YACnC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;YAC1D,GAAG,CAAC,2BAA2B,MAAM,MAAM,cAAc,cAAc,UAAU,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,eAAe,SAAS,CAAC,IAAI,aAAa,YAAY,CAAC,IAAI,iBAAiB,IAAI,CAAC,MAAM,sDAAsD,CAAC,CAAC;QAC3P,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,qEAAqE,IAAI,CAAC,MAAM,gFAAgF,CAAC,CAAC;QACxK,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACxC,GAAG,CAAC,YAAY,UAAU,CAAC,MAAM,oCAAoC,CAAC,CAAC;IAEvE,yEAAyE;IACzE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,KAAK,UAAU,OAAO,CAAC,OAAgB,EAAE,QAAkB;QACzD,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAC;QACvE,2EAA2E;QAC3E,6EAA6E;QAC7E,4BAA4B;QAC5B,IAAI,KAAK,GAAoC,SAAS,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,iBAAiB,CAAC;gBACzC,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO;aAC7F,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,CAAC;YAC1F,MAAM,IAAI,GAAS;gBACjB,EAAE,EAAE,QAAQ,CAAC,WAAW;gBACxB,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,iBAAiB;gBAC5C,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,oBAAoB;gBAChC,IAAI,EAAE,QAAQ,CAAC,IAA0C;gBACzD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,EAAE;aAC1D,CAAC;YACF,KAAK,GAAG,SAAS,CAAC;YAClB,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC;gBAClC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ;aAC9F,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,SAAS;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,iCAAiC,EAAE,CAAC;YAChG,IAAI,CAAC,GAAG,CAAC,QAAQ;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,CAAC;YAC9F,KAAK,GAAG,OAAO,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC;gBACpC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,eAAe,EAAE,EAAE,aAAa,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAC3F,GAAG,EAAE,QAAQ,CAAC,GAAG;aAClB,CAAC,CAAC;YACH,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2EAA2E;YAC3E,2EAA2E;YAC3E,qEAAqE;YACrE,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,2BAA2B,CAAC,IAAI,CAAC,GAAG,CAAC;gBAClD,CAAC,CAAC,GAAG,KAAK,2BAA2B;gBACrC,CAAC,CAAC,GAAG,KAAK,WAAW,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,qBAAqB,EAAE,MAAM;QACxD,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,GAAG,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,cAAc,GAAG,UAAU,MAAM,gBAAgB,CAAC,YAAY,EAAE,QAAQ,CAAC,EAAE,CAAC;IAClF,MAAM,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,+EAA+E;IAC/E,iFAAiF;IACjF,iFAAiF;IACjF,2DAA2D;IAC3D,MAAM,gBAAgB,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;QACvC,QAAQ;QACR,SAAS,EAAE,eAAe,CAAC;YACzB,SAAS,EAAE,MAAM,CAAC,WAAW,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB;YACvF,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,oBAAoB,EAAE,sBAAsB;SACxD,CAAC;KACH,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC;QAAE,GAAG,CAAC,sBAAsB,QAAQ,CAAC,IAAI,yDAAyD,CAAC,CAAC;IAEzH,MAAM,IAAI,GAAe;QACvB,GAAG;QACH,oBAAoB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE;gBAClC,OAAO,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc;gBACjF,gBAAgB,EAAE,sBAAsB,EAAE,eAAe;aAC1D,EAAE,EAAE,CAAC,CAAC;YACP,OAAO,CAAC,MAAM,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC;QACtG,CAAC;QACD,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAE,CAAC;QAC1E,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAE,CAAC;KAC/E,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,IAAI,EAAE;QACxD,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU;KAC3G,CAAC,CAAC;IAEH,2EAA2E;IAC3E,0EAA0E;IAC1E,+EAA+E;IAC/E,4EAA4E;IAC5E,4EAA4E;IAC5E,4CAA4C;IAC5C,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,aAAa,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI,CAC7D,CAAC,MAAM,CAAC;IACT,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,GAAG,CACD,gEAAgE,gBAAgB,gBAAgB;YAChG,sGAAsG;YACtG,yGAAyG,CAC1G,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,uEAAuE;IACvE,8EAA8E;IAC9E,gFAAgF;IAChF,iFAAiF;IACjF,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,eAAe,GAAG,CAAC,GAAG,gBAAgB,CAAC,oBAAoB,EAAE,aAAa,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;IAC/C,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,eAAe,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAExE,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAC/D,SAAS,CAAC,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,uCAAuC,CAAC,CAAC;IAC7E,aAAa,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,wDAAwD,CAAC,CAAC;IAC/F,aAAa,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1C,WAAW,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,cAAc;QACzE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU;QACpC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC;QAC1F,YAAY,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe;QACxF,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAEjB,gFAAgF;IAChF,oFAAoF;IACpF,oFAAoF;IACpF,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,KAAK,CAAC,gBAAgB,CAAC;gBACrB,cAAc,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa;gBACxF,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,cAAc;gBAC9E,SAAS,EAAE,OAAO,EAAE,gBAAgB,EAAE,2CAA2C;aAClF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC;AACjH,CAAC"}

package/dist/eval/screen.d.ts

@@ -0,0 +1,107 @@
+import type { PoolTask } from '../solver-types/_swe-rebench-v2-pool.js';
+import { type HeldOutSlateArtifact } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+/** Stratification / diversity key: the org prefix of an instance_id
+ *  (`tobymao__sqlglot-4661` → `tobymao`). Derivable without an HF fetch. */
+export declare function repoOf(task: PoolTask): string;
+/**
+ * Order candidates round-robin across repos so the first N base-fails span
+ * repos rather than clumping in alphabetically-early ones. Deterministic:
+ * instances sort by instance_id within each repo group; repo groups iterate in
+ * sorted repo order.
+ */
+export declare function stratifyByRepo(pool: PoolTask[]): PoolTask[];
+/** One frozen run's grade outcome. `null` = unscorable (Docker/grader/infra failure). */
+export interface ScreenCandidateRun {
+    passed: boolean | null;
+    /** When `passed === null`, the stage + error that made it unscorable
+     *  (e.g. `resolve: …`, `harness: no patch`, `grade-error: …`) — surfaced so an
+     *  exclusion is never an opaque black box. */
+    unscorableReason?: string;
+}
+/**
+ * The expensive, cacheable per-candidate outcome — gradeability + the base R-run
+ * loop + (if base is 0/R) the prover. Decoupled from the selection decision so a
+ * resumed run replays it from cache without re-spending inference. The decision
+ * (held-out / no-headroom / caps) is always recomputed fresh from this.
+ */
+export interface ScreenMeasurement {
+    gradeable: boolean;
+    basePasses: number;
+    baseRuns: number;
+    baseUnscorable: boolean;
+    baseUnscorableReason?: string;
+    /** Whether the prover was reached (only when base is gradeable + 0/R). */
+    proverRan: boolean;
+    /** Meaningful only when `proverRan`; `null` = prover unscorable. */
+    proverPassed: boolean | null;
+    proverUnscorableReason?: string;
+}
+export interface ScreenDeps {
+    /** Confirm gradeable at the current semantics version (idempotent; cheap/cached). */
+    ensureGradeable(task: PoolTask): Promise<boolean>;
+    /** Base Haiku, frozen, empty impl-state. `passed: null` = unscorable. */
+    runBaseFrozen(task: PoolTask): Promise<ScreenCandidateRun>;
+    /** Prover (Codex/GPT-5.5), frozen, empty impl-state. `passed: null` = unscorable. */
+    runProverFrozen(task: PoolTask): Promise<ScreenCandidateRun>;
+    /** Resumability (optional): return a cached measurement for this instance, or
+     *  undefined to measure live. A cache hit replays for free (no inference) and
+     *  does NOT consume the maxCandidates budget. */
+    getCachedMeasurement?(instance_id: string): ScreenMeasurement | undefined;
+    /** Resumability (optional): persist a freshly-measured candidate so a later
+     *  re-run of the same command resumes instead of restarting. */
+    recordMeasurement?(instance_id: string, m: ScreenMeasurement): void;
+    log?: (msg: string) => void;
+}
+export interface ScreenOpts {
+    /** Base runs per candidate (≥3). A candidate is a reliable fail iff 0/R passed. */
+    R: number;
+    /** Exam cap N. */
+    heldOutCount: number;
+    /** Budget: stop after this many candidates reach the base-run stage. */
+    maxCandidates: number;
+    /** Max held-out instances per repo (diversity). */
+    perRepoCap: number;
+}
+export type ScreenReason = 'held-out' | 'not-gradeable' | 'base-passes' | 'base-unscorable' | 'no-headroom' | 'per-repo-cap';
+export interface ScreenedCandidate {
+    instance_id: string;
+    repo: string;
+    gradeable: boolean;
+    baseRuns: number;
+    basePasses: number;
+    proverPassed: boolean | null;
+    heldOut: boolean;
+    reason: ScreenReason;
+    /** For `base-unscorable` / `no-headroom`-via-unscorable: the stage + error
+     *  the run reported, so the exclusion is diagnosable without transcript digs. */
+    unscorableReason?: string;
+}
+export interface ScreenResult {
+    heldOut: {
+        instance_id: string;
+        repo: string;
+        baseRuns: number;
+    }[];
+    screened: ScreenedCandidate[];
+}
+/**
+ * Partition a candidate stream into the held-out exam vs the rest, applying the
+ * three filter layers cheapest-first. `candidates` MUST already be ordered (use
+ * {@link stratifyByRepo}); selection order is the iteration order and is frozen.
+ *
+ * Resumable: if `deps.getCachedMeasurement` is provided, an already-measured
+ * candidate replays from cache (no inference, no budget cost), so re-running the
+ * same command resumes — the `maxCandidates` budget bounds only NEW measurements
+ * per invocation, letting a long screen proceed in budget-sized chunks. The
+ * selection decision (caps, held-out) is always recomputed fresh, so the cached
+ * measurements stay valid even if `heldOutCount`/`perRepoCap` change.
+ */
+export declare function screenBaseFailures(candidates: PoolTask[], deps: ScreenDeps, opts: ScreenOpts): Promise<ScreenResult>;
+/** The on-disk v2 slate file = the hashed artifact + a provenance `comment`
+ *  (the comment is outside the canonical hash). solverType matches the
+ *  `${solverType}.v1` key `jinn eval` loads with. */
+export interface V2SlateFile extends HeldOutSlateArtifact {
+    comment: string;
+    hash: `sha256:${string}`;
+}
+export declare function buildV2SlateFile(instanceIds: string[], generatedAt: string): V2SlateFile;