@jinn-network/client 0.1.7 → 0.1.8-canary.09a3b2f6

package/dist/eval/screen-runner.js

@@ -0,0 +1,289 @@
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadConfig } from '../config.js';
+import { Store } from '../store/store.js';
+import { hashImplStateDir } from '../harnesses/freeze.js';
+import { LearnerHarness } from '../harnesses/impls/learner/harness.js';
+import { ClaudeCodeHarnessAdapter } from '../harnesses/impls/learner/adapters/claude-code.js';
+import { CodexCodeHarnessAdapter } from '../harnesses/impls/learner/adapters/codex-code.js';
+import { CODEX_HARNESS } from '../harnesses/names.js';
+import { runHarnessForEval, resolveRuntimePluginsForSolverType } from './eval-harness-run.js';
+import { corpusEnvFromConfig } from '../cli/commands/eval.js';
+import { loadSweRebenchV2Pool, defaultStateDir, getSweRebenchV2ValidatedPoolStore, } from '../solver-types/swe-rebench-v2.js';
+import { PoolCacheStore, loadPoolWithCacheFallback } from '../solver-types/_swe-rebench-v2-pool-cache.js';
+import { validatePoolInstances, EVAL_SEMANTICS_VERSION, } from '../solver-types/_swe-rebench-v2-validated-pool.js';
+import { resolveSlateTasks } from './resolve-slate-tasks.js';
+import { loadActiveHeldOutSlateIds, ACTIVE_HELD_OUT_SLATE_VERSIONS, loadHeldOutSlate, } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+import { GeneratorStateStore } from '../solver-types/_swe-rebench-v2-state.js';
+import { fetchAttemptedInstanceIds } from './screen-discovery.js';
+import { DEFAULT_TESTNET_DISCOVERY_URL } from '../config.js';
+import { solverTypeFromJoinedContract } from '../solver-nets/registry.js';
+import { SweRebenchV2Evaluator } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import { HttpHfFetcher } from '../harnesses/impls/swe-rebench-v2-evaluator/hf-fetcher.js';
+import { PythonEvalRunner } from '../harnesses/impls/swe-rebench-v2-evaluator/eval-runner.js';
+import { readEnabledState, defaultSweRebenchV2EvaluatorImplStateDir } from '../harnesses/impls/swe-rebench-v2-evaluator/harness.js';
+import { stratifyByRepo, screenBaseFailures, buildV2SlateFile, } from './screen.js';
+import { ScreenProgressStore, screenSignature } from './screen-progress.js';
+const DISPATCH_SOLVER_TYPE = 'swe-rebench-v2.v1';
+const SLATE_VERSION = 'v2';
+const RUN_BUDGET_MS = 3_600_000;
+/** dist/src parity: the shipped slate JSON lives next to the compiled module. */
+function slatesDir() {
+    return join(dirname(fileURLToPath(import.meta.url)), '..', 'solver-types', 'slates');
+}
+export async function runScreenHeldOut(opts) {
+    const log = opts.log ?? (() => { });
+    const config = loadConfig(opts.configPath);
+    // Precondition: evaluator enabled (upstream repo cloned).
+    const enabled = readEnabledState(defaultSweRebenchV2EvaluatorImplStateDir());
+    if (!enabled) {
+        throw new Error('swe-rebench-v2 evaluator not enabled — run `jinn harnesses enable swe-rebench-v2-evaluator` first');
+    }
+    const upstreamRepoDir = enabled.upstreamRepoDir;
+    const stateDir = process.env['JINN_SWE_REBENCH_V2_STATE_DIR'] ?? defaultStateDir();
+    const fetcher = new HttpHfFetcher();
+    const evaluator = new SweRebenchV2Evaluator({ fetcher, runner: new PythonEvalRunner({ upstreamRepoDir }) });
+    const validatedStore = getSweRebenchV2ValidatedPoolStore();
+    const runtimePlugins = await resolveRuntimePluginsForSolverType(DISPATCH_SOLVER_TYPE, config.joinedSolverNets);
+    // Common adapter wiring (mirrors buildEvalHarness in cli/commands/eval.ts).
+    const daemonApiToken = process.env['DAEMON_API_TOKEN']?.trim();
+    const corpusEnv = corpusEnvFromConfig(config);
+    const common = {
+        claudePath: config.claudePath ?? 'claude',
+        storePath: config.dbPath,
+        daemonApiUrl: `http://127.0.0.1:${config.apiPort}`,
+        ...(daemonApiToken ? { daemonApiToken } : {}),
+        ...(corpusEnv ? { corpusEnv } : {}),
+    };
+    const baseHarness = new LearnerHarness({
+        adapter: new ClaudeCodeHarnessAdapter({ ...common, claudeModel: config.claudeModel }),
+        claudePath: common.claudePath,
+    });
+    // Prover harness: codex (default) or claude-code (e.g. an Opus prover via the
+    // working Claude auth, sidestepping a codex rate limit; same-family Haiku→Opus
+    // is a clean capability ladder for "proven headroom").
+    const proverKind = opts.proverHarness ?? 'codex';
+    const proverHarness = proverKind === 'claude-code'
+        ? new LearnerHarness({
+            adapter: new ClaudeCodeHarnessAdapter({ ...common, claudeModel: opts.proverModel ?? 'opus' }),
+            claudePath: common.claudePath,
+        })
+        : new LearnerHarness({
+            name: CODEX_HARNESS,
+            adapter: new CodexCodeHarnessAdapter({ ...common, ...(opts.proverModel ? { codexModel: opts.proverModel } : {}) }),
+            claudePath: common.claudePath,
+            ...(config.codexPath !== undefined ? { codexPath: config.codexPath } : {}),
+        });
+    // Candidate pool (whole gradeable pool by default; scopeable).
+    const cacheResult = await loadPoolWithCacheFallback({
+        loadPool: loadSweRebenchV2Pool, cache: new PoolCacheStore({ stateDir }), currentPool: [],
+    });
+    let pool = cacheResult.pool;
+    if (pool.length === 0)
+        throw new Error(`SWE-rebench v2 pool empty${cacheResult.error ? ` (${cacheResult.error.message})` : ''}`);
+    // Held-out discipline (#986): draw the exam from the never-trained, never-held-out
+    // remainder. Union three exclusion sources:
+    //  - active held-out slate (would overlap an existing exam);
+    //  - already-ATTEMPTED on-network (indexer verdictEnvelopeMeta, any verdict,
+    //    cross-operator, CURRENT) — the authoritative "the learner trained on it"
+    //    signal; and
+    //  - this box's posted ledger (belt — may be STALE when another generator is
+    //    the active poster, e.g. a hosted operator; hence the indexer is the truth).
+    // A trained instance held out later would make a trained-checkpoint pass count
+    // as memorization, not generalization.
+    const heldOutIds = loadActiveHeldOutSlateIds(DISPATCH_SOLVER_TYPE, ACTIVE_HELD_OUT_SLATE_VERSIONS);
+    const postedIds = await new GeneratorStateStore({ stateDir }).postedInstanceIds();
+    let attemptedIds = new Set();
+    const discoveryUrl = config.discovery?.url?.trim()
+        || (config.network === 'testnet' ? DEFAULT_TESTNET_DISCOVERY_URL : undefined);
+    const joinedNet = Object.values(config.joinedSolverNets ?? {}).find((n) => solverTypeFromJoinedContract(n) === DISPATCH_SOLVER_TYPE);
+    if (discoveryUrl && joinedNet?.manifestCid) {
+        attemptedIds = await fetchAttemptedInstanceIds(discoveryUrl, joinedNet.manifestCid);
+        log(`[screen] indexer: ${attemptedIds.size} instance(s) already attempted on-network (authoritative, cross-operator) → excluded`);
+    }
+    else {
+        log('[screen] WARNING: no discovery URL / manifestCid — cannot exclude already-attempted instances; relying on the local posted ledger, which may be stale');
+    }
+    const excludeIds = new Set([...heldOutIds, ...postedIds, ...attemptedIds]);
+    if (opts.instanceIds?.length) {
+        // Explicit operator override — screen exactly these, but warn if any are
+        // already contaminated (attempted/posted/held-out) so an intentional pick is informed.
+        const want = new Set(opts.instanceIds);
+        pool = pool.filter((t) => want.has(t.instance_id));
+        const tainted = pool.filter((t) => excludeIds.has(t.instance_id)).map((t) => t.instance_id);
+        if (tainted.length > 0) {
+            log(`[screen] WARNING: ${tainted.length} explicitly-named instance(s) are already attempted/posted/held-out (NOT clean held-out candidates): ${tainted.join(', ')}`);
+        }
+    }
+    else {
+        if (opts.repo)
+            pool = pool.filter((t) => t.instance_id.startsWith(`${opts.repo}__`));
+        const before = pool.length;
+        pool = pool.filter((t) => !excludeIds.has(t.instance_id));
+        // Restrict to ALREADY-VALIDATED-SCORABLE candidates (#986): the never-validated
+        // tail is mostly not-gradeable (~90% deeper in the pool), so base-screening it
+        // wastes inference. Discovery of gradeability is `validate-pool`'s job; the
+        // screen selects held-out FROM the scorable set. Falls back to the full
+        // remainder only when no validation data exists yet (and warns).
+        const scorableIds = await validatedStore.getScorableIds(EVAL_SEMANTICS_VERSION);
+        if (scorableIds) {
+            const beforeScorable = pool.length;
+            pool = pool.filter((t) => scorableIds.has(t.instance_id));
+            log(`[screen] candidate pool ${before} → ${beforeScorable} (excluded ${excludeIds.size}: ${heldOutIds.size} held-out ∪ ${postedIds.size} posted ∪ ${attemptedIds.size} attempted) → ${pool.length} validated-scorable (run validate-pool to grow this)`);
+        }
+        else {
+            log(`[screen] WARNING: no validated-scorable data — screening the full ${pool.length}-task remainder (mostly not-gradeable; run validate-pool first for efficiency)`);
+        }
+    }
+    const candidates = stratifyByRepo(pool);
+    log(`[screen] ${candidates.length} candidate(s) after stratification`);
+    // Resolve a single instance to the {task,row} the harness + grader need.
+    const byId = new Map(pool.map((t) => [t.instance_id, t]));
+    async function runOnce(harness, poolTask) {
+        const implStateDir = mkdtempSync(join(tmpdir(), 'jinn-screen-state-'));
+        // Track which stage we're in so an unscorable result names its cause (#476
+        // excludes infra failures from the denominator — but it must be diagnosable,
+        // not an opaque black box).
+        let stage = 'resolve';
+        try {
+            const [resolved] = await resolveSlateTasks({
+                poolTasks: [poolTask], hf_dataset: poolTask.hf_dataset, hf_split: poolTask.hf_split, fetcher,
+            });
+            if (!resolved)
+                return { passed: null, unscorableReason: 'resolve: instance not in pool' };
+            const task = {
+                id: poolTask.instance_id,
+                description: resolved.task.problem_statement,
+                role: 'restoration',
+                solverType: DISPATCH_SOLVER_TYPE,
+                spec: resolved.task,
+                window: { startTs: 0, endTs: Date.now() + RUN_BUDGET_MS },
+            };
+            stage = 'harness';
+            const run = await runHarnessForEval({
+                harness, task, solverType: DISPATCH_SOLVER_TYPE, runtimePlugins, implStateDir, mode: 'frozen',
+            });
+            if (run.violation)
+                return { passed: null, unscorableReason: 'harness: freeze-fence violation' };
+            if (!run.solution)
+                return { passed: null, unscorableReason: 'harness: no solution produced' };
+            stage = 'grade';
+            const verdict = await evaluator.grade({
+                task: resolved.task,
+                solutionPayload: { schemaVersion: 'swe-rebench-v2-solution.v1', patch: run.solution.patch },
+                row: resolved.row,
+            });
+            return { passed: verdict.passed_match };
+        }
+        catch (err) {
+            // Any harness/grader/infra failure ⇒ unscorable, never a fail (#476). Name
+            // the stage + error so the exclusion is diagnosable. An "agent produced no
+            // patch" throw is flagged distinctly from a true infra/grader error.
+            const msg = err instanceof Error ? err.message : String(err);
+            const reason = /produced no\b|no .*patch/i.test(msg)
+                ? `${stage}: agent produced no patch`
+                : `${stage}-error: ${msg.slice(0, 200)}`;
+            return { passed: null, unscorableReason: reason };
+        }
+        finally {
+            rmSync(implStateDir, { recursive: true, force: true });
+        }
+    }
+    const emptyBaseDir = mkdtempSync(join(tmpdir(), 'jinn-screen-base-'));
+    const hashOpts = baseHarness.freezeStateHashIgnore?.length
+        ? { ignoreRelPaths: [...baseHarness.freezeStateHashIgnore] } : undefined;
+    const baseCodeDigest = `sha256:${await hashImplStateDir(emptyBaseDir, hashOpts)}`;
+    rmSync(emptyBaseDir, { recursive: true, force: true });
+    // Resumability: cache each candidate's measurement under a config signature so
+    // an interrupted run resumes (re-run the same command). The base policy is fixed
+    // (empty impl-state) so its measurement is stable; the signature invalidates the
+    // cache if the base model / prover / R / semantics change.
+    const proverModelLabel = opts.proverModel ?? (proverKind === 'claude-code' ? 'opus' : 'codex-default');
+    const progress = new ScreenProgressStore({
+        stateDir,
+        signature: screenSignature({
+            baseModel: config.claudeModel, proverHarness: proverKind, proverModel: proverModelLabel,
+            R: opts.R, evalSemanticsVersion: EVAL_SEMANTICS_VERSION,
+        }),
+    });
+    if (progress.size > 0)
+        log(`[screen] resuming: ${progress.size} candidate(s) already measured (cached) for this config`);
+    const deps = {
+        log,
+        getCachedMeasurement: (id) => progress.get(id),
+        recordMeasurement: (id, m) => progress.record(id, m),
+        ensureGradeable: async (task) => {
+            await validatePoolInstances([task], {
+                fetcher, runner: new PythonEvalRunner({ upstreamRepoDir }), store: validatedStore,
+                semanticsVersion: EVAL_SEMANTICS_VERSION, upstreamRepoDir,
+            }, {});
+            return (await validatedStore.getEntry(task.instance_id, EVAL_SEMANTICS_VERSION))?.scorable === true;
+        },
+        runBaseFrozen: (task) => runOnce(baseHarness, byId.get(task.instance_id)),
+        runProverFrozen: (task) => runOnce(proverHarness, byId.get(task.instance_id)),
+    };
+    const result = await screenBaseFailures(candidates, deps, {
+        R: opts.R, heldOutCount: opts.heldOutCount, maxCandidates: opts.maxCandidates, perRepoCap: opts.perRepoCap,
+    });
+    // Diagnosability: a base-failing candidate routed to the prover that comes
+    // back `proverPassed: null` means the prover produced NO gradeable result
+    // (errored / no patch), not a clean "the prover can't solve it". That silently
+    // routes to `no-headroom` and can yield a misleadingly empty slate when the
+    // prover is simply unavailable (e.g. codex CLI < 0.133.0, or auth missing).
+    // Surface it loudly rather than swallow it.
+    const proverUnscorable = result.screened.filter((s) => s.reason === 'no-headroom' && s.proverPassed === null).length;
+    if (proverUnscorable > 0) {
+        log(`[screen] WARNING: the prover returned no gradeable result on ${proverUnscorable} base-failing ` +
+            `candidate(s) — excluded as no-headroom, but this likely means the prover is UNAVAILABLE rather than ` +
+            `unable. Verify the codex CLI (>=0.133.0) + auth, then re-run. See proverPassed=null rows in the report.`);
+    }
+    // Cumulative v2: union the EXISTING slate with this run's NEW held-out, so
+    // re-running to GROW the exam never drops already-reserved instances. (The
+    // active-slate exclusion keeps existing held-out OUT of candidates, so
+    // result.heldOut is only the new admits.) Growing changes the slate's content
+    // hash; re-recording the base arm for the FULL set below upserts the prior rows
+    // to the new hash, so the orchestrator's slate-hash-drift guard stays satisfied.
+    let existingHeldOut = [];
+    try {
+        existingHeldOut = [...loadHeldOutSlate(DISPATCH_SOLVER_TYPE, SLATE_VERSION).instanceIds];
+    }
+    catch {
+        /* no v2 slate yet — this is the first cut */
+    }
+    const newHeldOut = result.heldOut.map((h) => h.instance_id);
+    const allHeldOutIds = [...new Set([...existingHeldOut, ...newHeldOut])];
+    const generatedAt = new Date().toISOString();
+    const slateFile = buildV2SlateFile(allHeldOutIds, generatedAt);
+    mkdirSync(slatesDir(), { recursive: true });
+    const slatePath = join(slatesDir(), 'held-out-slate.swe-rebench-v2.v2.json');
+    writeFileSync(slatePath, `${JSON.stringify(slateFile, null, 2)}\n`);
+    const reportPath = join(slatesDir(), 'held-out-slate.swe-rebench-v2.v2.screening-report.json');
+    writeFileSync(reportPath, `${JSON.stringify({
+        generatedAt, evalSemanticsVersion: EVAL_SEMANTICS_VERSION, baseCodeDigest,
+        R: opts.R, proverHarness: proverKind,
+        proverModel: opts.proverModel ?? (proverKind === 'claude-code' ? 'opus' : 'codex-default'),
+        heldOutTotal: allHeldOutIds.length, newThisRun: newHeldOut, carriedOver: existingHeldOut,
+        screened: result.screened,
+    }, null, 2)}\n`);
+    // Persist the base arm (all-fail) for the FULL held-out set under the (possibly
+    // new) slate hash. The upsert is keyed by (checkpoint, slate_version, instance_id),
+    // so prior rows get their slate_hash refreshed to match the grown slate — no drift.
+    const store = new Store(config.dbPath);
+    try {
+        const runAtMs = Date.now();
+        for (const id of allHeldOutIds) {
+            store.recordEvalResult({
+                checkpoint_cid: baseCodeDigest, slate_hash: slateFile.hash, slate_version: SLATE_VERSION,
+                instance_id: id, passed: false, unscorable: false, code_digest: baseCodeDigest,
+                run_at_ms: runAtMs, test_log_excerpt: 'base arm (screening): consistent fail 0/R',
+            });
+        }
+    }
+    finally {
+        store.close?.();
+    }
+    return { result, baseCodeDigest, slatePath, reportPath, heldOutCount: allHeldOutIds.length, proverUnscorable };
+}
+//# sourceMappingURL=screen-runner.js.map

package/dist/eval/screen-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-runner.js","sourceRoot":"","sources":["../../src/eval/screen-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,oDAAoD,CAAC;AAC9F,OAAO,EAAE,uBAAuB,EAAE,MAAM,mDAAmD,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,kCAAkC,EAAE,MAAM,uBAAuB,CAAC;AAC9F,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EACL,oBAAoB,EAAE,eAAe,EAAE,iCAAiC,GACzE,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,+CAA+C,CAAC;AAC1G,OAAO,EACL,qBAAqB,EAAE,sBAAsB,GAC9C,MAAM,mDAAmD,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EACL,yBAAyB,EAAE,8BAA8B,EAAE,gBAAgB,GAC5E,MAAM,mDAAmD,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAC;AAC/E,OAAO,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,qBAAqB,EAAE,MAAM,sDAAsD,CAAC;AAC7F,OAAO,EAAE,aAAa,EAAE,MAAM,2DAA2D,CAAC;AAC1F,OAAO,EAAE,gBAAgB,EAAE,MAAM,4DAA4D,CAAC;AAC9F,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,MAAM,wDAAwD,CAAC;AACpI,OAAO,EACL,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,GAErD,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAE5E,MAAM,oBAAoB,GAAG,mBAAmB,CAAC;AACjD,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,aAAa,GAAG,SAAS,CAAC;AAgChC,iFAAiF;AACjF,SAAS,SAAS;IAChB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;AACvF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAsB;IAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE3C,0DAA0D;IAC1D,MAAM,OAAO,GAAG,gBAAgB,CAAC,wCAAwC,EAAE,CAAC,CAAC;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;IACJ,CAAC;IACD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAEhD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,IAAI,eAAe,EAAE,CAAC;IACnF,MAAM,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,IAAI,qBAAqB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5G,MAAM,cAAc,GAAG,iCAAiC,EAAE,CAAC;IAC3D,MAAM,cAAc,GAAoB,MAAM,kCAAkC,CAC9E,oBAAoB,EAAE,MAAM,CAAC,gBAAgB,CAC9C,CAAC;IAEF,4EAA4E;IAC5E,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,CAAC;IAC/D,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG;QACb,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,QAAQ;QACzC,SAAS,EAAE,MAAM,CAAC,MAAM;QACxB,YAAY,EAAE,oBAAoB,MAAM,CAAC,OAAO,EAAE;QAClD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpC,CAAC;IACF,MAAM,WAAW,GAAY,IAAI,cAAc,CAAC;QAC9C,OAAO,EAAE,IAAI,wBAAwB,CAAC,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;QACrF,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IACH,8EAA8E;IAC9E,+EAA+E;IAC/E,uDAAuD;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC;IACjD,MAAM,aAAa,GAAY,UAAU,KAAK,aAAa;QACzD,CAAC,CAAC,IAAI,cAAc,CAAC;YACjB,OAAO,EAAE,IAAI,wBAAwB,CAAC,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,MAAM,EAAE,CAAC;YAC7F,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;QACJ,CAAC,CAAC,IAAI,cAAc,CAAC;YACjB,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,IAAI,uBAAuB,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YAClH,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3E,CAAC,CAAC;IAEP,+DAA+D;IAC/D,MAAM,WAAW,GAAG,MAAM,yBAAyB,CAAC;QAClD,QAAQ,EAAE,oBAAoB,EAAE,KAAK,EAAE,IAAI,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE;KACzF,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;IAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjI,mFAAmF;IACnF,4CAA4C;IAC5C,6DAA6D;IAC7D,6EAA6E;IAC7E,8EAA8E;IAC9E,iBAAiB;IACjB,6EAA6E;IAC7E,iFAAiF;IACjF,+EAA+E;IAC/E,uCAAuC;IACvC,MAAM,UAAU,GAAG,yBAAyB,CAAC,oBAAoB,EAAE,8BAA8B,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,MAAM,IAAI,mBAAmB,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,iBAAiB,EAAE,CAAC;IAClF,IAAI,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE;WAC7C,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,CACjE,CAAC,CAAC,EAAE,EAAE,CAAC,4BAA4B,CAAC,CAAC,CAAC,KAAK,oBAAoB,CAChE,CAAC;IACF,IAAI,YAAY,IAAI,SAAS,EAAE,WAAW,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,YAAY,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;QACpF,GAAG,CAAC,qBAAqB,YAAY,CAAC,IAAI,sFAAsF,CAAC,CAAC;IACpI,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,uJAAuJ,CAAC,CAAC;IAC/J,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,UAAU,EAAE,GAAG,SAAS,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;IACnF,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QAC7B,yEAAyE;QACzE,uFAAuF;QACvF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,qBAAqB,OAAO,CAAC,MAAM,wGAAwG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvK,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,IAAI,CAAC,IAAI;YAAE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACrF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1D,gFAAgF;QAChF,+EAA+E;QAC/E,4EAA4E;QAC5E,wEAAwE;QACxE,iEAAiE;QACjE,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;QAChF,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;YACnC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;YAC1D,GAAG,CAAC,2BAA2B,MAAM,MAAM,cAAc,cAAc,UAAU,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,eAAe,SAAS,CAAC,IAAI,aAAa,YAAY,CAAC,IAAI,iBAAiB,IAAI,CAAC,MAAM,sDAAsD,CAAC,CAAC;QAC3P,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,qEAAqE,IAAI,CAAC,MAAM,gFAAgF,CAAC,CAAC;QACxK,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACxC,GAAG,CAAC,YAAY,UAAU,CAAC,MAAM,oCAAoC,CAAC,CAAC;IAEvE,yEAAyE;IACzE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,KAAK,UAAU,OAAO,CAAC,OAAgB,EAAE,QAAkB;QACzD,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAC;QACvE,2EAA2E;QAC3E,6EAA6E;QAC7E,4BAA4B;QAC5B,IAAI,KAAK,GAAoC,SAAS,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,iBAAiB,CAAC;gBACzC,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO;aAC7F,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,CAAC;YAC1F,MAAM,IAAI,GAAS;gBACjB,EAAE,EAAE,QAAQ,CAAC,WAAW;gBACxB,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,iBAAiB;gBAC5C,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,oBAAoB;gBAChC,IAAI,EAAE,QAAQ,CAAC,IAA0C;gBACzD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,EAAE;aAC1D,CAAC;YACF,KAAK,GAAG,SAAS,CAAC;YAClB,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC;gBAClC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,oBAAoB,EAAE,cAAc,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ;aAC9F,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,SAAS;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,iCAAiC,EAAE,CAAC;YAChG,IAAI,CAAC,GAAG,CAAC,QAAQ;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,CAAC;YAC9F,KAAK,GAAG,OAAO,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC;gBACpC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,eAAe,EAAE,EAAE,aAAa,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAC3F,GAAG,EAAE,QAAQ,CAAC,GAAG;aAClB,CAAC,CAAC;YACH,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2EAA2E;YAC3E,2EAA2E;YAC3E,qEAAqE;YACrE,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,2BAA2B,CAAC,IAAI,CAAC,GAAG,CAAC;gBAClD,CAAC,CAAC,GAAG,KAAK,2BAA2B;gBACrC,CAAC,CAAC,GAAG,KAAK,WAAW,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,qBAAqB,EAAE,MAAM;QACxD,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,GAAG,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,cAAc,GAAG,UAAU,MAAM,gBAAgB,CAAC,YAAY,EAAE,QAAQ,CAAC,EAAE,CAAC;IAClF,MAAM,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,+EAA+E;IAC/E,iFAAiF;IACjF,iFAAiF;IACjF,2DAA2D;IAC3D,MAAM,gBAAgB,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,IAAI,mBAAmB,CAAC;QACvC,QAAQ;QACR,SAAS,EAAE,eAAe,CAAC;YACzB,SAAS,EAAE,MAAM,CAAC,WAAW,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB;YACvF,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,oBAAoB,EAAE,sBAAsB;SACxD,CAAC;KACH,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC;QAAE,GAAG,CAAC,sBAAsB,QAAQ,CAAC,IAAI,yDAAyD,CAAC,CAAC;IAEzH,MAAM,IAAI,GAAe;QACvB,GAAG;QACH,oBAAoB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE;gBAClC,OAAO,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc;gBACjF,gBAAgB,EAAE,sBAAsB,EAAE,eAAe;aAC1D,EAAE,EAAE,CAAC,CAAC;YACP,OAAO,CAAC,MAAM,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC,EAAE,QAAQ,KAAK,IAAI,CAAC;QACtG,CAAC;QACD,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAE,CAAC;QAC1E,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAE,CAAC;KAC/E,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,IAAI,EAAE;QACxD,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU;KAC3G,CAAC,CAAC;IAEH,2EAA2E;IAC3E,0EAA0E;IAC1E,+EAA+E;IAC/E,4EAA4E;IAC5E,4EAA4E;IAC5E,4CAA4C;IAC5C,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,aAAa,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI,CAC7D,CAAC,MAAM,CAAC;IACT,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,GAAG,CACD,gEAAgE,gBAAgB,gBAAgB;YAChG,sGAAsG;YACtG,yGAAyG,CAC1G,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,uEAAuE;IACvE,8EAA8E;IAC9E,gFAAgF;IAChF,iFAAiF;IACjF,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,eAAe,GAAG,CAAC,GAAG,gBAAgB,CAAC,oBAAoB,EAAE,aAAa,CAAC,CAAC,WAAW,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,6CAA6C;IAC/C,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,eAAe,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAExE,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAC/D,SAAS,CAAC,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,uCAAuC,CAAC,CAAC;IAC7E,aAAa,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,wDAAwD,CAAC,CAAC;IAC/F,aAAa,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1C,WAAW,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,cAAc;QACzE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU;QACpC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC;QAC1F,YAAY,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe;QACxF,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAEjB,gFAAgF;IAChF,oFAAoF;IACpF,oFAAoF;IACpF,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,KAAK,CAAC,gBAAgB,CAAC;gBACrB,cAAc,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa;gBACxF,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,cAAc;gBAC9E,SAAS,EAAE,OAAO,EAAE,gBAAgB,EAAE,2CAA2C;aAClF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC;AACjH,CAAC"}

package/dist/eval/screen.d.ts

@@ -0,0 +1,107 @@
+import type { PoolTask } from '../solver-types/_swe-rebench-v2-pool.js';
+import { type HeldOutSlateArtifact } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+/** Stratification / diversity key: the org prefix of an instance_id
+ *  (`tobymao__sqlglot-4661` → `tobymao`). Derivable without an HF fetch. */
+export declare function repoOf(task: PoolTask): string;
+/**
+ * Order candidates round-robin across repos so the first N base-fails span
+ * repos rather than clumping in alphabetically-early ones. Deterministic:
+ * instances sort by instance_id within each repo group; repo groups iterate in
+ * sorted repo order.
+ */
+export declare function stratifyByRepo(pool: PoolTask[]): PoolTask[];
+/** One frozen run's grade outcome. `null` = unscorable (Docker/grader/infra failure). */
+export interface ScreenCandidateRun {
+    passed: boolean | null;
+    /** When `passed === null`, the stage + error that made it unscorable
+     *  (e.g. `resolve: …`, `harness: no patch`, `grade-error: …`) — surfaced so an
+     *  exclusion is never an opaque black box. */
+    unscorableReason?: string;
+}
+/**
+ * The expensive, cacheable per-candidate outcome — gradeability + the base R-run
+ * loop + (if base is 0/R) the prover. Decoupled from the selection decision so a
+ * resumed run replays it from cache without re-spending inference. The decision
+ * (held-out / no-headroom / caps) is always recomputed fresh from this.
+ */
+export interface ScreenMeasurement {
+    gradeable: boolean;
+    basePasses: number;
+    baseRuns: number;
+    baseUnscorable: boolean;
+    baseUnscorableReason?: string;
+    /** Whether the prover was reached (only when base is gradeable + 0/R). */
+    proverRan: boolean;
+    /** Meaningful only when `proverRan`; `null` = prover unscorable. */
+    proverPassed: boolean | null;
+    proverUnscorableReason?: string;
+}
+export interface ScreenDeps {
+    /** Confirm gradeable at the current semantics version (idempotent; cheap/cached). */
+    ensureGradeable(task: PoolTask): Promise<boolean>;
+    /** Base Haiku, frozen, empty impl-state. `passed: null` = unscorable. */
+    runBaseFrozen(task: PoolTask): Promise<ScreenCandidateRun>;
+    /** Prover (Codex/GPT-5.5), frozen, empty impl-state. `passed: null` = unscorable. */
+    runProverFrozen(task: PoolTask): Promise<ScreenCandidateRun>;
+    /** Resumability (optional): return a cached measurement for this instance, or
+     *  undefined to measure live. A cache hit replays for free (no inference) and
+     *  does NOT consume the maxCandidates budget. */
+    getCachedMeasurement?(instance_id: string): ScreenMeasurement | undefined;
+    /** Resumability (optional): persist a freshly-measured candidate so a later
+     *  re-run of the same command resumes instead of restarting. */
+    recordMeasurement?(instance_id: string, m: ScreenMeasurement): void;
+    log?: (msg: string) => void;
+}
+export interface ScreenOpts {
+    /** Base runs per candidate (≥3). A candidate is a reliable fail iff 0/R passed. */
+    R: number;
+    /** Exam cap N. */
+    heldOutCount: number;
+    /** Budget: stop after this many candidates reach the base-run stage. */
+    maxCandidates: number;
+    /** Max held-out instances per repo (diversity). */
+    perRepoCap: number;
+}
+export type ScreenReason = 'held-out' | 'not-gradeable' | 'base-passes' | 'base-unscorable' | 'no-headroom' | 'per-repo-cap';
+export interface ScreenedCandidate {
+    instance_id: string;
+    repo: string;
+    gradeable: boolean;
+    baseRuns: number;
+    basePasses: number;
+    proverPassed: boolean | null;
+    heldOut: boolean;
+    reason: ScreenReason;
+    /** For `base-unscorable` / `no-headroom`-via-unscorable: the stage + error
+     *  the run reported, so the exclusion is diagnosable without transcript digs. */
+    unscorableReason?: string;
+}
+export interface ScreenResult {
+    heldOut: {
+        instance_id: string;
+        repo: string;
+        baseRuns: number;
+    }[];
+    screened: ScreenedCandidate[];
+}
+/**
+ * Partition a candidate stream into the held-out exam vs the rest, applying the
+ * three filter layers cheapest-first. `candidates` MUST already be ordered (use
+ * {@link stratifyByRepo}); selection order is the iteration order and is frozen.
+ *
+ * Resumable: if `deps.getCachedMeasurement` is provided, an already-measured
+ * candidate replays from cache (no inference, no budget cost), so re-running the
+ * same command resumes — the `maxCandidates` budget bounds only NEW measurements
+ * per invocation, letting a long screen proceed in budget-sized chunks. The
+ * selection decision (caps, held-out) is always recomputed fresh, so the cached
+ * measurements stay valid even if `heldOutCount`/`perRepoCap` change.
+ */
+export declare function screenBaseFailures(candidates: PoolTask[], deps: ScreenDeps, opts: ScreenOpts): Promise<ScreenResult>;
+/** The on-disk v2 slate file = the hashed artifact + a provenance `comment`
+ *  (the comment is outside the canonical hash). solverType matches the
+ *  `${solverType}.v1` key `jinn eval` loads with. */
+export interface V2SlateFile extends HeldOutSlateArtifact {
+    comment: string;
+    hash: `sha256:${string}`;
+}
+export declare function buildV2SlateFile(instanceIds: string[], generatedAt: string): V2SlateFile;

package/dist/eval/screen.js

@@ -0,0 +1,159 @@
+import { HELD_OUT_SLATE_SCHEMA_VERSION, hashHeldOutSlateArtifact, } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+/** Stratification / diversity key: the org prefix of an instance_id
+ *  (`tobymao__sqlglot-4661` → `tobymao`). Derivable without an HF fetch. */
+export function repoOf(task) {
+    const idx = task.instance_id.indexOf('__');
+    return idx === -1 ? task.instance_id : task.instance_id.slice(0, idx);
+}
+/**
+ * Order candidates round-robin across repos so the first N base-fails span
+ * repos rather than clumping in alphabetically-early ones. Deterministic:
+ * instances sort by instance_id within each repo group; repo groups iterate in
+ * sorted repo order.
+ */
+export function stratifyByRepo(pool) {
+    const groups = new Map();
+    for (const task of pool) {
+        const repo = repoOf(task);
+        (groups.get(repo) ?? groups.set(repo, []).get(repo)).push(task);
+    }
+    const repos = [...groups.keys()].sort((a, b) => a.localeCompare(b));
+    for (const repo of repos) {
+        groups.get(repo).sort((a, b) => a.instance_id.localeCompare(b.instance_id));
+    }
+    const out = [];
+    let added = true;
+    for (let i = 0; added; i++) {
+        added = false;
+        for (const repo of repos) {
+            const g = groups.get(repo);
+            if (i < g.length) {
+                out.push(g[i]);
+                added = true;
+            }
+        }
+    }
+    return out;
+}
+/**
+ * The expensive part: gradeable → base R-runs (early-stop on first pass) →
+ * prover (only if base is 0/R). Pure measurement; no selection/caps. This is what
+ * gets cached for resumability.
+ */
+async function measureCandidate(task, deps, R) {
+    const none = { gradeable: false, basePasses: 0, baseRuns: 0, baseUnscorable: false, proverRan: false, proverPassed: null };
+    if (!(await deps.ensureGradeable(task)))
+        return none;
+    let basePasses = 0;
+    let baseUnscorable = false;
+    let baseUnscorableReason;
+    let r = 0;
+    for (; r < R; r++) {
+        const run = await deps.runBaseFrozen(task);
+        if (run.passed === null) {
+            baseUnscorable = true;
+            baseUnscorableReason = run.unscorableReason;
+            break;
+        }
+        if (run.passed) {
+            basePasses++;
+            break;
+        }
+    }
+    const baseRuns = r + (baseUnscorable || basePasses > 0 ? 1 : 0);
+    if (baseUnscorable) {
+        return { gradeable: true, basePasses: 0, baseRuns, baseUnscorable: true, ...(baseUnscorableReason ? { baseUnscorableReason } : {}), proverRan: false, proverPassed: null };
+    }
+    if (basePasses > 0) {
+        return { gradeable: true, basePasses, baseRuns, baseUnscorable: false, proverRan: false, proverPassed: null };
+    }
+    // Base reliably fails (0/R) → layer 3: prover (existence proof of headroom).
+    const prover = await deps.runProverFrozen(task);
+    return {
+        gradeable: true, basePasses: 0, baseRuns, baseUnscorable: false, proverRan: true, proverPassed: prover.passed,
+        ...(prover.passed === null && prover.unscorableReason ? { proverUnscorableReason: prover.unscorableReason } : {}),
+    };
+}
+/**
+ * Partition a candidate stream into the held-out exam vs the rest, applying the
+ * three filter layers cheapest-first. `candidates` MUST already be ordered (use
+ * {@link stratifyByRepo}); selection order is the iteration order and is frozen.
+ *
+ * Resumable: if `deps.getCachedMeasurement` is provided, an already-measured
+ * candidate replays from cache (no inference, no budget cost), so re-running the
+ * same command resumes — the `maxCandidates` budget bounds only NEW measurements
+ * per invocation, letting a long screen proceed in budget-sized chunks. The
+ * selection decision (caps, held-out) is always recomputed fresh, so the cached
+ * measurements stay valid even if `heldOutCount`/`perRepoCap` change.
+ */
+export async function screenBaseFailures(candidates, deps, opts) {
+    const log = deps.log ?? (() => { });
+    const heldOut = [];
+    const screened = [];
+    const perRepo = new Map();
+    let liveMeasured = 0;
+    for (const task of candidates) {
+        if (heldOut.length >= opts.heldOutCount)
+            break;
+        const repo = repoOf(task);
+        const base = { instance_id: task.instance_id, repo, basePasses: 0, proverPassed: null };
+        // Measure (from cache, or live — bounded by the per-invocation budget).
+        let m = deps.getCachedMeasurement?.(task.instance_id);
+        if (!m) {
+            if (liveMeasured >= opts.maxCandidates)
+                break; // budget bounds NEW (inference-spending) measurements
+            m = await measureCandidate(task, deps, opts.R);
+            liveMeasured += 1;
+            deps.recordMeasurement?.(task.instance_id, m);
+        }
+        // Decide from the measurement (always fresh; cap/diversity not cached).
+        if (!m.gradeable) {
+            screened.push({ ...base, baseRuns: 0, gradeable: false, heldOut: false, reason: 'not-gradeable' });
+            continue;
+        }
+        if (m.baseUnscorable) {
+            if (m.baseUnscorableReason)
+                log(`[screen] ${task.instance_id} base-unscorable: ${m.baseUnscorableReason}`);
+            screened.push({ ...base, baseRuns: m.baseRuns, gradeable: true, heldOut: false, reason: 'base-unscorable', ...(m.baseUnscorableReason ? { unscorableReason: m.baseUnscorableReason } : {}) });
+            continue;
+        }
+        if (m.basePasses > 0) {
+            screened.push({ ...base, baseRuns: m.baseRuns, basePasses: m.basePasses, gradeable: true, heldOut: false, reason: 'base-passes' });
+            continue;
+        }
+        // Base 0/R → prover outcome (layer 3).
+        if (m.proverPassed !== true) {
+            if (m.proverPassed === null && m.proverUnscorableReason)
+                log(`[screen] ${task.instance_id} prover-unscorable: ${m.proverUnscorableReason}`);
+            screened.push({
+                ...base, baseRuns: m.baseRuns, gradeable: true, proverPassed: m.proverPassed, heldOut: false, reason: 'no-headroom',
+                ...(m.proverPassed === null && m.proverUnscorableReason ? { unscorableReason: m.proverUnscorableReason } : {}),
+            });
+            continue;
+        }
+        if ((perRepo.get(repo) ?? 0) >= opts.perRepoCap) {
+            screened.push({ ...base, baseRuns: m.baseRuns, gradeable: true, proverPassed: true, heldOut: false, reason: 'per-repo-cap' });
+            continue;
+        }
+        perRepo.set(repo, (perRepo.get(repo) ?? 0) + 1);
+        heldOut.push({ instance_id: task.instance_id, repo, baseRuns: m.baseRuns });
+        screened.push({ ...base, baseRuns: m.baseRuns, gradeable: true, proverPassed: true, heldOut: true, reason: 'held-out' });
+        log(`[screen] held out ${task.instance_id} (${heldOut.length}/${opts.heldOutCount})`);
+    }
+    return { heldOut, screened };
+}
+const V2_SLATE_COMMENT = 'BASELINE-FAILURE REGRESSION BENCHMARK (issue #986). Screened: gradeable at the current ' +
+    'evalSemanticsVersion AND base claude-code/Haiku frozen fails 0/R (R≥3) AND a stronger Codex/GPT-5.5 ' +
+    'prover passes ≥1 (proven headroom). Baseline 0% by construction. Held out from the generator train ' +
+    'stream via the active-slate-version union. Content-addressed; scores comparable WITHIN this version only.';
+export function buildV2SlateFile(instanceIds, generatedAt) {
+    const artifact = {
+        schemaVersion: HELD_OUT_SLATE_SCHEMA_VERSION,
+        solverType: 'swe-rebench-v2.v1',
+        version: 'v2',
+        generatedAt,
+        instanceIds: [...instanceIds].sort((a, b) => a.localeCompare(b)),
+    };
+    return { comment: V2_SLATE_COMMENT, ...artifact, hash: hashHeldOutSlateArtifact(artifact) };
+}
+//# sourceMappingURL=screen.js.map

package/dist/eval/screen.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen.js","sourceRoot":"","sources":["../../src/eval/screen.ts"],"names":[],"mappings":"AACA,OAAO,EACL,6BAA6B,EAC7B,wBAAwB,GAEzB,MAAM,mDAAmD,CAAC;AAE3D;4EAC4E;AAC5E,MAAM,UAAU,MAAM,CAAC,IAAc;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACxE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,IAAgB;IAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,KAAK,GAAG,KAAK,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;gBAChB,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAgFD;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAAC,IAAc,EAAE,IAAgB,EAAE,CAAS;IACzE,MAAM,IAAI,GAAsB,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC9I,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAErD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,oBAAwC,CAAC;IAC7C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAAC,cAAc,GAAG,IAAI,CAAC;YAAC,oBAAoB,GAAG,GAAG,CAAC,gBAAgB,CAAC;YAAC,MAAM;QAAC,CAAC;QACvG,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAAC,UAAU,EAAE,CAAC;YAAC,MAAM;QAAC,CAAC;IAC1C,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,cAAc,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC7K,CAAC;IACD,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAChH,CAAC;IACD,6EAA6E;IAC7E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAChD,OAAO;QACL,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM;QAC7G,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClH,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAsB,EACtB,IAAgB,EAChB,IAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,OAAO,GAA4B,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,IAAsB,EAAE,CAAC;QAE1G,wEAAwE;QACxE,IAAI,CAAC,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,YAAY,IAAI,IAAI,CAAC,aAAa;gBAAE,MAAM,CAAC,sDAAsD;YACrG,CAAC,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/C,YAAY,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,iBAAiB,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;YACnG,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,oBAAoB;gBAAE,GAAG,CAAC,YAAY,IAAI,CAAC,WAAW,qBAAqB,CAAC,CAAC,oBAAoB,EAAE,CAAC,CAAC;YAC3G,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9L,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YACnI,SAAS;QACX,CAAC;QACD,uCAAuC;QACvC,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI,IAAI,CAAC,CAAC,sBAAsB;gBAAE,GAAG,CAAC,YAAY,IAAI,CAAC,WAAW,uBAAuB,CAAC,CAAC,sBAAsB,EAAE,CAAC,CAAC;YAC5I,QAAQ,CAAC,IAAI,CAAC;gBACZ,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa;gBACnH,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,IAAI,IAAI,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/G,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;YAC9H,SAAS;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACzH,GAAG,CAAC,qBAAqB,IAAI,CAAC,WAAW,KAAK,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAC/B,CAAC;AAUD,MAAM,gBAAgB,GACpB,yFAAyF;IACzF,sGAAsG;IACtG,qGAAqG;IACrG,2GAA2G,CAAC;AAE9G,MAAM,UAAU,gBAAgB,CAAC,WAAqB,EAAE,WAAmB;IACzE,MAAM,QAAQ,GAAyB;QACrC,aAAa,EAAE,6BAA6B;QAC5C,UAAU,EAAE,mBAAmB;QAC/B,OAAO,EAAE,IAAI;QACb,WAAW;QACX,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;KACjE,CAAC;IACF,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,wBAAwB,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC9F,CAAC"}

package/dist/eval/slope.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Ordinary least-squares slope of resolved-rate vs cycle index for the
+ * train-arm slope measurement (issue #822, AC#1).
+ *
+ * The train-arm e2e evaluates a checkpoint against the held-out slate (#817)
+ * at intervals via the eval orchestrator (#818), collecting one
+ * `{ cycleIndex, rate }` point per interval (`rate` = passed / scorable, the
+ * Wilson point estimate). The slope of the least-squares fit is the headline
+ * "is the learner improving across the training sequence" number.
+ *
+ * It is deliberately a thin helper over the closed-form OLS slope
+ * (`cov(x,y) / var(x)`); the per-point confidence intervals come from
+ * `wilson.ts` — this module does NOT reimplement them. The slope sign alone is
+ * never a verdict at small N: a flat or slightly negative slope is "within
+ * noise", which the e2e surfaces via the §4.1 honesty caveat.
+ */
+export interface RatePoint {
+    /** Training cycle index the eval ran at (0 = baseline, before any training). */
+    cycleIndex: number;
+    /** Observed resolved rate at that interval (passed / scorable, in [0, 1]). */
+    rate: number;
+}
+/**
+ * Least-squares slope of `rate` regressed on `cycleIndex`. Returns 0 for fewer
+ * than two points (no line to fit) and for a degenerate fit where every x is
+ * identical (zero variance — division would be NaN). A flat sequence yields
+ * exactly 0.
+ */
+export declare function leastSquaresSlope(points: RatePoint[]): number;