@jinn-network/client 0.1.8 → 0.1.9-canary.023faefb

package/dist/eval/resolve-slate-tasks.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Resolve a held-out slate's `instance_id`s into `SweRebenchV2Task` objects
+ * for the `jinn eval` orchestrator (issue #818, AC#1).
+ *
+ * A slate stores only `instance_id`s (`_swe-rebench-v2-held-out-slate.ts`). The
+ * evaluator grades against the full HuggingFace row, fetched by
+ * `(hf_dataset, hf_split, instance_id)`. The `HfRow` does NOT echo
+ * `hf_dataset`/`hf_split` (see `swe-rebench-v2-evaluator/index.ts`), so the
+ * caller supplies the slate-level dataset+split as args; the resolver verifies
+ * each id exists by fetching its row and returns the `{ task, row }` pair so the
+ * orchestrator reuses the row at grade time (avoids a second fetch).
+ *
+ * No retry logic here — `HttpHfFetcher` owns retries. A fetcher throw for any id
+ * propagates loudly (a missing slate instance is a hard error, never a silent
+ * drop).
+ */
+import { type SweRebenchV2Task } from '../vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js';
+import type { HfFetcher, HfRow } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import type { PoolTask } from '../solver-types/_swe-rebench-v2-pool.js';
+export interface ResolvedSlateTask {
+    task: SweRebenchV2Task;
+    row: HfRow;
+}
+export declare function resolveSlateTasks(args: {
+    /**
+     * The pool tasks for this slate group, each carrying the real
+     * `problem_statement` / `base_commit` / `language` the agent run needs to
+     * solve the instance (the slate stores only ids; the pool is the source of
+     * these fields).
+     */
+    poolTasks: PoolTask[];
+    hf_dataset: string;
+    hf_split: string;
+    fetcher: HfFetcher;
+}): Promise<ResolvedSlateTask[]>;

package/dist/eval/resolve-slate-tasks.js

@@ -0,0 +1,56 @@
+/**
+ * Resolve a held-out slate's `instance_id`s into `SweRebenchV2Task` objects
+ * for the `jinn eval` orchestrator (issue #818, AC#1).
+ *
+ * A slate stores only `instance_id`s (`_swe-rebench-v2-held-out-slate.ts`). The
+ * evaluator grades against the full HuggingFace row, fetched by
+ * `(hf_dataset, hf_split, instance_id)`. The `HfRow` does NOT echo
+ * `hf_dataset`/`hf_split` (see `swe-rebench-v2-evaluator/index.ts`), so the
+ * caller supplies the slate-level dataset+split as args; the resolver verifies
+ * each id exists by fetching its row and returns the `{ task, row }` pair so the
+ * orchestrator reuses the row at grade time (avoids a second fetch).
+ *
+ * No retry logic here — `HttpHfFetcher` owns retries. A fetcher throw for any id
+ * propagates loudly (a missing slate instance is a hard error, never a silent
+ * drop).
+ */
+import { SweRebenchV2TaskSchema } from '../vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js';
+export async function resolveSlateTasks(args) {
+    const tasks = [...args.poolTasks].sort((a, b) => a.instance_id.localeCompare(b.instance_id));
+    const out = [];
+    for (const poolTask of tasks) {
+        const row = await args.fetcher.fetchTaskRow({
+            hf_dataset: args.hf_dataset,
+            hf_split: args.hf_split,
+            instance_id: poolTask.instance_id,
+        });
+        out.push({ task: buildTask(poolTask, args.hf_dataset, args.hf_split, row), row });
+    }
+    return out;
+}
+/**
+ * Construct the `SweRebenchV2Task` the orchestrator hands to the harness +
+ * evaluator. The agent run needs the real `problem_statement` (what to solve)
+ * and `base_commit` (the repo state to check out) — these are threaded through
+ * from the pool task (the generator's mapping in `swe-rebench-v2.ts`).
+ * `hf_dataset`/`hf_split` are the evaluator's row-fetch key; `repo` comes from
+ * the fetched row. A pool task missing an optional field falls back the same
+ * way the generator does, so the object stays schema-valid.
+ */
+function buildTask(poolTask, hf_dataset, hf_split, row) {
+    const language = SweRebenchV2TaskSchema.shape.language.safeParse(poolTask.language);
+    return {
+        schemaVersion: 'swe-rebench-v2.v1',
+        instance_id: poolTask.instance_id,
+        repo: row.repo,
+        base_commit: poolTask.base_commit ?? '0'.repeat(40),
+        language: language.success ? language.data : 'python',
+        problem_statement: poolTask.problem_statement ?? '',
+        interface: poolTask.interface ?? '',
+        hf_dataset,
+        hf_split,
+        deadline_unix: 1,
+        round_month: hf_split.replace('_', '-'),
+    };
+}
+//# sourceMappingURL=resolve-slate-tasks.js.map

package/dist/eval/resolve-slate-tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolve-slate-tasks.js","sourceRoot":"","sources":["../../src/eval/resolve-slate-tasks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,sBAAsB,EAAyB,MAAM,6CAA6C,CAAC;AAS5G,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAWvC;IACC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7F,MAAM,GAAG,GAAwB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;YAC1C,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;SAClC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,SAAS,CAChB,QAAkB,EAClB,UAAkB,EAClB,QAAgB,EAChB,GAAU;IAEV,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpF,OAAO;QACL,aAAa,EAAE,mBAAmB;QAClC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;QACrD,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB,IAAI,EAAE;QACnD,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;QACnC,UAAU;QACV,QAAQ;QACR,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;KACxC,CAAC;AACJ,CAAC"}

package/dist/eval/screen-discovery.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Held-out screening's authoritative "already trained-on" exclusion source (#986).
+ *
+ * Returns the set of swe-rebench-v2 `instance_id`s that have been ATTEMPTED on
+ * the network for a SolverNet — any verdict envelope, passed OR failed,
+ * cross-operator. An attempted instance was executed by an operator, so the
+ * learner trained on it; holding it out later would make a trained-checkpoint
+ * pass count as memorization, not generalization.
+ *
+ * Why the indexer and not the local generator-state: the local ledger only
+ * reflects THIS box's posting and can be stale (a different active generator —
+ * e.g. a hosted operator — posts to its own ledger). The indexer's
+ * `verdictEnvelopeMeta` is the cross-operator, current record. (The on-chain
+ * task/attempt tables carry no instance_id; only the indexer's IPFS enrichment
+ * resolves it — same backing as `DiscoveryAPI.getInstanceSuccessCounts`, minus
+ * the `actualPassed: true` filter so failed attempts count too.)
+ *
+ * Throws on indexer failure — callers MUST abort rather than screen against an
+ * unknown attempted set, because a missing exclusion can silently contaminate
+ * the exam (the whole point of held-out discipline).
+ */
+export declare function fetchAttemptedInstanceIds(discoveryUrl: string, manifestCid: string, fetchImpl?: typeof fetch): Promise<Set<string>>;

package/dist/eval/screen-discovery.js

@@ -0,0 +1,71 @@
+/**
+ * Held-out screening's authoritative "already trained-on" exclusion source (#986).
+ *
+ * Returns the set of swe-rebench-v2 `instance_id`s that have been ATTEMPTED on
+ * the network for a SolverNet — any verdict envelope, passed OR failed,
+ * cross-operator. An attempted instance was executed by an operator, so the
+ * learner trained on it; holding it out later would make a trained-checkpoint
+ * pass count as memorization, not generalization.
+ *
+ * Why the indexer and not the local generator-state: the local ledger only
+ * reflects THIS box's posting and can be stale (a different active generator —
+ * e.g. a hosted operator — posts to its own ledger). The indexer's
+ * `verdictEnvelopeMeta` is the cross-operator, current record. (The on-chain
+ * task/attempt tables carry no instance_id; only the indexer's IPFS enrichment
+ * resolves it — same backing as `DiscoveryAPI.getInstanceSuccessCounts`, minus
+ * the `actualPassed: true` filter so failed attempts count too.)
+ *
+ * Throws on indexer failure — callers MUST abort rather than screen against an
+ * unknown attempted set, because a missing exclusion can silently contaminate
+ * the exam (the whole point of held-out discipline).
+ */
+const ATTEMPTED_QUERY = `
+query InstanceAttempted($cid: String!, $limit: Int!, $after: String) {
+  verdictEnvelopeMetas(
+    where: {
+      solverNetManifestCid: $cid,
+      solverType_starts_with: "swe-rebench-v2",
+      enrichmentStatus: "ok",
+      instanceId_not: ""
+    },
+    limit: $limit,
+    after: $after,
+    orderBy: "enrichedAtBlock",
+    orderDirection: "asc"
+  ) {
+    items { instanceId }
+    pageInfo { hasNextPage endCursor }
+  }
+}`;
+export async function fetchAttemptedInstanceIds(discoveryUrl, manifestCid, fetchImpl = fetch) {
+    const gqlUrl = discoveryUrl.endsWith('/graphql') ? discoveryUrl : `${discoveryUrl.replace(/\/$/, '')}/graphql`;
+    const ids = new Set();
+    let cursor = null;
+    const MAX_PAGES = 20;
+    const PAGE_LIMIT = 1000;
+    for (let page = 0; page < MAX_PAGES; page++) {
+        const res = await fetchImpl(gqlUrl, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ query: ATTEMPTED_QUERY, variables: { cid: manifestCid, limit: PAGE_LIMIT, after: cursor } }),
+        });
+        if (!res.ok) {
+            throw new Error(`held-out screening: indexer attempted-ids query failed (HTTP ${res.status}) at ${gqlUrl}`);
+        }
+        const json = (await res.json());
+        if (json.errors) {
+            throw new Error(`held-out screening: indexer attempted-ids query errors: ${JSON.stringify(json.errors).slice(0, 300)}`);
+        }
+        const conn = json.data?.verdictEnvelopeMetas;
+        for (const item of conn?.items ?? []) {
+            if (item.instanceId)
+                ids.add(item.instanceId);
+        }
+        const pageInfo = conn?.pageInfo;
+        if (!pageInfo?.hasNextPage || !pageInfo.endCursor)
+            break;
+        cursor = pageInfo.endCursor;
+    }
+    return ids;
+}
+//# sourceMappingURL=screen-discovery.js.map

package/dist/eval/screen-discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-discovery.js","sourceRoot":"","sources":["../../src/eval/screen-discovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,eAAe,GAAG;;;;;;;;;;;;;;;;;EAiBtB,CAAC;AAYH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,YAAoB,EACpB,WAAmB,EACnB,YAA0B,KAAK;IAE/B,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC;IAC/G,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,UAAU,GAAG,IAAI,CAAC;IAExB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE;YAClC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;SACpH,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gEAAgE,GAAG,CAAC,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC;QAC9G,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAkB,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,2DAA2D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1H,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC;QAC7C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,UAAU;gBAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,EAAE,QAAQ,CAAC;QAChC,IAAI,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAE,MAAM;QACzD,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/eval/screen-progress.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Resumability store for held-out screening (#986).
+ *
+ * A real exam cut runs base R-runs + a prover over dozens of candidates — many
+ * hours of inference. This store persists each candidate's {@link ScreenMeasurement}
+ * so an interrupted run (rate limit, crash, disk) resumes instead of restarting:
+ * re-running the same command replays cached candidates for free and the
+ * `maxCandidates` budget bounds only the NEW work, so a long screen proceeds in
+ * budget-sized chunks.
+ *
+ * Keyed by a `signature` of the measurement-determining config (base model,
+ * prover, R, eval-semantics version). A signature mismatch on load → fresh start:
+ * cached measurements are only valid for the exact config that produced them
+ * (e.g. changing R changes the 0/R determination; a stronger base model changes
+ * pass/fail). Stored at `<stateDir>/held-out-screen-progress.json`.
+ */
+import type { ScreenMeasurement } from './screen.js';
+export declare class ScreenProgressStore {
+    private readonly file;
+    private readonly signature;
+    private data;
+    constructor(opts: {
+        stateDir: string;
+        signature: string;
+    });
+    private load;
+    /** Cached measurement for this instance under the current signature, or undefined. */
+    get(instance_id: string): ScreenMeasurement | undefined;
+    /** Persist a freshly-measured candidate (atomic-enough: whole-file rewrite). */
+    record(instance_id: string, m: ScreenMeasurement): void;
+    /** Number of candidates already measured under the current signature. */
+    get size(): number;
+}
+/** Build the cache signature from the measurement-determining config. */
+export declare function screenSignature(args: {
+    baseModel: string;
+    proverHarness: string;
+    proverModel: string;
+    R: number;
+    evalSemanticsVersion: string;
+}): string;

package/dist/eval/screen-progress.js

@@ -0,0 +1,60 @@
+/**
+ * Resumability store for held-out screening (#986).
+ *
+ * A real exam cut runs base R-runs + a prover over dozens of candidates — many
+ * hours of inference. This store persists each candidate's {@link ScreenMeasurement}
+ * so an interrupted run (rate limit, crash, disk) resumes instead of restarting:
+ * re-running the same command replays cached candidates for free and the
+ * `maxCandidates` budget bounds only the NEW work, so a long screen proceeds in
+ * budget-sized chunks.
+ *
+ * Keyed by a `signature` of the measurement-determining config (base model,
+ * prover, R, eval-semantics version). A signature mismatch on load → fresh start:
+ * cached measurements are only valid for the exact config that produced them
+ * (e.g. changing R changes the 0/R determination; a stronger base model changes
+ * pass/fail). Stored at `<stateDir>/held-out-screen-progress.json`.
+ */
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+const SCHEMA_VERSION = 'held-out-screen-progress.v1';
+export class ScreenProgressStore {
+    file;
+    signature;
+    data;
+    constructor(opts) {
+        this.file = join(opts.stateDir, 'held-out-screen-progress.json');
+        this.signature = opts.signature;
+        this.data = this.load();
+    }
+    load() {
+        try {
+            const raw = JSON.parse(readFileSync(this.file, 'utf8'));
+            if (raw?.schemaVersion === SCHEMA_VERSION && raw.signature === this.signature && raw.measurements) {
+                return raw;
+            }
+        }
+        catch {
+            /* absent or corrupt → fresh */
+        }
+        return { schemaVersion: SCHEMA_VERSION, signature: this.signature, measurements: {} };
+    }
+    /** Cached measurement for this instance under the current signature, or undefined. */
+    get(instance_id) {
+        return this.data.measurements[instance_id];
+    }
+    /** Persist a freshly-measured candidate (atomic-enough: whole-file rewrite). */
+    record(instance_id, m) {
+        this.data.measurements[instance_id] = m;
+        mkdirSync(dirname(this.file), { recursive: true });
+        writeFileSync(this.file, `${JSON.stringify(this.data, null, 2)}\n`);
+    }
+    /** Number of candidates already measured under the current signature. */
+    get size() {
+        return Object.keys(this.data.measurements).length;
+    }
+}
+/** Build the cache signature from the measurement-determining config. */
+export function screenSignature(args) {
+    return `base=${args.baseModel}|prover=${args.proverHarness}:${args.proverModel}|R=${args.R}|sem=${args.evalSemanticsVersion}`;
+}
+//# sourceMappingURL=screen-progress.js.map

package/dist/eval/screen-progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-progress.js","sourceRoot":"","sources":["../../src/eval/screen-progress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,cAAc,GAAG,6BAAsC,CAAC;AAS9D,MAAM,OAAO,mBAAmB;IACb,IAAI,CAAS;IACb,SAAS,CAAS;IAC3B,IAAI,CAAe;IAE3B,YAAY,IAA6C;QACvD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QACjE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAiB,CAAC;YACxE,IAAI,GAAG,EAAE,aAAa,KAAK,cAAc,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBAClG,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;QACD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IACxF,CAAC;IAED,sFAAsF;IACtF,GAAG,CAAC,WAAmB;QACrB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAED,gFAAgF;IAChF,MAAM,CAAC,WAAmB,EAAE,CAAoB;QAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACtE,CAAC;IAED,yEAAyE;IACzE,IAAI,IAAI;QACN,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;IACpD,CAAC;CACF;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,IAM/B;IACC,OAAO,QAAQ,IAAI,CAAC,SAAS,WAAW,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,MAAM,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,oBAAoB,EAAE,CAAC;AAChI,CAAC"}

package/dist/eval/screen-runner.d.ts

@@ -0,0 +1,30 @@
+import { type ScreenResult } from './screen.js';
+export interface ScreenRunOptions {
+    R: number;
+    heldOutCount: number;
+    maxCandidates: number;
+    perRepoCap: number;
+    /** Prover agent harness: `codex` (default) or `claude-code` (e.g. an Opus prover
+     *  via the working Claude auth — useful when codex is rate-limited). */
+    proverHarness?: 'codex' | 'claude-code';
+    /** Prover model: `codexModel` for the codex harness, `claudeModel` for the
+     *  claude-code harness (defaults to `opus` there). */
+    proverModel?: string;
+    /** Restrict candidates to these instance ids (else whole gradeable pool). */
+    instanceIds?: string[];
+    /** Restrict candidates to one repo (org prefix), e.g. `tobymao`. */
+    repo?: string;
+    configPath?: string;
+    log?: (msg: string) => void;
+}
+export interface ScreenRunSummary {
+    result: ScreenResult;
+    baseCodeDigest: string;
+    slatePath: string;
+    reportPath: string;
+    heldOutCount: number;
+    /** Base-failing candidates whose prover run returned no gradeable result
+     *  (excluded as no-headroom) — a signal the prover may be unavailable. */
+    proverUnscorable: number;
+}
+export declare function runScreenHeldOut(opts: ScreenRunOptions): Promise<ScreenRunSummary>;

package/dist/eval/screen-runner.js

@@ -0,0 +1,289 @@
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadConfig } from '../config.js';
+import { Store } from '../store/store.js';
+import { hashImplStateDir } from '../harnesses/freeze.js';
+import { LearnerHarness } from '../harnesses/impls/learner/harness.js';
+import { ClaudeCodeHarnessAdapter } from '../harnesses/impls/learner/adapters/claude-code.js';
+import { CodexCodeHarnessAdapter } from '../harnesses/impls/learner/adapters/codex-code.js';
+import { CODEX_HARNESS } from '../harnesses/names.js';
+import { runHarnessForEval, resolveRuntimePluginsForSolverType } from './eval-harness-run.js';
+import { corpusEnvFromConfig } from '../cli/commands/eval.js';
+import { loadSweRebenchV2Pool, defaultStateDir, getSweRebenchV2ValidatedPoolStore, } from '../solver-types/swe-rebench-v2.js';
+import { PoolCacheStore, loadPoolWithCacheFallback } from '../solver-types/_swe-rebench-v2-pool-cache.js';
+import { validatePoolInstances, EVAL_SEMANTICS_VERSION, } from '../solver-types/_swe-rebench-v2-validated-pool.js';
+import { resolveSlateTasks } from './resolve-slate-tasks.js';
+import { loadActiveHeldOutSlateIds, ACTIVE_HELD_OUT_SLATE_VERSIONS, loadHeldOutSlate, } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+import { GeneratorStateStore } from '../solver-types/_swe-rebench-v2-state.js';
+import { fetchAttemptedInstanceIds } from './screen-discovery.js';
+import { DEFAULT_TESTNET_DISCOVERY_URL } from '../config.js';
+import { solverTypeFromJoinedContract } from '../solver-nets/registry.js';
+import { SweRebenchV2Evaluator } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
+import { HttpHfFetcher } from '../harnesses/impls/swe-rebench-v2-evaluator/hf-fetcher.js';
+import { PythonEvalRunner } from '../harnesses/impls/swe-rebench-v2-evaluator/eval-runner.js';
+import { readEnabledState, defaultSweRebenchV2EvaluatorImplStateDir } from '../harnesses/impls/swe-rebench-v2-evaluator/harness.js';
+import { stratifyByRepo, screenBaseFailures, buildV2SlateFile, } from './screen.js';
+import { ScreenProgressStore, screenSignature } from './screen-progress.js';
+const DISPATCH_SOLVER_TYPE = 'swe-rebench-v2.v1';
+const SLATE_VERSION = 'v2';
+const RUN_BUDGET_MS = 3_600_000;
+/** dist/src parity: the shipped slate JSON lives next to the compiled module. */
+function slatesDir() {
+    return join(dirname(fileURLToPath(import.meta.url)), '..', 'solver-types', 'slates');
+}
+export async function runScreenHeldOut(opts) {
+    const log = opts.log ?? (() => { });
+    const config = loadConfig(opts.configPath);
+    // Precondition: evaluator enabled (upstream repo cloned).
+    const enabled = readEnabledState(defaultSweRebenchV2EvaluatorImplStateDir());
+    if (!enabled) {
+        throw new Error('swe-rebench-v2 evaluator not enabled — run `jinn harnesses enable swe-rebench-v2-evaluator` first');
+    }
+    const upstreamRepoDir = enabled.upstreamRepoDir;
+    const stateDir = process.env['JINN_SWE_REBENCH_V2_STATE_DIR'] ?? defaultStateDir();
+    const fetcher = new HttpHfFetcher();
+    const evaluator = new SweRebenchV2Evaluator({ fetcher, runner: new PythonEvalRunner({ upstreamRepoDir }) });
+    const validatedStore = getSweRebenchV2ValidatedPoolStore();
+    const runtimePlugins = await resolveRuntimePluginsForSolverType(DISPATCH_SOLVER_TYPE, config.joinedSolverNets);
+    // Common adapter wiring (mirrors buildEvalHarness in cli/commands/eval.ts).
+    const daemonApiToken = process.env['DAEMON_API_TOKEN']?.trim();
+    const corpusEnv = corpusEnvFromConfig(config);
+    const common = {
+        claudePath: config.claudePath ?? 'claude',
+        storePath: config.dbPath,
+        daemonApiUrl: `http://127.0.0.1:${config.apiPort}`,
+        ...(daemonApiToken ? { daemonApiToken } : {}),
+        ...(corpusEnv ? { corpusEnv } : {}),
+    };
+    const baseHarness = new LearnerHarness({
+        adapter: new ClaudeCodeHarnessAdapter({ ...common, claudeModel: config.claudeModel }),
+        claudePath: common.claudePath,
+    });
+    // Prover harness: codex (default) or claude-code (e.g. an Opus prover via the
+    // working Claude auth, sidestepping a codex rate limit; same-family Haiku→Opus
+    // is a clean capability ladder for "proven headroom").
+    const proverKind = opts.proverHarness ?? 'codex';
+    const proverHarness = proverKind === 'claude-code'
+        ? new LearnerHarness({
+            adapter: new ClaudeCodeHarnessAdapter({ ...common, claudeModel: opts.proverModel ?? 'opus' }),
+            claudePath: common.claudePath,
+        })
+        : new LearnerHarness({
+            name: CODEX_HARNESS,
+            adapter: new CodexCodeHarnessAdapter({ ...common, ...(opts.proverModel ? { codexModel: opts.proverModel } : {}) }),
+            claudePath: common.claudePath,
+            ...(config.codexPath !== undefined ? { codexPath: config.codexPath } : {}),
+        });
+    // Candidate pool (whole gradeable pool by default; scopeable).
+    const cacheResult = await loadPoolWithCacheFallback({
+        loadPool: loadSweRebenchV2Pool, cache: new PoolCacheStore({ stateDir }), currentPool: [],
+    });
+    let pool = cacheResult.pool;
+    if (pool.length === 0)
+        throw new Error(`SWE-rebench v2 pool empty${cacheResult.error ? ` (${cacheResult.error.message})` : ''}`);
+    // Held-out discipline (#986): draw the exam from the never-trained, never-held-out
+    // remainder. Union three exclusion sources:
+    //  - active held-out slate (would overlap an existing exam);
+    //  - already-ATTEMPTED on-network (indexer verdictEnvelopeMeta, any verdict,
+    //    cross-operator, CURRENT) — the authoritative "the learner trained on it"
+    //    signal; and
+    //  - this box's posted ledger (belt — may be STALE when another generator is
+    //    the active poster, e.g. a hosted operator; hence the indexer is the truth).
+    // A trained instance held out later would make a trained-checkpoint pass count
+    // as memorization, not generalization.
+    const heldOutIds = loadActiveHeldOutSlateIds(DISPATCH_SOLVER_TYPE, ACTIVE_HELD_OUT_SLATE_VERSIONS);
+    const postedIds = await new GeneratorStateStore({ stateDir }).postedInstanceIds();
+    let attemptedIds = new Set();
+    const discoveryUrl = config.discovery?.url?.trim()
+        || (config.network === 'testnet' ? DEFAULT_TESTNET_DISCOVERY_URL : undefined);
+    const joinedNet = Object.values(config.joinedSolverNets ?? {}).find((n) => solverTypeFromJoinedContract(n) === DISPATCH_SOLVER_TYPE);
+    if (discoveryUrl && joinedNet?.manifestCid) {
+        attemptedIds = await fetchAttemptedInstanceIds(discoveryUrl, joinedNet.manifestCid);
+        log(`[screen] indexer: ${attemptedIds.size} instance(s) already attempted on-network (authoritative, cross-operator) → excluded`);
+    }
+    else {
+        log('[screen] WARNING: no discovery URL / manifestCid — cannot exclude already-attempted instances; relying on the local posted ledger, which may be stale');
+    }
+    const excludeIds = new Set([...heldOutIds, ...postedIds, ...attemptedIds]);
+    if (opts.instanceIds?.length) {
+        // Explicit operator override — screen exactly these, but warn if any are
+        // already contaminated (attempted/posted/held-out) so an intentional pick is informed.
+        const want = new Set(opts.instanceIds);
+        pool = pool.filter((t) => want.has(t.instance_id));
+        const tainted = pool.filter((t) => excludeIds.has(t.instance_id)).map((t) => t.instance_id);
+        if (tainted.length > 0) {
+            log(`[screen] WARNING: ${tainted.length} explicitly-named instance(s) are already attempted/posted/held-out (NOT clean held-out candidates): ${tainted.join(', ')}`);
+        }
+    }
+    else {
+        if (opts.repo)
+            pool = pool.filter((t) => t.instance_id.startsWith(`${opts.repo}__`));
+        const before = pool.length;
+        pool = pool.filter((t) => !excludeIds.has(t.instance_id));
+        // Restrict to ALREADY-VALIDATED-SCORABLE candidates (#986): the never-validated
+        // tail is mostly not-gradeable (~90% deeper in the pool), so base-screening it
+        // wastes inference. Discovery of gradeability is `validate-pool`'s job; the
+        // screen selects held-out FROM the scorable set. Falls back to the full
+        // remainder only when no validation data exists yet (and warns).
+        const scorableIds = await validatedStore.getScorableIds(EVAL_SEMANTICS_VERSION);
+        if (scorableIds) {
+            const beforeScorable = pool.length;
+            pool = pool.filter((t) => scorableIds.has(t.instance_id));
+            log(`[screen] candidate pool ${before} → ${beforeScorable} (excluded ${excludeIds.size}: ${heldOutIds.size} held-out ∪ ${postedIds.size} posted ∪ ${attemptedIds.size} attempted) → ${pool.length} validated-scorable (run validate-pool to grow this)`);
+        }
+        else {
+            log(`[screen] WARNING: no validated-scorable data — screening the full ${pool.length}-task remainder (mostly not-gradeable; run validate-pool first for efficiency)`);
+        }
+    }
+    const candidates = stratifyByRepo(pool);
+    log(`[screen] ${candidates.length} candidate(s) after stratification`);
+    // Resolve a single instance to the {task,row} the harness + grader need.
+    const byId = new Map(pool.map((t) => [t.instance_id, t]));
+    async function runOnce(harness, poolTask) {
+        const implStateDir = mkdtempSync(join(tmpdir(), 'jinn-screen-state-'));
+        // Track which stage we're in so an unscorable result names its cause (#476
+        // excludes infra failures from the denominator — but it must be diagnosable,
+        // not an opaque black box).
+        let stage = 'resolve';
+        try {
+            const [resolved] = await resolveSlateTasks({
+                poolTasks: [poolTask], hf_dataset: poolTask.hf_dataset, hf_split: poolTask.hf_split, fetcher,
+            });
+            if (!resolved)
+                return { passed: null, unscorableReason: 'resolve: instance not in pool' };
+            const task = {
+                id: poolTask.instance_id,
+                description: resolved.task.problem_statement,
+                role: 'restoration',
+                solverType: DISPATCH_SOLVER_TYPE,
+                spec: resolved.task,
+                window: { startTs: 0, endTs: Date.now() + RUN_BUDGET_MS },
+            };
+            stage = 'harness';
+            const run = await runHarnessForEval({
+                harness, task, solverType: DISPATCH_SOLVER_TYPE, runtimePlugins, implStateDir, mode: 'frozen',
+            });
+            if (run.violation)
+                return { passed: null, unscorableReason: 'harness: freeze-fence violation' };
+            if (!run.solution)
+                return { passed: null, unscorableReason: 'harness: no solution produced' };
+            stage = 'grade';
+            const verdict = await evaluator.grade({
+                task: resolved.task,
+                solutionPayload: { schemaVersion: 'swe-rebench-v2-solution.v1', patch: run.solution.patch },
+                row: resolved.row,
+            });
+            return { passed: verdict.passed_match };
+        }
+        catch (err) {
+            // Any harness/grader/infra failure ⇒ unscorable, never a fail (#476). Name
+            // the stage + error so the exclusion is diagnosable. An "agent produced no
+            // patch" throw is flagged distinctly from a true infra/grader error.
+            const msg = err instanceof Error ? err.message : String(err);
+            const reason = /produced no\b|no .*patch/i.test(msg)
+                ? `${stage}: agent produced no patch`
+                : `${stage}-error: ${msg.slice(0, 200)}`;
+            return { passed: null, unscorableReason: reason };
+        }
+        finally {
+            rmSync(implStateDir, { recursive: true, force: true });
+        }
+    }
+    const emptyBaseDir = mkdtempSync(join(tmpdir(), 'jinn-screen-base-'));
+    const hashOpts = baseHarness.freezeStateHashIgnore?.length
+        ? { ignoreRelPaths: [...baseHarness.freezeStateHashIgnore] } : undefined;
+    const baseCodeDigest = `sha256:${await hashImplStateDir(emptyBaseDir, hashOpts)}`;
+    rmSync(emptyBaseDir, { recursive: true, force: true });
+    // Resumability: cache each candidate's measurement under a config signature so
+    // an interrupted run resumes (re-run the same command). The base policy is fixed
+    // (empty impl-state) so its measurement is stable; the signature invalidates the
+    // cache if the base model / prover / R / semantics change.
+    const proverModelLabel = opts.proverModel ?? (proverKind === 'claude-code' ? 'opus' : 'codex-default');
+    const progress = new ScreenProgressStore({
+        stateDir,
+        signature: screenSignature({
+            baseModel: config.claudeModel, proverHarness: proverKind, proverModel: proverModelLabel,
+            R: opts.R, evalSemanticsVersion: EVAL_SEMANTICS_VERSION,
+        }),
+    });
+    if (progress.size > 0)
+        log(`[screen] resuming: ${progress.size} candidate(s) already measured (cached) for this config`);
+    const deps = {
+        log,
+        getCachedMeasurement: (id) => progress.get(id),
+        recordMeasurement: (id, m) => progress.record(id, m),
+        ensureGradeable: async (task) => {
+            await validatePoolInstances([task], {
+                fetcher, runner: new PythonEvalRunner({ upstreamRepoDir }), store: validatedStore,
+                semanticsVersion: EVAL_SEMANTICS_VERSION, upstreamRepoDir,
+            }, {});
+            return (await validatedStore.getEntry(task.instance_id, EVAL_SEMANTICS_VERSION))?.scorable === true;
+        },
+        runBaseFrozen: (task) => runOnce(baseHarness, byId.get(task.instance_id)),
+        runProverFrozen: (task) => runOnce(proverHarness, byId.get(task.instance_id)),
+    };
+    const result = await screenBaseFailures(candidates, deps, {
+        R: opts.R, heldOutCount: opts.heldOutCount, maxCandidates: opts.maxCandidates, perRepoCap: opts.perRepoCap,
+    });
+    // Diagnosability: a base-failing candidate routed to the prover that comes
+    // back `proverPassed: null` means the prover produced NO gradeable result
+    // (errored / no patch), not a clean "the prover can't solve it". That silently
+    // routes to `no-headroom` and can yield a misleadingly empty slate when the
+    // prover is simply unavailable (e.g. codex CLI < 0.133.0, or auth missing).
+    // Surface it loudly rather than swallow it.
+    const proverUnscorable = result.screened.filter((s) => s.reason === 'no-headroom' && s.proverPassed === null).length;
+    if (proverUnscorable > 0) {
+        log(`[screen] WARNING: the prover returned no gradeable result on ${proverUnscorable} base-failing ` +
+            `candidate(s) — excluded as no-headroom, but this likely means the prover is UNAVAILABLE rather than ` +
+            `unable. Verify the codex CLI (>=0.133.0) + auth, then re-run. See proverPassed=null rows in the report.`);
+    }
+    // Cumulative v2: union the EXISTING slate with this run's NEW held-out, so
+    // re-running to GROW the exam never drops already-reserved instances. (The
+    // active-slate exclusion keeps existing held-out OUT of candidates, so
+    // result.heldOut is only the new admits.) Growing changes the slate's content
+    // hash; re-recording the base arm for the FULL set below upserts the prior rows
+    // to the new hash, so the orchestrator's slate-hash-drift guard stays satisfied.
+    let existingHeldOut = [];
+    try {
+        existingHeldOut = [...loadHeldOutSlate(DISPATCH_SOLVER_TYPE, SLATE_VERSION).instanceIds];
+    }
+    catch {
+        /* no v2 slate yet — this is the first cut */
+    }
+    const newHeldOut = result.heldOut.map((h) => h.instance_id);
+    const allHeldOutIds = [...new Set([...existingHeldOut, ...newHeldOut])];
+    const generatedAt = new Date().toISOString();
+    const slateFile = buildV2SlateFile(allHeldOutIds, generatedAt);
+    mkdirSync(slatesDir(), { recursive: true });
+    const slatePath = join(slatesDir(), 'held-out-slate.swe-rebench-v2.v2.json');
+    writeFileSync(slatePath, `${JSON.stringify(slateFile, null, 2)}\n`);
+    const reportPath = join(slatesDir(), 'held-out-slate.swe-rebench-v2.v2.screening-report.json');
+    writeFileSync(reportPath, `${JSON.stringify({
+        generatedAt, evalSemanticsVersion: EVAL_SEMANTICS_VERSION, baseCodeDigest,
+        R: opts.R, proverHarness: proverKind,
+        proverModel: opts.proverModel ?? (proverKind === 'claude-code' ? 'opus' : 'codex-default'),
+        heldOutTotal: allHeldOutIds.length, newThisRun: newHeldOut, carriedOver: existingHeldOut,
+        screened: result.screened,
+    }, null, 2)}\n`);
+    // Persist the base arm (all-fail) for the FULL held-out set under the (possibly
+    // new) slate hash. The upsert is keyed by (checkpoint, slate_version, instance_id),
+    // so prior rows get their slate_hash refreshed to match the grown slate — no drift.
+    const store = new Store(config.dbPath);
+    try {
+        const runAtMs = Date.now();
+        for (const id of allHeldOutIds) {
+            store.recordEvalResult({
+                checkpoint_cid: baseCodeDigest, slate_hash: slateFile.hash, slate_version: SLATE_VERSION,
+                instance_id: id, passed: false, unscorable: false, code_digest: baseCodeDigest,
+                run_at_ms: runAtMs, test_log_excerpt: 'base arm (screening): consistent fail 0/R',
+            });
+        }
+    }
+    finally {
+        store.close?.();
+    }
+    return { result, baseCodeDigest, slatePath, reportPath, heldOutCount: allHeldOutIds.length, proverUnscorable };
+}
+//# sourceMappingURL=screen-runner.js.map