@tangle-network/agent-bench 0.3.7 → 0.4.0

package/src/swe-arena/run-experiment.mts

@@ -24,9 +24,14 @@
  */
 
 import { appendFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
-import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { dirname, isAbsolute, join, resolve } from 'node:path'
 import { pathToFileURL, fileURLToPath } from 'node:url'
 import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
+import { exportBaseTree } from './factory-judge-child.mts'
+import { runFactoryCommand } from './factory-command-container.ts'
+import { loadFactoryInstances, type LoadedFactoryInstance } from './fixtures.ts'
+import { run, runOk } from './proc.ts'
 import {
   extractPatch,
   loadExcludes,
@@ -39,6 +44,7 @@ import {
   type SupervisorArmResult,
   type SupervisorArmSpec,
 } from './arms.ts'
+import type { FactoryJudgeResult } from './factory-judge-child.mts'
 import { applyPatchWithFallback } from './calibrate.ts'
 import { gatesForArmKind, waitForCapacity } from './capacity.ts'
 import { materializeWorkspace } from './materialize.ts'
@@ -47,6 +53,10 @@ import {
   type JudgeVerdict,
   type SerializedJudge,
 } from './serialized-judge.ts'
+import {
+  reportSupervisorRound,
+  writeSupervisorRunReportSafe,
+} from '@tangle-network/agent-eval/supervisor-run'
 import type { LedgerRow } from './types.ts'
 
 const fixturesDir = fileURLToPath(new URL('./fixtures', import.meta.url))
@@ -163,6 +173,12 @@ export interface ExperimentConfig {
   capacityModel?: string
   /** Pause between instances (orchestrate.sh: 15s, gentle on the shared key). */
   cooldownMs?: number
+  /**
+   * Run log the per-cell run-report headline is appended to. Defaults to
+   * `<outDir>/run.log`; the headline is always echoed to stdout as well, so a
+   * shell-redirected log gets it either way.
+   */
+  runLogPath?: string
 }
 
 function armPair(arms: ExecutableArmSpec[]): { solo: SoloArmSpec; sup: SupervisorArmSpec } {
@@ -242,8 +258,23 @@ export async function runExperiment(config: ExperimentConfig): Promise<void> {
     const row = buildLedgerRow(soloResult, soloVerdict, supResult, supVerdict)
     await appendFile(config.ledgerPath, JSON.stringify(row) + '\n')
     log(`LEDGER_ROW ${iid} solo=${row.solo_resolved} sup=${row.sup_resolved}`)
+
+    // Deterministic run observability: never hand-grep a journal for steers/waves/
+    // idle/cost again. Best-effort — a reporting failure can't lose a finished cell.
+    await writeSupervisorRunReportSafe(join(config.outDir, 'runs', iid, sup.name), {
+      appendHeadlineTo: config.runLogPath ?? join(config.outDir, 'run.log'),
+      ledgerPath: config.ledgerPath,
+    })
+
     await new Promise((r) => setTimeout(r, config.cooldownMs ?? 15_000))
   }
+
+  await reportSupervisorRound(config.outDir, {
+    appendHeadlineTo: config.runLogPath ?? join(config.outDir, 'run.log'),
+    ledgerPath: config.ledgerPath,
+    title: 'Round rollup — paired solo/supervisor experiment',
+    echo: true,
+  })
 }
 
 // ---------------------------------------------------------------------------
@@ -330,6 +361,320 @@ export async function replayPatchParity(
   return results
 }
 
+// ---------------------------------------------------------------------------
+// Factory-bench: worker workspace + experiment loop.
+//
+// The worker cell for a factory instance is the `git archive` export of the
+// base commit re-initialized as a FRESH git repo with one synthetic commit —
+// worker tooling that expects git works, but `git log`/refs cannot leak the
+// real repo's future history (the PR's impl and tests live only on the
+// judge-side mirror). SPEC.md (the rewritten PM-ticket spec) is part of that
+// initial commit. Everything downstream — arm runners, budgets, serialized
+// judge queue/ceiling, ledger resume — is the same machinery as swe-arena.
+// ---------------------------------------------------------------------------
+
+/** Ref name the synthetic initial commit is pinned to; the arm's diff base. */
+export const FACTORY_BASE_REF = 'factory-base'
+
+export interface FactoryWorkspace {
+  /** Sha of the synthetic initial commit (== FACTORY_BASE_REF). */
+  syntheticBase: string
+}
+
+/**
+ * Materialize a worker workspace for a factory instance: archive-export the
+ * base tree, add SPEC.md, re-init as a synthetic-history repo (single commit,
+ * no remotes), then pre-run `setup_cmds` so the worker starts on installed
+ * deps. The real repo's refs/objects are unreachable by construction — the
+ * leak test greps the workspace for them after setup.
+ */
+export async function materializeFactoryWorkspace(
+  inst: LoadedFactoryInstance,
+  dest: string,
+  opts: { setup?: boolean } = {},
+): Promise<FactoryWorkspace> {
+  await rm(dest, { recursive: true, force: true })
+  await mkdir(dirname(dest), { recursive: true })
+  await exportBaseTree(inst.repo_local_mirror, inst.base_commit, dest)
+  await writeFile(join(dest, 'SPEC.md'), inst.spec)
+
+  await runOk('git', ['-C', dest, 'init', '-q', '-b', 'work'])
+  await runOk('git', ['-C', dest, 'config', 'user.email', 'factory-bench@local'])
+  await runOk('git', ['-C', dest, 'config', 'user.name', 'factory-bench'])
+  await runOk('git', ['-C', dest, 'add', '-A'])
+  await runOk('git', ['-C', dest, 'commit', '-q', '-m', 'baseline workspace'])
+  await runOk('git', ['-C', dest, 'branch', '-f', FACTORY_BASE_REF, 'HEAD'])
+  const syntheticBase = (await runOk('git', ['-C', dest, 'rev-parse', 'HEAD'])).stdout.trim()
+  if (syntheticBase === inst.base_commit) {
+    throw new Error(`factory workspace ${inst.id}: synthetic base equals the real base commit — history leaked`)
+  }
+
+  if (opts.setup !== false && inst.setup_cmds.length > 0) {
+    for (const cmd of inst.setup_cmds) {
+      const res = await runFactoryCommand(dest, cmd, {
+        image: inst.command_image,
+        network: 'enabled',
+        timeoutMs: inst.timeout_s * 1000,
+      })
+      if (res.code !== 0) {
+        throw new Error(
+          `factory workspace ${inst.id}: setup_cmd failed (rc=${res.code}): ${cmd}\n${(res.stderr || res.stdout).slice(-2000)}`,
+        )
+      }
+    }
+  }
+  return { syntheticBase }
+}
+
+/** One appended line of the factory ledger (JSONL, resume key = iid + rep). */
+export interface FactoryLedgerRow {
+  at: string
+  iid: string
+  rep: number
+  arm: string
+  resolved: boolean
+  /** passed / calibrated total — the dense partial-credit signal. */
+  score: number
+  passed: number | null
+  total: number | null
+  verify_pass: boolean
+  patch_lines: number
+  wall_s: number
+  judge_secs: number | null
+  judge_attempts: number | null
+  driver_rc: number
+  sup_status: string | null
+  sup_verdict: string | null
+  delivered: boolean | null
+  spentTokens: number | null
+  spentUsd: number | null
+  spawned: number
+  workers: number
+  settled: number
+  patchPath: string
+  runDir: string
+}
+
+/** `iid#r<rep>` keys already in a factory ledger (resume-skip). */
+export async function factoryLedgerKeys(ledgerPath: string): Promise<Set<string>> {
+  const raw = await readFile(ledgerPath, 'utf8').catch(() => '')
+  const keys = new Set<string>()
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue
+    let row: { iid?: string; rep?: number }
+    try {
+      row = JSON.parse(line) as { iid?: string; rep?: number }
+    } catch {
+      throw new Error(`corrupt factory ledger line in ${ledgerPath}: ${line.slice(0, 120)}`)
+    }
+    if (typeof row.iid === 'string' && typeof row.rep === 'number') keys.add(`${row.iid}#r${row.rep}`)
+  }
+  return keys
+}
+
+export interface FactoryArmConfig {
+  workerModel: string
+  driverModel: string
+  budget?: number
+  maxSandboxes?: number
+  maxUsd?: number
+  maxDepth?: number
+  timeoutMs?: number
+  envKnobs?: Record<string, string>
+}
+
+export interface FactoryExperimentConfig {
+  /** Instance-dir root; relative paths resolve against the config file. */
+  instancesDir: string
+  /** Manifest ids to run (subset of instancesDir). */
+  instances: string[]
+  repsPerInstance: number
+  armName: string
+  arm: FactoryArmConfig
+  /** The loops checkout in the supervisor seat (baseline = loops main). */
+  loopsRepo: string
+  ledgerPath: string
+  outDir: string
+  secretsDir: string
+  envFiles: string[]
+  /**
+   * Worker-side self-check per instance (the supervisor's internal verify
+   * gate). NEVER the judge tests — those stay hidden. Default `true` (no gate).
+   */
+  verifyCmds?: Record<string, string>
+  judgeTimeoutMs?: number
+  cooldownMs?: number
+  gateWaitCeilingMs?: number
+  capacityModel?: string
+  /** Run log the per-cell run-report headline is appended to (default `<outDir>/run.log`). */
+  runLogPath?: string
+}
+
+const factoryJudgeChildPath = fileURLToPath(new URL('./factory-judge-child.mts', import.meta.url))
+const benchRootDir = fileURLToPath(new URL('../..', import.meta.url))
+
+/**
+ * Serialized judge whose child is factory-judge-child.mts — same JUDGE_RESULT
+ * line protocol, queue, retry, and SIGKILL ceiling as the swebench judge. The
+ * 1800s ceiling floor is kept as the backstop; the child self-enforces the
+ * manifest's (much smaller) per-command timeout_s inside it.
+ */
+export function createFactoryJudge(
+  instances: LoadedFactoryInstance[],
+  opts: { timeoutMs?: number } = {},
+): SerializedJudge {
+  const dirById = new Map(instances.map((i) => [i.id, i.dir]))
+  return createSerializedJudge({
+    ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
+    lockFile: join(tmpdir(), 'factory-arena-judge.lock'),
+    command: (iid, patchPath) => {
+      const dir = dirById.get(iid)
+      if (!dir) throw new Error(`factory judge: unknown instance id ${iid}`)
+      return { bin: 'node', argv: ['--import', 'tsx', factoryJudgeChildPath, dir, patchPath], cwd: benchRootDir }
+    },
+  })
+}
+
+/**
+ * Factory gen0 loop: per (instance × rep), sequentially — ledger resume →
+ * capacity gates → supervisor arm on a factory workspace → factory judge →
+ * one FactoryLedgerRow appended. Structure mirrors runExperiment.
+ */
+export async function runFactoryExperiment(
+  config: FactoryExperimentConfig,
+  opts: { configDir?: string; only?: string[]; repsOverride?: number } = {},
+): Promise<void> {
+  const log = (msg: string): void => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`)
+  const baseDir = opts.configDir ?? process.cwd()
+  const instancesDir = isAbsolute(config.instancesDir) ? config.instancesDir : resolve(baseDir, config.instancesDir)
+  const all = loadFactoryInstances(instancesDir)
+  const byId = new Map(all.map((i) => [i.id, i]))
+  const wanted = (opts.only ?? config.instances).map((id) => {
+    const inst = byId.get(id)
+    if (!inst) throw new Error(`factory config: instance ${id} not found under ${instancesDir}`)
+    return inst
+  })
+  const reps = opts.repsOverride ?? config.repsPerInstance
+  if (!Number.isInteger(reps) || reps < 1) throw new Error(`repsPerInstance must be an integer ≥ 1, got ${reps}`)
+
+  const secrets: SecretsEnv = { secretsDir: config.secretsDir, envFiles: config.envFiles }
+  const judge = createFactoryJudge(all, {
+    ...(config.judgeTimeoutMs !== undefined ? { timeoutMs: config.judgeTimeoutMs } : {}),
+  })
+  await mkdir(config.outDir, { recursive: true })
+  await mkdir(dirname(config.ledgerPath), { recursive: true })
+  const done = await factoryLedgerKeys(config.ledgerPath)
+
+  for (const inst of wanted) {
+    for (let rep = 0; rep < reps; rep += 1) {
+      const key = `${inst.id}#r${rep}`
+      if (done.has(key)) {
+        log(`SKIP ${key} (already in ledger)`)
+        continue
+      }
+
+      const gateOpts = {
+        ...(config.gateWaitCeilingMs !== undefined ? { waitCeilingMs: config.gateWaitCeilingMs } : {}),
+        ...(config.capacityModel !== undefined ? { model: config.capacityModel } : {}),
+        onStatus: log,
+      }
+      for (const gate of gatesForArmKind('supervisor', secrets, gateOpts)) {
+        if (!(await waitForCapacity(gate))) {
+          log(`NO CAPACITY on ${gate.name} within ceiling — stopping before ${key} (resume later)`)
+          return
+        }
+      }
+
+      const spec: SupervisorArmSpec = {
+        kind: 'supervisor',
+        name: config.armName,
+        workerModel: config.arm.workerModel,
+        driverModel: config.arm.driverModel,
+        budget: config.arm.budget,
+        maxSandboxes: config.arm.maxSandboxes,
+        maxUsd: config.arm.maxUsd,
+        maxDepth: config.arm.maxDepth,
+        ...(config.arm.envKnobs ? { envKnobs: config.arm.envKnobs } : {}),
+        loopsRepo: config.loopsRepo,
+        timeoutMs: config.arm.timeoutMs,
+      }
+      const armOutDir = join(config.outDir, `rep-${rep}`)
+      log(`>>> ${config.armName} ${inst.id} rep=${rep}`)
+      const armRes = await runSupervisorArm(spec, {
+        instanceId: inst.id,
+        image: `factory-archive:${inst.id}`,
+        // The synthetic-history ref, NOT the real base sha: patch extraction
+        // diffs against the workspace's own single commit.
+        baseCommit: FACTORY_BASE_REF,
+        materialize: async (dest) => {
+          await materializeFactoryWorkspace(inst, dest)
+        },
+        problemStatement: inst.spec,
+        verifyCmd: config.verifyCmds?.[inst.id] ?? 'true',
+        outDir: armOutDir,
+        secrets,
+        // SPEC.md is workspace furniture, not worker product.
+        excludes: [':(exclude)SPEC.md'],
+      })
+
+      const factoryRunDir = join(armOutDir, 'runs', inst.id, config.armName)
+      const { ws: _ws, ...armSummary } = armRes
+      await writeFile(join(factoryRunDir, 'result.json'), JSON.stringify(armSummary, null, 1)).catch(() => {})
+
+      const verdict = await judge.judge(inst.id, armRes.patchPath, `${config.armName}-r${rep}`)
+      log(`${inst.id} r${rep} judged: ${JSON.stringify(verdict)}`)
+      await writeFile(join(factoryRunDir, 'judge.json'), JSON.stringify(verdict, null, 1)).catch(() => {})
+      if (verdict.resolved === null) {
+        throw new Error(`inconclusive factory judge verdict for ${key} (${verdict.error ?? 'unknown'}) — not writing a fabricated boolean`)
+      }
+      const fv = verdict as JudgeVerdict & Partial<FactoryJudgeResult>
+      const row: FactoryLedgerRow = {
+        at: new Date().toISOString(),
+        iid: inst.id,
+        rep,
+        arm: config.armName,
+        resolved: verdict.resolved,
+        score: typeof fv.score === 'number' ? fv.score : 0,
+        passed: typeof fv.passed === 'number' ? fv.passed : null,
+        total: typeof fv.total === 'number' ? fv.total : null,
+        verify_pass: armRes.verify_pass,
+        patch_lines: armRes.patch_lines,
+        wall_s: armRes.wall_s,
+        judge_secs: fv.secs ?? null,
+        judge_attempts: fv.attempts ?? null,
+        driver_rc: armRes.driver_rc,
+        sup_status: armRes.sup_status,
+        sup_verdict: armRes.sup_verdict,
+        delivered: armRes.delivered,
+        spentTokens: armRes.spentTokens,
+        spentUsd: armRes.spentUsd,
+        spawned: armRes.spawned,
+        workers: armRes.workers,
+        settled: armRes.settled,
+        patchPath: armRes.patchPath,
+        runDir: factoryRunDir,
+      }
+      await appendFile(config.ledgerPath, JSON.stringify(row) + '\n')
+      log(`LEDGER_ROW ${key} resolved=${row.resolved} score=${row.score} (${row.passed}/${row.total})`)
+
+      await writeSupervisorRunReportSafe(row.runDir, {
+        appendHeadlineTo: config.runLogPath ?? join(config.outDir, 'run.log'),
+        ledgerPath: config.ledgerPath,
+        patchPath: armRes.patchPath,
+      })
+
+      await new Promise((r) => setTimeout(r, config.cooldownMs ?? 15_000))
+    }
+  }
+
+  await reportSupervisorRound(config.outDir, {
+    appendHeadlineTo: config.runLogPath ?? join(config.outDir, 'run.log'),
+    ledgerPath: config.ledgerPath,
+    title: `Round rollup — factory ${config.armName}`,
+    echo: true,
+  })
+}
+
 // ---------------------------------------------------------------------------
 // CLI.
 // ---------------------------------------------------------------------------
@@ -348,12 +693,33 @@ if (isMain) {
           `files(fixture=${r.fixtureFiles.join(',')} extracted=${r.extractedFiles.join(',')})`,
       )
     }
+  } else if (arg === '--factory') {
+    if (!extra) {
+      console.error('usage: tsx src/swe-arena/run-experiment.mts --factory <config.json> [--only <iid> ...] [--reps <n>]')
+      process.exit(2)
+    }
+    const rest = process.argv.slice(4)
+    const only: string[] = []
+    let repsOverride: number | undefined
+    for (let i = 0; i < rest.length; i += 1) {
+      if (rest[i] === '--only' && rest[i + 1]) only.push(rest[(i += 1)]!)
+      else if (rest[i] === '--reps' && rest[i + 1]) repsOverride = Number(rest[(i += 1)])
+      else throw new Error(`unknown --factory flag: ${rest[i]}`)
+    }
+    const configPath = resolve(extra)
+    const config = JSON.parse(await readFile(configPath, 'utf8')) as FactoryExperimentConfig
+    await runFactoryExperiment(config, {
+      configDir: dirname(configPath),
+      ...(only.length > 0 ? { only } : {}),
+      ...(repsOverride !== undefined ? { repsOverride } : {}),
+    })
   } else if (arg && !arg.startsWith('--')) {
     const config = JSON.parse(await readFile(arg, 'utf8')) as ExperimentConfig
     await runExperiment(config)
   } else {
     console.error(
       'usage: tsx src/swe-arena/run-experiment.mts <config.json>\n' +
+        '       tsx src/swe-arena/run-experiment.mts --factory <config.json> [--only <iid> ...] [--reps <n>]\n' +
         '       tsx src/swe-arena/run-experiment.mts --dry-run-parity [workDir]',
     )
     process.exit(2)

package/src/swe-arena/run-report.mts

@@ -0,0 +1,75 @@
+/**
+ * CLI over the supervisor-run reader. NO metric logic lives here.
+ *
+ * The reader itself is `@tangle-network/agent-eval/supervisor-run` — a
+ * supervision tree is a rollout trace with one more dimension, so it sits in
+ * the trace-analysis layer next to single-rollout analysis, beside the
+ * `tangle.rollout.v1` ledger whose row type its tree nodes ARE. This file is
+ * only argv parsing plus the entry points the arenas already call.
+ *
+ * Harness-session view of the same run (model calls, latency, stuck loops,
+ * tool errors):
+ * `npx @tangle-network/traces analyze --harness opencode --cwd <worker-clone-cwd>`.
+ */
+
+import { pathToFileURL } from 'node:url'
+import {
+  renderSupervisorRunMarkdown,
+  reportSupervisorRound,
+  type WriteSupervisorRunOptions,
+  writeSupervisorRunReport,
+} from '@tangle-network/agent-eval/supervisor-run'
+
+const isMain =
+  process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const args = process.argv.slice(2)
+  const usage =
+    'usage:\n' +
+    '  tsx run-report.mts <cellDir> [--log <run.log>] [--patch <file>] [--ledger <ledger.jsonl>] [--report-dir <dir>] [--no-opencode]\n' +
+    '  tsx run-report.mts --round <outDir> [--log <run.log>] [--ledger <ledger.jsonl>] [--report-dir <dir>] [--no-opencode]\n' +
+    '\n--report-dir writes the reports outside the run directory (use it when the run dir is READ-ONLY).\n'
+  const noOpencode = args.includes('--no-opencode')
+  const flagValue = (flag: string): string | undefined => {
+    const i = args.indexOf(flag)
+    return i >= 0 ? args[i + 1] : undefined
+  }
+  const appendHeadlineTo = flagValue('--log')
+  const patchPath = flagValue('--patch')
+  const reportDir = flagValue('--report-dir')
+  const ledgerPath = flagValue('--ledger')
+  const flagValueIndices = new Set(
+    ['--log', '--patch', '--report-dir', '--ledger', '--round']
+      .map((f) => args.indexOf(f))
+      .filter((i) => i >= 0)
+      .map((i) => i + 1),
+  )
+  const positional = args.filter((a, i) => !a.startsWith('--') && !flagValueIndices.has(i))
+  const roundIdx = args.indexOf('--round')
+  const opts: WriteSupervisorRunOptions = {
+    ...(appendHeadlineTo !== undefined ? { appendHeadlineTo } : {}),
+    ...(patchPath !== undefined ? { patchPath } : {}),
+    ...(reportDir !== undefined ? { reportDir } : {}),
+    ...(ledgerPath !== undefined ? { ledgerPath } : {}),
+    ...(noOpencode ? { opencodeDb: null } : {}),
+    echo: true,
+  }
+  if (roundIdx >= 0) {
+    const outDir = args[roundIdx + 1]
+    if (outDir === undefined) {
+      console.error(usage)
+      process.exit(2)
+    }
+    await reportSupervisorRound(outDir, opts)
+  } else {
+    const cellDir = positional[0]
+    if (cellDir === undefined) {
+      console.error(usage)
+      process.exit(2)
+    }
+    const report = await writeSupervisorRunReport(cellDir, opts)
+    console.log('')
+    console.log(renderSupervisorRunMarkdown(report))
+  }
+}

package/src/swe-arena/run-supervisor.test.mts

@@ -28,10 +28,10 @@ describe('supervisor driver cancellation', () => {
     const extension = join(dir, 'fake-extension.mjs')
     await mkdir(workspace)
     await writeFile(params, '{}')
-    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); setInterval(()=>{},1000)`
+    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; const ready=process.argv[2]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); fs.writeFileSync(ready,'ready'); setInterval(()=>{},1000)`
     await writeFile(extension, `
 import { spawn } from 'node:child_process'
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
+import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
 
 const id = 'sup-1-fake12'
@@ -47,11 +47,13 @@ export default function fakeExtension(pi) {
       writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running', progress: 'driving' }))
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'worker-cleaned.txt')], {
+      const readyPath = join(ctx.cwd, 'worker-ready.txt')
+      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'worker-cleaned.txt'), readyPath], {
         detached: true,
         stdio: 'ignore',
       })
       writeFileSync(join(ctx.cwd, 'worker.pid'), String(child.pid))
+      while (!existsSync(readyPath)) await new Promise((resolve) => setTimeout(resolve, 5))
       return text('spawned supervisor ' + id)
     },
   })

package/src/swe-arena/score-split.mts

@@ -0,0 +1,140 @@
+/**
+ * Gen-5 public/private score split (SOTA adoption #1, AIDE2-style).
+ *
+ * Per generation the improvement instances are DETERMINISTICALLY split
+ * (seeded by runId) into a PUBLIC set — whose scores/evidence proposers and
+ * the pre-filter may see — and a PRIVATE set that is scored identically but
+ * NEVER surfaced to proposers. Winner selection stays on the COMBINED
+ * public+private score (the lib judge already averages every scenario), so
+ * the split changes only what feedback the search sees, exactly the
+ * anti-overfitting mechanism: a candidate that games the visible instances
+ * still has to survive the instances it never saw named.
+ *
+ * SMALL-N CAVEAT: with only 6 improvement instances the split is 4 public /
+ * 2 private. Two private instances give a coarse overfit signal (0, 1/2 or
+ * 2/2) and a single flake flips it — treat the private sub-score as a
+ * direction check, not a certification; the pre-registered 6-instance holdout
+ * remains the only promotion evidence.
+ *
+ * RESUME STABILITY: runId embeds a launch timestamp, so a resumed run gets a
+ * fresh runId. The split is therefore PERSISTED to <outDir>/score-split.json
+ * on first computation and reloaded verbatim afterwards — a resume can never
+ * rotate previously-private instances into view.
+ */
+
+import { createHash } from 'node:crypto'
+import { existsSync } from 'node:fs'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+
+export const SCORE_SPLIT_SCHEMA = 'swe-arena.score-split.v1'
+export const SCORE_SPLIT_FILENAME = 'score-split.json'
+
+export interface ScoreSplitConfig {
+  /** Number of PUBLIC instances (the rest are private). Must satisfy
+   *  0 < publicCount < instances.length. */
+  publicCount: number
+}
+
+export interface ScoreSplit {
+  schema: typeof SCORE_SPLIT_SCHEMA
+  /** The runId that seeded the split (provenance; a resumed run keeps it). */
+  seededBy: string
+  publicInstances: string[]
+  privateInstances: string[]
+}
+
+/** Deterministic split: instances are ranked by sha256(runId + ':' + iid) and
+ *  the first `publicCount` become public. Pure — same (runId, instances,
+ *  publicCount) always yields the same split; instance-list ORDER does not
+ *  matter (ranking is content-derived). */
+export function splitInstances(runId: string, instances: readonly string[], publicCount: number): ScoreSplit {
+  if (!Number.isInteger(publicCount) || publicCount <= 0 || publicCount >= instances.length) {
+    throw new Error(
+      `score-split: publicCount must be an integer in (0, ${instances.length}), got ${JSON.stringify(publicCount)}`,
+    )
+  }
+  if (new Set(instances).size !== instances.length) {
+    throw new Error('score-split: duplicate instance ids')
+  }
+  const ranked = [...instances]
+    .map((iid) => ({ iid, rank: createHash('sha256').update(`${runId}:${iid}`).digest('hex') }))
+    .sort((a, b) => (a.rank < b.rank ? -1 : a.rank > b.rank ? 1 : 0))
+  const pub = ranked.slice(0, publicCount).map((r) => r.iid)
+  const priv = ranked.slice(publicCount).map((r) => r.iid)
+  return {
+    schema: SCORE_SPLIT_SCHEMA,
+    seededBy: runId,
+    publicInstances: [...pub].sort(),
+    privateInstances: [...priv].sort(),
+  }
+}
+
+/** Load the persisted split for this outDir, or compute + persist it. The
+ *  persisted file always wins (resume stability); it is validated against the
+ *  current instance set so a config edit mid-outDir fails loud. */
+export async function loadOrCreateScoreSplit(args: {
+  outDir: string
+  runId: string
+  instances: readonly string[]
+  publicCount: number
+}): Promise<ScoreSplit> {
+  const path = join(args.outDir, SCORE_SPLIT_FILENAME)
+  if (existsSync(path)) {
+    const split = JSON.parse(await readFile(path, 'utf8')) as ScoreSplit
+    if (split.schema !== SCORE_SPLIT_SCHEMA) {
+      throw new Error(`score-split: ${path} has unknown schema ${JSON.stringify(split.schema)}`)
+    }
+    const persisted = [...split.publicInstances, ...split.privateInstances].sort()
+    const expected = [...args.instances].sort()
+    if (JSON.stringify(persisted) !== JSON.stringify(expected)) {
+      throw new Error(
+        `score-split: persisted split at ${path} covers [${persisted.join(', ')}] but the config ` +
+          `names [${expected.join(', ')}] — refusing to silently re-split; move the outDir or fix the config`,
+      )
+    }
+    if (split.publicInstances.length !== args.publicCount) {
+      throw new Error(
+        `score-split: persisted split has ${split.publicInstances.length} public instances, config wants ` +
+          `${args.publicCount} — refusing to silently re-split`,
+      )
+    }
+    return split
+  }
+  const split = splitInstances(args.runId, args.instances, args.publicCount)
+  await mkdir(dirname(path), { recursive: true })
+  await writeFile(path, JSON.stringify(split, null, 2))
+  return split
+}
+
+/** True when `text` mentions any private instance id — the NEVER-SURFACED
+ *  invariant's checkable form. Used by tests and by the evidence-index writer
+ *  as a final guard on proposer-visible text. */
+export function leaksPrivateInstance(text: string, split: Pick<ScoreSplit, 'privateInstances'>): string[] {
+  return split.privateInstances.filter((iid) => text.includes(iid))
+}
+
+/** Filter per-instance evidence records to the PUBLIC set. Identity function
+ *  when no split is configured (split === null). */
+export function publicOnly<T>(
+  records: readonly T[],
+  iidOf: (record: T) => string,
+  split: Pick<ScoreSplit, 'privateInstances'> | null,
+): T[] {
+  if (split === null) return [...records]
+  const priv = new Set(split.privateInstances)
+  return records.filter((r) => !priv.has(iidOf(r)))
+}
+
+/** Public/private sub-scores for one candidate, from per-instance AND-verdicts
+ *  (fail-closed: an instance with no conclusive full-reps verdict counts 0). */
+export function subScores(
+  verdicts: Record<string, boolean>,
+  split: Pick<ScoreSplit, 'publicInstances' | 'privateInstances'>,
+): { publicResolvedCount: number; privateResolvedCount: number } {
+  const count = (iids: readonly string[]): number => iids.filter((iid) => verdicts[iid] === true).length
+  return {
+    publicResolvedCount: count(split.publicInstances),
+    privateResolvedCount: count(split.privateInstances),
+  }
+}