@tangle-network/agent-bench 0.11.2 → 0.13.0

package/src/swe-arena/run-supervisor.test.mts

@@ -1,539 +0,0 @@
-import { access, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterAll, describe, expect, it } from 'vitest'
-import { runSupervisorShim, supervisorDriverKillGraceMs } from './arms.ts'
-import { ABORT_RC, run } from './proc.ts'
-
-const dirs: string[] = []
-const workerPids: number[] = []
-
-async function waitForFile(path: string, timeoutMs = 10_000): Promise<void> {
-  const deadline = Date.now() + timeoutMs
-  while (Date.now() < deadline) {
-    try {
-      await access(path)
-      return
-    } catch (error) {
-      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
-    }
-    await new Promise((resolve) => setTimeout(resolve, 10))
-  }
-  throw new Error(`timed out waiting for ${path}`)
-}
-
-afterAll(async () => {
-  for (const pid of workerPids) {
-    try {
-      process.kill(-pid, 'SIGKILL')
-    } catch {
-      // The expected path already reaped the process group.
-    }
-  }
-  await Promise.all(dirs.map((dir) => rm(dir, { recursive: true, force: true })))
-})
-
-describe('supervisor driver cancellation', () => {
-  it('waits for worker cleanup before returning its deadline result', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-driver-cancel-'))
-    dirs.push(dir)
-    const workspace = join(dir, 'workspace')
-    const params = join(dir, 'params.json')
-    const extension = join(dir, 'fake-extension.mjs')
-    await mkdir(workspace)
-    await writeFile(params, '{}')
-    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; const ready=process.argv[2]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); fs.writeFileSync(ready,'ready'); setInterval(()=>{},1000)`
-    await writeFile(extension, `
-import { spawn } from 'node:child_process'
-import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-1-fake12'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-let child
-
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running', progress: 'driving' }))
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      const readyPath = join(ctx.cwd, 'worker-ready.txt')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'worker-cleaned.txt'), readyPath], {
-        detached: true,
-        stdio: 'ignore',
-      })
-      writeFileSync(join(ctx.cwd, 'worker.pid'), String(child.pid))
-      while (!existsSync(readyPath)) await new Promise((resolve) => setTimeout(resolve, 5))
-      return text('spawned supervisor ' + id)
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute(_call, _params, _signal, _update, ctx) {
-      appendFileSync(join(ctx.cwd, 'cancel-count.txt'), '1\\n')
-      process.kill(-child.pid, 'SIGTERM')
-      child.once('close', () => {
-        const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-        appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'cancelled', id: id + ':s1', reason: 'test cancellation' }) + '\\n')
-        writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'cancelled', progress: 'cancelled' }))
-        writeFileSync(join(ctx.cwd, 'cancel-settled.txt'), 'settled')
-      })
-      return text('cancellation signalled before worker cleanup')
-    },
-  })
-}
-`)
-
-    const result = await run(process.execPath, [
-      '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-    ], {
-      timeoutMs: 5_000,
-      killGraceMs: 500,
-      env: {
-        ...process.env,
-        DRIVER_DEADLINE_MS: '40',
-        DRIVER_POLL_MS: '10',
-        DRIVER_CANCEL_TIMEOUT_MS: '2000',
-      },
-    })
-    const workerPid = Number(await readFile(join(workspace, 'worker.pid'), 'utf8'))
-    workerPids.push(workerPid)
-
-    expect(result.code, result.stdout + result.stderr).toBe(3)
-    expect(result.timedOut).toBe(false)
-    expect(await readFile(join(workspace, 'cancel-count.txt'), 'utf8')).toBe('1\n')
-    expect(await readFile(join(workspace, 'worker-cleaned.txt'), 'utf8')).toBe('clean')
-    expect(await readFile(join(workspace, 'cancel-settled.txt'), 'utf8')).toBe('settled')
-    expect(JSON.parse(await readFile(join(workspace, '.loops', 'supervisor', 'sup-1-fake12', 'state.json'), 'utf8')))
-      .toMatchObject({ status: 'cancelled' })
-    await expect(access(`/proc/${workerPid}`)).rejects.toMatchObject({ code: 'ENOENT' })
-  })
-
-  it('lets near-deadline cancellation settle after outer SIGTERM', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-driver-outer-term-'))
-    dirs.push(dir)
-    const workspace = join(dir, 'workspace')
-    const params = join(dir, 'params.json')
-    const extension = join(dir, 'fake-extension.mjs')
-    await mkdir(workspace)
-    await writeFile(params, '{}')
-    await writeFile(extension, `
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-2-fake34'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      writeFileSync(join(ctx.cwd, 'outer-driver-ready.txt'), 'ready')
-      return text('spawned supervisor ' + id)
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute(_call, _params, _signal, _update, ctx) {
-      await new Promise((resolve) => setTimeout(resolve, 1200))
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'cancelled', id: id + ':s1' }) + '\\n')
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'cancelled' }))
-      writeFileSync(join(ctx.cwd, 'outer-cancel-settled.txt'), 'settled')
-      return text('cancellation settled')
-    },
-  })
-}
-`)
-
-    const cancelTimeoutMs = 1_500
-    const controller = new AbortController()
-    const resultPromise = run(process.execPath, [
-      '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-    ], {
-      signal: controller.signal,
-      killGraceMs: supervisorDriverKillGraceMs({
-        DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
-      }),
-      env: {
-        ...process.env,
-        DRIVER_DEADLINE_MS: '60000',
-        DRIVER_POLL_MS: '10',
-        DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
-      },
-    })
-    try {
-      await waitForFile(join(workspace, 'outer-driver-ready.txt'))
-    } catch (error) {
-      controller.abort()
-      await resultPromise
-      throw error
-    }
-    controller.abort()
-    const result = await resultPromise
-
-    expect(result.code, result.stdout + result.stderr).toBe(ABORT_RC)
-    expect(result.aborted).toBe(true)
-    expect(result.timedOut).toBe(false)
-    expect(await readFile(join(workspace, 'outer-cancel-settled.txt'), 'utf8')).toBe('settled')
-    expect(JSON.parse(await readFile(join(workspace, '.loops', 'supervisor', 'sup-2-fake34', 'state.json'), 'utf8')))
-      .toMatchObject({ status: 'cancelled' })
-  }, 15_000)
-
-  it('latches SIGTERM while spawn_supervisor is still returning and cancels its detached worker', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-driver-signal-during-spawn-'))
-    dirs.push(dir)
-    const workspace = join(dir, 'workspace')
-    const params = join(dir, 'params.json')
-    const extension = join(dir, 'fake-extension.mjs')
-    await mkdir(workspace)
-    await writeFile(params, '{}')
-    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; const ready=process.argv[2]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); fs.writeFileSync(ready,'ready'); setInterval(()=>{},1000)`
-    await writeFile(extension, `
-import { spawn } from 'node:child_process'
-import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-3-signal56'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-let child
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      const readyPath = join(ctx.cwd, 'signal-worker-ready.txt')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'signal-worker-cleaned.txt'), readyPath], {
-        detached: true,
-        stdio: 'ignore',
-      })
-      writeFileSync(join(ctx.cwd, 'signal-worker.pid'), String(child.pid))
-      while (!existsSync(readyPath)) await new Promise((resolve) => setTimeout(resolve, 5))
-      writeFileSync(join(ctx.cwd, 'spawn-entered.txt'), 'entered')
-      await new Promise((resolve) => setTimeout(resolve, 30_000))
-      return text('spawned supervisor ' + id)
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute(_call, _params, _signal, _update, ctx) {
-      process.kill(-child.pid, 'SIGTERM')
-      await new Promise((resolve) => child.once('close', resolve))
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'cancelled', id: id + ':s1' }) + '\\n')
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'cancelled' }))
-      writeFileSync(join(ctx.cwd, 'signal-cancel-settled.txt'), 'settled')
-      return text('cancellation settled')
-    },
-  })
-}
-`)
-
-    const cancelTimeoutMs = 2_000
-    const controller = new AbortController()
-    const resultPromise = run(process.execPath, [
-      '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-    ], {
-      signal: controller.signal,
-      killGraceMs: supervisorDriverKillGraceMs({
-        DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
-      }),
-      env: {
-        ...process.env,
-        DRIVER_DEADLINE_MS: '60000',
-        DRIVER_POLL_MS: '10',
-        DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
-      },
-    })
-    try {
-      await waitForFile(join(workspace, 'spawn-entered.txt'))
-    } catch (error) {
-      controller.abort()
-      await resultPromise
-      throw error
-    }
-    controller.abort()
-    const result = await resultPromise
-    const workerPid = Number(await readFile(join(workspace, 'signal-worker.pid'), 'utf8'))
-    workerPids.push(workerPid)
-
-    expect(result.code, result.stdout + result.stderr).toBe(ABORT_RC)
-    expect(result.aborted).toBe(true)
-    expect(result.timedOut).toBe(false)
-    expect(await readFile(join(workspace, 'spawn-entered.txt'), 'utf8')).toBe('entered')
-    expect(await readFile(join(workspace, 'signal-worker-cleaned.txt'), 'utf8')).toBe('clean')
-    expect(await readFile(join(workspace, 'signal-cancel-settled.txt'), 'utf8')).toBe('settled')
-    await expect(access(`/proc/${workerPid}`)).rejects.toMatchObject({ code: 'ENOENT' })
-  }, 15_000)
-
-  it('recovers an on-disk supervisor id from an unparseable spawn response and cancels it', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-driver-id-recovery-'))
-    dirs.push(dir)
-    const workspace = join(dir, 'workspace')
-    const params = join(dir, 'params.json')
-    const extension = join(dir, 'fake-extension.mjs')
-    await mkdir(workspace)
-    await writeFile(params, '{}')
-    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); setInterval(()=>{},1000)`
-    await writeFile(extension, `
-import { spawn } from 'node:child_process'
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-4-recover78'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-let child
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'recovered-worker-cleaned.txt')], {
-        detached: true,
-        stdio: 'ignore',
-      })
-      writeFileSync(join(ctx.cwd, 'recovered-worker.pid'), String(child.pid))
-      return text('the supervisor started, but this response omitted its identifier')
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute(_call, params, _signal, _update, ctx) {
-      writeFileSync(join(ctx.cwd, 'recovered-id.txt'), params.id)
-      process.kill(-child.pid, 'SIGTERM')
-      await new Promise((resolve) => child.once('close', resolve))
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'cancelled', id: id + ':s1' }) + '\\n')
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'cancelled' }))
-      return text('cancellation settled')
-    },
-  })
-}
-`)
-
-    const result = await run(process.execPath, [
-      '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-    ], {
-      timeoutMs: 5_000,
-      killGraceMs: 500,
-      env: {
-        ...process.env,
-        DRIVER_DEADLINE_MS: '60000',
-        DRIVER_POLL_MS: '10',
-        DRIVER_CANCEL_TIMEOUT_MS: '2000',
-      },
-    })
-    const workerPid = Number(await readFile(join(workspace, 'recovered-worker.pid'), 'utf8'))
-    workerPids.push(workerPid)
-
-    expect(result.code, result.stdout + result.stderr).toBe(1)
-    expect(result.timedOut).toBe(false)
-    expect(result.stderr).toMatch(/cancelling recovered run\(s\): sup-4-recover78/)
-    expect(await readFile(join(workspace, 'recovered-id.txt'), 'utf8')).toBe('sup-4-recover78')
-    await expect(access(`/proc/${workerPid}`)).rejects.toMatchObject({ code: 'ENOENT' })
-  }, 15_000)
-
-  it('waits for a late worker terminal record before accepting completed state', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-driver-late-settlement-'))
-    dirs.push(dir)
-    const workspace = join(dir, 'workspace')
-    const params = join(dir, 'params.json')
-    const extension = join(dir, 'fake-extension.mjs')
-    await mkdir(workspace)
-    await writeFile(params, '{}')
-    await writeFile(extension, `
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-5-late90'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'completed', verdict: 'delivered' }))
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      setTimeout(() => {
-        appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'settled', id: id + ':s1', status: 'done' }) + '\\n')
-      }, 40)
-      return text('spawned supervisor ' + id)
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute() { return text('already completed') },
-  })
-}
-`)
-
-    const result = await run(process.execPath, [
-      '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-    ], {
-      timeoutMs: 5_000,
-      killGraceMs: 500,
-      env: {
-        ...process.env,
-        DRIVER_DEADLINE_MS: '60000',
-        DRIVER_POLL_MS: '10',
-        DRIVER_CANCEL_TIMEOUT_MS: '2000',
-        DRIVER_SETTLEMENT_TIMEOUT_MS: '500',
-      },
-    })
-
-    expect(result.code, result.stdout + result.stderr).toBe(0)
-    expect(result.timedOut).toBe(false)
-    expect(result.stdout).toMatch(/worker settlement: workers=1, terminal=1/)
-  })
-
-  it('refuses completed state while a journal worker remains live', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-driver-live-completed-worker-'))
-    dirs.push(dir)
-    const workspace = join(dir, 'workspace')
-    const params = join(dir, 'params.json')
-    const extension = join(dir, 'fake-extension.mjs')
-    await mkdir(workspace)
-    await writeFile(params, '{}')
-    await writeFile(extension, `
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-5-live90'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'completed', verdict: 'delivered' }))
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      return text('spawned supervisor ' + id)
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute() { return text('already completed') },
-  })
-}
-`)
-
-    const result = await run(process.execPath, [
-      '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-    ], {
-      timeoutMs: 5_000,
-      killGraceMs: 500,
-      env: {
-        ...process.env,
-        DRIVER_DEADLINE_MS: '60000',
-        DRIVER_POLL_MS: '10',
-        DRIVER_CANCEL_TIMEOUT_MS: '2000',
-        DRIVER_SETTLEMENT_TIMEOUT_MS: '80',
-      },
-    })
-
-    expect(result.code, result.stdout + result.stderr).toBe(4)
-    expect(result.timedOut).toBe(false)
-    expect(result.stderr).toMatch(/refusing completed supervisor/)
-    expect(result.stderr).toMatch(/workers=1, terminal=0, unsettled=1/)
-    expect(result.stdout).not.toContain('[done] id=')
-  })
-
-  it.each(['failed', 'cancelled'] as const)(
-    'settles a detached worker before returning terminal status $status',
-    async (status) => {
-      const dir = await mkdtemp(join(tmpdir(), `swe-driver-${status}-worker-`))
-      dirs.push(dir)
-      const workspace = join(dir, 'workspace')
-      const params = join(dir, 'params.json')
-      const extension = join(dir, 'fake-extension.mjs')
-      await mkdir(workspace)
-      await writeFile(params, '{}')
-      const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; const ready=process.argv[2]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); fs.writeFileSync(ready,'ready'); setInterval(()=>{},1000)`
-      await writeFile(extension, `
-import { spawn } from 'node:child_process'
-import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-const id = 'sup-6-${status}12'
-const text = (value) => ({ content: [{ type: 'text', text: value }] })
-let child
-export default function fakeExtension(pi) {
-  pi.registerTool({
-    name: 'spawn_supervisor',
-    async execute(_call, _params, _signal, _update, ctx) {
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      mkdirSync(runDir, { recursive: true })
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      const readyPath = join(ctx.cwd, 'terminal-worker-ready.txt')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'terminal-worker-cleaned.txt'), readyPath], {
-        detached: true,
-        stdio: 'ignore',
-      })
-      writeFileSync(join(ctx.cwd, 'terminal-worker.pid'), String(child.pid))
-      while (!existsSync(readyPath)) await new Promise((resolve) => setTimeout(resolve, 5))
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: '${status}', error: 'fixture terminal state' }))
-      return text('spawned supervisor ' + id)
-    },
-  })
-  pi.registerTool({
-    name: 'supervisor_cancel',
-    async execute(_call, _params, _signal, _update, ctx) {
-      appendFileSync(join(ctx.cwd, 'terminal-cancel-count.txt'), '1\\n')
-      process.kill(-child.pid, 'SIGTERM')
-      await new Promise((resolve) => child.once('close', resolve))
-      const runDir = join(ctx.cwd, '.loops', 'supervisor', id)
-      appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'cancelled', id: id + ':s1' }) + '\\n')
-      writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'cancelled' }))
-      return text('terminal worker settled')
-    },
-  })
-}
-`)
-
-      const result = await run(process.execPath, [
-        '--import', 'tsx', runSupervisorShim, extension, workspace, params,
-      ], {
-        timeoutMs: 5_000,
-        killGraceMs: 500,
-        env: {
-          ...process.env,
-          DRIVER_DEADLINE_MS: '60000',
-          DRIVER_POLL_MS: '10',
-          DRIVER_CANCEL_TIMEOUT_MS: '2000',
-        },
-      })
-      const workerPid = Number(await readFile(join(workspace, 'terminal-worker.pid'), 'utf8'))
-      workerPids.push(workerPid)
-
-      expect(result.code, result.stdout + result.stderr).toBe(1)
-      expect(result.timedOut).toBe(false)
-      expect(result.stdout).toContain(`[done] id=sup-6-${status}12 status=${status}`)
-      expect(result.stdout).toContain('worker settlement: workers=1, terminal=1')
-      expect(await readFile(join(workspace, 'terminal-cancel-count.txt'), 'utf8')).toBe('1\n')
-      expect(await readFile(join(workspace, 'terminal-worker-cleaned.txt'), 'utf8')).toBe('clean')
-      await expect(access(`/proc/${workerPid}`)).rejects.toMatchObject({ code: 'ENOENT' })
-    },
-    15_000,
-  )
-})

package/src/swe-arena/score-split.mts

@@ -1,140 +0,0 @@
-/**
- * Gen-5 public/private score split (SOTA adoption #1, AIDE2-style).
- *
- * Per generation the improvement instances are DETERMINISTICALLY split
- * (seeded by runId) into a PUBLIC set — whose scores/evidence proposers and
- * the pre-filter may see — and a PRIVATE set that is scored identically but
- * NEVER surfaced to proposers. Winner selection stays on the COMBINED
- * public+private score (the lib judge already averages every scenario), so
- * the split changes only what feedback the search sees, exactly the
- * anti-overfitting mechanism: a candidate that games the visible instances
- * still has to survive the instances it never saw named.
- *
- * SMALL-N CAVEAT: with only 6 improvement instances the split is 4 public /
- * 2 private. Two private instances give a coarse overfit signal (0, 1/2 or
- * 2/2) and a single flake flips it — treat the private sub-score as a
- * direction check, not a certification; the pre-registered 6-instance holdout
- * remains the only promotion evidence.
- *
- * RESUME STABILITY: runId embeds a launch timestamp, so a resumed run gets a
- * fresh runId. The split is therefore PERSISTED to <outDir>/score-split.json
- * on first computation and reloaded verbatim afterwards — a resume can never
- * rotate previously-private instances into view.
- */
-
-import { createHash } from 'node:crypto'
-import { existsSync } from 'node:fs'
-import { mkdir, readFile, writeFile } from 'node:fs/promises'
-import { dirname, join } from 'node:path'
-
-export const SCORE_SPLIT_SCHEMA = 'swe-arena.score-split.v1'
-export const SCORE_SPLIT_FILENAME = 'score-split.json'
-
-export interface ScoreSplitConfig {
-  /** Number of PUBLIC instances (the rest are private). Must satisfy
-   *  0 < publicCount < instances.length. */
-  publicCount: number
-}
-
-export interface ScoreSplit {
-  schema: typeof SCORE_SPLIT_SCHEMA
-  /** The runId that seeded the split (provenance; a resumed run keeps it). */
-  seededBy: string
-  publicInstances: string[]
-  privateInstances: string[]
-}
-
-/** Deterministic split: instances are ranked by sha256(runId + ':' + iid) and
- *  the first `publicCount` become public. Pure — same (runId, instances,
- *  publicCount) always yields the same split; instance-list ORDER does not
- *  matter (ranking is content-derived). */
-export function splitInstances(runId: string, instances: readonly string[], publicCount: number): ScoreSplit {
-  if (!Number.isInteger(publicCount) || publicCount <= 0 || publicCount >= instances.length) {
-    throw new Error(
-      `score-split: publicCount must be an integer in (0, ${instances.length}), got ${JSON.stringify(publicCount)}`,
-    )
-  }
-  if (new Set(instances).size !== instances.length) {
-    throw new Error('score-split: duplicate instance ids')
-  }
-  const ranked = [...instances]
-    .map((iid) => ({ iid, rank: createHash('sha256').update(`${runId}:${iid}`).digest('hex') }))
-    .sort((a, b) => (a.rank < b.rank ? -1 : a.rank > b.rank ? 1 : 0))
-  const pub = ranked.slice(0, publicCount).map((r) => r.iid)
-  const priv = ranked.slice(publicCount).map((r) => r.iid)
-  return {
-    schema: SCORE_SPLIT_SCHEMA,
-    seededBy: runId,
-    publicInstances: [...pub].sort(),
-    privateInstances: [...priv].sort(),
-  }
-}
-
-/** Load the persisted split for this outDir, or compute + persist it. The
- *  persisted file always wins (resume stability); it is validated against the
- *  current instance set so a config edit mid-outDir fails loud. */
-export async function loadOrCreateScoreSplit(args: {
-  outDir: string
-  runId: string
-  instances: readonly string[]
-  publicCount: number
-}): Promise<ScoreSplit> {
-  const path = join(args.outDir, SCORE_SPLIT_FILENAME)
-  if (existsSync(path)) {
-    const split = JSON.parse(await readFile(path, 'utf8')) as ScoreSplit
-    if (split.schema !== SCORE_SPLIT_SCHEMA) {
-      throw new Error(`score-split: ${path} has unknown schema ${JSON.stringify(split.schema)}`)
-    }
-    const persisted = [...split.publicInstances, ...split.privateInstances].sort()
-    const expected = [...args.instances].sort()
-    if (JSON.stringify(persisted) !== JSON.stringify(expected)) {
-      throw new Error(
-        `score-split: persisted split at ${path} covers [${persisted.join(', ')}] but the config ` +
-          `names [${expected.join(', ')}] — refusing to silently re-split; move the outDir or fix the config`,
-      )
-    }
-    if (split.publicInstances.length !== args.publicCount) {
-      throw new Error(
-        `score-split: persisted split has ${split.publicInstances.length} public instances, config wants ` +
-          `${args.publicCount} — refusing to silently re-split`,
-      )
-    }
-    return split
-  }
-  const split = splitInstances(args.runId, args.instances, args.publicCount)
-  await mkdir(dirname(path), { recursive: true })
-  await writeFile(path, JSON.stringify(split, null, 2))
-  return split
-}
-
-/** True when `text` mentions any private instance id — the NEVER-SURFACED
- *  invariant's checkable form. Used by tests and by the evidence-index writer
- *  as a final guard on proposer-visible text. */
-export function leaksPrivateInstance(text: string, split: Pick<ScoreSplit, 'privateInstances'>): string[] {
-  return split.privateInstances.filter((iid) => text.includes(iid))
-}
-
-/** Filter per-instance evidence records to the PUBLIC set. Identity function
- *  when no split is configured (split === null). */
-export function publicOnly<T>(
-  records: readonly T[],
-  iidOf: (record: T) => string,
-  split: Pick<ScoreSplit, 'privateInstances'> | null,
-): T[] {
-  if (split === null) return [...records]
-  const priv = new Set(split.privateInstances)
-  return records.filter((r) => !priv.has(iidOf(r)))
-}
-
-/** Public/private sub-scores for one candidate, from per-instance AND-verdicts
- *  (fail-closed: an instance with no conclusive full-reps verdict counts 0). */
-export function subScores(
-  verdicts: Record<string, boolean>,
-  split: Pick<ScoreSplit, 'publicInstances' | 'privateInstances'>,
-): { publicResolvedCount: number; privateResolvedCount: number } {
-  const count = (iids: readonly string[]): number => iids.filter((iid) => verdicts[iid] === true).length
-  return {
-    publicResolvedCount: count(split.publicInstances),
-    privateResolvedCount: count(split.privateInstances),
-  }
-}