@tangle-network/agent-bench 0.3.6 → 0.3.7

package/src/swe-arena/parity.test.mts

@@ -0,0 +1,87 @@
+/**
+ * PARITY GATE for M2: the typed execution path (materialize → patch apply →
+ * extractPatch → serialized-judge → official swebench verdict) must reproduce
+ * the pinned M1 fixture verdicts for two committed arm patches — one resolved
+ * (pallets__flask-5014 SOLO), one unresolved (pydata__xarray-4687 SUP) —
+ * WITHOUT executing any arm. Zero model tokens; docker time only (it pulls the
+ * two instance images if absent and runs the official judge, ~5-25 min each).
+ *
+ * Opt-in because of that docker cost:
+ *
+ *   SWE_ARENA_PARITY=1 ../node_modules/.bin/vitest run src/swe-arena/parity.test.mts
+ *
+ * Expected verdicts are DERIVED from the pinned fixtures (ledger + rejudge
+ * reconciliation), not hardcoded — parity means agreeing with the record.
+ *
+ * TODO(operator approval): the full 12-instance ARM parity re-run (typed path
+ * vs the bash harness on the same 12 ids, both arms) costs ~$1 model spend +
+ * ~4h wall. Do not run without an explicit operator go — this 2-patch gate is
+ * the milestone check.
+ */
+
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { loadLedger, loadRejudge } from './fixtures.ts'
+import { reconcile } from './reconcile.ts'
+import { createSerializedJudge } from './serialized-judge.ts'
+import { PARITY_CASES, replayPatchParity, type ParityCaseResult } from './run-experiment.mts'
+
+const RUN_PARITY = process.env.SWE_ARENA_PARITY === '1'
+// materialize + image pull + official judge, twice; generous but finite.
+const PARITY_BUDGET_MS = 5_400_000
+
+describe.runIf(RUN_PARITY)('dry-run parity vs pinned M1 verdicts (docker, no tokens)', () => {
+  let results: ParityCaseResult[] = []
+  let workDir = ''
+
+  beforeAll(async () => {
+    workDir = await mkdtemp(join(tmpdir(), 'swe-arena-parity-'))
+    const judge = createSerializedJudge() // real judge child, 1800s ceiling, serialized
+    results = await replayPatchParity(PARITY_CASES, { workDir, judge })
+    for (const r of results) {
+      console.log(`PARITY ${r.iid} [${r.arm}] verdict=${JSON.stringify(r.verdict)}`)
+    }
+  }, PARITY_BUDGET_MS)
+
+  afterAll(async () => {
+    if (workDir) await rm(workDir, { recursive: true, force: true })
+  })
+
+  it('re-extraction preserves each committed patch’s changed-file set', () => {
+    expect(results).toHaveLength(2)
+    for (const r of results) {
+      expect(r.applyRc).toBe(0)
+      expect(r.extractedFiles, `${r.iid} [${r.arm}]`).toEqual(r.fixtureFiles)
+      expect(r.extractedPatchLines).toBeGreaterThan(0)
+    }
+  })
+
+  it('pallets__flask-5014 SOLO: typed judge verdict == pinned ledger verdict (resolved)', () => {
+    const pinned = loadLedger().find((r) => r.iid === 'pallets__flask-5014')
+    expect(pinned?.solo_resolved).toBe(true) // sanity: the fixture side of the parity claim
+    const r = results.find((x) => x.iid === 'pallets__flask-5014')
+    expect(r?.verdict.attempts).toBeGreaterThan(0)
+    expect(r?.verdict.resolved).toBe(pinned?.solo_resolved)
+  })
+
+  it('pydata__xarray-4687 SUP: typed judge verdict == reconciled verdict (unresolved)', () => {
+    // The authoritative record for this arm is the personal re-judge
+    // (sup-final), which M1's reconciliation applies over the ledger.
+    const reconciled = reconcile(loadLedger(), loadRejudge()).valid.find(
+      (r) => r.iid === 'pydata__xarray-4687',
+    )
+    expect(reconciled?.sup.source).toBe('sup-final')
+    expect(reconciled?.sup.resolved).toBe(false)
+    const r = results.find((x) => x.iid === 'pydata__xarray-4687')
+    expect(r?.verdict.attempts).toBeGreaterThan(0)
+    expect(r?.verdict.resolved).toBe(reconciled?.sup.resolved)
+  })
+})
+
+describe.runIf(!RUN_PARITY)('dry-run parity (skipped)', () => {
+  it('is opt-in: set SWE_ARENA_PARITY=1 to run the docker-backed parity gate', () => {
+    expect(RUN_PARITY).toBe(false)
+  })
+})

package/src/swe-arena/proc.test.mts

@@ -0,0 +1,174 @@
+import { access, mkdtemp, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, describe, expect, it } from 'vitest'
+import { waitForCapacity, type EndpointCapacityGate } from './capacity.ts'
+import { installProcessSignalAbort, run, TIMEOUT_RC } from './proc.ts'
+
+const dirs: string[] = []
+const cleanupPids: number[] = []
+
+afterAll(async () => {
+  for (const pid of cleanupPids) {
+    try {
+      process.kill(pid, 'SIGKILL')
+    } catch {
+      // The expected path already removed it.
+    }
+  }
+  await Promise.all(dirs.map((dir) => rm(dir, { recursive: true, force: true })))
+})
+
+describe('run process-group timeout', () => {
+  it('does not spawn when the caller signal is already aborted', () => {
+    const controller = new AbortController()
+    controller.abort(new Error('stop before spawn'))
+    expect(() => run(process.execPath, ['-e', 'process.exit(99)'], { signal: controller.signal })).toThrow(
+      'stop before spawn',
+    )
+  })
+
+  it('allows a SIGTERM handler to finish cleanup before returning timeout', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-term-'))
+    dirs.push(dir)
+    const marker = join(dir, 'cleanup.txt')
+    const result = await run(process.execPath, [
+      '-e',
+      `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>{fs.writeFileSync(marker,'settled'); setTimeout(()=>process.exit(0),25)}); setInterval(()=>{},1000)`,
+      marker,
+    ], { timeoutMs: 60, killGraceMs: 500 })
+
+    expect(result.code).toBe(TIMEOUT_RC)
+    expect(result.timedOut).toBe(true)
+    expect(await readFile(marker, 'utf8')).toBe('settled')
+  })
+
+  it('force-kills a process group that ignores SIGTERM after the grace period', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-kill-'))
+    dirs.push(dir)
+    const marker = join(dir, 'should-not-exist.txt')
+    const started = Date.now()
+    const result = await run(process.execPath, [
+      '-e',
+      `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>{}); setTimeout(()=>fs.writeFileSync(marker,'escaped'),5000); setInterval(()=>{},1000)`,
+      marker,
+    ], { timeoutMs: 60, killGraceMs: 80 })
+
+    expect(result.code).toBe(TIMEOUT_RC)
+    expect(result.timedOut).toBe(true)
+    expect(Date.now() - started).toBeLessThan(2_000)
+    await expect(access(marker)).rejects.toMatchObject({ code: 'ENOENT' })
+  })
+
+  it('kills a TERM-ignoring grandchild when its direct parent exits on TERM', async () => {
+    if (process.platform === 'win32') return
+    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-tree-'))
+    dirs.push(dir)
+    const pidFile = join(dir, 'grandchild.pid')
+    const grandchild = `process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)`
+    const parent = [
+      `const {spawn}=require('node:child_process')`,
+      `const fs=require('node:fs')`,
+      `const child=spawn(process.execPath,['-e',${JSON.stringify(grandchild)}],{stdio:'ignore'})`,
+      `fs.writeFileSync(process.argv[1],String(child.pid))`,
+      `process.on('SIGTERM',()=>process.exit(0))`,
+      `setInterval(()=>{},1000)`,
+    ].join(';')
+
+    const result = await run(process.execPath, ['-e', parent, pidFile], {
+      timeoutMs: 100,
+      killGraceMs: 50,
+    })
+    const grandchildPid = Number(await readFile(pidFile, 'utf8'))
+    cleanupPids.push(grandchildPid)
+
+    expect(result.code).toBe(TIMEOUT_RC)
+    expect(result.timedOut).toBe(true)
+    expect(() => process.kill(grandchildPid, 0)).toThrow(expect.objectContaining({ code: 'ESRCH' }))
+  })
+
+  it('kills a TERM-ignoring grandchild when its direct parent exits normally', async () => {
+    if (process.platform === 'win32') return
+    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-normal-exit-tree-'))
+    dirs.push(dir)
+    const pidFile = join(dir, 'grandchild.pid')
+    const grandchild = [
+      `const fs=require('node:fs')`,
+      `process.on('SIGTERM',()=>{})`,
+      `fs.writeFileSync(process.argv[1],String(process.pid))`,
+      `setInterval(()=>{},1000)`,
+    ].join(';')
+    const parent = [
+      `const {spawn}=require('node:child_process')`,
+      `const fs=require('node:fs')`,
+      `spawn(process.execPath,['-e',${JSON.stringify(grandchild)},process.argv[1]],{stdio:'ignore'})`,
+      `const ready=setInterval(()=>{if(fs.existsSync(process.argv[1])){clearInterval(ready);process.exit(0)}},5)`,
+    ].join(';')
+
+    const result = await run(process.execPath, ['-e', parent, pidFile], { killGraceMs: 50 })
+    const grandchildPid = Number(await readFile(pidFile, 'utf8'))
+    cleanupPids.push(grandchildPid)
+
+    expect(result.code).toBe(0)
+    expect(result.timedOut).toBe(false)
+    expect(result.aborted).toBe(false)
+    expect(() => process.kill(grandchildPid, 0)).toThrow(expect.objectContaining({ code: 'ESRCH' }))
+  })
+})
+
+describe('cooperative process cancellation', () => {
+  it('maps the first terminal signal to cancellation and removes its handlers', () => {
+    const oldExitCode = process.exitCode
+    const beforeInt = new Set(process.rawListeners('SIGINT'))
+    const beforeTerm = process.listenerCount('SIGTERM')
+    const interrupt = installProcessSignalAbort('test-run')
+    const listener = process.rawListeners('SIGINT').find((candidate) => !beforeInt.has(candidate))
+    expect(listener).toBeDefined()
+    listener?.call(process, 'SIGINT')
+    expect(interrupt.signal.aborted).toBe(true)
+    expect((interrupt.signal.reason as Error).message).toBe('test-run interrupted by SIGINT')
+    expect(interrupt.receivedSignal).toBe('SIGINT')
+    expect(process.exitCode).toBe(130)
+    interrupt.dispose()
+    expect(process.rawListeners('SIGINT').filter((candidate) => !beforeInt.has(candidate))).toHaveLength(0)
+    expect(process.listenerCount('SIGTERM')).toBe(beforeTerm)
+    process.exitCode = oldExitCode
+  })
+
+  it('cancels during a capacity retry sleep and never starts another probe', async () => {
+    const controller = new AbortController()
+    let probes = 0
+    const gate: EndpointCapacityGate = {
+      name: 'cancel-test',
+      probe: async () => {
+        probes += 1
+        return false
+      },
+      kOfN: { k: 1, n: 1 },
+      waitCeilingMs: 10_000,
+      retryDelayMs: 10_000,
+    }
+    const waiting = waitForCapacity(gate, controller.signal)
+    await new Promise((resolve) => setTimeout(resolve, 20))
+    controller.abort(new Error('capacity cancelled'))
+    await expect(waiting).rejects.toThrow('capacity cancelled')
+    expect(probes).toBe(1)
+  })
+
+  it('runs zero capacity probes when already aborted', async () => {
+    const controller = new AbortController()
+    controller.abort(new Error('cancelled before probing'))
+    let probes = 0
+    const gate: EndpointCapacityGate = {
+      name: 'pre-abort',
+      probe: async () => {
+        probes += 1
+        return true
+      },
+      kOfN: { k: 1, n: 1 },
+      waitCeilingMs: 1_000,
+    }
+    await expect(waitForCapacity(gate, controller.signal)).rejects.toThrow('cancelled before probing')
+    expect(probes).toBe(0)
+  })
+})

package/src/swe-arena/proc.ts

@@ -0,0 +1,260 @@
+/**
+ * Process helpers shared by the swe-arena execution path (materialize /
+ * calibrate / arms / serialized-judge). One place owns spawn semantics so
+ * every port of a bash script gets identical rc/timeout behavior:
+ *
+ *  - `run` never throws on a nonzero exit — the bash scripts branched on `$?`,
+ *    so callers get `{ code, stdout, stderr, timedOut }` and decide.
+ *  - timeouts signal the whole process GROUP (detached spawn), because the
+ *    arm commands are `dotenvx → bash -c → opencode/node` chains; killing only
+ *    the leader would orphan the model call exactly like a dropped `timeout`.
+ *    SIGTERM gets a short cleanup window before SIGKILL so children can cancel
+ *    their own detached workers and flush terminal evidence.
+ *  - `code` is 124 on timeout, mirroring coreutils `timeout` so ledger rc
+ *    fields keep the bash experiment's encoding.
+ */
+
+import { spawn } from 'node:child_process'
+
+export interface RunResult {
+  code: number
+  stdout: string
+  stderr: string
+  timedOut: boolean
+  aborted: boolean
+}
+
+export interface RunOptions {
+  cwd?: string
+  env?: NodeJS.ProcessEnv
+  /** 0 / undefined = no timeout. On expiry the process group gets TERM then KILL. */
+  timeoutMs?: number
+  /** Grace between timeout SIGTERM and fallback SIGKILL. Default 1 second. */
+  killGraceMs?: number
+  /** Cancels the complete process group and waits until it is gone. */
+  signal?: AbortSignal
+  /** Cap captured stdout/stderr (bytes); beyond it output is dropped, not buffered. */
+  maxBuffer?: number
+  /** Piped to the child's stdin, then stdin is closed. */
+  stdin?: string
+}
+
+const DEFAULT_MAX_BUFFER = 64 * 1024 * 1024
+const PROCESS_GROUP_EXIT_CONFIRM_MS = 1_000
+const PROCESS_GROUP_EXIT_POLL_MS = 10
+
+const exitKillGroups = new Set<number>()
+const killRegisteredGroupsOnExit = (): void => {
+  for (const processGroupId of exitKillGroups) {
+    try {
+      process.kill(process.platform === 'win32' ? processGroupId : -processGroupId, 'SIGKILL')
+    } catch {
+      // The group may have settled between its final async check and process exit.
+    }
+  }
+}
+
+function registerExitKill(processGroupId: number | undefined): () => void {
+  if (processGroupId === undefined) return () => {}
+  if (exitKillGroups.size === 0) process.once('exit', killRegisteredGroupsOnExit)
+  exitKillGroups.add(processGroupId)
+  return () => {
+    exitKillGroups.delete(processGroupId)
+    if (exitKillGroups.size === 0) process.removeListener('exit', killRegisteredGroupsOnExit)
+  }
+}
+
+/** coreutils `timeout` exit code — kept so ledger rc columns stay comparable. */
+export const TIMEOUT_RC = 124
+export const ABORT_RC = 130
+
+export interface ProcessSignalAbort {
+  signal: AbortSignal
+  /** First signal received; null until interrupted. */
+  readonly receivedSignal: NodeJS.Signals | null
+  /** Removes handlers after the caller has awaited cleanup. */
+  dispose: () => void
+}
+
+/**
+ * Translate terminal interrupts into cooperative cancellation. The handler
+ * never exits immediately: callers await their normal cleanup in `finally`,
+ * then dispose it. Repeated interrupts remain cooperative rather than
+ * bypassing cleanup.
+ */
+export function installProcessSignalAbort(label: string): ProcessSignalAbort {
+  const controller = new AbortController()
+  let receivedSignal: NodeJS.Signals | null = null
+  const onSignal = (signal: NodeJS.Signals): void => {
+    if (receivedSignal !== null) return
+    receivedSignal = signal
+    process.exitCode = signal === 'SIGINT' ? 130 : 143
+    controller.abort(new Error(`${label} interrupted by ${signal}`))
+  }
+  const onSigint = (): void => onSignal('SIGINT')
+  const onSigterm = (): void => onSignal('SIGTERM')
+  process.on('SIGINT', onSigint)
+  process.on('SIGTERM', onSigterm)
+  return {
+    signal: controller.signal,
+    get receivedSignal() {
+      return receivedSignal
+    },
+    dispose: () => {
+      process.removeListener('SIGINT', onSigint)
+      process.removeListener('SIGTERM', onSigterm)
+    },
+  }
+}
+
+function processGroupExists(processGroupId: number): boolean {
+  try {
+    process.kill(-processGroupId, 0)
+    return true
+  } catch (err) {
+    return (err as NodeJS.ErrnoException).code !== 'ESRCH'
+  }
+}
+
+async function waitUntil(deadline: number, predicate: () => boolean): Promise<boolean> {
+  while (predicate()) {
+    if (Date.now() >= deadline) return false
+    await new Promise((resolve) => setTimeout(resolve, PROCESS_GROUP_EXIT_POLL_MS))
+  }
+  return true
+}
+
+export function run(bin: string, argv: string[], opts: RunOptions = {}): Promise<RunResult> {
+  opts.signal?.throwIfAborted()
+  return new Promise((resolve, reject) => {
+    const child = spawn(bin, argv, {
+      cwd: opts.cwd,
+      env: opts.env ?? process.env,
+      detached: true,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    })
+    const unregisterExitKill = registerExitKill(child.pid)
+    const cap = opts.maxBuffer ?? DEFAULT_MAX_BUFFER
+    let stdout = ''
+    let stderr = ''
+    let outBytes = 0
+    let errBytes = 0
+    let timedOut = false
+    let aborted = false
+    let settled = false
+    let timer: NodeJS.Timeout | undefined
+    const signalGroup = (signal: NodeJS.Signals) => {
+      try {
+        process.kill(-child.pid!, signal)
+      } catch {
+        try {
+          child.kill(signal)
+        } catch {
+          /* already dead */
+        }
+      }
+    }
+    let termination: Promise<void> | undefined
+    const terminate = (): Promise<void> => {
+      termination ??= (async () => {
+        signalGroup('SIGTERM')
+        if (process.platform === 'win32' || typeof child.pid !== 'number') return
+        const processGroupId = child.pid
+        const exitedOnTerm = await waitUntil(
+          Date.now() + Math.max(0, opts.killGraceMs ?? 1_000),
+          () => processGroupExists(processGroupId),
+        )
+        if (exitedOnTerm) return
+        signalGroup('SIGKILL')
+        const exitedOnKill = await waitUntil(
+          Date.now() + PROCESS_GROUP_EXIT_CONFIRM_MS,
+          () => processGroupExists(processGroupId),
+        )
+        if (!exitedOnKill) throw new Error(`process group ${processGroupId} survived SIGKILL`)
+      })()
+      // The close handler owns the rejection. Attaching this handler prevents
+      // an early unhandled rejection while the direct child is still closing.
+      void termination.catch(() => undefined)
+      return termination
+    }
+    const onAbort = () => {
+      aborted = true
+      void terminate()
+    }
+    if (opts.timeoutMs && opts.timeoutMs > 0) {
+      timer = setTimeout(() => {
+        timedOut = true
+        void terminate()
+      }, opts.timeoutMs)
+    }
+    opts.signal?.addEventListener('abort', onAbort, { once: true })
+    if (opts.signal?.aborted) onAbort()
+    child.stdout.on('data', (c: Buffer) => {
+      outBytes += c.length
+      if (outBytes <= cap) stdout += c.toString('utf8')
+    })
+    child.stderr.on('data', (c: Buffer) => {
+      errBytes += c.length
+      if (errBytes <= cap) stderr += c.toString('utf8')
+    })
+    child.on('error', (err) => {
+      if (settled) return
+      settled = true
+      if (timer) clearTimeout(timer)
+      opts.signal?.removeEventListener('abort', onAbort)
+      unregisterExitKill()
+      reject(err)
+    })
+    child.on('close', (code, signal) => {
+      if (settled) return
+      if (timer) clearTimeout(timer)
+      opts.signal?.removeEventListener('abort', onAbort)
+      void (async () => {
+        try {
+          if (
+            !termination &&
+            process.platform !== 'win32' &&
+            typeof child.pid === 'number' &&
+            processGroupExists(child.pid)
+          ) {
+            await terminate()
+          }
+          if (termination) await termination
+          if (settled) return
+          settled = true
+          unregisterExitKill()
+          resolve({
+            code: timedOut ? TIMEOUT_RC : aborted ? ABORT_RC : (code ?? (signal ? 1 : 0)),
+            stdout,
+            stderr,
+            timedOut,
+            aborted,
+          })
+        } catch (err) {
+          if (settled) return
+          settled = true
+          unregisterExitKill()
+          reject(err)
+        }
+      })()
+    })
+    if (opts.stdin !== undefined) child.stdin.end(opts.stdin)
+    else child.stdin.end()
+  })
+}
+
+/** `run` that throws (with captured stderr) on nonzero exit — for steps the bash aborted on. */
+export async function runOk(bin: string, argv: string[], opts: RunOptions = {}): Promise<RunResult> {
+  const res = await run(bin, argv, opts)
+  opts.signal?.throwIfAborted()
+  if (res.code !== 0) {
+    const detail = (res.stderr || res.stdout).slice(0, 1500)
+    throw new Error(`${bin} ${argv.join(' ')} exited ${res.code}${res.timedOut ? ' (timeout)' : ''}: ${detail}`)
+  }
+  return res
+}
+
+/** Single-quote shell escaping — identical to the quoting the bash scripts relied on. */
+export function shq(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`
+}

package/src/swe-arena/profiles/default-author.profile.json

@@ -0,0 +1,4 @@
+{
+  "name": "swe-arena-default-author",
+  "description": "The gen-2 winning author seat, codified: bare claude CLI invocation. No systemPrompt, no instructions, no model override — harnessInvocation() with this profile is byte-identical to the legacy prompt-only buildArgs path, so provenance pins the author identity without changing gen-2 behavior."
+}