@tangle-network/agent-bench 0.11.3 → 0.13.0

package/src/swe-arena/proc.test.mts

@@ -1,172 +0,0 @@
-import { access, mkdtemp, readFile, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterAll, describe, expect, it } from 'vitest'
-import { waitForCapacity, type EndpointCapacityGate } from './capacity.ts'
-import { installProcessSignalAbort, run, TIMEOUT_RC } from './proc.ts'
-
-const dirs: string[] = []
-const cleanupPids: number[] = []
-
-afterAll(async () => {
-  for (const pid of cleanupPids) {
-    try {
-      process.kill(pid, 'SIGKILL')
-    } catch {
-      // The expected path already removed it.
-    }
-  }
-  await Promise.all(dirs.map((dir) => rm(dir, { recursive: true, force: true })))
-})
-
-describe('run process-group timeout', () => {
-  it('does not spawn when the caller signal is already aborted', () => {
-    const controller = new AbortController()
-    controller.abort(new Error('stop before spawn'))
-    expect(() => run(process.execPath, ['-e', 'process.exit(99)'], { signal: controller.signal })).toThrow(
-      'stop before spawn',
-    )
-  })
-
-  it('allows a SIGTERM handler to finish cleanup before returning timeout', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-term-'))
-    dirs.push(dir)
-    const marker = join(dir, 'cleanup.txt')
-    const result = await run(process.execPath, [
-      '-e',
-      `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>{fs.writeFileSync(marker,'settled'); setTimeout(()=>process.exit(0),25)}); setInterval(()=>{},1000)`,
-      marker,
-    ], { timeoutMs: 1_000, killGraceMs: 500 })
-
-    expect(result.code).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
-    expect(await readFile(marker, 'utf8')).toBe('settled')
-  })
-
-  it('force-kills a process group that ignores SIGTERM after the grace period', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-kill-'))
-    dirs.push(dir)
-    const marker = join(dir, 'should-not-exist.txt')
-    const started = Date.now()
-    const result = await run(process.execPath, [
-      '-e',
-      `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>{}); setTimeout(()=>fs.writeFileSync(marker,'escaped'),5000); setInterval(()=>{},1000)`,
-      marker,
-    ], { timeoutMs: 60, killGraceMs: 80 })
-
-    expect(result.code).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
-    expect(Date.now() - started).toBeLessThan(2_000)
-    await expect(access(marker)).rejects.toMatchObject({ code: 'ENOENT' })
-  })
-
-  it('kills a TERM-ignoring grandchild when its direct parent exits on TERM', async () => {
-    if (process.platform === 'win32') return
-    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-tree-'))
-    dirs.push(dir)
-    const pidFile = join(dir, 'grandchild.pid')
-    const grandchild = `process.on('SIGTERM',()=>{}); require('node:fs').writeFileSync(process.argv[1],String(process.pid)); setInterval(()=>{},1000)`
-    const parent = [
-      `const {spawn}=require('node:child_process')`,
-      `spawn(process.execPath,['-e',${JSON.stringify(grandchild)},process.argv[1]],{stdio:'ignore'})`,
-      `process.on('SIGTERM',()=>process.exit(0))`,
-      `setInterval(()=>{},1000)`,
-    ].join(';')
-
-    const result = await run(process.execPath, ['-e', parent, pidFile], {
-      timeoutMs: 2_000,
-      killGraceMs: 50,
-    })
-    const grandchildPid = Number(await readFile(pidFile, 'utf8'))
-    cleanupPids.push(grandchildPid)
-
-    expect(result.code).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
-    expect(() => process.kill(grandchildPid, 0)).toThrow(expect.objectContaining({ code: 'ESRCH' }))
-  })
-
-  it('kills a TERM-ignoring grandchild when its direct parent exits normally', async () => {
-    if (process.platform === 'win32') return
-    const dir = await mkdtemp(join(tmpdir(), 'swe-proc-normal-exit-tree-'))
-    dirs.push(dir)
-    const pidFile = join(dir, 'grandchild.pid')
-    const grandchild = [
-      `const fs=require('node:fs')`,
-      `process.on('SIGTERM',()=>{})`,
-      `fs.writeFileSync(process.argv[1],String(process.pid))`,
-      `setInterval(()=>{},1000)`,
-    ].join(';')
-    const parent = [
-      `const {spawn}=require('node:child_process')`,
-      `const fs=require('node:fs')`,
-      `spawn(process.execPath,['-e',${JSON.stringify(grandchild)},process.argv[1]],{stdio:'ignore'})`,
-      `const ready=setInterval(()=>{if(fs.existsSync(process.argv[1])){clearInterval(ready);process.exit(0)}},5)`,
-    ].join(';')
-
-    const result = await run(process.execPath, ['-e', parent, pidFile], { killGraceMs: 50 })
-    const grandchildPid = Number(await readFile(pidFile, 'utf8'))
-    cleanupPids.push(grandchildPid)
-
-    expect(result.code).toBe(0)
-    expect(result.timedOut).toBe(false)
-    expect(result.aborted).toBe(false)
-    expect(() => process.kill(grandchildPid, 0)).toThrow(expect.objectContaining({ code: 'ESRCH' }))
-  })
-})
-
-describe('cooperative process cancellation', () => {
-  it('maps the first terminal signal to cancellation and removes its handlers', () => {
-    const oldExitCode = process.exitCode
-    const beforeInt = new Set(process.rawListeners('SIGINT'))
-    const beforeTerm = process.listenerCount('SIGTERM')
-    const interrupt = installProcessSignalAbort('test-run')
-    const listener = process.rawListeners('SIGINT').find((candidate) => !beforeInt.has(candidate))
-    expect(listener).toBeDefined()
-    listener?.call(process, 'SIGINT')
-    expect(interrupt.signal.aborted).toBe(true)
-    expect((interrupt.signal.reason as Error).message).toBe('test-run interrupted by SIGINT')
-    expect(interrupt.receivedSignal).toBe('SIGINT')
-    expect(process.exitCode).toBe(130)
-    interrupt.dispose()
-    expect(process.rawListeners('SIGINT').filter((candidate) => !beforeInt.has(candidate))).toHaveLength(0)
-    expect(process.listenerCount('SIGTERM')).toBe(beforeTerm)
-    process.exitCode = oldExitCode
-  })
-
-  it('cancels during a capacity retry sleep and never starts another probe', async () => {
-    const controller = new AbortController()
-    let probes = 0
-    const gate: EndpointCapacityGate = {
-      name: 'cancel-test',
-      probe: async () => {
-        probes += 1
-        return false
-      },
-      kOfN: { k: 1, n: 1 },
-      waitCeilingMs: 10_000,
-      retryDelayMs: 10_000,
-    }
-    const waiting = waitForCapacity(gate, controller.signal)
-    await new Promise((resolve) => setTimeout(resolve, 20))
-    controller.abort(new Error('capacity cancelled'))
-    await expect(waiting).rejects.toThrow('capacity cancelled')
-    expect(probes).toBe(1)
-  })
-
-  it('runs zero capacity probes when already aborted', async () => {
-    const controller = new AbortController()
-    controller.abort(new Error('cancelled before probing'))
-    let probes = 0
-    const gate: EndpointCapacityGate = {
-      name: 'pre-abort',
-      probe: async () => {
-        probes += 1
-        return true
-      },
-      kOfN: { k: 1, n: 1 },
-      waitCeilingMs: 1_000,
-    }
-    await expect(waitForCapacity(gate, controller.signal)).rejects.toThrow('cancelled before probing')
-    expect(probes).toBe(0)
-  })
-})

package/src/swe-arena/proc.ts

@@ -1,260 +0,0 @@
-/**
- * Process helpers shared by the swe-arena execution path (materialize /
- * calibrate / arms / serialized-judge). One place owns spawn semantics so
- * every port of a bash script gets identical rc/timeout behavior:
- *
- *  - `run` never throws on a nonzero exit — the bash scripts branched on `$?`,
- *    so callers get `{ code, stdout, stderr, timedOut }` and decide.
- *  - timeouts signal the whole process GROUP (detached spawn), because the
- *    arm commands are `dotenvx → bash -c → opencode/node` chains; killing only
- *    the leader would orphan the model call exactly like a dropped `timeout`.
- *    SIGTERM gets a short cleanup window before SIGKILL so children can cancel
- *    their own detached workers and flush terminal evidence.
- *  - `code` is 124 on timeout, mirroring coreutils `timeout` so ledger rc
- *    fields keep the bash experiment's encoding.
- */
-
-import { spawn } from 'node:child_process'
-
-export interface RunResult {
-  code: number
-  stdout: string
-  stderr: string
-  timedOut: boolean
-  aborted: boolean
-}
-
-export interface RunOptions {
-  cwd?: string
-  env?: NodeJS.ProcessEnv
-  /** 0 / undefined = no timeout. On expiry the process group gets TERM then KILL. */
-  timeoutMs?: number
-  /** Grace between timeout SIGTERM and fallback SIGKILL. Default 1 second. */
-  killGraceMs?: number
-  /** Cancels the complete process group and waits until it is gone. */
-  signal?: AbortSignal
-  /** Cap captured stdout/stderr (bytes); beyond it output is dropped, not buffered. */
-  maxBuffer?: number
-  /** Piped to the child's stdin, then stdin is closed. */
-  stdin?: string
-}
-
-const DEFAULT_MAX_BUFFER = 64 * 1024 * 1024
-const PROCESS_GROUP_EXIT_CONFIRM_MS = 1_000
-const PROCESS_GROUP_EXIT_POLL_MS = 10
-
-const exitKillGroups = new Set<number>()
-const killRegisteredGroupsOnExit = (): void => {
-  for (const processGroupId of exitKillGroups) {
-    try {
-      process.kill(process.platform === 'win32' ? processGroupId : -processGroupId, 'SIGKILL')
-    } catch {
-      // The group may have settled between its final async check and process exit.
-    }
-  }
-}
-
-function registerExitKill(processGroupId: number | undefined): () => void {
-  if (processGroupId === undefined) return () => {}
-  if (exitKillGroups.size === 0) process.once('exit', killRegisteredGroupsOnExit)
-  exitKillGroups.add(processGroupId)
-  return () => {
-    exitKillGroups.delete(processGroupId)
-    if (exitKillGroups.size === 0) process.removeListener('exit', killRegisteredGroupsOnExit)
-  }
-}
-
-/** coreutils `timeout` exit code — kept so ledger rc columns stay comparable. */
-export const TIMEOUT_RC = 124
-export const ABORT_RC = 130
-
-export interface ProcessSignalAbort {
-  signal: AbortSignal
-  /** First signal received; null until interrupted. */
-  readonly receivedSignal: NodeJS.Signals | null
-  /** Removes handlers after the caller has awaited cleanup. */
-  dispose: () => void
-}
-
-/**
- * Translate terminal interrupts into cooperative cancellation. The handler
- * never exits immediately: callers await their normal cleanup in `finally`,
- * then dispose it. Repeated interrupts remain cooperative rather than
- * bypassing cleanup.
- */
-export function installProcessSignalAbort(label: string): ProcessSignalAbort {
-  const controller = new AbortController()
-  let receivedSignal: NodeJS.Signals | null = null
-  const onSignal = (signal: NodeJS.Signals): void => {
-    if (receivedSignal !== null) return
-    receivedSignal = signal
-    process.exitCode = signal === 'SIGINT' ? 130 : 143
-    controller.abort(new Error(`${label} interrupted by ${signal}`))
-  }
-  const onSigint = (): void => onSignal('SIGINT')
-  const onSigterm = (): void => onSignal('SIGTERM')
-  process.on('SIGINT', onSigint)
-  process.on('SIGTERM', onSigterm)
-  return {
-    signal: controller.signal,
-    get receivedSignal() {
-      return receivedSignal
-    },
-    dispose: () => {
-      process.removeListener('SIGINT', onSigint)
-      process.removeListener('SIGTERM', onSigterm)
-    },
-  }
-}
-
-function processGroupExists(processGroupId: number): boolean {
-  try {
-    process.kill(-processGroupId, 0)
-    return true
-  } catch (err) {
-    return (err as NodeJS.ErrnoException).code !== 'ESRCH'
-  }
-}
-
-async function waitUntil(deadline: number, predicate: () => boolean): Promise<boolean> {
-  while (predicate()) {
-    if (Date.now() >= deadline) return false
-    await new Promise((resolve) => setTimeout(resolve, PROCESS_GROUP_EXIT_POLL_MS))
-  }
-  return true
-}
-
-export function run(bin: string, argv: string[], opts: RunOptions = {}): Promise<RunResult> {
-  opts.signal?.throwIfAborted()
-  return new Promise((resolve, reject) => {
-    const child = spawn(bin, argv, {
-      cwd: opts.cwd,
-      env: opts.env ?? process.env,
-      detached: true,
-      stdio: ['pipe', 'pipe', 'pipe'],
-    })
-    const unregisterExitKill = registerExitKill(child.pid)
-    const cap = opts.maxBuffer ?? DEFAULT_MAX_BUFFER
-    let stdout = ''
-    let stderr = ''
-    let outBytes = 0
-    let errBytes = 0
-    let timedOut = false
-    let aborted = false
-    let settled = false
-    let timer: NodeJS.Timeout | undefined
-    const signalGroup = (signal: NodeJS.Signals) => {
-      try {
-        process.kill(-child.pid!, signal)
-      } catch {
-        try {
-          child.kill(signal)
-        } catch {
-          /* already dead */
-        }
-      }
-    }
-    let termination: Promise<void> | undefined
-    const terminate = (): Promise<void> => {
-      termination ??= (async () => {
-        signalGroup('SIGTERM')
-        if (process.platform === 'win32' || typeof child.pid !== 'number') return
-        const processGroupId = child.pid
-        const exitedOnTerm = await waitUntil(
-          Date.now() + Math.max(0, opts.killGraceMs ?? 1_000),
-          () => processGroupExists(processGroupId),
-        )
-        if (exitedOnTerm) return
-        signalGroup('SIGKILL')
-        const exitedOnKill = await waitUntil(
-          Date.now() + PROCESS_GROUP_EXIT_CONFIRM_MS,
-          () => processGroupExists(processGroupId),
-        )
-        if (!exitedOnKill) throw new Error(`process group ${processGroupId} survived SIGKILL`)
-      })()
-      // The close handler owns the rejection. Attaching this handler prevents
-      // an early unhandled rejection while the direct child is still closing.
-      void termination.catch(() => undefined)
-      return termination
-    }
-    const onAbort = () => {
-      aborted = true
-      void terminate()
-    }
-    if (opts.timeoutMs && opts.timeoutMs > 0) {
-      timer = setTimeout(() => {
-        timedOut = true
-        void terminate()
-      }, opts.timeoutMs)
-    }
-    opts.signal?.addEventListener('abort', onAbort, { once: true })
-    if (opts.signal?.aborted) onAbort()
-    child.stdout.on('data', (c: Buffer) => {
-      outBytes += c.length
-      if (outBytes <= cap) stdout += c.toString('utf8')
-    })
-    child.stderr.on('data', (c: Buffer) => {
-      errBytes += c.length
-      if (errBytes <= cap) stderr += c.toString('utf8')
-    })
-    child.on('error', (err) => {
-      if (settled) return
-      settled = true
-      if (timer) clearTimeout(timer)
-      opts.signal?.removeEventListener('abort', onAbort)
-      unregisterExitKill()
-      reject(err)
-    })
-    child.on('close', (code, signal) => {
-      if (settled) return
-      if (timer) clearTimeout(timer)
-      opts.signal?.removeEventListener('abort', onAbort)
-      void (async () => {
-        try {
-          if (
-            !termination &&
-            process.platform !== 'win32' &&
-            typeof child.pid === 'number' &&
-            processGroupExists(child.pid)
-          ) {
-            await terminate()
-          }
-          if (termination) await termination
-          if (settled) return
-          settled = true
-          unregisterExitKill()
-          resolve({
-            code: timedOut ? TIMEOUT_RC : aborted ? ABORT_RC : (code ?? (signal ? 1 : 0)),
-            stdout,
-            stderr,
-            timedOut,
-            aborted,
-          })
-        } catch (err) {
-          if (settled) return
-          settled = true
-          unregisterExitKill()
-          reject(err)
-        }
-      })()
-    })
-    if (opts.stdin !== undefined) child.stdin.end(opts.stdin)
-    else child.stdin.end()
-  })
-}
-
-/** `run` that throws (with captured stderr) on nonzero exit — for steps the bash aborted on. */
-export async function runOk(bin: string, argv: string[], opts: RunOptions = {}): Promise<RunResult> {
-  const res = await run(bin, argv, opts)
-  opts.signal?.throwIfAborted()
-  if (res.code !== 0) {
-    const detail = (res.stderr || res.stdout).slice(0, 1500)
-    throw new Error(`${bin} ${argv.join(' ')} exited ${res.code}${res.timedOut ? ' (timeout)' : ''}: ${detail}`)
-  }
-  return res
-}
-
-/** Single-quote shell escaping — identical to the quoting the bash scripts relied on. */
-export function shq(value: string): string {
-  return `'${value.replace(/'/g, `'\\''`)}'`
-}

package/src/swe-arena/profiles/deepseek-author.profile.json

@@ -1,12 +0,0 @@
-{
-  "name": "swe-arena-deepseek-author",
-  "description": "Pi author using DeepSeek V4 Flash through Tangle Router.",
-  "harness": "pi",
-  "model": {
-    "provider": "tangle-router",
-    "default": "deepseek-v4-flash"
-  },
-  "prompt": {
-    "systemPrompt": "Improve the candidate worktree from measured failure evidence. Make the smallest coherent change, verify it, and leave the edit uncommitted."
-  }
-}

package/src/swe-arena/profiles/default-author.profile.json

@@ -1,12 +0,0 @@
-{
-  "name": "swe-arena-default-author",
-  "description": "Pi author for measured SWE arena code improvement.",
-  "harness": "pi",
-  "model": {
-    "provider": "tangle-router",
-    "default": "glm-5.2"
-  },
-  "prompt": {
-    "systemPrompt": "Improve the candidate worktree from measured failure evidence. Make the smallest coherent change, verify it, and leave the edit uncommitted."
-  }
-}