@tangle-network/agent-bench 0.8.23 → 0.9.0

package/src/run-benchmarks.ts

@@ -35,7 +35,7 @@ import type {
   Deliverable,
   OpenSandboxRunOptions,
 } from '@tangle-network/agent-runtime/kernel'
-import { openSandboxRun } from '@tangle-network/agent-runtime/kernel'
+import { openSandboxRun, SandboxRunAbortError, sumSandboxUsage } from '@tangle-network/agent-runtime/kernel'
 import type { SandboxEvent } from '@tangle-network/sandbox'
 import { resolveAdapter } from './adapters'
 import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types'
@@ -60,8 +60,17 @@ export interface BenchCell {
   readonly profile?: AgentProfile
 }
 
-/** Runs one (adapter, task, cell) shot and returns the deliverable text. The default uses
- *  `openSandboxRun`; tests inject a deterministic stub so the matrix runs offline. */
+/** A worker's artifact and observed execution evidence, before external grading. */
+export interface BenchShotResult {
+  readonly artifact: string
+  readonly ok: boolean
+  readonly detail?: string
+  /** Provider observations, including explicit unknown counters. Omitted when the shot reports none. */
+  readonly usage?: ReturnType<typeof sumSandboxUsage>
+  readonly events?: readonly SandboxEvent[]
+}
+
+/** Runs one (adapter, task, cell) shot. Defaults to `openSandboxRun`. */
 export type BenchShot = (input: {
   readonly adapter: BenchmarkAdapter
   readonly task: BenchTask
@@ -72,12 +81,15 @@ export type BenchShot = (input: {
   readonly attempt?: number
   readonly routerBaseUrl: string
   readonly routerKey: string
+  /** Optional inference credential for the box; routerKey continues to authorize sandbox control. */
+  readonly modelApiKey?: string
   readonly bridgeUrl?: string
   readonly bridgeBearer?: string
   readonly sandboxBaseUrl?: string
   readonly timeoutMs?: number
+  readonly signal?: AbortSignal
   readonly resolveClient?: typeof resolveBenchClient
-}) => Promise<{ artifact: string; ok: boolean; detail?: string }>
+}) => Promise<BenchShotResult>
 
 export interface RunBenchmarksOptions {
   /** Registry keys (`resolveAdapter`) — the benchmark subset to run. */
@@ -86,6 +98,8 @@ export interface RunBenchmarksOptions {
   readonly cells: readonly BenchCell[]
   readonly routerBaseUrl: string
   readonly routerKey: string
+  /** Optional inference credential for the box; never used for sandbox creation or deletion. */
+  readonly modelApiKey?: string
   readonly bridgeUrl?: string
   readonly bridgeBearer?: string
   readonly sandboxBaseUrl?: string
@@ -99,6 +113,8 @@ export interface RunBenchmarksOptions {
   readonly concurrency?: number
   /** Per-shot wall-clock (ms). */
   readonly timeoutMs?: number
+  /** Cancels active shots and prevents queued shots from starting. */
+  readonly signal?: AbortSignal
   /** Test seam: resolve the runtime transport. Defaults to `resolveBenchClient`. */
   readonly resolveClient?: typeof resolveBenchClient
   /** Max attempts per (benchmark × cell × task). Default 1. Attempts after the first receive
@@ -126,6 +142,11 @@ export interface BenchCellTaskResult {
   readonly ok: boolean
   readonly detail?: string
   readonly wallMs: number
+  /** Exact bytes given to the benchmark judge, retained even when judging fails. */
+  readonly artifact?: string
+  readonly usage?: ReturnType<typeof sumSandboxUsage>
+  /** Worker events only; benchmark grading remains outside this trace. */
+  readonly events?: readonly SandboxEvent[]
 }
 
 export interface BenchLeaderboardRow {
@@ -173,7 +194,8 @@ function finalText(events: readonly SandboxEvent[]): string {
 
 /** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable
  *  extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */
-const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, resolveClient }) => {
+const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, modelApiKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, signal, resolveClient }) => {
+  signal?.throwIfAborted()
   const client = (resolveClient ?? resolveBenchClient)({
     backend: cell.backend ?? 'router',
     routerBaseUrl,
@@ -206,7 +228,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
     sandboxOverrides: {
       name: `bench-${adapter.name}-${task.id}-${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
       environment: 'universal',
-      backend: { type: harness as never, model: { provider: profileProvider, model: cell.model, baseUrl: routerBaseUrl } },
+      backend: { type: harness as never, model: { provider: profileProvider, model: cell.model, baseUrl: routerBaseUrl, ...(modelApiKey === undefined ? {} : { apiKey: modelApiKey }) } },
     },
   }
   const deliverable: Deliverable<string> = {
@@ -217,7 +239,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
   const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : undefined
   const runOptions: OpenSandboxRunOptions = {
     agentRun,
-    signal: controller.signal,
+    signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal,
     runId: `bench:${adapter.name}:${task.id}:${uniq}`,
     scenarioId: task.id,
   }
@@ -236,9 +258,12 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
         )
     }
   }
-  const run = await openSandboxRun(client, runOptions, deliverable)
+  let run: Awaited<ReturnType<typeof openSandboxRun<string>>> | undefined
+  let result: BenchShotResult = { artifact: '', ok: false }
   try {
+    run = await openSandboxRun(client, runOptions, deliverable)
     const turn = await run.start(prompt ?? task.prompt)
+    result = { artifact: '', ok: false, usage: sumSandboxUsage(turn.events), events: turn.events }
     // Event-stream deliverable (adapter.output ?? finalText) — the FALLBACK.
     let artifact = (turn.out ?? '').trim()
     let boxExtractError: string | undefined
@@ -289,7 +314,9 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
       }
     }
     const detail =
-      turn.readError !== undefined
+      !turn.outcome.success
+        ? turn.outcome.error ?? `agent ended with status ${turn.outcome.status}`
+        : turn.readError !== undefined
         ? `read: ${turn.readError.slice(0, 160)}`
         : boxExtractError !== undefined
           ? `boxExtract: ${boxExtractError}`
@@ -306,15 +333,30 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB
         // debug-only; never fail the shot on a dump error
       }
     }
-    return {
+    result = {
       artifact,
-      ok: artifact.length > 0,
+      ok: turn.outcome.success && artifact.length > 0 && turn.readError === undefined && boxExtractError === undefined,
+      usage: result.usage,
+      events: turn.events,
       ...(detail ? { detail } : {}),
     }
+  } catch (err) {
+    result = {
+      ...result,
+      ok: false,
+      detail: err instanceof Error ? err.message : String(err),
+      ...(err instanceof SandboxRunAbortError ? { usage: sumSandboxUsage(err.events), events: err.events } : {}),
+    }
   } finally {
     if (timer) clearTimeout(timer)
-    await run.close()
+    try {
+      await run?.close()
+    } catch (err) {
+      const cleanup = `cleanup: ${err instanceof Error ? err.message : String(err)}`
+      result = { ...result, detail: combineDetails(result.detail, cleanup) }
+    }
   }
+  return result
 }
 
 function parseMaybeJson(value: string): unknown {
@@ -380,23 +422,44 @@ async function loopedShot(
   input: Parameters<BenchShot>[0],
   shot: BenchShot,
   attempts: number,
-): Promise<{ artifact: string; ok: boolean; detail?: string }> {
+): Promise<BenchShotResult> {
   const scores = new Map<number, BenchScore>()
-  const result = await runRefineLoop<string>({
-    rounds: attempts,
-    prompt: (round, history) => (round === 1 ? input.task.prompt : retryPrompt(input.task, history, scores)),
-    runShot: async (prompt, round) => {
-      const out = await shot({ ...input, prompt, attempt: round })
-      return { artifact: out.artifact, note: out.detail }
-    },
-    judge: async (artifact, round) => {
-      const score = await input.adapter.judge(input.task, artifact)
-      scores.set(round, score)
-      return { valid: score.resolved, score: score.score }
-    },
-  })
+  const shots = new Map<number, BenchShotResult>()
+  let pendingShot = false
+  let result: Awaited<ReturnType<typeof runRefineLoop<string>>>
+  try {
+    result = await runRefineLoop<string>({
+      rounds: attempts,
+      prompt: (round, history) => (round === 1 ? input.task.prompt : retryPrompt(input.task, history, scores)),
+      runShot: async (prompt, round) => {
+        input.signal?.throwIfAborted()
+        pendingShot = true
+        const out = await shot({ ...input, prompt, attempt: round })
+        shots.set(round, out)
+        pendingShot = false
+        return { artifact: out.artifact, note: out.detail }
+      },
+      judge: async (artifact, round) => {
+        const score = await input.adapter.judge(input.task, artifact)
+        scores.set(round, score)
+        const succeeded = shots.get(round)?.ok === true
+        return { valid: succeeded && score.resolved, score: succeeded ? score.score : 0 }
+      },
+    })
+  } catch (err) {
+    const completed = [...shots.values()]
+    return {
+      artifact: completed.at(-1)?.artifact ?? '',
+      ok: false,
+      usage: combinedUsage(pendingShot ? [...completed, { artifact: '', ok: false }] : completed),
+      events: completed.flatMap((shot) => shot.events ?? []),
+      detail: err instanceof Error ? err.message : String(err),
+    }
+  }
 
   const best = result.rounds.reduce((winner, candidate) => {
+    if (shots.get(candidate.round)?.ok !== true) return winner
+    if (shots.get(winner.round)?.ok !== true) return candidate
     const a = scores.get(winner.round)
     const b = scores.get(candidate.round)
     if (!a) return candidate
@@ -408,7 +471,9 @@ async function loopedShot(
   const bestScore = scores.get(best.round)
   return {
     artifact: best.artifact,
-    ok: best.artifact.trim().length > 0,
+    ok: shots.get(best.round)?.ok === true && best.artifact.trim().length > 0,
+    usage: combinedUsage([...shots.values()]),
+    events: [...shots.values()].flatMap((shot) => shot.events ?? []),
     detail: JSON.stringify({
       mode: 'refine-loop',
       attempts: result.rounds.length,
@@ -425,6 +490,30 @@ async function loopedShot(
   }
 }
 
+function combinedUsage(shots: readonly BenchShotResult[]): ReturnType<typeof sumSandboxUsage> {
+  const usage = { input: 0, output: 0, costUsd: 0 }
+  let tokensKnown = shots.length > 0
+  let usdKnown = shots.length > 0
+  let estimate: number | undefined
+  let unknownReason: string | undefined
+  for (const shot of shots) {
+    usage.input += shot.usage?.input ?? 0
+    usage.output += shot.usage?.output ?? 0
+    usage.costUsd += shot.usage?.costUsd ?? 0
+    tokensKnown &&= shot.usage !== undefined && shot.usage.tokensKnown !== false
+    usdKnown &&= shot.usage !== undefined && shot.usage.usdKnown !== false
+    unknownReason ??= shot.usage?.tokensUnknownReason
+    if (shot.usage?.estimatedCostUsd !== undefined) estimate = (estimate ?? 0) + shot.usage.estimatedCostUsd
+  }
+  return {
+    ...usage,
+    ...(tokensKnown ? {} : { tokensKnown: false as const }),
+    ...(usdKnown ? {} : { usdKnown: false as const }),
+    ...(estimate === undefined ? {} : { estimatedCostUsd: estimate }),
+    ...(unknownReason === undefined ? {} : { tokensUnknownReason: unknownReason }),
+  }
+}
+
 function combineDetails(runDetail: string | undefined, scoreDetail: string | undefined): string | undefined {
   if (runDetail && scoreDetail) {
     return JSON.stringify({ run: parseMaybeJson(runDetail), score: parseMaybeJson(scoreDetail) })
@@ -481,6 +570,7 @@ async function prepareBenchmarks(
 }
 
 export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenchmarksReport> {
+  opts.signal?.throwIfAborted()
   if (opts.benchmarks.length === 0) throw new Error('runBenchmarks: no benchmarks selected')
   if (opts.cells.length === 0) throw new Error('runBenchmarks: no cells to run')
   const reps = Math.max(1, opts.reps ?? 1)
@@ -497,20 +587,24 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
   await runPool(jobs, Math.max(1, opts.concurrency ?? 4), async (job, index) => {
     const startedAt = Date.now()
     let result: BenchCellTaskResult
+    let out: BenchShotResult | undefined
     try {
+      opts.signal?.throwIfAborted()
       const shotInput = {
         adapter: job.adapter,
         task: job.task,
         cell: job.cell,
         routerBaseUrl: opts.routerBaseUrl,
         routerKey: opts.routerKey,
+        ...(opts.modelApiKey === undefined ? {} : { modelApiKey: opts.modelApiKey }),
         ...(opts.bridgeUrl ? { bridgeUrl: opts.bridgeUrl } : {}),
         ...(opts.bridgeBearer ? { bridgeBearer: opts.bridgeBearer } : {}),
         ...(opts.sandboxBaseUrl ? { sandboxBaseUrl: opts.sandboxBaseUrl } : {}),
         ...(opts.timeoutMs ? { timeoutMs: opts.timeoutMs } : {}),
+        ...(opts.signal ? { signal: opts.signal } : {}),
         ...(opts.resolveClient ? { resolveClient: opts.resolveClient } : {}),
       }
-      const out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
+      out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput)
       const score: BenchScore = await job.adapter.judge(job.task, out.artifact)
       result = {
         benchmark: job.benchmark,
@@ -522,6 +616,9 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ok: out.ok,
         ...(out.detail ?? score.detail ? { detail: combineDetails(out.detail, score.detail) } : {}),
         wallMs: Date.now() - startedAt,
+        artifact: out.artifact,
+        ...(out.usage === undefined ? {} : { usage: out.usage }),
+        ...(out.events === undefined ? {} : { events: out.events }),
       }
     } catch (err) {
       // A thrown shot/judge is infra error for THIS cell-task: ok=false excludes it from the
@@ -536,6 +633,9 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise<RunBenc
         ok: false,
         detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
         wallMs: Date.now() - startedAt,
+        ...(out === undefined ? {} : { artifact: out.artifact }),
+        ...(out?.usage === undefined ? {} : { usage: out.usage }),
+        ...(out?.events === undefined ? {} : { events: out.events }),
       }
     }
     void index

package/src/sandbox-run.ts

@@ -14,7 +14,8 @@ import {
   type AgentRunSpec,
   type OutputAdapter,
 } from '@tangle-network/agent-runtime/kernel'
-import { parseExactAgentProfile } from '@tangle-network/agent-runtime'
+import { parseExactAgentProfile } from '@tangle-network/agent-runtime/candidate-execution'
+
 // `BackendType` is the sandbox SDK's harness union and its canonical home. Runtime consumes it
 // from there too; benchmark profiles use the same values as their exact harness identity.
 import type { BackendType } from '@tangle-network/sandbox'

package/src/swe-arena/activation.test.mts

@@ -211,6 +211,7 @@ describe('activation-predicate prefilter', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'act-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'act-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
     await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
     await writeFile(join(loopsRepo, 'src.ts'), 'base\n')

package/src/swe-arena/briefing.test.mts

@@ -152,6 +152,7 @@ describe('briefing text', () => {
     const repo = await mkdtemp(join(tmpdir(), 'briefing-repo-'))
     try {
       await runOk('git', ['init', '-q', '-b', 'main', repo])
+      await runOk('git', ['-C', repo, 'config', 'core.hooksPath', '/dev/null'])
       await runOk('git', ['-C', repo, 'config', 'user.email', 't@t.dev'])
       await runOk('git', ['-C', repo, 'config', 'user.name', 'T'])
       await writeFile(join(repo, 'base.txt'), 'x\n')

package/src/swe-arena/execution.test.mts

@@ -344,6 +344,7 @@ describe('arms: patch extraction (real git, no docker)', () => {
     const ws = await scratch('swe-arena-git-')
     const git = (...argv: string[]) => runOk('git', ['-C', ws, ...argv])
     await runOk('git', ['init', '-q', ws])
+    await runOk('git', ['-C', ws, 'config', 'core.hooksPath', '/dev/null'])
     await git('config', 'user.email', 't@t')
     await git('config', 'user.name', 't')
     await writeFile(join(ws, 'lib.py'), 'x = 1\n')
@@ -904,7 +905,7 @@ describe('serialized-judge', () => {
     const dir = await scratch('swe-arena-judge-flock-stale-bytes-')
     const lockFile = join(dir, 'lock')
     await writeFile(lockFile, 'half-written-or-dead-owner\n')
-    await expect(withJudgeLock(lockFile, async () => 'acquired', { timeoutMs: 1_000 })).resolves.toBe('acquired')
+    await expect(withJudgeLock(lockFile, async () => 'acquired', { timeoutMs: 5_000 })).resolves.toBe('acquired')
     expect(await readFile(lockFile, 'utf8')).toBe('half-written-or-dead-owner\n')
   })
 
@@ -926,7 +927,7 @@ describe('serialized-judge', () => {
 
     await expect(result).rejects.toThrow('cancelled active judge')
     expect((await readFile(attempts, 'utf8')).trim().split('\n')).toEqual(['attempt'])
-    await expect(withJudgeLock(lockFile, async () => 'released', { timeoutMs: 1_000 })).resolves.toBe('released')
+    await expect(withJudgeLock(lockFile, async () => 'released', { timeoutMs: 5_000 })).resolves.toBe('released')
   }, 15_000)
 
   it('bounds stale-container cleanup before starting the judge child', async () => {

package/src/swe-arena/factory.test.mts

@@ -246,6 +246,7 @@ async function makeSyntheticMirror(root: string): Promise<SyntheticMirror> {
   const mirror = join(root, 'mirror')
   await mkdir(mirror, { recursive: true })
   await runOk('git', ['-C', mirror, 'init', '-q', '-b', 'main'])
+  await runOk('git', ['-C', mirror, 'config', 'core.hooksPath', '/dev/null'])
   await writeFile(join(mirror, 'README.md'), '# synthetic\n')
   await writeFile(
     join(mirror, 'package.json'),
@@ -359,7 +360,8 @@ describe('parseVitestSummary', () => {
   })
 })
 
-describe('factory command credential isolation', () => {
+// The isolated Docker controller uses the Linux daemon socket, never personal Docker contexts.
+describe.skipIf(process.platform !== 'linux')('factory command credential isolation', () => {
   it('blocks arbitrary env, auth sockets, npm config, and home credentials from a package lifecycle script', async () => {
     const ambientHome = join(root, 'ambient-home')
     const ambientXdg = join(root, 'ambient-xdg')
@@ -506,7 +508,7 @@ describe('loadFactoryInstance', () => {
 // Judge child pipeline on the synthetic fixture.
 // ---------------------------------------------------------------------------
 
-describe('judgeFactoryPatch', () => {
+describe.skipIf(process.platform !== 'linux')('judgeFactoryPatch', () => {
   it('gold (impl-only PR diff) resolves with full score', async () => {
     const { result } = await judgeFactoryPatch(goodInst, await goldImplPatch(goodInst))
     expect(result).toMatchObject({ resolved: true, score: 1, passed: 2, total: 2 })
@@ -597,7 +599,7 @@ describe('materializeFactoryWorkspace', () => {
 // Calibration admission gate — both rejection directions.
 // ---------------------------------------------------------------------------
 
-describe('calibrateFactoryInstance', () => {
+describe.skipIf(process.platform !== 'linux')('calibrateFactoryInstance', () => {
   it('admits a well-formed instance (gold passes, base fails)', async () => {
     const r = await calibrateFactoryInstance(goodInst)
     expect(r).toMatchObject({

package/src/swe-arena/gepa-seat.test.mts

@@ -535,6 +535,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'gepa-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await git(['config', 'user.email', 't@t.dev'], loopsRepo)
     await git(['config', 'user.name', 'T'], loopsRepo)
     await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })
@@ -995,6 +996,7 @@ describe('integration: real adapter roundtrip', () => {
     const outDir = await mkdtemp(join(tmpdir(), 'gepa-int-out-'))
     try {
       await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+      await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
       await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
       await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
       await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })

package/src/swe-arena/premeasured-from-cells.test.mts

@@ -182,6 +182,7 @@ describe('loadFullCampaignCells + incumbentSurfaceHash (real fs/git)', () => {
 
   it('computes the same incumbent hash for the unchanged tip (baseCommit == candidateCommit, empty patch)', async () => {
     await runOk('git', ['init', '-q', '-b', 'main', repo])
+    await runOk('git', ['-C', repo, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', repo, 'config', 'user.email', 't@t.dev'])
     await runOk('git', ['-C', repo, 'config', 'user.name', 'T'])
     await writeFile(join(repo, 'a.txt'), 'x\n')

package/src/swe-arena/proc.test.mts

@@ -65,18 +65,16 @@ describe('run process-group timeout', () => {
     const dir = await mkdtemp(join(tmpdir(), 'swe-proc-tree-'))
     dirs.push(dir)
     const pidFile = join(dir, 'grandchild.pid')
-    const grandchild = `process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)`
+    const grandchild = `process.on('SIGTERM',()=>{}); require('node:fs').writeFileSync(process.argv[1],String(process.pid)); setInterval(()=>{},1000)`
     const parent = [
       `const {spawn}=require('node:child_process')`,
-      `const fs=require('node:fs')`,
-      `const child=spawn(process.execPath,['-e',${JSON.stringify(grandchild)}],{stdio:'ignore'})`,
-      `fs.writeFileSync(process.argv[1],String(child.pid))`,
+      `spawn(process.execPath,['-e',${JSON.stringify(grandchild)},process.argv[1]],{stdio:'ignore'})`,
       `process.on('SIGTERM',()=>process.exit(0))`,
       `setInterval(()=>{},1000)`,
     ].join(';')
 
     const result = await run(process.execPath, ['-e', parent, pidFile], {
-      timeoutMs: 100,
+      timeoutMs: 2_000,
       killGraceMs: 50,
     })
     const grandchildPid = Number(await readFile(pidFile, 'utf8'))

package/src/swe-arena/proposer-fanout.test.mts

@@ -358,6 +358,7 @@ describe('fanOutLoopsGenerator', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'fanout-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'fanout-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await git(['config', 'user.email', 't@t.dev'], loopsRepo)
     await git(['config', 'user.name', 'T'], loopsRepo)
     await writeFile(join(loopsRepo, 'src.ts'), 'base\n')

package/src/swe-arena/run-experiment.mts

@@ -399,6 +399,8 @@ export async function materializeFactoryWorkspace(
   await writeFile(join(dest, 'SPEC.md'), inst.spec)
 
   await runOk('git', ['-C', dest, 'init', '-q', '-b', 'work'])
+  // This synthetic benchmark repository must not execute the host's personal Git hooks.
+  await runOk('git', ['-C', dest, 'config', 'core.hooksPath', '/dev/null'])
   await runOk('git', ['-C', dest, 'config', 'user.email', 'factory-bench@local'])
   await runOk('git', ['-C', dest, 'config', 'user.name', 'factory-bench'])
   await runOk('git', ['-C', dest, 'add', '-A'])

package/src/swe-arena/scratch-worktree.test.mts

@@ -21,6 +21,7 @@ describe('scratch worktrees', () => {
     const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-'))
     roots.push(repository, output)
     await runOk('git', ['init', '-q', '-b', 'main', repository])
+    await runOk('git', ['-C', repository, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com'])
     await runOk('git', ['-C', repository, 'config', 'user.name', 'Test'])
     await writeFile(join(repository, 'seed.txt'), 'seed\n')
@@ -51,5 +52,5 @@ describe('scratch worktrees', () => {
       await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain'])
     ).stdout
     expect(listed.match(/^worktree /gmu)).toHaveLength(1)
-  })
+  }, 60_000)
 })

package/src/swe-bench-env.test.ts

@@ -93,9 +93,13 @@ describe('SWE worker prompts', () => {
 
 describe('SWE temporary directory', () => {
   it('stays absolute when model temperature is configured through TEMPERATURE', () => {
+    const priorTmpdir = process.env.TMPDIR
+    const priorTmp = process.env.TMP
     const priorTemp = process.env.TEMP
     const priorTemperature = process.env.TEMPERATURE
     try {
+      delete process.env.TMPDIR
+      delete process.env.TMP
       delete process.env.TEMP
       process.env.TEMPERATURE = '0.8'
       assert.equal(isAbsolute(absoluteSweTempDir()), true)
@@ -103,6 +107,10 @@ describe('SWE temporary directory', () => {
       process.env.TEMP = '0.8'
       assert.throws(() => absoluteSweTempDir(), /must be absolute.*TEMPERATURE/)
     } finally {
+      if (priorTmpdir === undefined) delete process.env.TMPDIR
+      else process.env.TMPDIR = priorTmpdir
+      if (priorTmp === undefined) delete process.env.TMP
+      else process.env.TMP = priorTmp
       if (priorTemp === undefined) delete process.env.TEMP
       else process.env.TEMP = priorTemp
       if (priorTemperature === undefined) delete process.env.TEMPERATURE
@@ -197,7 +205,7 @@ describe('isInsideJail (realpath containment)', () => {
     symlinkSync('/etc', link)
     // `resolveInJail` does `realpathSync(join(ws.dir, relPath))` then this containment check.
     const real = realpathSync(join(dir, 'escape/passwd'))
-    assert.equal(real, '/etc/passwd')
+    assert.equal(real, realpathSync('/etc/passwd'))
     assert.equal(isInsideJail(jailRoot, real), false)
   })
 

package/src/tb-container-executor.test.mts

@@ -89,7 +89,10 @@ printf 'argv:%s\\n' "$*"
   assert.match(meteredResult.out.command, /'tangle-router\/deepseek-v4-flash'/)
   assert.match(
     meteredResult.out.stdout,
-    /argv:exec -i --workdir \/work -e OPENAI_BASE_URL=http:\/\/router\.test cid \/bin\/sh -c exec 'opencode' 'run' 'echo hello' '-m' 'tangle-router\/deepseek-v4-flash'/,
+    // `--auto` is opencode's permission bypass, emitted for an unattended container run the
+    // same way claude-code and codex emit theirs. Without it the harness denies writes outside
+    // the working directory and reports "The user rejected permission".
+    /argv:exec -i --workdir \/work -e OPENAI_BASE_URL=http:\/\/router\.test cid \/bin\/sh -c exec 'opencode' 'run' 'echo hello' '--auto' '-m' 'tangle-router\/deepseek-v4-flash'/,
   )
   assert.deepEqual(meteredResult.spent.tokens, { input: 7, output: 11 })
   assert.equal(meteredResult.spent.usd, 0.004)

package/src/tb-supervisor-sidecar.mts

@@ -113,6 +113,7 @@ async function main(): Promise<void> {
         blobs,
         makeWorkerAgent,
         perWorker: { maxIterations: 40, maxTokens: 200_000 },
+        toolNames: ['spawn_worker', 'observe_agent', 'await_event', 'stop'],
         host: '0.0.0.0',
         onEvent: (event) => logEvent('bus', event),
       })

package/tb_agents/opencode_supervisor_agent.py

@@ -200,11 +200,11 @@ class OpenCodeSupervisorAgent(OpenCodeRouterAgent):
         try:
             supervised_instruction = (
                 "You are the supervisor. You have an MCP server named \"coordination\" with "
-                "tools including spawn_agent(profile, task), observe_agent(workerId), "
+                "tools including spawn_worker(profile, task), observe_agent(workerId), "
                 "await_event(), and stop(). A spawned worker runs a full coding agent INSIDE "
                 "THIS SAME container and its file changes persist here, so you may delegate "
                 "concrete sub-tasks (e.g. \"install perl and 7z\", \"crack the hash with john\") "
-                "to workers via spawn_agent, then observe_agent/await_event for their results. "
+                "to workers via spawn_worker, then observe_agent/await_event for their results. "
                 "Delegate independent or heavy sub-tasks to workers when useful; do the rest "
                 "yourself. Complete the task fully.\n\n"
                 f"TASK:\n{instruction}"