@tangle-network/agent-bench 0.8.32 → 0.9.1

package/src/swe-arena/briefing.test.mts

@@ -152,6 +152,7 @@ describe('briefing text', () => {
     const repo = await mkdtemp(join(tmpdir(), 'briefing-repo-'))
     try {
       await runOk('git', ['init', '-q', '-b', 'main', repo])
+      await runOk('git', ['-C', repo, 'config', 'core.hooksPath', '/dev/null'])
       await runOk('git', ['-C', repo, 'config', 'user.email', 't@t.dev'])
       await runOk('git', ['-C', repo, 'config', 'user.name', 'T'])
       await writeFile(join(repo, 'base.txt'), 'x\n')

package/src/swe-arena/execution.test.mts

@@ -344,6 +344,7 @@ describe('arms: patch extraction (real git, no docker)', () => {
     const ws = await scratch('swe-arena-git-')
     const git = (...argv: string[]) => runOk('git', ['-C', ws, ...argv])
     await runOk('git', ['init', '-q', ws])
+    await runOk('git', ['-C', ws, 'config', 'core.hooksPath', '/dev/null'])
     await git('config', 'user.email', 't@t')
     await git('config', 'user.name', 't')
     await writeFile(join(ws, 'lib.py'), 'x = 1\n')
@@ -904,7 +905,7 @@ describe('serialized-judge', () => {
     const dir = await scratch('swe-arena-judge-flock-stale-bytes-')
     const lockFile = join(dir, 'lock')
     await writeFile(lockFile, 'half-written-or-dead-owner\n')
-    await expect(withJudgeLock(lockFile, async () => 'acquired', { timeoutMs: 1_000 })).resolves.toBe('acquired')
+    await expect(withJudgeLock(lockFile, async () => 'acquired', { timeoutMs: 5_000 })).resolves.toBe('acquired')
     expect(await readFile(lockFile, 'utf8')).toBe('half-written-or-dead-owner\n')
   })
 
@@ -926,7 +927,7 @@ describe('serialized-judge', () => {
 
     await expect(result).rejects.toThrow('cancelled active judge')
     expect((await readFile(attempts, 'utf8')).trim().split('\n')).toEqual(['attempt'])
-    await expect(withJudgeLock(lockFile, async () => 'released', { timeoutMs: 1_000 })).resolves.toBe('released')
+    await expect(withJudgeLock(lockFile, async () => 'released', { timeoutMs: 5_000 })).resolves.toBe('released')
   }, 15_000)
 
   it('bounds stale-container cleanup before starting the judge child', async () => {

package/src/swe-arena/factory.test.mts

@@ -246,6 +246,7 @@ async function makeSyntheticMirror(root: string): Promise<SyntheticMirror> {
   const mirror = join(root, 'mirror')
   await mkdir(mirror, { recursive: true })
   await runOk('git', ['-C', mirror, 'init', '-q', '-b', 'main'])
+  await runOk('git', ['-C', mirror, 'config', 'core.hooksPath', '/dev/null'])
   await writeFile(join(mirror, 'README.md'), '# synthetic\n')
   await writeFile(
     join(mirror, 'package.json'),
@@ -359,7 +360,8 @@ describe('parseVitestSummary', () => {
   })
 })
 
-describe('factory command credential isolation', () => {
+// The isolated Docker controller uses the Linux daemon socket, never personal Docker contexts.
+describe.skipIf(process.platform !== 'linux')('factory command credential isolation', () => {
   it('blocks arbitrary env, auth sockets, npm config, and home credentials from a package lifecycle script', async () => {
     const ambientHome = join(root, 'ambient-home')
     const ambientXdg = join(root, 'ambient-xdg')
@@ -506,7 +508,7 @@ describe('loadFactoryInstance', () => {
 // Judge child pipeline on the synthetic fixture.
 // ---------------------------------------------------------------------------
 
-describe('judgeFactoryPatch', () => {
+describe.skipIf(process.platform !== 'linux')('judgeFactoryPatch', () => {
   it('gold (impl-only PR diff) resolves with full score', async () => {
     const { result } = await judgeFactoryPatch(goodInst, await goldImplPatch(goodInst))
     expect(result).toMatchObject({ resolved: true, score: 1, passed: 2, total: 2 })
@@ -597,7 +599,7 @@ describe('materializeFactoryWorkspace', () => {
 // Calibration admission gate — both rejection directions.
 // ---------------------------------------------------------------------------
 
-describe('calibrateFactoryInstance', () => {
+describe.skipIf(process.platform !== 'linux')('calibrateFactoryInstance', () => {
   it('admits a well-formed instance (gold passes, base fails)', async () => {
     const r = await calibrateFactoryInstance(goodInst)
     expect(r).toMatchObject({

package/src/swe-arena/gepa-seat.mts

@@ -130,7 +130,7 @@ export type GepaSeatRecipe = Extract<
 /** Build the bounded recipe for a seat. The TOTAL inner-evaluation budget is
  *  exactly `maxMetricCalls`. The adapter's local callback enforces the sum
  *  of per-run limits, and the seat's own dispatch wrapper re-enforces it. */
-export function recipeForSeat(spec: GepaSeatSpec): GepaSeatRecipe {
+export function recipeForSeat(spec: GepaSeatSpec, optimizerModel?: string): GepaSeatRecipe {
   const calls = spec.maxMetricCalls ?? DEFAULT_MAX_METRIC_CALLS
   const cost = spec.maxProposerCostUsd ?? DEFAULT_MAX_PROPOSER_COST_USD
   if (spec.engine === 'gepa') {
@@ -145,6 +145,7 @@ export function recipeForSeat(spec: GepaSeatSpec): GepaSeatRecipe {
     engine,
     maxEvaluations: perExplore,
     maxProposerCostUsd: perRunCost,
+    ...(engine !== 'gepa' && optimizerModel ? { engineConfig: { model: optimizerModel } } : {}),
   }))
   return {
     kind: 'omni',
@@ -758,6 +759,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut
         : officialOptimizerModel({
             env: process.env,
             envPrefix: 'GEPA_OPTIMIZER',
+            anthropicEndpoint: spec.engine === 'omni',
             model: process.env.GEPA_OPTIMIZER_MODEL ?? config.arm.driverModel,
             baseUrl:
               process.env.GEPA_OPTIMIZER_BASE_URL ??
@@ -771,7 +773,7 @@ export function gepaSeatAuthor(config: OuterLoopConfig, deps: GepaSeatDeps): Aut
           }))
     const method = factory({
       name: `gepa-seat:${spec.name}`,
-      recipe,
+      recipe: recipeForSeat(spec, optimizer?.model),
       objective,
       evaluationId: gepaSeatEvaluationId({
         smokeInstanceId: deps.smokeInstanceId,

package/src/swe-arena/gepa-seat.test.mts

@@ -4,9 +4,12 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promis
 import { createServer } from 'node:http'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type {
-  DispatchContext,
-  OptimizationMethodProvenance,
+import {
+  gepaOptimizationMethod,
+  createRunCostLedger,
+  fsCampaignStorage,
+  type DispatchContext,
+  type OptimizationMethodProvenance,
 } from '@tangle-network/agent-eval/campaign'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import { ACTIVATION_PREDICATE_RELPATH, parseActivationPredicate } from './activation.mts'
@@ -89,6 +92,68 @@ describe('validateGepaSeat', () => {
 // ---------------------------------------------------------------------------
 
 describe('recipeForSeat', () => {
+  it.each(['gepa', 'omni'] as const)('transports the real %s optimizer configuration', async (engine) => {
+    const runDir = await mkdtemp(join(tmpdir(), 'gepa-seat-transport-'))
+    const requests: unknown[] = []
+    try {
+      const optimizer = officialOptimizerModel({
+        env: { OPT_INPUT_USD_PER_MILLION: '1', OPT_CACHED_INPUT_USD_PER_MILLION: '1', OPT_CACHE_WRITE_USD_PER_MILLION: '1', OPT_OUTPUT_USD_PER_MILLION: '1' },
+        envPrefix: 'OPT', model: 'fixture-model', baseUrl: 'http://127.0.0.1:1/v1', apiKey: 'fixture-key',
+        maxCostUsd: 1, maxOutputTokensPerRequest: 100, anthropicEndpoint: engine === 'omni',
+        complete: async (request) => {
+          requests.push(request)
+          return { model: 'fixture-model', choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 2, completion_tokens: 1, cost: 0.000003 } }
+        },
+      })
+      const spec = seat({ engine })
+      validateGepaSeat(spec)
+      const runtime = {
+        python: { implementation: 'CPython', version: '3.12.0' },
+        bridge: { package: 'agent-eval-rpc', version: 'fixture', sourceUrl: 'https://github.com/tangle-network/agent-eval', revision: 'fixture', sourceSha256: 'a'.repeat(64) },
+        optimizer: { package: 'gepa', version: 'fixture', sourceUrl: 'https://github.com/gepa-ai/gepa', revision: 'fixture', sourceSha256: 'b'.repeat(64) },
+        engineModules: [],
+      }
+      // Only the child optimizer is substituted; the real Eval proxy calls Runtime.
+      const bridge = join(runDir, 'transport.mjs')
+      await writeFile(bridge, `
+import fs from 'node:fs'
+const input = JSON.parse(fs.readFileSync(process.argv[process.argv.indexOf('--input') + 1], 'utf8'))
+if (input.operation === 'inspect') {
+  fs.writeFileSync(process.argv[process.argv.indexOf('--output') + 1], JSON.stringify({ runtime: ${JSON.stringify(runtime)} }))
+} else {
+  const omni = input.recipe.kind === 'omni'
+  if (omni && input.recipe.explore.some(run => run.engine !== 'gepa' && run.engineConfig.model !== input.modelProxy.model)) throw new Error('wrong engine model')
+  const send = maxTokens => fetch(input.modelProxy.baseUrl + (omni ? '/messages' : '/chat/completions'), {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', authorization: 'Bearer ' + input.modelProxy.apiKey, 'x-api-key': input.modelProxy.apiKey, 'anthropic-version': '2023-06-01' },
+    body: JSON.stringify({ model: input.modelProxy.model, max_tokens: maxTokens, messages: [{ role: 'user', content: 'transport proof' }] }),
+  })
+  const overBudget = await send(101)
+  if (overBudget.ok) throw new Error('output ceiling was not enforced')
+  await overBudget.text()
+  const response = await send(10)
+  if (!response.ok) throw new Error('model transport failed: ' + response.status + ' ' + await response.text())
+  await response.json()
+  throw new Error('transport-handshake-complete')
+}
+`)
+      const method = gepaOptimizationMethod({
+        recipe: recipeForSeat(spec, optimizer.model), optimizer,
+        objective: 'Check transport only', evaluationId: 'bench-transport',
+        runner: { command: process.execPath, args: [bridge] },
+      })
+      await expect(method.optimize({
+        baselineSurface: 'seed', trainScenarios: [{ id: 'train', kind: 'fixture' }], selectionScenarios: [{ id: 'selection', kind: 'fixture' }],
+        dispatchWithSurface: async () => 'unused',
+        judges: [{ name: 'fixture', dimensions: [{ key: 'score', description: 'fixture' }], score: async () => ({ dimensions: { score: 1 }, composite: 1 }) }],
+        runDir, seed: 42, runOptions: {}, costLedger: createRunCostLedger({ storage: fsCampaignStorage(), runDir: join(runDir, 'cost') }),
+      })).rejects.toThrow('transport-handshake-complete')
+      expect(requests).toHaveLength(1)
+    } finally {
+      await rm(runDir, { recursive: true, force: true })
+    }
+  })
+
   it("'gepa' is one bounded engine run carrying the full budget (default 10)", () => {
     const recipe = recipeForSeat(seat() as GepaSeatSpec)
     expect(recipe).toMatchObject({ kind: 'engine', run: { engine: 'gepa', maxEvaluations: DEFAULT_MAX_METRIC_CALLS } })
@@ -535,6 +600,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'gepa-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await git(['config', 'user.email', 't@t.dev'], loopsRepo)
     await git(['config', 'user.name', 'T'], loopsRepo)
     await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })
@@ -995,6 +1061,7 @@ describe('integration: real adapter roundtrip', () => {
     const outDir = await mkdtemp(join(tmpdir(), 'gepa-int-out-'))
     try {
       await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+      await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
       await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
       await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
       await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })

package/src/swe-arena/premeasured-from-cells.test.mts

@@ -182,6 +182,7 @@ describe('loadFullCampaignCells + incumbentSurfaceHash (real fs/git)', () => {
 
   it('computes the same incumbent hash for the unchanged tip (baseCommit == candidateCommit, empty patch)', async () => {
     await runOk('git', ['init', '-q', '-b', 'main', repo])
+    await runOk('git', ['-C', repo, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', repo, 'config', 'user.email', 't@t.dev'])
     await runOk('git', ['-C', repo, 'config', 'user.name', 'T'])
     await writeFile(join(repo, 'a.txt'), 'x\n')

package/src/swe-arena/proc.test.mts

@@ -65,18 +65,16 @@ describe('run process-group timeout', () => {
     const dir = await mkdtemp(join(tmpdir(), 'swe-proc-tree-'))
     dirs.push(dir)
     const pidFile = join(dir, 'grandchild.pid')
-    const grandchild = `process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)`
+    const grandchild = `process.on('SIGTERM',()=>{}); require('node:fs').writeFileSync(process.argv[1],String(process.pid)); setInterval(()=>{},1000)`
     const parent = [
       `const {spawn}=require('node:child_process')`,
-      `const fs=require('node:fs')`,
-      `const child=spawn(process.execPath,['-e',${JSON.stringify(grandchild)}],{stdio:'ignore'})`,
-      `fs.writeFileSync(process.argv[1],String(child.pid))`,
+      `spawn(process.execPath,['-e',${JSON.stringify(grandchild)},process.argv[1]],{stdio:'ignore'})`,
       `process.on('SIGTERM',()=>process.exit(0))`,
       `setInterval(()=>{},1000)`,
     ].join(';')
 
     const result = await run(process.execPath, ['-e', parent, pidFile], {
-      timeoutMs: 100,
+      timeoutMs: 2_000,
       killGraceMs: 50,
     })
     const grandchildPid = Number(await readFile(pidFile, 'utf8'))

package/src/swe-arena/proposer-fanout.test.mts

@@ -358,6 +358,7 @@ describe('fanOutLoopsGenerator', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'fanout-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'fanout-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await git(['config', 'user.email', 't@t.dev'], loopsRepo)
     await git(['config', 'user.name', 'T'], loopsRepo)
     await writeFile(join(loopsRepo, 'src.ts'), 'base\n')

package/src/swe-arena/run-experiment.mts

@@ -399,6 +399,8 @@ export async function materializeFactoryWorkspace(
   await writeFile(join(dest, 'SPEC.md'), inst.spec)
 
   await runOk('git', ['-C', dest, 'init', '-q', '-b', 'work'])
+  // This synthetic benchmark repository must not execute the host's personal Git hooks.
+  await runOk('git', ['-C', dest, 'config', 'core.hooksPath', '/dev/null'])
   await runOk('git', ['-C', dest, 'config', 'user.email', 'factory-bench@local'])
   await runOk('git', ['-C', dest, 'config', 'user.name', 'factory-bench'])
   await runOk('git', ['-C', dest, 'add', '-A'])

package/src/swe-arena/scratch-worktree.test.mts

@@ -21,6 +21,7 @@ describe('scratch worktrees', () => {
     const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-'))
     roots.push(repository, output)
     await runOk('git', ['init', '-q', '-b', 'main', repository])
+    await runOk('git', ['-C', repository, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com'])
     await runOk('git', ['-C', repository, 'config', 'user.name', 'Test'])
     await writeFile(join(repository, 'seed.txt'), 'seed\n')
@@ -51,5 +52,5 @@ describe('scratch worktrees', () => {
       await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain'])
     ).stdout
     expect(listed.match(/^worktree /gmu)).toHaveLength(1)
-  })
+  }, 60_000)
 })

package/src/swe-bench-env.test.ts

@@ -93,9 +93,13 @@ describe('SWE worker prompts', () => {
 
 describe('SWE temporary directory', () => {
   it('stays absolute when model temperature is configured through TEMPERATURE', () => {
+    const priorTmpdir = process.env.TMPDIR
+    const priorTmp = process.env.TMP
     const priorTemp = process.env.TEMP
     const priorTemperature = process.env.TEMPERATURE
     try {
+      delete process.env.TMPDIR
+      delete process.env.TMP
       delete process.env.TEMP
       process.env.TEMPERATURE = '0.8'
       assert.equal(isAbsolute(absoluteSweTempDir()), true)
@@ -103,6 +107,10 @@ describe('SWE temporary directory', () => {
       process.env.TEMP = '0.8'
       assert.throws(() => absoluteSweTempDir(), /must be absolute.*TEMPERATURE/)
     } finally {
+      if (priorTmpdir === undefined) delete process.env.TMPDIR
+      else process.env.TMPDIR = priorTmpdir
+      if (priorTmp === undefined) delete process.env.TMP
+      else process.env.TMP = priorTmp
       if (priorTemp === undefined) delete process.env.TEMP
       else process.env.TEMP = priorTemp
       if (priorTemperature === undefined) delete process.env.TEMPERATURE
@@ -197,7 +205,7 @@ describe('isInsideJail (realpath containment)', () => {
     symlinkSync('/etc', link)
     // `resolveInJail` does `realpathSync(join(ws.dir, relPath))` then this containment check.
     const real = realpathSync(join(dir, 'escape/passwd'))
-    assert.equal(real, '/etc/passwd')
+    assert.equal(real, realpathSync('/etc/passwd'))
     assert.equal(isInsideJail(jailRoot, real), false)
   })