@tangle-network/agent-bench 0.8.32 → 0.9.0

package/src/swe-arena/factory.test.mts

@@ -246,6 +246,7 @@ async function makeSyntheticMirror(root: string): Promise<SyntheticMirror> {
   const mirror = join(root, 'mirror')
   await mkdir(mirror, { recursive: true })
   await runOk('git', ['-C', mirror, 'init', '-q', '-b', 'main'])
+  await runOk('git', ['-C', mirror, 'config', 'core.hooksPath', '/dev/null'])
   await writeFile(join(mirror, 'README.md'), '# synthetic\n')
   await writeFile(
     join(mirror, 'package.json'),
@@ -359,7 +360,8 @@ describe('parseVitestSummary', () => {
   })
 })
 
-describe('factory command credential isolation', () => {
+// The isolated Docker controller uses the Linux daemon socket, never personal Docker contexts.
+describe.skipIf(process.platform !== 'linux')('factory command credential isolation', () => {
   it('blocks arbitrary env, auth sockets, npm config, and home credentials from a package lifecycle script', async () => {
     const ambientHome = join(root, 'ambient-home')
     const ambientXdg = join(root, 'ambient-xdg')
@@ -506,7 +508,7 @@ describe('loadFactoryInstance', () => {
 // Judge child pipeline on the synthetic fixture.
 // ---------------------------------------------------------------------------
 
-describe('judgeFactoryPatch', () => {
+describe.skipIf(process.platform !== 'linux')('judgeFactoryPatch', () => {
   it('gold (impl-only PR diff) resolves with full score', async () => {
     const { result } = await judgeFactoryPatch(goodInst, await goldImplPatch(goodInst))
     expect(result).toMatchObject({ resolved: true, score: 1, passed: 2, total: 2 })
@@ -597,7 +599,7 @@ describe('materializeFactoryWorkspace', () => {
 // Calibration admission gate — both rejection directions.
 // ---------------------------------------------------------------------------
 
-describe('calibrateFactoryInstance', () => {
+describe.skipIf(process.platform !== 'linux')('calibrateFactoryInstance', () => {
   it('admits a well-formed instance (gold passes, base fails)', async () => {
     const r = await calibrateFactoryInstance(goodInst)
     expect(r).toMatchObject({

package/src/swe-arena/gepa-seat.test.mts

@@ -535,6 +535,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'gepa-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await git(['config', 'user.email', 't@t.dev'], loopsRepo)
     await git(['config', 'user.name', 'T'], loopsRepo)
     await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })
@@ -995,6 +996,7 @@ describe('integration: real adapter roundtrip', () => {
     const outDir = await mkdtemp(join(tmpdir(), 'gepa-int-out-'))
     try {
       await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+      await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
       await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
       await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
       await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })

package/src/swe-arena/premeasured-from-cells.test.mts

@@ -182,6 +182,7 @@ describe('loadFullCampaignCells + incumbentSurfaceHash (real fs/git)', () => {
 
   it('computes the same incumbent hash for the unchanged tip (baseCommit == candidateCommit, empty patch)', async () => {
     await runOk('git', ['init', '-q', '-b', 'main', repo])
+    await runOk('git', ['-C', repo, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', repo, 'config', 'user.email', 't@t.dev'])
     await runOk('git', ['-C', repo, 'config', 'user.name', 'T'])
     await writeFile(join(repo, 'a.txt'), 'x\n')

package/src/swe-arena/proc.test.mts

@@ -65,18 +65,16 @@ describe('run process-group timeout', () => {
     const dir = await mkdtemp(join(tmpdir(), 'swe-proc-tree-'))
     dirs.push(dir)
     const pidFile = join(dir, 'grandchild.pid')
-    const grandchild = `process.on('SIGTERM',()=>{}); setInterval(()=>{},1000)`
+    const grandchild = `process.on('SIGTERM',()=>{}); require('node:fs').writeFileSync(process.argv[1],String(process.pid)); setInterval(()=>{},1000)`
     const parent = [
       `const {spawn}=require('node:child_process')`,
-      `const fs=require('node:fs')`,
-      `const child=spawn(process.execPath,['-e',${JSON.stringify(grandchild)}],{stdio:'ignore'})`,
-      `fs.writeFileSync(process.argv[1],String(child.pid))`,
+      `spawn(process.execPath,['-e',${JSON.stringify(grandchild)},process.argv[1]],{stdio:'ignore'})`,
       `process.on('SIGTERM',()=>process.exit(0))`,
       `setInterval(()=>{},1000)`,
     ].join(';')
 
     const result = await run(process.execPath, ['-e', parent, pidFile], {
-      timeoutMs: 100,
+      timeoutMs: 2_000,
       killGraceMs: 50,
     })
     const grandchildPid = Number(await readFile(pidFile, 'utf8'))

package/src/swe-arena/proposer-fanout.test.mts

@@ -358,6 +358,7 @@ describe('fanOutLoopsGenerator', () => {
     loopsRepo = await mkdtemp(join(tmpdir(), 'fanout-repo-'))
     outDir = await mkdtemp(join(tmpdir(), 'fanout-out-'))
     await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
+    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
     await git(['config', 'user.email', 't@t.dev'], loopsRepo)
     await git(['config', 'user.name', 'T'], loopsRepo)
     await writeFile(join(loopsRepo, 'src.ts'), 'base\n')

package/src/swe-arena/run-experiment.mts

@@ -399,6 +399,8 @@ export async function materializeFactoryWorkspace(
   await writeFile(join(dest, 'SPEC.md'), inst.spec)
 
   await runOk('git', ['-C', dest, 'init', '-q', '-b', 'work'])
+  // This synthetic benchmark repository must not execute the host's personal Git hooks.
+  await runOk('git', ['-C', dest, 'config', 'core.hooksPath', '/dev/null'])
   await runOk('git', ['-C', dest, 'config', 'user.email', 'factory-bench@local'])
   await runOk('git', ['-C', dest, 'config', 'user.name', 'factory-bench'])
   await runOk('git', ['-C', dest, 'add', '-A'])

package/src/swe-arena/scratch-worktree.test.mts

@@ -21,6 +21,7 @@ describe('scratch worktrees', () => {
     const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-'))
     roots.push(repository, output)
     await runOk('git', ['init', '-q', '-b', 'main', repository])
+    await runOk('git', ['-C', repository, 'config', 'core.hooksPath', '/dev/null'])
     await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com'])
     await runOk('git', ['-C', repository, 'config', 'user.name', 'Test'])
     await writeFile(join(repository, 'seed.txt'), 'seed\n')
@@ -51,5 +52,5 @@ describe('scratch worktrees', () => {
       await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain'])
     ).stdout
     expect(listed.match(/^worktree /gmu)).toHaveLength(1)
-  })
+  }, 60_000)
 })

package/src/swe-bench-env.test.ts

@@ -93,9 +93,13 @@ describe('SWE worker prompts', () => {
 
 describe('SWE temporary directory', () => {
   it('stays absolute when model temperature is configured through TEMPERATURE', () => {
+    const priorTmpdir = process.env.TMPDIR
+    const priorTmp = process.env.TMP
     const priorTemp = process.env.TEMP
     const priorTemperature = process.env.TEMPERATURE
     try {
+      delete process.env.TMPDIR
+      delete process.env.TMP
       delete process.env.TEMP
       process.env.TEMPERATURE = '0.8'
       assert.equal(isAbsolute(absoluteSweTempDir()), true)
@@ -103,6 +107,10 @@ describe('SWE temporary directory', () => {
       process.env.TEMP = '0.8'
       assert.throws(() => absoluteSweTempDir(), /must be absolute.*TEMPERATURE/)
     } finally {
+      if (priorTmpdir === undefined) delete process.env.TMPDIR
+      else process.env.TMPDIR = priorTmpdir
+      if (priorTmp === undefined) delete process.env.TMP
+      else process.env.TMP = priorTmp
       if (priorTemp === undefined) delete process.env.TEMP
       else process.env.TEMP = priorTemp
       if (priorTemperature === undefined) delete process.env.TEMPERATURE
@@ -197,7 +205,7 @@ describe('isInsideJail (realpath containment)', () => {
     symlinkSync('/etc', link)
     // `resolveInJail` does `realpathSync(join(ws.dir, relPath))` then this containment check.
     const real = realpathSync(join(dir, 'escape/passwd'))
-    assert.equal(real, '/etc/passwd')
+    assert.equal(real, realpathSync('/etc/passwd'))
     assert.equal(isInsideJail(jailRoot, real), false)
   })