@tangle-network/agent-bench 0.4.3 → 0.4.5

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.4.5
+
+- Allow the zero-model Pier proof to complete a cold separate-verifier image build before its task and overall execution deadlines.
+- Consume Runtime 0.107.5 with Eval 0.133.3, Interface 0.36.0, Knowledge 6.1.5, and Sandbox 0.15.1.
+
+## 0.4.4
+
+- Consume Runtime 0.107.2 and Sandbox 0.15.0 with the current Eval, Interface, and Knowledge packages.
+- Publish fake Pier terminal results atomically so the full test suite cannot observe partial JSON.
+
 ## 0.4.3
 
 - Consume Runtime 0.107.1 with Eval 0.133.0, Interface 0.36.0, and Knowledge 6.1.2.

package/fixtures/pier-agent/no-model-task/task.toml

@@ -16,7 +16,7 @@ tags = ["no-model", "separate-verifier"]
 timeout_sec = 60.0
 
 [verifier]
-timeout_sec = 60.0
+timeout_sec = 300.0
 environment_mode = "separate"
 
 [verifier.environment]

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "type": "module",
-  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/tangle-network/agent-runtime.git",
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.133.0",
+    "@tangle-network/agent-eval": "0.133.3",
     "@tangle-network/agent-interface": "0.36.0",
-    "@tangle-network/agent-knowledge": "6.1.2",
-    "@tangle-network/sandbox": "0.14.0",
-    "@tangle-network/agent-runtime": "0.107.1"
+    "@tangle-network/agent-knowledge": "6.1.5",
+    "@tangle-network/sandbox": "0.15.1",
+    "@tangle-network/agent-runtime": "0.107.5"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/verify-pier-agent.mts

@@ -435,7 +435,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
     workspace: taskWorkspace,
     evaluatorTaskContainer: container,
     limits: {
-      timeoutMs: 180_000,
+      timeoutMs: 600_000,
       maxSteps: 8,
       maxModelCalls: 0,
       maxInputTokens: 0,

package/src/pier-trial-controller.test.mts

@@ -85,18 +85,20 @@ test('the Pier result wait does not add time beyond an expired deadline', async
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream } from 'node:fs'
-import { writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 const directory = process.argv[2]
 process.on('SIGTERM', () => {
-  writeFileSync(path.join(directory, 'terminal.json'), JSON.stringify({
+  const target = path.join(directory, 'terminal.json')
+  const temporary = path.join(directory, '.terminal.json.tmp')
+  writeFileSync(temporary, JSON.stringify({
     schemaVersion: 1,
     kind: 'pier-trial-terminal',
     status: 'stopped',
     processExited: true,
     containersRemoved: true,
   }))
+  renameSync(temporary, target)
   process.exit(0)
 })
 for await (const _chunk of createReadStream('', { fd: 3 })) {}
@@ -156,7 +158,7 @@ test('the supervisor receives only launch data and no inherited evaluator enviro
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream, writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 const chunks = []
 for await (const chunk of createReadStream('', { fd: 3 })) chunks.push(Buffer.from(chunk))
@@ -165,7 +167,9 @@ const isolated = process.env.PIER_SUPERVISOR_MODE === 'explicit' &&
   process.env.PIER_PARENT_SECRET === undefined &&
   launch.env.PIER_PARENT_SECRET === undefined &&
   launch.env.PIER_CHILD_SECRET === 'child-only'
-writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
+const target = path.join(process.argv[2], 'terminal.json')
+const temporary = path.join(process.argv[2], '.terminal.json.tmp')
+writeFileSync(temporary, JSON.stringify({
   schemaVersion: 1,
   kind: 'pier-trial-terminal',
   status: isolated ? 'completed' : 'failed',
@@ -173,6 +177,7 @@ writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
   containersRemoved: true,
   ...(isolated ? {} : { error: 'supervisor inherited evaluator environment' }),
 }))
+renameSync(temporary, target)
 `,
     )
     const controller = new FilePierCandidateTrialController({
@@ -224,10 +229,12 @@ test('terminal acknowledgements reject unknown fields', async () => {
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream, writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 for await (const _chunk of createReadStream('', { fd: 3 })) {}
-writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
+const target = path.join(process.argv[2], 'terminal.json')
+const temporary = path.join(process.argv[2], '.terminal.json.tmp')
+writeFileSync(temporary, JSON.stringify({
   schemaVersion: 1,
   kind: 'pier-trial-terminal',
   status: 'completed',
@@ -235,6 +242,7 @@ writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
   containersRemoved: true,
   containerRemoved: true,
 }))
+renameSync(temporary, target)
 `,
     )
     const controller = new FilePierCandidateTrialController({

package/src/swe-arena/run-supervisor.test.mts

@@ -3,11 +3,25 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterAll, describe, expect, it } from 'vitest'
 import { runSupervisorShim, supervisorDriverKillGraceMs } from './arms.ts'
-import { run, TIMEOUT_RC } from './proc.ts'
+import { ABORT_RC, run } from './proc.ts'
 
 const dirs: string[] = []
 const workerPids: number[] = []
 
+async function waitForFile(path: string, timeoutMs = 10_000): Promise<void> {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    try {
+      await access(path)
+      return
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
+    }
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+  throw new Error(`timed out waiting for ${path}`)
+}
+
 afterAll(async () => {
   for (const pid of workerPids) {
     try {
@@ -122,6 +136,7 @@ export default function fakeExtension(pi) {
       writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
+      writeFileSync(join(ctx.cwd, 'outer-driver-ready.txt'), 'ready')
       return text('spawned supervisor ' + id)
     },
   })
@@ -140,10 +155,11 @@ export default function fakeExtension(pi) {
 `)
 
     const cancelTimeoutMs = 1_500
-    const result = await run(process.execPath, [
+    const controller = new AbortController()
+    const resultPromise = run(process.execPath, [
       '--import', 'tsx', runSupervisorShim, extension, workspace, params,
     ], {
-      timeoutMs: 1_000,
+      signal: controller.signal,
       killGraceMs: supervisorDriverKillGraceMs({
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       }),
@@ -154,9 +170,19 @@ export default function fakeExtension(pi) {
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       },
     })
+    try {
+      await waitForFile(join(workspace, 'outer-driver-ready.txt'))
+    } catch (error) {
+      controller.abort()
+      await resultPromise
+      throw error
+    }
+    controller.abort()
+    const result = await resultPromise
 
-    expect(result.code, result.stdout + result.stderr).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
+    expect(result.code, result.stdout + result.stderr).toBe(ABORT_RC)
+    expect(result.aborted).toBe(true)
+    expect(result.timedOut).toBe(false)
     expect(await readFile(join(workspace, 'outer-cancel-settled.txt'), 'utf8')).toBe('settled')
     expect(JSON.parse(await readFile(join(workspace, '.loops', 'supervisor', 'sup-2-fake34', 'state.json'), 'utf8')))
       .toMatchObject({ status: 'cancelled' })
@@ -170,10 +196,10 @@ export default function fakeExtension(pi) {
     const extension = join(dir, 'fake-extension.mjs')
     await mkdir(workspace)
     await writeFile(params, '{}')
-    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); setInterval(()=>{},1000)`
+    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; const ready=process.argv[2]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); fs.writeFileSync(ready,'ready'); setInterval(()=>{},1000)`
     await writeFile(extension, `
 import { spawn } from 'node:child_process'
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
+import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
 
 const id = 'sup-3-signal56'
@@ -188,11 +214,13 @@ export default function fakeExtension(pi) {
       writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'signal-worker-cleaned.txt')], {
+      const readyPath = join(ctx.cwd, 'signal-worker-ready.txt')
+      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'signal-worker-cleaned.txt'), readyPath], {
         detached: true,
         stdio: 'ignore',
       })
       writeFileSync(join(ctx.cwd, 'signal-worker.pid'), String(child.pid))
+      while (!existsSync(readyPath)) await new Promise((resolve) => setTimeout(resolve, 5))
       writeFileSync(join(ctx.cwd, 'spawn-entered.txt'), 'entered')
       await new Promise((resolve) => setTimeout(resolve, 30_000))
       return text('spawned supervisor ' + id)
@@ -214,10 +242,11 @@ export default function fakeExtension(pi) {
 `)
 
     const cancelTimeoutMs = 2_000
-    const result = await run(process.execPath, [
+    const controller = new AbortController()
+    const resultPromise = run(process.execPath, [
       '--import', 'tsx', runSupervisorShim, extension, workspace, params,
     ], {
-      timeoutMs: 1_000,
+      signal: controller.signal,
       killGraceMs: supervisorDriverKillGraceMs({
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       }),
@@ -228,11 +257,21 @@ export default function fakeExtension(pi) {
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       },
     })
+    try {
+      await waitForFile(join(workspace, 'spawn-entered.txt'))
+    } catch (error) {
+      controller.abort()
+      await resultPromise
+      throw error
+    }
+    controller.abort()
+    const result = await resultPromise
     const workerPid = Number(await readFile(join(workspace, 'signal-worker.pid'), 'utf8'))
     workerPids.push(workerPid)
 
-    expect(result.code, result.stdout + result.stderr).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
+    expect(result.code, result.stdout + result.stderr).toBe(ABORT_RC)
+    expect(result.aborted).toBe(true)
+    expect(result.timedOut).toBe(false)
     expect(await readFile(join(workspace, 'spawn-entered.txt'), 'utf8')).toBe('entered')
     expect(await readFile(join(workspace, 'signal-worker-cleaned.txt'), 'utf8')).toBe('clean')
     expect(await readFile(join(workspace, 'signal-cancel-settled.txt'), 'utf8')).toBe('settled')