@tangle-network/agent-bench 0.1.0 → 0.3.5

package/scripts/verify-pier-pair.mts

@@ -0,0 +1,74 @@
+import { execFileSync } from 'node:child_process'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+interface PierProofResult {
+  readonly arm: 'failure' | 'success'
+  readonly reward: number
+  readonly patchApplied: number
+  readonly modelCalls: number
+  readonly inputTokens: number
+  readonly outputTokens: number
+  readonly costUsd: number
+  readonly runReceiptDigest: string
+}
+
+const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const script = path.join(benchDir, 'scripts', 'verify-pier-agent.mts')
+const recoveryScript = path.join(benchDir, 'scripts', 'verify-pier-recovery.mts')
+
+function runArm(arm: PierProofResult['arm']): PierProofResult {
+  const stdout = execFileSync(process.execPath, ['--import', 'tsx', script], {
+    cwd: benchDir,
+    env: { ...process.env, PIER_PROOF_ARM: arm },
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'inherit'],
+    timeout: 20 * 60_000,
+    maxBuffer: 20 * 1024 * 1024,
+  })
+  return JSON.parse(stdout) as PierProofResult
+}
+
+const failure = runArm('failure')
+const recovery = JSON.parse(
+  execFileSync(process.execPath, ['--import', 'tsx', recoveryScript], {
+    cwd: benchDir,
+    env: { ...process.env },
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'inherit'],
+    timeout: 120_000,
+  }),
+) as { freshProcess?: boolean; processExited?: boolean; containersRemoved?: boolean }
+if (
+  recovery.freshProcess !== true ||
+  recovery.processExited !== true ||
+  recovery.containersRemoved !== true
+) {
+  throw new Error(`Pier fresh-process recovery failed: ${JSON.stringify(recovery)}`)
+}
+const success = runArm('success')
+if (
+  failure.arm !== 'failure' ||
+  failure.reward !== 0 ||
+  failure.patchApplied !== 0 ||
+  success.arm !== 'success' ||
+  success.reward !== 1 ||
+  success.patchApplied !== 1
+) {
+  throw new Error(`Pier controls did not separate: ${JSON.stringify({ failure, success })}`)
+}
+for (const result of [failure, success]) {
+  if (
+    result.modelCalls !== 0 ||
+    result.inputTokens !== 0 ||
+    result.outputTokens !== 0 ||
+    result.costUsd !== 0
+  ) {
+    throw new Error(`Pier ${result.arm} control spent model budget: ${JSON.stringify(result)}`)
+  }
+  if (!/^sha256:[a-f0-9]{64}$/.test(result.runReceiptDigest)) {
+    throw new Error(`Pier ${result.arm} control omitted its run receipt digest`)
+  }
+}
+
+console.log(JSON.stringify({ recovery, controls: [failure, success] }, null, 2))

package/scripts/verify-pier-recovery.mts

@@ -0,0 +1,139 @@
+import { execFileSync } from 'node:child_process'
+import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime'
+import { InMemoryTraceStore } from '@tangle-network/agent-eval'
+
+import { createStagedPierCandidateExecutionFixture } from '../src/pier-agent.test-fixtures.mts'
+import { FilePierCandidateTrialController } from '../src/pier-trial-controller'
+
+const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const root = mkdtempSync(path.join(tmpdir(), 'pier-fresh-recovery-'))
+const controlRoot = path.join(root, 'control')
+const jobsDirectory = path.join(root, 'jobs')
+const jobName = 'fresh-recovery'
+const trialName = 'fresh-recovery-trial'
+const executionId = 'pier-fresh-process-recovery'
+const executionPlanDigest = `sha256:${'e'.repeat(64)}` as const
+const image = 'ghcr.io/tangle-network/devcontainers/universal:latest'
+let containerId: string | undefined
+let handle: ReturnType<FilePierCandidateTrialController['start']> | undefined
+let resultSettled: Promise<unknown> | undefined
+
+try {
+  containerId = execFileSync(
+    'docker',
+    [
+      'run',
+      '-d',
+      '--label',
+      `com.docker.compose.project=${trialName}`,
+      image,
+      'sleep',
+      'infinity',
+    ],
+    { encoding: 'utf8', timeout: 120_000 },
+  ).trim()
+  if (!/^[a-f0-9]{12,64}$/.test(containerId)) {
+    throw new Error(`Docker returned an invalid recovery container id: ${containerId}`)
+  }
+
+  const controller = new FilePierCandidateTrialController({
+    directory: controlRoot,
+    launch: () => ({
+      command: process.execPath,
+      args: [
+        '-e',
+        `require('node:fs').mkdirSync(${JSON.stringify(path.join(jobsDirectory, jobName, trialName))}, { recursive: true }); setInterval(() => undefined, 1_000)`,
+      ],
+      cwd: root,
+      env: { ...process.env },
+      jobsDirectory,
+      jobName,
+      readResult: async () => {
+        throw new Error('recovery probe must not complete normally')
+      },
+    }),
+  })
+  handle = controller.start(
+    createStagedPierCandidateExecutionFixture(executionId),
+    {
+      request: {
+        executionId,
+        executionPlan: { value: { digest: executionPlanDigest } },
+      } as AgentCandidateExecutorRequest,
+      traceStore: new InMemoryTraceStore(),
+      signal: new AbortController().signal,
+      deadlineAtMs: Date.now() + 30_000,
+    },
+  )
+  resultSettled = handle.result.catch(() => undefined)
+
+  const [slot] = readdirSync(controlRoot)
+  if (!slot) throw new Error('Pier controller omitted durable state')
+  const identityPath = path.join(controlRoot, slot, 'identity.json')
+  let pierPid: number | undefined
+  for (let attempt = 0; attempt < 200; attempt++) {
+    const identity = JSON.parse(readFileSync(identityPath, 'utf8'))
+    if (identity.state === 'running' && Number.isSafeInteger(identity.pier?.pid)) {
+      pierPid = identity.pier.pid
+      break
+    }
+    await new Promise((resolveWait) => setTimeout(resolveWait, 10))
+  }
+  if (!pierPid) throw new Error('Pier controller never persisted the child process identity')
+  const trialDirectory = path.join(jobsDirectory, jobName, trialName)
+  for (let attempt = 0; attempt < 200 && !existsSync(trialDirectory); attempt++) {
+    await new Promise((resolveWait) => setTimeout(resolveWait, 10))
+  }
+  if (!existsSync(trialDirectory)) {
+    throw new Error('recovery probe never created its evaluator-owned trial directory')
+  }
+
+  const recovery = JSON.parse(
+    execFileSync(
+      process.execPath,
+      [
+        '--import',
+        'tsx',
+        path.join(benchDir, 'scripts', 'terminate-pier-trial.mts'),
+        controlRoot,
+        executionId,
+        executionPlanDigest,
+      ],
+      { cwd: benchDir, encoding: 'utf8', timeout: 20_000 },
+    ),
+  )
+  if (recovery.processExited !== true || recovery.containersRemoved !== true) {
+    throw new Error(`fresh recovery returned an incomplete acknowledgement: ${JSON.stringify(recovery)}`)
+  }
+  try {
+    process.kill(pierPid, 0)
+    throw new Error(`recovered Pier process ${pierPid} is still alive`)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error
+  }
+  let containerPresent = true
+  try {
+    execFileSync('docker', ['inspect', containerId], { stdio: 'ignore', timeout: 30_000 })
+  } catch {
+    containerPresent = false
+  }
+  if (containerPresent) throw new Error(`recovered Pier container ${containerId} is still present`)
+  await resultSettled
+  process.stdout.write(
+    `${JSON.stringify({ freshProcess: true, processExited: true, containersRemoved: true })}\n`,
+  )
+} finally {
+  if (handle) await handle.terminateAndWait().catch(() => undefined)
+  if (resultSettled) await resultSettled
+  if (containerId) {
+    try {
+      execFileSync('docker', ['rm', '-f', containerId], { stdio: 'ignore', timeout: 30_000 })
+    } catch {}
+  }
+  rmSync(root, { recursive: true, force: true })
+}

package/src/adapters.ts

@@ -6,22 +6,35 @@
  */
 
 import { createAecBenchAdapter } from './benchmarks/aec-bench'
+import { createAgentBenchAdapter } from './benchmarks/agentbench'
 import { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'
+import { createBfclAdapter } from './benchmarks/bfcl'
 import { createCadBenchAdapter } from './benchmarks/cadbench'
 import { createCadDesignAdapter } from './benchmarks/cad-design'
 import { createCadGenBenchAdapter } from './benchmarks/cadgenbench'
 import { createCommit0Adapter } from './benchmarks/commit0'
+import { createCragAdapter } from './benchmarks/crag'
+import { createDabstepAdapter } from './benchmarks/dabstep'
 import { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'
+import { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'
 import { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'
 import { createFramesAdapter } from './benchmarks/frames'
 import { createHotpotqaAdapter } from './benchmarks/hotpotqa'
 import { createHumanEvalAdapter } from './benchmarks/humaneval'
 import { createMind2WebAdapter } from './benchmarks/mind2web'
+import { createNoMiraclAdapter } from './benchmarks/nomiracl'
+import { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'
 import { createProgrambenchAdapter } from './benchmarks/programbench'
+import { createRagBenchAdapter } from './benchmarks/ragbench'
 import { createSimpleQaAdapter } from './benchmarks/simpleqa'
 import { createSweBenchAdapter } from './benchmarks/swe-bench'
+import { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'
+import { createTau2BenchAdapter } from './benchmarks/tau2-bench'
+import { createTau3BankingAdapter } from './benchmarks/tau3-banking'
 import { createTerminalBenchAdapter } from './benchmarks/terminal-bench'
+import { createToolLlmAdapter } from './benchmarks/toollm'
 import { createTrataHedgeAdapter } from './benchmarks/trata-hedge'
+import { createWebArenaVerifiedAdapter } from './benchmarks/webarena-verified'
 import type { BenchmarkAdapter } from './benchmarks/types'
 
 export const ADAPTERS: Record<string, () => BenchmarkAdapter> = {
@@ -32,7 +45,14 @@ export const ADAPTERS: Record<string, () => BenchmarkAdapter> = {
   // delegates to the benchmark's own harness and fails loud when it/Docker is absent.
   'aec-bench': createAecBenchAdapter,
   commit0: createCommit0Adapter,
+  dabstep: createDabstepAdapter,
   programbench: createProgrambenchAdapter,
+  'webarena-verified': createWebArenaVerifiedAdapter,
+  'tau2-bench': createTau2BenchAdapter,
+  'tau3-banking': createTau3BankingAdapter,
+  agentbench: createAgentBenchAdapter,
+  bfcl: createBfclAdapter,
+  toollm: createToolLlmAdapter,
   appworld: createAppWorldAdapter,
   // AppWorld's native interactive protocol — the worker is the in-engine ReAct
   // episode (execution feedback every turn), the mode published baselines use.
@@ -42,6 +62,12 @@ export const ADAPTERS: Record<string, () => BenchmarkAdapter> = {
   cadbench: createCadBenchAdapter,
   cadgenbench: createCadGenBenchAdapter,
   frames: createFramesAdapter,
+  ragbench: createRagBenchAdapter,
+  crag: createCragAdapter,
+  nomiracl: createNoMiraclAdapter,
+  'open-rag-bench': createOpenRagBenchAdapter,
+  't2-ragbench': createT2RagBenchAdapter,
+  finresearchbench: createFinResearchBenchAdapter,
   finsearchcomp: createFinsearchcompAdapter,
   simpleqa: createSimpleQaAdapter,
   hotpotqa: createHotpotqaAdapter,

package/src/benchmarks/_harness.test.mts

@@ -0,0 +1,178 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import test from 'node:test'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { resolveBenchPython, runStagedJudge } from './_harness'
+
+const digest = (bytes: Uint8Array): `sha256:${string}` =>
+  `sha256:${createHash('sha256').update(bytes).digest('hex')}`
+
+test('resolveBenchPython defaults to the bench-local virtual environment', () => {
+  assert.equal(resolveBenchPython({}, '/opt/agent-bench'), join('/opt/agent-bench', '.venv', 'bin', 'python'))
+})
+
+test('resolveBenchPython accepts an absolute consumer-managed interpreter', () => {
+  assert.equal(
+    resolveBenchPython({ AGENT_BENCH_PYTHON: '/srv/bench-venv/bin/python' }, '/opt/agent-bench'),
+    '/srv/bench-venv/bin/python',
+  )
+})
+
+test('resolveBenchPython rejects relative and empty interpreter paths', () => {
+  assert.throws(
+    () => resolveBenchPython({ AGENT_BENCH_PYTHON: '.venv/bin/python' }, '/opt/agent-bench'),
+    /must be an absolute path/,
+  )
+  assert.throws(
+    () => resolveBenchPython({ AGENT_BENCH_PYTHON: '' }, '/opt/agent-bench'),
+    /must be an absolute path/,
+  )
+})
+
+test('runStagedJudge terminates a hung evaluator at its explicit timeout', async () => {
+  const started = Date.now()
+  await assert.rejects(
+    () => runStagedJudge({
+      tmpPrefix: 'timeout-test-',
+      async stage() {},
+      bin: process.execPath,
+      argv: () => ['-e', 'setInterval(() => {}, 1_000)'],
+      timeoutMs: 100,
+      async parseReport() {
+        throw new Error('timed-out evaluator must not reach report parsing')
+      },
+    }),
+    /evaluator failed/,
+  )
+  assert.ok(Date.now() - started < 5_000)
+})
+
+test('runStagedJudge durably captures exact evaluator and process bytes with hashes', async () => {
+  const scratch = await mkdtemp(join(tmpdir(), 'staged-judge-capture-test-'))
+  const destination = join(scratch, 'attempt-001')
+  const perTestBytes = Buffer.from([0x00, 0xff, 0x80, 0x0a, 0x41])
+  const stdoutBytes = Buffer.from([0x53, 0x00, 0xff, 0x0a])
+  const stderrBytes = Buffer.from([0x45, 0x80, 0x0a])
+  try {
+    const score = await runStagedJudge({
+      tmpPrefix: 'capture-test-',
+      async stage(dir) {
+        await writeFile(join(dir, 'preds.json'), Buffer.from([0x7b, 0x7d]))
+      },
+      bin: process.execPath,
+      argv: () => [
+        '-e',
+        [
+          "const fs = require('node:fs')",
+          "fs.mkdirSync('logs/run_evaluation/run/model/task', { recursive: true })",
+          `fs.writeFileSync('logs/run_evaluation/run/model/task/test_output.txt', Buffer.from(${JSON.stringify([...perTestBytes])}))`,
+          "fs.writeFileSync('official.report.json', Buffer.from('{\"resolved\":true}'))",
+          `process.stdout.write(Buffer.from(${JSON.stringify([...stdoutBytes])}))`,
+          `process.stderr.write(Buffer.from(${JSON.stringify([...stderrBytes])}))`,
+        ].join(';'),
+      ],
+      capture: { destination },
+      async parseReport(dir) {
+        assert.deepEqual(
+          await readFile(join(dir, 'logs/run_evaluation/run/model/task/test_output.txt')),
+          perTestBytes,
+        )
+        return { resolved: true, score: 1 }
+      },
+    })
+
+    const receipt = score.judgeArtifacts
+    assert.ok(receipt)
+    assert.equal(receipt.schema, 'agent-bench/judge-artifacts/v1')
+    assert.equal(receipt.evaluatorSucceeded, true)
+    assert.equal(receipt.directory, destination)
+    assert.deepEqual(
+      await readFile(join(destination, 'evaluator/logs/run_evaluation/run/model/task/test_output.txt')),
+      perTestBytes,
+    )
+    assert.deepEqual(await readFile(join(destination, 'process/stdout.bin')), stdoutBytes)
+    assert.deepEqual(await readFile(join(destination, 'process/stderr.bin')), stderrBytes)
+    assert.deepEqual(
+      JSON.parse(await readFile(receipt.manifestPath, 'utf8')),
+      receipt,
+    )
+
+    const expected = new Map([
+      ['evaluator/logs/run_evaluation/run/model/task/test_output.txt', perTestBytes],
+      ['evaluator/official.report.json', Buffer.from('{"resolved":true}')],
+      ['evaluator/preds.json', Buffer.from('{}')],
+      ['process/stderr.bin', stderrBytes],
+      ['process/stdout.bin', stdoutBytes],
+    ])
+    assert.deepEqual(receipt.files.map((file) => file.path), [...expected.keys()])
+    for (const file of receipt.files) {
+      const bytes = expected.get(file.path)
+      assert.ok(bytes)
+      assert.equal(file.kind, 'file')
+      assert.equal(file.byteLength, bytes.byteLength)
+      assert.equal(file.sha256, digest(bytes))
+    }
+    const treeBytes = Buffer.from(
+      receipt.files
+        .map((file) => `${file.path}\0${file.kind}\0${file.byteLength}\0${file.sha256}\n`)
+        .join(''),
+    )
+    assert.equal(receipt.treeSha256, digest(treeBytes))
+    assert.equal(receipt.fileCount, expected.size)
+    assert.equal(
+      receipt.byteLength,
+      [...expected.values()].reduce((total, bytes) => total + bytes.byteLength, 0),
+    )
+  } finally {
+    await rm(scratch, { recursive: true, force: true })
+  }
+})
+
+test('runStagedJudge captures partial evaluator evidence before throwing', async () => {
+  const scratch = await mkdtemp(join(tmpdir(), 'staged-judge-failure-capture-test-'))
+  const destination = join(scratch, 'attempt-failed')
+  try {
+    let caught: unknown
+    try {
+      await runStagedJudge({
+        tmpPrefix: 'capture-failure-test-',
+        async stage() {},
+        bin: process.execPath,
+        argv: () => [
+          '-e',
+          [
+            "const fs = require('node:fs')",
+            "fs.mkdirSync('logs/run_evaluation/run/model/task', { recursive: true })",
+            "fs.writeFileSync('logs/run_evaluation/run/model/task/run_instance.log', Buffer.from([1, 2, 255]))",
+            'process.stderr.write(Buffer.from([69, 82, 82, 0, 255]))',
+            'process.exit(9)',
+          ].join(';'),
+        ],
+        capture: { destination },
+        async parseReport() {
+          throw new Error('failed evaluator must not reach report parsing')
+        },
+      })
+    } catch (error) {
+      caught = error
+    }
+    assert.ok(caught instanceof Error)
+    assert.match(caught.message, /evaluator failed/)
+    const receipt = (caught as Error & { judgeArtifacts?: { evaluatorSucceeded: boolean } })
+      .judgeArtifacts
+    assert.ok(receipt)
+    assert.equal(receipt.evaluatorSucceeded, false)
+    assert.deepEqual(
+      await readFile(join(destination, 'evaluator/logs/run_evaluation/run/model/task/run_instance.log')),
+      Buffer.from([1, 2, 255]),
+    )
+    assert.deepEqual(
+      await readFile(join(destination, 'process/stderr.bin')),
+      Buffer.from([69, 82, 82, 0, 255]),
+    )
+  } finally {
+    await rm(scratch, { recursive: true, force: true })
+  }
+})