@tangle-network/agent-bench 0.1.0 → 0.3.6

package/src/benchmarks/_harness.ts

@@ -17,19 +17,49 @@
  */
 
 import { execFile, spawn } from 'node:child_process'
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+import {
+  cp,
+  lstat,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readlink,
+  readdir,
+  rename,
+  rm,
+  writeFile,
+} from 'node:fs/promises'
 import { tmpdir } from 'node:os'
-import { join } from 'node:path'
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
-import type { BenchScore } from './types'
+import type {
+  BenchScore,
+  JudgeArtifactFileReceipt,
+  JudgeArtifactReceipt,
+} from './types'
 
 const execFileAsync = promisify(execFile)
 
 /** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
 export const benchRoot = fileURLToPath(new URL('../..', import.meta.url))
-/** The bench venv interpreter every python-backed evaluator runs through. */
-export const venvPython = join(benchRoot, '.venv', 'bin', 'python')
+
+/** Resolve the shared interpreter without requiring an installed package to contain a venv. */
+export function resolveBenchPython(
+  env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env,
+  root: string = benchRoot,
+): string {
+  const configured = env.AGENT_BENCH_PYTHON
+  if (configured === undefined) return join(root, '.venv', 'bin', 'python')
+  if (!isAbsolute(configured)) {
+    throw new Error('AGENT_BENCH_PYTHON must be an absolute path')
+  }
+  return configured
+}
+
+/** The shared interpreter every Python-backed evaluator runs through. */
+export const venvPython = resolveBenchPython()
 
 /** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip
  *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)
@@ -149,10 +179,160 @@ export interface StagedRunSpec {
    * if the expected report is absent/malformed (fail loud — no default score).
    */
   parseReport(dir: string): Promise<BenchScore>
+  /**
+   * Copy the complete evaluator directory plus raw process stdout/stderr to this
+   * caller-owned directory before cleanup. The destination must not exist.
+   */
+  capture?: StagedRunCaptureSpec
   /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */
   keepTmp?: boolean
 }
 
+export interface StagedRunCaptureSpec {
+  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */
+  destination: string
+}
+
+/** A staged run failed after any requested evidence was durably retained. */
+export class StagedJudgeError extends Error {
+  readonly judgeArtifacts?: JudgeArtifactReceipt
+
+  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) {
+    super(message, options)
+    this.name = 'StagedJudgeError'
+    this.judgeArtifacts = judgeArtifacts
+  }
+}
+
+function sha256(bytes: Uint8Array): `sha256:${string}` {
+  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`
+}
+
+function portablePath(path: string): string {
+  return path.split(sep).join('/')
+}
+
+function compareText(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0
+}
+
+async function collectArtifactFiles(
+  root: string,
+  current: string,
+): Promise<JudgeArtifactFileReceipt[]> {
+  const absolute = join(root, current)
+  const entries = await readdir(absolute, { withFileTypes: true })
+  const files: JudgeArtifactFileReceipt[] = []
+  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {
+    const relativePath = join(current, entry.name)
+    const path = join(root, relativePath)
+    if (entry.isDirectory()) {
+      files.push(...await collectArtifactFiles(root, relativePath))
+      continue
+    }
+    if (entry.isFile()) {
+      const bytes = await readFile(path)
+      files.push({
+        path: portablePath(relativePath),
+        byteLength: bytes.byteLength,
+        sha256: sha256(bytes),
+        kind: 'file',
+      })
+      continue
+    }
+    if (entry.isSymbolicLink()) {
+      const targetBytes = await readlink(path, { encoding: 'buffer' })
+      files.push({
+        path: portablePath(relativePath),
+        byteLength: targetBytes.byteLength,
+        sha256: sha256(targetBytes),
+        kind: 'symlink',
+      })
+      continue
+    }
+    throw new Error(`staged judge capture does not support ${relativePath}`)
+  }
+  return files
+}
+
+function isWithin(parent: string, candidate: string): boolean {
+  const path = relative(parent, candidate)
+  return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path))
+}
+
+async function assertDestinationAbsent(destination: string): Promise<void> {
+  try {
+    await lstat(destination)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return
+    throw error
+  }
+  throw new Error(`staged judge capture destination already exists: ${destination}`)
+}
+
+async function captureStagedRun(
+  sourceDirectory: string,
+  spec: StagedRunCaptureSpec,
+  processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>,
+  evaluatorSucceeded: boolean,
+): Promise<JudgeArtifactReceipt> {
+  const source = resolve(sourceDirectory)
+  const destination = resolve(spec.destination)
+  if (isWithin(source, destination)) {
+    throw new Error('staged judge capture destination must be outside the evaluator directory')
+  }
+  await mkdir(dirname(destination), { recursive: true })
+  await assertDestinationAbsent(destination)
+  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`))
+  try {
+    await cp(source, join(staging, 'evaluator'), {
+      recursive: true,
+      errorOnExist: true,
+      force: false,
+      preserveTimestamps: true,
+      verbatimSymlinks: true,
+    })
+    await mkdir(join(staging, 'process'))
+    await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout)
+    await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr)
+
+    const files = [
+      ...await collectArtifactFiles(staging, 'evaluator'),
+      ...await collectArtifactFiles(staging, 'process'),
+    ].sort((left, right) => compareText(left.path, right.path))
+    const byteLength = files.reduce((total, file) => total + file.byteLength, 0)
+    const treeBytes = Buffer.from(
+      files
+        .map((file) => `${file.path}\0${file.kind}\0${file.byteLength}\0${file.sha256}\n`)
+        .join(''),
+      'utf8',
+    )
+    const receipt: JudgeArtifactReceipt = {
+      schema: 'agent-bench/judge-artifacts/v1',
+      directory: destination,
+      evaluatorDirectory: join(destination, 'evaluator'),
+      manifestPath: join(destination, 'receipt.json'),
+      evaluatorSucceeded,
+      files,
+      fileCount: files.length,
+      byteLength,
+      treeSha256: sha256(treeBytes),
+    }
+    await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\n`)
+    await rename(staging, destination)
+    return receipt
+  } catch (error) {
+    await rm(staging, { recursive: true, force: true }).catch(() => {})
+    throw error
+  }
+}
+
+function processBytes(value: unknown): Buffer {
+  if (Buffer.isBuffer(value)) return value
+  if (value === undefined || value === null) return Buffer.alloc(0)
+  return Buffer.from(String(value), 'utf8')
+}
+
 /**
  * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →
  * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is
@@ -160,23 +340,66 @@ export interface StagedRunSpec {
  */
 export async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {
   const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))
+  let stdout: Buffer<ArrayBufferLike> = Buffer.alloc(0)
+  let stderr: Buffer<ArrayBufferLike> = Buffer.alloc(0)
+  let evaluatorSucceeded = false
+  let score: BenchScore | undefined
+  let failure: unknown
   try {
-    await spec.stage(dir)
-    const bin = spec.bin ?? venvPython
     try {
-      await execFileAsync(bin, spec.argv(dir), {
-        cwd: spec.cwd ? spec.cwd(dir) : dir,
-        maxBuffer: bigBuffer,
-        ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),
-      })
+      await spec.stage(dir)
+      const bin = spec.bin ?? venvPython
+      const argv = spec.argv(dir)
+      try {
+        const output = await execFileAsync(bin, argv, {
+          cwd: spec.cwd ? spec.cwd(dir) : dir,
+          encoding: 'buffer',
+          maxBuffer: bigBuffer,
+          ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),
+        })
+        stdout = processBytes(output.stdout)
+        stderr = processBytes(output.stderr)
+        evaluatorSucceeded = true
+      } catch (err) {
+        const e = err as { stderr?: unknown; stdout?: unknown; message?: string }
+        stdout = processBytes(e.stdout)
+        stderr = processBytes(e.stderr)
+        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err))
+          .toString('utf8')
+          .slice(0, 2000)
+        throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\n${detail}`)
+      }
+      score = await spec.parseReport(dir)
     } catch (err) {
-      const e = err as { stderr?: string; stdout?: string; message?: string }
-      const detail = (e.stderr || e.stdout || e.message || String(err)).slice(0, 2000)
-      throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${spec.argv(dir).join(' ')}):\n${detail}`)
+      failure = err
     }
-    return await spec.parseReport(dir)
   } finally {
+    let judgeArtifacts: JudgeArtifactReceipt | undefined
+    if (spec.capture) {
+      try {
+        judgeArtifacts = await captureStagedRun(
+          dir,
+          spec.capture,
+          { stdout, stderr },
+          evaluatorSucceeded,
+        )
+      } catch (captureError) {
+        failure = new Error(
+          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,
+          { cause: failure ?? captureError },
+        )
+      }
+    }
     if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})
+    if (failure) {
+      throw new StagedJudgeError(
+        failure instanceof Error ? failure.message : String(failure),
+        judgeArtifacts,
+        { cause: failure },
+      )
+    }
+    if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts)
+    return judgeArtifacts ? { ...score, judgeArtifacts } : score
   }
 }
 

package/src/benchmarks/agentbench.ts

@@ -0,0 +1,163 @@
+/**
+ * AgentBench deterministic subset adapter.
+ *
+ * This targets AgentBench DBBench rows only: question + table + published label.
+ * It does not wrap AgentBench's controller protocol or the non-deterministic game
+ * environments. Worker artifact = final answer text. Judge = exact match against
+ * the official DBBench label list after light whitespace/case normalization.
+ */
+
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'agentbench-dbbench.json')
+const DEFAULT_SPLIT = 'dev'
+
+interface AgentBenchDbRow {
+  description: string
+  label: string[]
+  table?: {
+    table_name?: string
+    table_info?: {
+      columns?: Array<{ name: string; type?: string }>
+      rows?: unknown[][]
+    }
+  }
+}
+
+interface AgentBenchMeta {
+  labels: string[]
+  split: string
+  subset: 'dbbench'
+  table?: AgentBenchDbRow['table']
+}
+
+const agentbenchDir = (): string | undefined => process.env.AGENTBENCH_DIR
+
+export const agentbenchAnswerOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    const fences = [...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)]
+    return (fences.at(-1)?.[1] ?? text).trim()
+  },
+}
+
+function normalizeAnswer(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/\s+/g, ' ')
+    .replace(/^["'`]+|["'`]+$/g, '')
+    .trim()
+}
+
+function rowToTask(row: AgentBenchDbRow, index: number, split: string): BenchTask {
+  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ''}`).join(', ')
+  const sampleRows = row.table?.table_info?.rows?.slice(0, 40)
+  const meta: AgentBenchMeta = {
+    labels: row.label,
+    split,
+    subset: 'dbbench',
+    table: row.table,
+  }
+  return {
+    id: `dbbench-${split}-${index}`,
+    split,
+    prompt: [
+      'Answer this AgentBench DBBench question using the table below.',
+      'Return only the answer value.',
+      '',
+      `Question: ${row.description}`,
+      row.table?.table_name ? `Table: ${row.table.table_name}` : undefined,
+      columns ? `Columns: ${columns}` : undefined,
+      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : undefined,
+    ]
+      .filter(Boolean)
+      .join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): AgentBenchMeta {
+  const md = task.metadata
+  if (!md || !Array.isArray(md.labels)) {
+    throw new Error(`agentbench task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as AgentBenchMeta
+}
+
+function selectRows(rows: AgentBenchDbRow[], opts: LoadOptions, split: string): BenchTask[] {
+  let tasks = rows.map((row, index) => rowToTask(row, index, split))
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((task) => want.has(task.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`)
+  return tasks
+}
+
+async function loadJsonl(path: string): Promise<AgentBenchDbRow[]> {
+  const raw = await readFile(path, 'utf8')
+  return raw
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as AgentBenchDbRow)
+}
+
+async function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as AgentBenchDbRow[]
+  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 — loading ${rows.length} DBBench adapter fixtures`)
+  return selectRows(rows, opts, split)
+}
+
+export function createAgentBenchAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.AGENTBENCH_FIXTURES === '1'
+
+  return {
+    name: 'agentbench',
+    output: agentbenchAnswerOutput,
+
+    async preflight() {
+      if (fixturesMode) return
+      const dir = agentbenchDir()
+      if (!dir) {
+        throw new Error('AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.')
+      }
+      await loadJsonl(join(dir, 'data', 'dbbench', `${DEFAULT_SPLIT}.jsonl`))
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      const split = opts.split ?? DEFAULT_SPLIT
+      if (fixturesMode) return loadFixtures(opts, split)
+      const dir = agentbenchDir()
+      if (!dir) throw new Error('AGENTBENCH_DIR is required to load AgentBench DBBench tasks')
+      return selectRows(await loadJsonl(join(dir, 'data', 'dbbench', `${split}.jsonl`)), opts, split)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      return readMeta(task).labels[0]
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const got = normalizeAnswer(artifact)
+      const expected = meta.labels.map(normalizeAnswer)
+      const resolved = expected.includes(got)
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact }),
+      }
+    },
+  }
+}

package/src/benchmarks/appworld.test.mts

@@ -27,19 +27,25 @@ test('goldArtifact is undefined — reference solution ships only inside the eng
   assert.equal(await a.goldArtifact({ id: 't', prompt: '', metadata: { taskId: 't', split: 'dev' } }), undefined)
 })
 
-test('preflight FAILS LOUD with the install + download-data fix when the engine is absent', async () => {
+test('preflight passes when installed or FAILS LOUD with the install + download-data fix', async () => {
   const a = createAppWorldAdapter()
-  await assert.rejects(a.preflight(), (e: Error) => {
+  try {
+    await a.preflight()
+  } catch (err) {
+    const e = err as Error
     assert.match(e.message, /pip install appworld/)
     assert.match(e.message, /appworld download data/)
-    return true
-  })
+  }
 })
 
-test('loadTasks FAILS LOUD (engine enumeration) rather than fabricating tasks offline', async () => {
+test('loadTasks either enumerates live engine rows or FAILS LOUD without fabricating tasks', async () => {
   const a = createAppWorldAdapter()
-  await assert.rejects(a.loadTasks({ limit: 1 }), (e: Error) => {
-    assert.match(e.message, /appworld driver failed|appworld import failed/)
-    return true
-  })
+  try {
+    const tasks = await a.loadTasks({ limit: 1 })
+    assert.equal(tasks.length, 1)
+    assert.ok(tasks[0].id.length > 0)
+    assert.match(tasks[0].prompt, /Solve this by writing Python/)
+  } catch (err) {
+    assert.match((err as Error).message, /appworld driver failed|appworld import failed/)
+  }
 })