@tangle-network/agent-bench 0.1.0 → 0.3.6

package/src/benchmarks/tau3-banking.ts

@@ -0,0 +1,29 @@
+/**
+ * tau3-banking adapter.
+ *
+ * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with
+ * package namespace `tau2` and a new `banking_knowledge` domain. This adapter
+ * reuses the shared tau reward recomputation path and only changes the default
+ * domain/env names.
+ */
+
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import { createTauBenchAdapter } from './tau-bench-shared'
+import type { BenchmarkAdapter } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'tau3-banking.json')
+
+export function createTau3BankingAdapter(): BenchmarkAdapter {
+  return createTauBenchAdapter({
+    name: 'tau3-banking',
+    fixturePath: FIXTURES,
+    fixturesEnv: 'TAU3_FIXTURES',
+    dirEnv: 'TAU3_BENCH_DIR',
+    domainEnv: 'TAU3_DOMAIN',
+    defaultDomain: 'banking_knowledge',
+    taskIntro: 'Run this tau3 banking task in the official tau3 knowledge benchmark.',
+    installHint:
+      'clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench.',
+  })
+}

package/src/benchmarks/terminal-bench.test.mts

@@ -0,0 +1,33 @@
+/**
+ * Offline Terminal-Bench adapter test. Live judging needs the official
+ * terminal-bench harness + Docker, so this asserts the offline surfaces and the
+ * fail-loud preflight path. Run: npx tsx --test src/benchmarks/terminal-bench.test.mts
+ */
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { createTerminalBenchAdapter } from './terminal-bench'
+
+test('goldArtifact returns portable shell solutions and rejects missing ones', async () => {
+  const a = createTerminalBenchAdapter()
+  assert.equal(await a.goldArtifact({ id: 'with-solution', prompt: '', metadata: { solution: 'echo ok\n' } }), 'echo ok\n')
+  assert.equal(await a.goldArtifact({ id: 'without-solution', prompt: '', metadata: {} }), undefined)
+})
+
+test('preflight FAILS LOUD with isolated Terminal-Bench venv instructions when harness is absent', async () => {
+  const prev = process.env.TERMINAL_BENCH_VENV
+  process.env.TERMINAL_BENCH_VENV = '.venv-terminal-bench-does-not-exist'
+  try {
+    const a = createTerminalBenchAdapter()
+    await assert.rejects(a.preflight(), (e: Error) => {
+      assert.match(e.message, /pip install terminal-bench/)
+      assert.match(e.message, /ISOLATED venv/)
+      assert.match(e.message, /TERMINAL_BENCH_VENV/)
+      assert.match(e.message, /Docker daemon/)
+      return true
+    })
+  } finally {
+    if (prev === undefined) delete process.env.TERMINAL_BENCH_VENV
+    else process.env.TERMINAL_BENCH_VENV = prev
+  }
+})
+

package/src/benchmarks/terminal-bench.ts

@@ -6,9 +6,10 @@
  * our ScriptAgent), then the task's own verifier scores the resulting state.
  * Fully deterministic — no LLM judge, no self-authored score.
  *
- * Requires: the bench `.venv` with `terminal-bench` installed + a running Docker
- * daemon (per-task images are built on first run). loadTasks caches the dataset
- * from the Terminal-Bench registry on first run.
+ * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`
+ * installed + a running Docker daemon (per-task images are built on first run).
+ * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the
+ * Terminal-Bench registry on first run.
  *
  * Process/Docker/report plumbing is shared via ./_harness; this file owns the
  * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay
@@ -24,11 +25,16 @@ import {
   runVenvPython,
   safeRunId,
   stageFile,
-  venvBin,
+  venvPythonAt,
 } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
-const TB = venvBin('tb')
+// Terminal-Bench imports LiteLLM/Pydantic-2 APIs, while AppWorld pins Pydantic 1.
+// Keep it out of the shared bench .venv. Resolved at call-time so tests/runs can
+// override the env without reloading this module.
+const terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'
+const terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())
+const terminalBenchBin = (): string => join(benchRoot, terminalBenchVenvDir(), 'bin', 'tb')
 
 // Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the
 // installed CLI) and is the published launch task set. name==version is what `tb
@@ -90,7 +96,12 @@ for task_dir in ds:
     })
 print(json.dumps(out))
 `
-  const stdout = await runVenvPython(script, [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''])
+  const stdout = await runVenvPython(
+    script,
+    [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''],
+    0,
+    terminalBenchPython(),
+  )
   return JSON.parse(stdout) as TbTaskRow[]
 }
 
@@ -102,8 +113,12 @@ export function createTerminalBenchAdapter(): BenchmarkAdapter {
       await preflightVenvImports({
         modules: ['terminal_bench'],
         requireDocker: true,
+        python: terminalBenchPython(),
         fix:
-          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install terminal-bench ; ` +
+          `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && ` +
+          `bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench ` +
+          `(an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; ` +
+          `override the dir with TERMINAL_BENCH_VENV) ; ` +
           `(2) ensure the Docker daemon is running (the judge builds per-task images on first run). ` +
           `The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`,
       })
@@ -146,7 +161,7 @@ export function createTerminalBenchAdapter(): BenchmarkAdapter {
       const runId = safeRunId('bench', `${task.id}-${Date.now()}`)
       return runStagedJudge({
         tmpPrefix: 'tbench-',
-        bin: TB,
+        bin: terminalBenchBin(),
         cwd: () => benchRoot,
         async stage(dir) {
           await stageFile(join(dir, 'attempt.sh'), artifact)

package/src/benchmarks/toollm.ts

@@ -0,0 +1,254 @@
+/**
+ * ToolLLM/ToolBench adapter.
+ *
+ * ToolBench task loading is useful for breadth, but the official ToolEval pass
+ * rate evaluator is LLM-based and stochastic. This adapter therefore scores
+ * only ToolBench's deterministic API-selection labels (`relevant APIs`). It
+ * never records a full ToolEval pass-rate score.
+ */
+
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')
+const DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')
+
+interface ToolApi {
+  category_name?: string
+  tool_name: string
+  api_name: string
+  api_description?: string
+  required_parameters?: unknown[]
+  optional_parameters?: unknown[]
+  method?: string
+}
+
+interface ToolBenchRow {
+  query_id: number | string
+  query: string
+  api_list?: ToolApi[]
+  'relevant APIs'?: Array<[string, string]>
+}
+
+interface ToolBenchMeta {
+  queryId: string
+  apiList: ToolApi[]
+  relevantApis: Array<[string, string]>
+  deterministicJudge: 'api-selection'
+}
+
+const toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR
+const queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)
+
+export const toollmOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    return text.trim()
+  },
+}
+
+function rowToTask(row: ToolBenchRow): BenchTask {
+  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])
+  const meta: ToolBenchMeta = {
+    queryId: String(row.query_id),
+    apiList: row.api_list ?? [],
+    relevantApis,
+    deterministicJudge: 'api-selection',
+  }
+  return {
+    id: String(row.query_id),
+    prompt: [
+      'Solve this ToolLLM/ToolBench API-use task.',
+      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',
+      '',
+      `Query: ${row.query}`,
+      '',
+      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,
+      '',
+      'Return the APIs you used as JSON: {"api_calls":[{"tool_name":"...","api_name":"..."}]}.',
+    ].join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function normalizeApiPart(value: string): string {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')
+}
+
+function apiKey(pair: readonly [string, string]): string {
+  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`
+}
+
+function normalizeApiPairs(value: unknown): Array<[string, string]> {
+  if (!Array.isArray(value)) return []
+  const out: Array<[string, string]> = []
+  for (const item of value) {
+    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {
+      out.push([item[0], item[1]])
+    } else if (
+      item && typeof item === 'object'
+      && typeof (item as { tool_name?: unknown }).tool_name === 'string'
+      && typeof (item as { api_name?: unknown }).api_name === 'string'
+    ) {
+      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])
+    }
+  }
+  return out
+}
+
+function readMeta(task: BenchTask): ToolBenchMeta {
+  const md = task.metadata
+  if (!md || !Array.isArray(md.relevantApis)) {
+    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)
+  }
+  return md as unknown as ToolBenchMeta
+}
+
+function selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {
+  let tasks = rows.map(rowToTask)
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((task) => want.has(task.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)
+  return tasks
+}
+
+function assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {
+  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)
+  if (missing.length > 0) {
+    throw new Error(
+      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +
+        'Use a ToolBench query file that includes "relevant APIs" labels, or do not score ToolLLM in agent-bench.',
+    )
+  }
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]
+  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)
+  const tasks = selectRows(rows, opts)
+  assertDeterministicSubset(tasks, FIXTURES)
+  return tasks
+}
+
+async function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {
+  const source = queryFile(dir)
+  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)
+  assertDeterministicSubset(tasks, source)
+  return tasks
+}
+
+function extractJsonBlock(text: string): unknown {
+  const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)]
+  const raw = (fences.at(-1)?.[1] ?? text).trim()
+  try {
+    return JSON.parse(raw)
+  } catch {
+    return undefined
+  }
+}
+
+interface ExtractedApis {
+  pairs: Array<[string, string]>
+  source: 'structured-json' | 'text-mention'
+}
+
+function extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {
+  const parsed = extractJsonBlock(text)
+  if (parsed && typeof parsed === 'object') {
+    const raw = parsed as Record<string, unknown>
+    const fromApiCalls = normalizeApiPairs(raw.api_calls)
+    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }
+    const fromCalls = normalizeApiPairs(raw.calls)
+    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }
+  }
+
+  const lower = text.toLowerCase()
+  return {
+    pairs: expected.filter(([tool, api]) => {
+      const toolNeedle = normalizeApiPart(tool)
+      const apiNeedle = normalizeApiPart(api)
+      const compactText = lower.replace(/[^a-z0-9]+/g, '')
+      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))
+    }),
+    source: 'text-mention',
+  }
+}
+
+function scoreApiSelection(task: BenchTask, artifact: string): BenchScore {
+  const meta = readMeta(task)
+  if (meta.relevantApis.length === 0) {
+    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)
+  }
+  const expected = new Set(meta.relevantApis.map(apiKey))
+  const extracted = extractCalledApis(artifact, meta.relevantApis)
+  const calledPairs = extracted.pairs
+  const called = new Set(calledPairs.map(apiKey))
+  const truePositives = [...called].filter((key) => expected.has(key)).length
+  const precision = called.size === 0 ? 0 : truePositives / called.size
+  const recall = truePositives / expected.size
+  const score = expected.size === 0 ? 0 : recall
+  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1
+  return {
+    resolved,
+    score,
+    detail: JSON.stringify({
+      scoring: 'api-selection-only',
+      extractionSource: extracted.source,
+      queryId: meta.queryId,
+      expected: meta.relevantApis,
+      called: calledPairs,
+      precision,
+      recall,
+      fullToolEvalScore: null,
+    }),
+  }
+}
+
+export function createToolLlmAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'
+
+  return {
+    name: 'toollm',
+    output: toollmOutput,
+
+    async preflight() {
+      if (fixturesMode) return
+      const dir = toolbenchDir()
+      if (!dir) {
+        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')
+      }
+      await loadOfficialTasks(dir, { limit: 1 })
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const dir = toolbenchDir()
+      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')
+      return loadOfficialTasks(dir, opts)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      const meta = readMeta(task)
+      if (meta.relevantApis.length === 0) return undefined
+      return JSON.stringify({
+        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),
+      }, null, 2)
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      return scoreApiSelection(task, artifact)
+    },
+  }
+}

package/src/benchmarks/types.ts

@@ -20,12 +20,40 @@ export interface BenchTask {
   metadata?: Record<string, unknown>
 }
 
+export interface JudgeArtifactFileReceipt {
+  /** POSIX path relative to the capture directory. */
+  path: string
+  /** Exact byte length of the retained file or symbolic-link target. */
+  byteLength: number
+  /** SHA-256 over the retained file bytes or UTF-8 symbolic-link target. */
+  sha256: `sha256:${string}`
+  kind: 'file' | 'symlink'
+}
+
+/** Durable evidence written before a staged evaluator's temporary directory is removed. */
+export interface JudgeArtifactReceipt {
+  schema: 'agent-bench/judge-artifacts/v1'
+  /** Absolute directory containing `evaluator/`, `process/`, and `receipt.json`. */
+  directory: string
+  /** Exact copy of the evaluator working directory. */
+  evaluatorDirectory: string
+  manifestPath: string
+  evaluatorSucceeded: boolean
+  files: JudgeArtifactFileReceipt[]
+  fileCount: number
+  byteLength: number
+  /** SHA-256 over every sorted path, kind, byte length, and content hash. */
+  treeSha256: `sha256:${string}`
+}
+
 export interface BenchScore {
   /** Did the deterministic judge pass (tests resolved / state correct)? */
   resolved: boolean
   /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */
   score: number
   detail?: string
+  /** Present only when the caller explicitly requested durable judge evidence. */
+  judgeArtifacts?: JudgeArtifactReceipt
 }
 
 export interface LoadOptions {
@@ -49,6 +77,20 @@ export interface BenchmarkAdapter {
    *  so the gate runner (`runGate` / `runBenchmark`) needs no
    *  per-benchmark branching. */
   output?: OutputAdapter<string>
+  /** Post-shot deliverable extraction from the box FILESYSTEM, not the event stream.
+   *  When set, the shot runner execs `command` in the STILL-ALIVE box after the agent
+   *  turn drains and uses its stdout as the judged artifact — the durable way to capture
+   *  a git diff of the agent's in-box edits (standard SWE-bench practice: SWE-agent /
+   *  OpenHands read the diff from repo STATE), instead of hoping the model printed a
+   *  fenced diff in its reply. Empty stdout ⇒ the runner falls back to `output` (the
+   *  event-stream parse). `cwd` defaults to the box root. */
+  boxExtract?(task: BenchTask): { command: string; cwd?: string }
+  /** Optional workspace pre-stage run in the box BEFORE the agent shot (same
+   *  session as `boxExtract`). For repo-state benchmarks (SWE-bench) this clones
+   *  the instance repo at `base_commit` into a fixed path so the agent only edits
+   *  — the harness owns the checkout, not the (stochastic) model. A non-zero exit
+   *  fails the shot loud rather than letting the agent run against an empty box. */
+  boxSetup?(task: BenchTask): { command: string; cwd?: string }
   /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the
    *  worker (e.g. AppWorld's interactive ReAct episode runs inside the engine,
    *  not as a chat completion) — the experiment uses this instead of the

package/src/benchmarks/webarena-verified.ts

@@ -0,0 +1,200 @@
+/**
+ * WebArena-Verified adapter (ServiceNow/webarena-verified).
+ *
+ * Worker artifact = a WebArena-Verified run output directory, not final chat text.
+ * Judge = the official `webarena_verified eval-tasks` evaluator over that output
+ * directory. The adapter refuses to score a plain answer so we do not turn a DOM
+ * benchmark into a fake text benchmark.
+ */
+
+import { access, readFile, stat } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import { benchRoot, runVenvPython } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'webarena-verified.json')
+const DATASET_REL = join('assets', 'dataset', 'webarena-verified.json')
+
+interface WebArenaRow {
+  task_id: number
+  intent: string
+  intent_template_id?: number
+  sites?: string[]
+  start_urls?: string[]
+  eval?: unknown[]
+  revision?: number
+}
+
+interface WebArenaMeta {
+  taskId: number
+  intentTemplateId?: number
+  sites: string[]
+  startUrls: string[]
+  revision?: number
+  eval?: unknown[]
+}
+
+const webarenaDir = (): string | undefined => process.env.WEBARENA_VERIFIED_DIR
+
+export const webarenaOutputDirOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    const fences = [...text.matchAll(/```(?:text|path)?\s*\n([\s\S]*?)```/g)]
+    return (fences.at(-1)?.[1] ?? text).trim()
+  },
+}
+
+async function assertFile(path: string, label: string): Promise<void> {
+  try {
+    await access(path)
+  } catch (err) {
+    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)
+  }
+}
+
+async function assertDir(path: string, label: string): Promise<void> {
+  try {
+    const s = await stat(path)
+    if (!s.isDirectory()) throw new Error('not a directory')
+  } catch (err) {
+    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)
+  }
+}
+
+function rowToTask(row: WebArenaRow): BenchTask {
+  const meta: WebArenaMeta = {
+    taskId: row.task_id,
+    intentTemplateId: row.intent_template_id,
+    sites: row.sites ?? [],
+    startUrls: row.start_urls ?? [],
+    revision: row.revision,
+    eval: row.eval,
+  }
+  return {
+    id: String(row.task_id),
+    prompt: [
+      'Run this WebArena-Verified browser task in the official environment.',
+      `Goal: ${row.intent}`,
+      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(', ')}` : undefined,
+      row.sites?.length ? `Sites: ${row.sites.join(', ')}` : undefined,
+      '',
+      'Return the path to the official WebArena-Verified run output directory for this task.',
+      'The judge expects that directory to contain the task response and network trace files.',
+    ]
+      .filter(Boolean)
+      .join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): WebArenaMeta {
+  const md = task.metadata
+  if (!md || typeof md.taskId !== 'number') {
+    throw new Error(`webarena-verified task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as WebArenaMeta
+}
+
+function selectRows(rows: WebArenaRow[], opts: LoadOptions): BenchTask[] {
+  let tasks = rows.map(rowToTask)
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((task) => want.has(task.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`)
+  return tasks
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as WebArenaRow[]
+  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 — loading ${rows.length} adapter fixtures`)
+  return selectRows(rows, opts)
+}
+
+async function loadOfficialRows(dir: string, opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), 'utf8')) as WebArenaRow[]
+  return selectRows(rows, opts)
+}
+
+async function runOfficialEval(root: string, taskId: number, outputDir: string): Promise<Record<string, unknown>> {
+  const script = `
+import json, os, subprocess, sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+task_id = sys.argv[2]
+output_dir = Path(sys.argv[3])
+env = os.environ.copy()
+env["PYTHONPATH"] = str(root / "src") + os.pathsep + env.get("PYTHONPATH", "")
+cmd = [sys.executable, "-m", "webarena_verified", "eval-tasks", "--task-ids", task_id, "--output-dir", str(output_dir)]
+proc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)
+if proc.returncode != 0:
+    raise SystemExit((proc.stderr or proc.stdout or f"exit {proc.returncode}")[:2000])
+result_path = output_dir / task_id / "eval_result.json"
+if not result_path.exists():
+    raise SystemExit(f"official evaluator did not write {result_path}")
+print(json.dumps(json.loads(result_path.read_text())))
+`
+  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0)
+  return JSON.parse(stdout.trim().split('\n').at(-1) ?? '{}') as Record<string, unknown>
+}
+
+export function createWebArenaVerifiedAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === '1'
+
+  return {
+    name: 'webarena-verified',
+    output: webarenaOutputDirOutput,
+
+    async preflight() {
+      if (fixturesMode) return
+      const dir = webarenaDir()
+      if (!dir) {
+        throw new Error(
+          'WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified.',
+        )
+      }
+      await assertFile(join(dir, DATASET_REL), 'official dataset')
+      await assertFile(join(dir, 'src', 'webarena_verified', '__main__.py'), 'official CLI module')
+      await runVenvPython(
+        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print("ok")',
+        [join(dir, 'src')],
+      )
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const dir = webarenaDir()
+      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks')
+      return loadOfficialRows(dir, opts)
+    },
+
+    async goldArtifact() {
+      return undefined
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const dir = webarenaDir()
+      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts')
+      const outputDir = resolve(artifact.trim())
+      await assertDir(outputDir, 'run output directory')
+      const meta = readMeta(task)
+      const report = await runOfficialEval(dir, meta.taskId, outputDir)
+      const score = typeof report.score === 'number' ? report.score : 0
+      const status = typeof report.status === 'string' ? report.status : 'unknown'
+      return {
+        resolved: score === 1,
+        score,
+        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir }),
+      }
+    },
+  }
+}