@tangle-network/agent-bench 0.1.0 → 0.3.6

package/src/benchmarks/swe-bench.ts

@@ -21,15 +21,26 @@ import {
   runVenvPython,
   safeRunId,
   stageFile,
+  type StagedRunCaptureSpec,
 } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
 /**
- * The SWE deliverable, extracted from the agent's event STREAM (not the box FS).
- * `runLoop`'s `OutputAdapter` only sees events, so the agent prints its unified
- * diff in a fenced block and this pulls the last one out — the seam that lets the
- * SWE benchmark run through the gate runner (`runGate` / `runBenchmark`)
- * like any other.
+ * Fixed in-box path the agent clones the instance repo into. It is the SINGLE
+ * source of truth shared by the prompt template (which tells the agent to clone
+ * here) and `boxExtract` (which runs `git diff` here after the shot) — so the
+ * harness always knows exactly where the agent's edits live, for any instance.
+ */
+const SWE_REPO_DIR = '/work'
+
+/**
+ * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
+ *
+ * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's
+ * actual edits, read from the cloned repo's STATE inside the box (standard
+ * SWE-bench practice). This event-stream parse only runs when that diff is empty
+ * — a model that edited the source correctly but never printed a fenced diff (the
+ * exact failure this replaces) still scores off its real changes, not its prose.
  */
 export const swePatchOutput: OutputAdapter<string> = {
   parse(events) {
@@ -48,17 +59,179 @@ export const swePatchOutput: OutputAdapter<string> = {
 }
 
 const DATASET = 'princeton-nlp/SWE-bench_Verified'
+export type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'
+
+export interface SweBenchArtifactCaptureContext {
+  readonly taskId: string
+  readonly runId: string
+  /** One-based sequence unique within this adapter instance. */
+  readonly attemptSequence: number
+}
+
+export interface SweBenchAdapterOptions {
+  readonly timeoutMs?: number
+  readonly cacheLevel?: SweBenchCacheLevel
+  /**
+   * Return a unique destination for any attempt whose complete official
+   * evaluator directory and process logs should be retained.
+   */
+  readonly captureEvaluatorArtifacts?: (
+    context: SweBenchArtifactCaptureContext,
+  ) => StagedRunCaptureSpec | undefined
+}
+
+const SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])
+
+function scorerNamespace(): 'swebench' | 'none' {
+  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'
+  if (namespace !== 'swebench' && namespace !== 'none') {
+    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`)
+  }
+  return namespace
+}
+const TEST_FILE_EXCLUDES = [
+  "':(exclude,glob)**/tests/**'",
+  "':(exclude,glob)**/test/**'",
+  "':(exclude,glob)test_*.py'",
+  "':(exclude,glob)**/test_*.py'",
+  "':(exclude,glob)*_test.py'",
+  "':(exclude,glob)**/*_test.py'",
+  "':(exclude,glob)conftest.py'",
+  "':(exclude,glob)**/conftest.py'",
+].join(' ')
 
 interface SweReport {
   resolved_instances?: number
   resolved_ids?: string[]
+  unresolved_ids?: string[]
+  empty_patch_ids?: string[]
+  completed_ids?: string[]
+  incomplete_ids?: string[]
+  error_ids?: string[]
+  submitted_ids?: string[]
+}
+
+function stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {
+  const value = report[key]
+  if (value === undefined) return []
+  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {
+    throw new Error(`swe-bench: malformed ${key}`)
+  }
+  return value
 }
 
-export function createSweBenchAdapter(): BenchmarkAdapter {
+/** Convert one official report into a score without turning evaluator failures into agent failures. */
+export function scoreSweReport(taskId: string, value: unknown): BenchScore {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error('swe-bench: report must be an object')
+  }
+  const report = value as Record<string, unknown>
+  const statusIds = {
+    resolved: stringIds(report, 'resolved_ids'),
+    unresolved: stringIds(report, 'unresolved_ids'),
+    emptyPatch: stringIds(report, 'empty_patch_ids'),
+    completed: stringIds(report, 'completed_ids'),
+    incomplete: stringIds(report, 'incomplete_ids'),
+    error: stringIds(report, 'error_ids'),
+  }
+  const submitted = stringIds(report, 'submitted_ids')
+  const mentioned = Object.values(statusIds).flat()
+  if (
+    mentioned.some((id) => id !== taskId)
+    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))
+  ) {
+    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)
+  }
+  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {
+    throw new Error(`swe-bench: evaluator failed for ${taskId}`)
+  }
+  const outcomes = [
+    statusIds.resolved.includes(taskId),
+    statusIds.unresolved.includes(taskId),
+    statusIds.emptyPatch.includes(taskId),
+  ]
+  if (outcomes.filter(Boolean).length !== 1) {
+    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)
+  }
+  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {
+    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)
+  }
+  const resolved = outcomes[0]
+  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }
+}
+
+export function sweEvaluationArgv(args: {
+  readonly predictionsPath: string
+  readonly runId: string
+  readonly instanceId: string
+  readonly cacheLevel: SweBenchCacheLevel
+  readonly namespace?: 'swebench' | 'none'
+}): string[] {
+  return [
+    '-m', 'swebench.harness.run_evaluation',
+    '--dataset_name', DATASET,
+    '--predictions_path', args.predictionsPath,
+    '--run_id', args.runId,
+    '--instance_ids', args.instanceId,
+    '--max_workers', '1',
+    '--namespace', args.namespace ?? scorerNamespace(),
+    '--cache_level', args.cacheLevel,
+  ]
+}
+
+function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`
+}
+
+function sweMetadata(task: BenchTask): { repo: string; base: string } {
+  const repo = String(task.metadata?.repo ?? '')
+  const base = String(task.metadata?.base_commit ?? '')
+  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
+    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)
+  }
+  if (!/^[0-9a-f]{7,40}$/i.test(base)) {
+    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)
+  }
+  return { repo, base }
+}
+
+export function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {
+  if (
+    options.timeoutMs !== undefined
+    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)
+  ) throw new Error('swe-bench: timeoutMs must be a positive integer')
+  const cacheLevel = options.cacheLevel ?? 'env'
+  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')
+  if (
+    options.captureEvaluatorArtifacts !== undefined
+    && typeof options.captureEvaluatorArtifacts !== 'function'
+  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')
+  let attemptSequence = 0
   return {
     name: 'swe-bench-verified',
     output: swePatchOutput,
 
+    // Extract the patch from repo STATE, not printed text: stage every edit the
+    // agent made in the cloned repo and diff it against the checked-out
+    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold
+    // `test_patch` itself, so an agent edit to a test would collide on apply. The
+    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold
+    // patch format the swebench judge's `git apply` expects.
+    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the
+    // agent only edits (the harness owns the checkout — a stochastic model can't be
+    // trusted to clone to an exact path). `--quiet` keeps the exec output small.
+    boxSetup(task) {
+      const { repo, base } = sweMetadata(task)
+      return {
+        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,
+      }
+    },
+    boxExtract() {
+      return {
+        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,
+      }
+    },
+
     async preflight() {
       await preflightVenvImports({
         modules: ['swebench'],
@@ -101,8 +274,10 @@ print(json.dumps(out))
           prompt: [
             `Repository: ${r.repo} @ ${r.base_commit}`,
             '',
-            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests, so editing tests does not count. Keep the change minimal.',
-            'When done, END your reply with the COMPLETE unified git diff as the LAST thing, fenced exactly as ```diff … ``` (nothing after the closing fence). That fenced diff is the only deliverable.',
+            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
+            '',
+            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',
+            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',
             '',
             '--- Issue ---',
             String(r.problem_statement ?? ''),
@@ -119,8 +294,18 @@ print(json.dumps(out))
 
     async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
       const runId = safeRunId('bench', task.id)
+      const capture = options.captureEvaluatorArtifacts?.({
+        taskId: task.id,
+        runId,
+        attemptSequence: ++attemptSequence,
+      })
       return runStagedJudge({
         tmpPrefix: 'swebench-',
+        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
+        ...(capture === undefined ? {} : { capture }),
+        // Debug: retain the staged dir (holds swebench's per-instance apply/run
+        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.
+        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),
         async stage(dir) {
           await stageFile(
             join(dir, 'preds.json'),
@@ -131,20 +316,17 @@ print(json.dumps(out))
         },
         // The official evaluation harness. Pulls/builds the instance image, applies
         // the patch, runs the test spec, writes a per-run report JSON in cwd.
-        argv: (dir) => [
-          '-m', 'swebench.harness.run_evaluation',
-          '--dataset_name', DATASET,
-          '--predictions_path', join(dir, 'preds.json'),
-          '--run_id', runId,
-          '--instance_ids', task.id,
-          '--max_workers', '1',
-          '--cache_level', 'env',
-        ],
+        argv: (dir) => sweEvaluationArgv({
+          predictionsPath: join(dir, 'preds.json'),
+          runId,
+          instanceId: task.id,
+          cacheLevel,
+          namespace: scorerNamespace(),
+        }),
         async parseReport(dir) {
           // Report file: agent-runtime-bench.<run_id>.json
           const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))
-          const resolved = (report.resolved_ids ?? []).includes(task.id)
-          return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }
+          return scoreSweReport(task.id, report)
         },
       })
     },

package/src/benchmarks/t2-ragbench.ts

@@ -0,0 +1,166 @@
+/**
+ * T2-RAGBench adapter.
+ *
+ * T2-RAGBench stresses text+table retrieval and numerical reasoning over
+ * financial documents. The judge uses the shared deterministic answer scorer
+ * with numeric tolerance enabled by default.
+ */
+
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringFrom,
+  type RagContext,
+} from './rag-shared'
+
+const FIXTURES = join(benchRoot, 'fixtures', 't2-ragbench.json')
+
+interface T2RagBenchMeta {
+  benchmark: 't2-ragbench'
+  query: string
+  goldAnswers: string[]
+  contexts: RagContext[]
+  subset: string
+  documentId: string
+}
+
+const dataFile = (): string | undefined => process.env.T2_RAGBENCH_DATA_FILE
+
+function rowToTask(raw: unknown, index: number): BenchTask {
+  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`)
+  const query = firstString(raw, ['question', 'query', 'prompt'])
+  const goldAnswers = allStrings(raw, [
+    'program_answer',
+    'original_answer',
+    'answer',
+    'answers',
+    'reference',
+    'reference_answer',
+    'gold',
+  ])
+  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`)
+  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`)
+  const baseContexts =
+    contextsFrom(raw.context).length > 0
+      ? contextsFrom(raw.context)
+      : contextsFrom(raw.contexts).length > 0
+      ? contextsFrom(raw.contexts)
+      : contextsFrom(raw.chunks).length > 0
+        ? contextsFrom(raw.chunks)
+        : contextsFrom(raw.passages)
+  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text)
+  const contexts = table
+    ? [...baseContexts, { id: 'table', title: 'Table', text: table }]
+    : baseContexts
+  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? 'unknown'
+  const documentId =
+    stringFrom(raw.context_id) ??
+    stringFrom(raw.document_id) ??
+    stringFrom(raw.doc_id) ??
+    stringFrom(raw.file_name) ??
+    'unknown'
+  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`
+  const meta: T2RagBenchMeta = {
+    benchmark: 't2-ragbench',
+    query,
+    goldAnswers,
+    contexts,
+    subset,
+    documentId,
+  }
+  return {
+    id,
+    split: stringFrom(raw.split) ?? subset,
+    prompt: [
+      'Answer this T2-RAGBench text-and-table financial question.',
+      'Do the required numerical reasoning from the supplied context before giving the final value.',
+      'End with a single final line: `FINAL ANSWER: <answer>`.',
+      '',
+      `Question: ${query}`,
+      `Document: ${documentId}`,
+      `Subset: ${subset}`,
+      contexts.length > 0 ? `\nContext:\n${contextBlock(contexts)}` : undefined,
+    ]
+      .filter(Boolean)
+      .join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): T2RagBenchMeta {
+  const md = task.metadata
+  if (!md || !Array.isArray(md.goldAnswers)) {
+    throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as T2RagBenchMeta
+}
+
+async function loadRows(path: string): Promise<unknown[]> {
+  const rows = await readJsonRows(path)
+  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`)
+  return rows
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]
+  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)
+  return selectTasks(rows.map(rowToTask), opts, 'T2-RAGBench')
+}
+
+export function createT2RagBenchAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === '1'
+
+  return {
+    name: 't2-ragbench',
+    output: ragAnswerOutput,
+
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, 'utf8')
+        return
+      }
+      const path = dataFile()
+      if (!path) {
+        throw new Error(
+          'T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.',
+        )
+      }
+      await loadRows(path)
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const path = dataFile()
+      if (!path) throw new Error('T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks')
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'T2-RAGBench')
+    },
+
+    async goldArtifact(task: BenchTask) {
+      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 })
+      return answerScoreToBenchScore(score, {
+        benchmark: meta.benchmark,
+        subset: meta.subset,
+        documentId: meta.documentId,
+        contextCount: meta.contexts.length,
+      })
+    },
+  }
+}

package/src/benchmarks/tau-bench-shared.ts

@@ -0,0 +1,214 @@
+/**
+ * Shared tau-bench adapter spine.
+ *
+ * tau2 and tau3 live in the same upstream repository/package namespace today:
+ * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation
+ * for task loading and reward recomputation so the domain/version adapters only
+ * choose env names, default domain, and fixture file.
+ */
+
+import { readFile, stat } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import { runVenvPython } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+export interface TauBenchConfig {
+  name: string
+  fixturePath: string
+  fixturesEnv: string
+  dirEnv: string
+  domainEnv: string
+  defaultDomain: string
+  installHint: string
+  taskIntro: string
+}
+
+interface TauRow {
+  id: string
+  domain: string
+  user_scenario?: unknown
+  description?: unknown
+  evaluation_criteria?: unknown
+}
+
+interface TauMeta {
+  taskId: string
+  domain: string
+  split?: string
+  userScenario?: unknown
+  description?: unknown
+  evaluationCriteria?: unknown
+}
+
+export const tauResultsOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    const fences = [...text.matchAll(/```(?:text|path|json)?\s*\n([\s\S]*?)```/g)]
+    return (fences.at(-1)?.[1] ?? text).trim()
+  },
+}
+
+async function assertPath(path: string, label: string, benchName: string): Promise<void> {
+  try {
+    await stat(path)
+  } catch (err) {
+    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)
+  }
+}
+
+function benchDir(config: TauBenchConfig): string | undefined {
+  return process.env[config.dirEnv]
+}
+
+function benchDomain(config: TauBenchConfig): string {
+  return process.env[config.domainEnv] ?? config.defaultDomain
+}
+
+function rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {
+  const meta: TauMeta = {
+    taskId: row.id,
+    domain: row.domain,
+    split,
+    userScenario: row.user_scenario,
+    description: row.description,
+    evaluationCriteria: row.evaluation_criteria,
+  }
+  return {
+    id: row.id,
+    split,
+    prompt: [
+      config.taskIntro,
+      `Run this task in the official ${row.domain} domain.`,
+      'The benchmark is a simulated multi-turn user/tool conversation.',
+      '',
+      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),
+      '',
+      'Return the path to the official tau results.json or trajectory file containing this task run.',
+    ].join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask, benchName: string): TauMeta {
+  const md = task.metadata
+  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {
+    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as TauMeta
+}
+
+function selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {
+  let tasks = rows.map((row) => rowToTask(row, config, split))
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((task) => want.has(task.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)
+  return tasks
+}
+
+async function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]
+  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)
+  return selectRows(rows, opts, config, opts.split)
+}
+
+async function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {
+  const domain = benchDomain(config)
+  const script = `
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1])
+domain = sys.argv[2]
+split = sys.argv[3] or None
+sys.path.insert(0, str(root / "src"))
+from tau2.registry import registry
+loader = registry.get_tasks_loader(domain)
+tasks = loader(split)
+rows = []
+for task in tasks:
+    row = task.model_dump(mode="json")
+    row["domain"] = domain
+    rows.append(row)
+print(json.dumps(rows))
+`
+  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])
+  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)
+}
+
+async function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {
+  const script = `
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1])
+task_id = sys.argv[2]
+artifact = Path(sys.argv[3])
+sys.path.insert(0, str(root / "src"))
+from tau2.data_model.simulation import Results
+from tau2.scripts.evaluate_trajectories import compute_simulation_rewards
+results = Results.load(artifact)
+updated = compute_simulation_rewards(results)
+scores = []
+for sim in updated.simulations:
+    if sim.task_id == task_id and sim.reward_info is not None:
+        scores.append(float(sim.reward_info.reward))
+if not scores:
+    raise SystemExit(f"no scored simulations for task_id={task_id} in {artifact}")
+print(json.dumps({"count": len(scores), "score": sum(scores) / len(scores), "scores": scores}))
+`
+  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)
+  return JSON.parse(stdout.trim().split('\n').at(-1) ?? '{}') as Record<string, unknown>
+}
+
+export function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {
+  const fixturesMode = process.env[config.fixturesEnv] === '1'
+
+  return {
+    name: config.name,
+    output: tauResultsOutput,
+
+    async preflight() {
+      if (fixturesMode) return
+      const dir = benchDir(config)
+      if (!dir) {
+        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)
+      }
+      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)
+      await loadOfficialTasks(config, dir, { limit: 1 })
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(config, opts)
+      const dir = benchDir(config)
+      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)
+      return loadOfficialTasks(config, dir, opts)
+    },
+
+    async goldArtifact() {
+      return undefined
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const dir = benchDir(config)
+      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)
+      const meta = readMeta(task, config.name)
+      const artifactPath = resolve(artifact.trim())
+      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)
+      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)
+      const score = typeof report.score === 'number' ? report.score : 0
+      return {
+        resolved: score === 1,
+        score,
+        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),
+      }
+    },
+  }
+}

package/src/benchmarks/tau2-bench.ts

@@ -0,0 +1,30 @@
+/**
+ * tau2-bench adapter (Sierra tau2/tau-bench successor).
+ *
+ * Worker artifact = a tau2 `results.json`/trajectory file. Judge = tau2's own
+ * reward recomputation over that trajectory. A normal final-answer transcript is
+ * not a valid artifact for this benchmark.
+ */
+
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import { createTauBenchAdapter, tauResultsOutput } from './tau-bench-shared'
+import type { BenchmarkAdapter } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'tau2-bench.json')
+
+export const tau2ResultsOutput = tauResultsOutput
+
+export function createTau2BenchAdapter(): BenchmarkAdapter {
+  return createTauBenchAdapter({
+    name: 'tau2-bench',
+    fixturePath: FIXTURES,
+    fixturesEnv: 'TAU2_FIXTURES',
+    dirEnv: 'TAU2_BENCH_DIR',
+    domainEnv: 'TAU2_DOMAIN',
+    defaultDomain: 'retail',
+    taskIntro: 'Run this tau2 task in the official tau2 text benchmark.',
+    installHint:
+      'clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench.',
+  })
+}