@tangle-network/agent-bench 0.1.0 → 0.3.5

package/src/benchmarks/dabstep.ts

@@ -0,0 +1,212 @@
+/**
+ * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic
+ * payment files. Worker artifact = final answer text. Judge = the official
+ * DABStep `grade.py` normalization/matching function. No LLM judge.
+ *
+ * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that
+ * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter
+ * exposes `metadata.resourceRoot` so runners can mount the benchmark files into
+ * AgentProfile.resources.files; it does not paste the dataset into prompt text.
+ */
+
+import { join } from 'node:path'
+import { access, readFile, stat } from 'node:fs/promises'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'dabstep.json')
+const DEFAULT_SPLIT = 'easy'
+
+interface DabstepFixtureRow {
+  task_id: number
+  instructions: string
+  all_golds_by_task: Array<Record<string, unknown>>
+}
+
+interface DabstepMeta {
+  taskId: number
+  split: string
+  golds: Array<Record<string, unknown>>
+  resourceRoot?: string
+}
+
+const dabstepDir = (): string | undefined => process.env.DABSTEP_DIR
+const gradeFile = (dir: string): string => join(dir, 'grade.py')
+const resourceRoot = (dir: string): string => join(dir, 'files')
+
+async function assertFile(path: string, label: string): Promise<void> {
+  try {
+    await access(path)
+  } catch (err) {
+    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)
+  }
+}
+
+async function assertOfficialFiles(dir: string, split: string): Promise<void> {
+  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')
+  await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)
+  await assertFile(gradeFile(dir), 'official grade.py')
+  const files = resourceRoot(dir)
+  try {
+    const s = await stat(files)
+    if (!s.isDirectory()) throw new Error('not a directory')
+  } catch (err) {
+    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`)
+  }
+}
+
+export const dabstepAnswerOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    const fences = [...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)]
+    return (fences.at(-1)?.[1] ?? text).trim()
+  },
+}
+
+function rowToTask(row: DabstepFixtureRow, split: string, dir?: string): BenchTask {
+  const meta: DabstepMeta = {
+    taskId: row.task_id,
+    split,
+    golds: row.all_golds_by_task,
+    ...(dir ? { resourceRoot: resourceRoot(dir) } : {}),
+  }
+  return {
+    id: String(row.task_id),
+    split,
+    prompt: [
+      'Solve this DABStep data-analysis task using the mounted payment files.',
+      'Use code or shell commands as needed, then return only the final answer.',
+      '',
+      row.instructions,
+    ].join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): DabstepMeta {
+  const md = task.metadata
+  if (!md || typeof md.taskId !== 'number' || !Array.isArray(md.golds)) {
+    throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as DabstepMeta
+}
+
+function selectRows(rows: DabstepFixtureRow[], opts: LoadOptions, split: string, dir?: string): BenchTask[] {
+  let tasks = rows.map((row) => rowToTask(row, split, dir))
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((task) => want.has(task.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`)
+  return tasks
+}
+
+async function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as DabstepFixtureRow[]
+  console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`)
+  return selectRows(rows, opts, split)
+}
+
+async function loadOfficialTasks(dir: string, opts: LoadOptions, split: string): Promise<BenchTask[]> {
+  const script = `
+import ast, csv, json, sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+split = sys.argv[2]
+limit = None if sys.argv[3] == "" else int(sys.argv[3])
+ids = set(json.loads(sys.argv[4]))
+dataset = root / "dataset.csv"
+split_file = root / "splits" / f"{split}.txt"
+if not dataset.exists():
+    raise SystemExit(f"missing official DABStep dataset.csv at {dataset}")
+if not split_file.exists():
+    raise SystemExit(f"missing official DABStep split file at {split_file}")
+split_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}
+out = []
+with dataset.open(newline="") as f:
+    for row in csv.DictReader(f):
+        task_id = int(row["task_id"])
+        if task_id not in split_ids:
+            continue
+        if ids and str(task_id) not in ids:
+            continue
+        out.append({
+            "task_id": task_id,
+            "instructions": f"{row['question']}\\n{row['guidelines']}",
+            "all_golds_by_task": ast.literal_eval(str(row["all_golds_by_task"])),
+        })
+        if limit is not None and len(out) >= limit:
+            break
+if not out:
+    raise SystemExit(f"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}")
+print(json.dumps(out))
+`
+  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])
+  return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)
+}
+
+export function createDabstepAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.DABSTEP_FIXTURES === '1'
+
+  return {
+    name: 'dabstep',
+    output: dabstepAnswerOutput,
+
+    async preflight() {
+      if (fixturesMode) return
+      const dir = dabstepDir()
+      if (!dir) {
+        throw new Error(
+          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',
+        )
+      }
+      await assertOfficialFiles(dir, DEFAULT_SPLIT)
+      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT)
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      const split = opts.split ?? DEFAULT_SPLIT
+      if (fixturesMode) return loadFixtures(opts, split)
+      const dir = dabstepDir()
+      if (!dir) throw new Error('DABSTEP_DIR is required to load official DABStep tasks')
+      return loadOfficialTasks(dir, opts, split)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      const meta = readMeta(task)
+      const first = meta.golds[0]
+      if (!first) return undefined
+      const value = first.value
+      return value === undefined ? undefined : String(value)
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const dir = dabstepDir()
+      if (!dir) throw new Error('DABSTEP_DIR is required to judge DABStep tasks with the official grade.py')
+      const stdout = await runVenvScriptStdin(
+        join(benchRoot, 'scripts', 'dabstep_judge.py'),
+        ['--grade-file', gradeFile(dir)],
+        JSON.stringify({ prediction: artifact, golds: meta.golds }),
+        { cwd: benchRoot },
+      )
+      const report = JSON.parse(stdout.trim().split('\n').at(-1) ?? '{}') as { correct?: boolean; score?: number; error?: string }
+      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`)
+      const score = typeof report.score === 'number' ? report.score : 0
+      return {
+        resolved: report.correct === true,
+        score,
+        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct }),
+      }
+    },
+  }
+}

package/src/benchmarks/external-adapters.test.mts

@@ -0,0 +1,150 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { createAgentBenchAdapter } from './agentbench'
+import { createBfclAdapter } from './bfcl'
+import { createFinResearchBenchAdapter } from './finresearchbench'
+import { createTau2BenchAdapter } from './tau2-bench'
+import { createTau3BankingAdapter } from './tau3-banking'
+import { createToolLlmAdapter } from './toollm'
+import { createWebArenaVerifiedAdapter } from './webarena-verified'
+
+async function withEnv<T>(patch: Record<string, string | undefined>, fn: () => Promise<T>): Promise<T> {
+  const old: Record<string, string | undefined> = {}
+  for (const [key, value] of Object.entries(patch)) {
+    old[key] = process.env[key]
+    if (value === undefined) delete process.env[key]
+    else process.env[key] = value
+  }
+  try {
+    return await fn()
+  } finally {
+    for (const [key, value] of Object.entries(old)) {
+      if (value === undefined) delete process.env[key]
+      else process.env[key] = value
+    }
+  }
+}
+
+test('WebArena-Verified fixture mode loads tasks and live mode fails loud without checkout', async () => {
+  await withEnv({ WEBARENA_VERIFIED_FIXTURES: '1', WEBARENA_VERIFIED_DIR: undefined }, async () => {
+    const adapter = createWebArenaVerifiedAdapter()
+    await adapter.preflight()
+    const tasks = await adapter.loadTasks({ limit: 1 })
+    assert.equal(tasks.length, 1)
+    assert.equal(tasks[0].id, '0')
+    assert.match(tasks[0].prompt, /WebArena-Verified/)
+  })
+
+  await withEnv({ WEBARENA_VERIFIED_FIXTURES: undefined, WEBARENA_VERIFIED_DIR: undefined }, async () => {
+    await assert.rejects(() => createWebArenaVerifiedAdapter().preflight(), /WEBARENA_VERIFIED_DIR is required/)
+  })
+})
+
+test('tau2-bench fixture mode loads tasks and live mode fails loud without checkout', async () => {
+  await withEnv({ TAU2_FIXTURES: '1', TAU2_BENCH_DIR: undefined }, async () => {
+    const adapter = createTau2BenchAdapter()
+    await adapter.preflight()
+    const tasks = await adapter.loadTasks({ limit: 1 })
+    assert.equal(tasks.length, 1)
+    assert.equal(tasks[0].id, 'retail-fixture-0')
+    assert.match(tasks[0].prompt, /tau2 task/)
+  })
+
+  await withEnv({ TAU2_FIXTURES: undefined, TAU2_BENCH_DIR: undefined }, async () => {
+    await assert.rejects(() => createTau2BenchAdapter().preflight(), /TAU2_BENCH_DIR is required/)
+  })
+})
+
+test('tau3-banking fixture mode loads tasks and live mode fails loud without checkout', async () => {
+  await withEnv({ TAU3_FIXTURES: '1', TAU3_BENCH_DIR: undefined }, async () => {
+    const adapter = createTau3BankingAdapter()
+    await adapter.preflight()
+    const tasks = await adapter.loadTasks({ limit: 1 })
+    assert.equal(tasks.length, 1)
+    assert.equal(tasks[0].id, 'banking-knowledge-fixture-0')
+    assert.match(tasks[0].prompt, /tau3 banking task/)
+  })
+
+  await withEnv({ TAU3_FIXTURES: undefined, TAU3_BENCH_DIR: undefined }, async () => {
+    await assert.rejects(() => createTau3BankingAdapter().preflight(), /TAU3_BENCH_DIR is required/)
+  })
+})
+
+test('AgentBench DBBench fixture mode loads and exact-label judge scores deterministically', async () => {
+  await withEnv({ AGENTBENCH_FIXTURES: '1', AGENTBENCH_DIR: undefined }, async () => {
+    const adapter = createAgentBenchAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    assert.equal(await adapter.goldArtifact(task), 'Women +60kg Bronze')
+    assert.equal((await adapter.judge(task, 'women +60kg bronze')).score, 1)
+    assert.equal((await adapter.judge(task, 'wrong')).score, 0)
+  })
+
+  await withEnv({ AGENTBENCH_FIXTURES: undefined, AGENTBENCH_DIR: undefined }, async () => {
+    await assert.rejects(() => createAgentBenchAdapter().preflight(), /AGENTBENCH_DIR is required/)
+  })
+})
+
+test('BFCL fixture mode loads official-shaped rows and scores function calls', async () => {
+  await withEnv({ BFCL_FIXTURES: '1', BFCL_DIR: undefined }, async () => {
+    const adapter = createBfclAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    assert.equal(task.id, 'simple_python_fixture_0')
+    const gold = await adapter.goldArtifact(task)
+    assert.match(gold ?? '', /calculate_triangle_area/)
+    assert.equal((await adapter.judge(task, gold ?? '')).score, 1)
+    assert.equal((await adapter.judge(task, '{"function_calls":[{"name":"wrong","arguments":{}}]}')).score, 0)
+  })
+
+  await withEnv({ BFCL_FIXTURES: undefined, BFCL_DIR: undefined }, async () => {
+    await assert.rejects(() => createBfclAdapter().preflight(), /BFCL_DIR is required/)
+  })
+})
+
+test('ToolLLM scores only deterministic API-selection labels and rejects unlabeled rows', async () => {
+  await withEnv({ TOOLLM_FIXTURES: '1', TOOLBENCH_DIR: undefined }, async () => {
+    const adapter = createToolLlmAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    assert.equal(task.id, '1')
+    const gold = await adapter.goldArtifact(task)
+    assert.match(gold ?? '', /Checkhealth/)
+    assert.equal((await adapter.judge(task, '')).score, 0)
+    assert.equal((await adapter.judge(task, gold ?? '')).score, 1)
+    assert.equal((await adapter.judge(task, '{"api_calls":[{"tool_name":"SQUAKE","api_name":"Checkhealth"}]}')).score, 0.5)
+    const freeText = await adapter.judge(task, 'Use SQUAKE Checkhealth, then call SQUAKE Projects.')
+    assert.equal(freeText.score, 1)
+    assert.equal(freeText.resolved, false)
+  })
+
+  const dir = await mkdtemp(join(tmpdir(), 'toollm-'))
+  try {
+    const query = join(dir, 'queries.json')
+    await writeFile(query, JSON.stringify([{ query_id: 1, query: 'Call a tool.', api_list: [] }]))
+    await withEnv({ TOOLLM_FIXTURES: undefined, TOOLBENCH_DIR: dir, TOOLLM_QUERY_FILE: query }, async () => {
+      await assert.rejects(() => createToolLlmAdapter().preflight(), /deterministic API-selection labels missing/)
+    })
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+})
+
+test('FinResearchBench fixture mode is explicit and live mode requires exported judge rows', async () => {
+  await withEnv({ FINRESEARCHBENCH_FIXTURES: '1', FINRESEARCHBENCH_DATA_FILE: undefined }, async () => {
+    const adapter = createFinResearchBenchAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    const gold = await adapter.goldArtifact(task)
+    assert.match(gold ?? '', /Margin expansion/)
+    assert.equal((await adapter.judge(task, gold ?? '')).score, 1)
+    assert.equal((await adapter.judge(task, 'unrelated answer')).score, 0)
+  })
+
+  await withEnv({ FINRESEARCHBENCH_FIXTURES: undefined, FINRESEARCHBENCH_DATA_FILE: undefined }, async () => {
+    await assert.rejects(() => createFinResearchBenchAdapter().preflight(), /FINRESEARCHBENCH_DATA_FILE is required/)
+  })
+})

package/src/benchmarks/finresearchbench.ts

@@ -0,0 +1,269 @@
+/**
+ * FinResearchBench-compatible adapter.
+ *
+ * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial
+ * research reports, but there is no stable public scorer package wired here.
+ * Live mode therefore requires a local data export whose rows carry the official
+ * judge prompt/template/logic tree. The adapter refuses to invent a judge.
+ */
+
+import { readFile, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')
+
+interface FinResearchRecord {
+  id: string
+  question: string
+  category?: string
+  reference_answer?: string
+  reference_report?: string
+  logic_tree?: unknown
+  rubric?: unknown
+  judge_system_prompt?: string
+  judge_prompt_template?: string
+}
+
+interface FinResearchMeta {
+  id: string
+  category: string
+  question: string
+  referenceAnswer: string
+  referenceReport: string
+  logicTree: unknown
+  rubric: unknown
+  judgeSystemPrompt?: string
+  judgePromptTemplate?: string
+  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'
+}
+
+const dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE
+
+function routerConfig(): { baseUrl: string; key: string; model: string } {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')
+  return {
+    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+    key,
+    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',
+  }
+}
+
+async function assertReadable(path: string, label: string): Promise<void> {
+  try {
+    await stat(path)
+  } catch (err) {
+    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)
+  }
+}
+
+function readRecords(raw: string): FinResearchRecord[] {
+  const trimmed = raw.trim()
+  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]
+  return trimmed
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .map((line) => JSON.parse(line) as FinResearchRecord)
+}
+
+function assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {
+  const missing = records
+    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)
+    .map((row) => row.id)
+  if (missing.length > 0) {
+    throw new Error(
+      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +
+        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',
+    )
+  }
+}
+
+function rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {
+  const referenceAnswer = row.reference_answer ?? ''
+  const referenceReport = row.reference_report ?? referenceAnswer
+  const meta: FinResearchMeta = {
+    id: row.id,
+    category: row.category ?? 'unknown',
+    question: row.question,
+    referenceAnswer,
+    referenceReport,
+    logicTree: row.logic_tree ?? null,
+    rubric: row.rubric ?? null,
+    judgeSystemPrompt: row.judge_system_prompt,
+    judgePromptTemplate: row.judge_prompt_template,
+    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',
+  }
+  return {
+    id: row.id,
+    split: row.category,
+    prompt: [
+      'Complete this FinResearchBench financial research task.',
+      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',
+      '',
+      row.question,
+    ].join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): FinResearchMeta {
+  const md = task.metadata
+  if (!md || typeof md.question !== 'string') {
+    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as FinResearchMeta
+}
+
+function selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {
+  let tasks = rows.map((row) => rowToTask(row, fixturesMode))
+  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((task) => want.has(task.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)
+  return tasks
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const records = readRecords(await readFile(FIXTURES, 'utf8'))
+  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)
+  return selectRows(records, opts, true)
+}
+
+async function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {
+  const records = readRecords(await readFile(path, 'utf8'))
+  assertLiveJudgeFields(records, path)
+  return selectRows(records, opts, false)
+}
+
+function fillTemplate(meta: FinResearchMeta, response: string): string {
+  const template = meta.judgePromptTemplate
+  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)
+  return template
+    .replaceAll('{question}', meta.question)
+    .replaceAll('{response}', response)
+    .replaceAll('{reference_answer}', meta.referenceAnswer)
+    .replaceAll('{reference_report}', meta.referenceReport)
+    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))
+    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))
+}
+
+function parseJudgeScore(content: string): { score: number; raw: unknown } {
+  const candidates: string[] = []
+  for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim())
+  for (const m of content.matchAll(/\{[\s\S]*?\}/g)) candidates.push(m[0])
+  candidates.push(content.trim())
+  for (const candidate of candidates) {
+    let parsed: Record<string, unknown>
+    try {
+      parsed = JSON.parse(candidate) as Record<string, unknown>
+    } catch {
+      continue
+    }
+    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score
+    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN
+    if (!Number.isFinite(n)) continue
+    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)
+    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100
+    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)
+    return { score, raw: parsed }
+  }
+  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)
+}
+
+async function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {
+  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)
+  const router = routerConfig()
+  const res = await fetch(`${router.baseUrl}/chat/completions`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
+    body: JSON.stringify({
+      model: router.model,
+      temperature: 0,
+      messages: [
+        { role: 'system', content: meta.judgeSystemPrompt },
+        { role: 'user', content: fillTemplate(meta, response) },
+      ],
+    }),
+  })
+  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  const content = body.choices?.[0]?.message?.content
+  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)
+  const { score, raw } = parseJudgeScore(content)
+  return {
+    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),
+    score,
+    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),
+  }
+}
+
+function normalizeText(value: string): string {
+  return value.toLowerCase().replace(/\s+/g, ' ').trim()
+}
+
+function scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {
+  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)
+  const response = normalizeText(artifact)
+  const score = answer.length > 0 && response.includes(answer) ? 1 : 0
+  return {
+    resolved: score === 1,
+    score,
+    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),
+  }
+}
+
+export function createFinResearchBenchAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'
+
+  return {
+    name: 'finresearchbench',
+
+    async preflight() {
+      if (fixturesMode) {
+        await assertReadable(FIXTURES, 'fixture file')
+        return
+      }
+      const file = dataFile()
+      if (!file) {
+        throw new Error(
+          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',
+        )
+      }
+      routerConfig()
+      await assertReadable(file, 'data file')
+      await loadOfficialTasks(file, { limit: 1 })
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const file = dataFile()
+      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')
+      return loadOfficialTasks(file, opts)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      const meta = readMeta(task)
+      return meta.referenceReport || meta.referenceAnswer || undefined
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      if (fixturesMode) return scoreFixture(meta, artifact)
+      if (artifact.trim().length === 0) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),
+        }
+      }
+      return runOfficialJudge(meta, artifact)
+    },
+  }
+}

package/src/benchmarks/humaneval.ts

@@ -17,7 +17,7 @@
  */
 
 import { execFile } from 'node:child_process'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { gunzipSync } from 'node:zlib'
@@ -41,9 +41,17 @@ export interface HumanEvalTask {
  *  selects a deeper slice (the later tasks are harder) so the worker has a
  *  correctable middle band rather than a saturated easy prefix. */
 export async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {
-  const res = await fetch(humanevalUrl)
-  if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
-  const gz = Buffer.from(await res.arrayBuffer())
+  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits
+  // (429) under repeated runs. Fall back to the network fetch when unset.
+  const localGz = process.env.HUMANEVAL_GZ
+  let gz: Buffer
+  if (localGz) {
+    gz = readFileSync(localGz)
+  } else {
+    const res = await fetch(humanevalUrl)
+    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
+    gz = Buffer.from(await res.arrayBuffer())
+  }
   const text = gunzipSync(gz).toString('utf8')
   const tasks: HumanEvalTask[] = []
   for (const line of text.split('\n')) {
@@ -242,10 +250,14 @@ export function createHumanEvalAdapter(): BenchmarkAdapter {
       return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }
     },
     async goldArtifact(task: BenchTask) {
-      const sol = (task.metadata as HumanEvalMeta | undefined)?.canonicalSolution
-      // The canonical solution is the function BODY; the program header supplies the
-      // signature, so the body alone runs against check().
-      return sol ? sol : undefined
+      const m = task.metadata as HumanEvalMeta | undefined
+      const sol = m?.canonicalSolution
+      // Return the COMPLETE function (signature header + canonical body), i.e. what a
+      // real worker emits — NOT the body alone. The judge runs `extractCode`, whose
+      // unfenced fallback is `reply.trim()`; trimming a body-only string strips its
+      // leading indent and breaks it, so a body-only gold fails its own judge. A full
+      // def starts at column 0, trims safely, and self-verifies.
+      return sol ? `${m!.promptHeader}${sol}` : undefined
     },
   }
 }