@tangle-network/agent-bench 0.1.0

package/src/benchmarks/simpleqa.ts

@@ -0,0 +1,390 @@
+/**
+ * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact
+ * = a single free-text final answer string (optionally preceded by a CITATIONS:
+ * block). Each item ships a short gold `answer` plus metadata (topic,
+ * answer_type, source urls).
+ *
+ * Judge is the official SimpleQA grader — an LLM classifier that maps
+ * (question, gold target, predicted answer) to exactly one of:
+ *   A = CORRECT       — fully contains the gold, no contradiction
+ *   B = INCORRECT     — contradicts / contains a different factual value
+ *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given
+ * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter
+ * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from
+ * INCORRECT so the scorecard can separate abstention from error.
+ *
+ * There is no deterministic tier: SimpleQA's rubric (containment + abstention)
+ * is the grader's job by design, so judge() always calls the pinned grader model
+ * (temperature 0) and fails loud on unparseable grader output. The final-answer
+ * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').
+ *
+ * Requires for a live run: the bench `.venv` with `datasets`/`requests` not
+ * needed — the test set is a single public CSV fetched over HTTP — plus a
+ * grader key (TANGLE_API_KEY). For offline/CI verification set
+ * SIMPLEQA_FIXTURES=1 to load the committed fixtures
+ * (bench/fixtures/simpleqa.json) — no network.
+ */
+
+import { execFile } from 'node:child_process'
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const execFileAsync = promisify(execFile)
+const BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const PY = join(BENCH_ROOT, '.venv', 'bin', 'python')
+const FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')
+
+const DATASET_URL =
+  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'
+const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'
+
+/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */
+const WORKER_CONTRACT = [
+  '',
+  'Research the question using live web sources and answer it with a short, specific factual value.',
+  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',
+  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+  'The answer after the sentinel must be the bare value only (no explanation on that line).',
+  'If you do not know the answer, state that you do not know rather than guessing.',
+].join('\n')
+
+/**
+ * Typed seam for the research worker. The benchmark adapter scores a plain
+ * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its
+ * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an
+ * optional `CITATIONS:` block) into that string before judging.
+ */
+export interface ResearchTask {
+  id: string
+  question: string
+  /** Gold short answer — the grader's target. */
+  gold: string
+  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */
+  goldSources: string[]
+  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */
+  topic: string
+  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */
+  answerType: string
+}
+
+export interface ResearchAnswer {
+  finalAnswer: string
+  citations: string[]
+}
+
+/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */
+interface SimpleQaRow {
+  problem: string
+  answer: string
+  topic: string
+  answer_type: string
+  urls: string[]
+}
+
+interface SimpleQaMeta {
+  gold: string
+  goldSources: string[]
+  topic: string
+  answerType: string
+  question: string
+}
+
+/** The three official SimpleQA grades. */
+type Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'
+
+/** Run the bench venv python with a script; return stdout (throws on nonzero). */
+async function py(script: string, args: string[] = []): Promise<string> {
+  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {
+    maxBuffer: 1024 * 1024 * 256,
+  })
+  return stdout
+}
+
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the last non-empty line. Returns '' when nothing is parseable
+ * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.
+ */
+export function parseFinalAnswer(artifact: string): string {
+  const lines = artifact.split(/\r?\n/)
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const line = lines[i] ?? ''
+    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)
+    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()
+  }
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const t = (lines[i] ?? '').trim()
+    if (t.length > 0) return t
+  }
+  return ''
+}
+
+/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */
+export function parseCitations(artifact: string): string[] {
+  const urls = new Set<string>()
+  for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
+    urls.add(m[0].replace(/[.,;]+$/, ''))
+  }
+  return [...urls]
+}
+
+/**
+ * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's
+ * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the
+ * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted
+ * answer must fully contain the gold value with no contradicting statement;
+ * hedged or non-committal answers are NOT_ATTEMPTED.
+ */
+const GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>
+  [
+    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',
+    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',
+    'First, follow these grading rules carefully.',
+    '',
+    'CORRECT:',
+    '  The predicted answer fully contains the gold target without contradicting it.',
+    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',
+    '  Hedging that still commits to the gold value (e.g. "I think it is X" where X is the gold) is CORRECT.',
+    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',
+    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',
+    '',
+    'INCORRECT:',
+    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',
+    '  A prediction that states a different factual value than the gold is INCORRECT.',
+    '',
+    'NOT_ATTEMPTED:',
+    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',
+    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',
+    '',
+    'Also note the following rules:',
+    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',
+    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',
+    '',
+    `Question: ${question}`,
+    `Gold target: ${gold}`,
+    `Predicted answer: ${predicted}`,
+    '',
+    'Respond with ONLY a fenced JSON block and nothing else:',
+    '```json',
+    '{"grade": "CORRECT" | "INCORRECT" | "NOT_ATTEMPTED"}',
+    '```',
+  ].join('\n')
+
+interface GraderRouter {
+  baseUrl: string
+  key: string
+  model: string
+}
+
+function graderRouter(): GraderRouter {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')
+  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'
+  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  return { baseUrl, key, model }
+}
+
+/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */
+async function gradeAnswer(
+  question: string,
+  gold: string,
+  predicted: string,
+  router: GraderRouter,
+): Promise<Grade> {
+  const res = await fetch(`${router.baseUrl}/chat/completions`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },
+    body: JSON.stringify({
+      model: router.model,
+      temperature: 0,
+      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],
+    }),
+  })
+  if (!res.ok) {
+    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+  }
+  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  const content = body.choices?.[0]?.message?.content
+  if (typeof content !== 'string') {
+    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)
+  }
+  const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/)
+  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''
+  let parsed: { grade?: unknown }
+  try {
+    parsed = JSON.parse(raw) as { grade?: unknown }
+  } catch {
+    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)
+  }
+  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {
+    return parsed.grade
+  }
+  throw new Error(
+    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,
+  )
+}
+
+function rowToTask(row: SimpleQaRow, index: number): BenchTask {
+  const meta: SimpleQaMeta = {
+    gold: row.answer,
+    goldSources: row.urls,
+    topic: row.topic,
+    answerType: row.answer_type,
+    question: row.problem,
+  }
+  return {
+    id: `simpleqa-${index}`,
+    split: 'test',
+    prompt: row.problem + WORKER_CONTRACT,
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): SimpleQaMeta {
+  const md = task.metadata
+  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {
+    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)
+  }
+  return md as unknown as SimpleQaMeta
+}
+
+function selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    return tasks.filter((t) => want.has(t.id))
+  }
+  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)
+  return tasks
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]
+  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)
+  return selectTasks(rows.map(rowToTask), opts)
+}
+
+/**
+ * Load the live test set: fetch the public CSV via the bench venv python, parse
+ * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for
+ * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape
+ * the fixtures use, so loadTasks/judge share one parse path.
+ */
+async function loadLive(opts: LoadOptions): Promise<BenchTask[]> {
+  const script = `
+import csv, ast, json, io, sys
+from urllib.request import urlopen
+with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
+    text = resp.read().decode('utf-8')
+rows = list(csv.DictReader(io.StringIO(text)))
+out = []
+for r in rows:
+    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}
+    out.append({
+        'problem': r.get('problem', ''),
+        'answer': r.get('answer', ''),
+        'topic': str(meta.get('topic', '')),
+        'answer_type': str(meta.get('answer_type', '')),
+        'urls': list(meta.get('urls', [])),
+    })
+print(json.dumps(out))
+`
+  const stdout = await py(script)
+  const rows = JSON.parse(stdout) as SimpleQaRow[]
+  return selectTasks(rows.map(rowToTask), opts)
+}
+
+export function createSimpleQaAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'
+
+  return {
+    name: 'simpleqa',
+
+    async preflight() {
+      // The grader router must be configured in both modes — SimpleQA's score is
+      // defined by the grader, so a run without it is meaningless.
+      graderRouter()
+      if (fixturesMode) {
+        await readFile(FIXTURES, 'utf8').catch((err) => {
+          throw new Error(
+            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,
+          )
+        })
+        return
+      }
+      try {
+        await py(
+          `from urllib.request import urlopen
+with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
+    head = resp.read(64).decode('utf-8', 'replace')
+assert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]
+print('ok')`,
+        )
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        throw new Error(
+          `simpleqa preflight failed: ${msg}\n` +
+            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +
+            `(2) ensure network access to ${DATASET_URL} ; ` +
+            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,
+        )
+      }
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      return loadLive(opts)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      // Gold artifact = the worker-contract serialization of the gold answer, so
+      // verify-judge proves gold→CORRECT through the SAME parse path the real
+      // artifact takes.
+      const meta = readMeta(task)
+      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const finalAnswer = parseFinalAnswer(artifact)
+      const citations = parseCitations(artifact)
+
+      if (finalAnswer.length === 0) {
+        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({
+            grade: 'NOT_ATTEMPTED',
+            gradeLetter: 'C',
+            reason: 'no parseable answer',
+            gold: meta.gold,
+            topic: meta.topic,
+            answerType: meta.answerType,
+            citationCount: citations.length,
+          }),
+        }
+      }
+
+      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())
+      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'
+      const resolved = grade === 'CORRECT'
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({
+          grade,
+          gradeLetter,
+          gold: meta.gold,
+          predicted: finalAnswer,
+          topic: meta.topic,
+          answerType: meta.answerType,
+          citationCount: citations.length,
+        }),
+      }
+    },
+  }
+}

package/src/benchmarks/swe-bench.ts

@@ -0,0 +1,152 @@
+/**
+ * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =
+ * the official `swebench` harness: apply the patch in the instance's Docker
+ * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic
+ * — no LLM judge.
+ *
+ * Requires: the bench `.venv` with `swebench` installed + a running Docker
+ * daemon (per-instance images are pulled/built on first run).
+ *
+ * Process/Docker/report plumbing is shared via ./_harness; this file owns the
+ * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the
+ * predictions-file → run_evaluation argv → report-shape mapping.
+ */
+
+import { join } from 'node:path'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import {
+  preflightVenvImports,
+  readJsonReport,
+  runStagedJudge,
+  runVenvPython,
+  safeRunId,
+  stageFile,
+} from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+/**
+ * The SWE deliverable, extracted from the agent's event STREAM (not the box FS).
+ * `runLoop`'s `OutputAdapter` only sees events, so the agent prints its unified
+ * diff in a fenced block and this pulls the last one out — the seam that lets the
+ * SWE benchmark run through the gate runner (`runGate` / `runBenchmark`)
+ * like any other.
+ */
+export const swePatchOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    // Last ```diff/```patch fenced block (the contract the prompt asks for);
+    // fall back to the raw text so a fence-less but valid diff still reaches the judge.
+    const fences = [...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)]
+    const last = fences.at(-1)?.[1]
+    return (last ?? text).trim()
+  },
+}
+
+const DATASET = 'princeton-nlp/SWE-bench_Verified'
+
+interface SweReport {
+  resolved_instances?: number
+  resolved_ids?: string[]
+}
+
+export function createSweBenchAdapter(): BenchmarkAdapter {
+  return {
+    name: 'swe-bench-verified',
+    output: swePatchOutput,
+
+    async preflight() {
+      await preflightVenvImports({
+        modules: ['swebench'],
+        requireDocker: true,
+        fix:
+          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +
+          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,
+      })
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      const limit = opts.limit ?? 10
+      const split = opts.split ?? 'test'
+      // Dump instances as JSON via the datasets loader (HF download on first run).
+      const script = `
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for r in ds:
+    if ids is not None and r["instance_id"] not in ids:
+        continue
+    out.append({
+        "instance_id": r["instance_id"], "repo": r["repo"], "base_commit": r["base_commit"],
+        "problem_statement": r["problem_statement"], "patch": r["patch"], "test_patch": r["test_patch"],
+        "FAIL_TO_PASS": r["FAIL_TO_PASS"], "PASS_TO_PASS": r["PASS_TO_PASS"],
+        "version": r.get("version"), "environment_setup_commit": r.get("environment_setup_commit"),
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`
+      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])
+      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>
+      return rows.map(
+        (r): BenchTask => ({
+          id: String(r.instance_id),
+          split,
+          prompt: [
+            `Repository: ${r.repo} @ ${r.base_commit}`,
+            '',
+            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests, so editing tests does not count. Keep the change minimal.',
+            'When done, END your reply with the COMPLETE unified git diff as the LAST thing, fenced exactly as ```diff … ``` (nothing after the closing fence). That fenced diff is the only deliverable.',
+            '',
+            '--- Issue ---',
+            String(r.problem_statement ?? ''),
+          ].join('\n'),
+          metadata: r,
+        }),
+      )
+    },
+
+    async goldArtifact(task: BenchTask) {
+      const gold = task.metadata?.patch
+      return typeof gold === 'string' ? gold : undefined
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const runId = safeRunId('bench', task.id)
+      return runStagedJudge({
+        tmpPrefix: 'swebench-',
+        async stage(dir) {
+          await stageFile(
+            join(dir, 'preds.json'),
+            JSON.stringify([
+              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },
+            ]),
+          )
+        },
+        // The official evaluation harness. Pulls/builds the instance image, applies
+        // the patch, runs the test spec, writes a per-run report JSON in cwd.
+        argv: (dir) => [
+          '-m', 'swebench.harness.run_evaluation',
+          '--dataset_name', DATASET,
+          '--predictions_path', join(dir, 'preds.json'),
+          '--run_id', runId,
+          '--instance_ids', task.id,
+          '--max_workers', '1',
+          '--cache_level', 'env',
+        ],
+        async parseReport(dir) {
+          // Report file: agent-runtime-bench.<run_id>.json
+          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))
+          const resolved = (report.resolved_ids ?? []).includes(task.id)
+          return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }
+        },
+      })
+    },
+  }
+}