@tangle-network/agent-bench 0.1.0

package/src/benchmarks/hotpotqa.ts

@@ -0,0 +1,320 @@
+/**
+ * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').
+ * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.
+ *
+ * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no
+ * LLM. Both the predicted final answer and the gold are normalized (lowercase,
+ * strip articles a/an/the, strip punctuation, collapse whitespace), then scored
+ * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS
+ * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the
+ * suite a judge that needs no model tokens at all.
+ *
+ * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of
+ * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,
+ * never part of the score.
+ *
+ * Requires for a live run: the bench `.venv` with `datasets` installed + network
+ * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the
+ * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.
+ */
+
+import { execFile } from 'node:child_process'
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const execFileAsync = promisify(execFile)
+const BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const PY = join(BENCH_ROOT, '.venv', 'bin', 'python')
+const FIXTURES = join(BENCH_ROOT, 'fixtures', 'hotpotqa.json')
+
+const DATASET = 'hotpotqa/hotpot_qa'
+const DATASET_CONFIG = 'distractor'
+const DATASET_SPLIT = 'validation'
+const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'
+
+/** Default F1 pass threshold; HotpotQA's leaderboard reports EM and F1 separately. */
+const DEFAULT_F1_PASS = 0.6
+
+/** Worker contract appended to every task prompt. The judge keys off the sentinel. */
+const WORKER_CONTRACT = [
+  '',
+  'Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.',
+  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+  'The answer after the sentinel must be the bare value only (no explanation on that line).',
+].join('\n')
+
+interface SupportingFacts {
+  title: string[]
+  sent_id: number[]
+}
+
+interface HotpotRow {
+  id: string
+  question: string
+  answer: string
+  type: string
+  level: string
+  supporting_facts: SupportingFacts
+}
+
+interface HotpotMeta {
+  gold: string
+  supportingFacts: SupportingFacts
+  type: string
+  level: string
+  rawQuestion: string
+}
+
+/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */
+async function py(script: string, args: string[] = []): Promise<string> {
+  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {
+    maxBuffer: 1024 * 1024 * 256,
+  })
+  return stdout
+}
+
+const ARTICLES = new Set(['a', 'an', 'the'])
+
+/**
+ * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,
+ * drop articles (a/an/the), collapse whitespace. Token comparisons run on the
+ * output of this exactly as the published evaluator does.
+ */
+export function normalizeAnswer(input: string): string {
+  const lower = input.toLowerCase()
+  // strip punctuation: keep word chars + whitespace only
+  const noPunct = lower.replace(/[^\w\s]/g, ' ')
+  const tokens = noPunct
+    .split(/\s+/)
+    .filter((t) => t.length > 0)
+    .filter((t) => !ARTICLES.has(t))
+  return tokens.join(' ').trim()
+}
+
+/** Normalized-token list (the F1 bag-of-tokens unit). */
+function answerTokens(input: string): string[] {
+  const n = normalizeAnswer(input)
+  return n.length === 0 ? [] : n.split(' ')
+}
+
+/** Exact match on the normalized strings. */
+export function exactMatch(prediction: string, gold: string): boolean {
+  return normalizeAnswer(prediction) === normalizeAnswer(gold)
+}
+
+/**
+ * Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of
+ * normalized tokens. Mirrors the published evaluator's special-case handling of
+ * yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both
+ * are empty, else 0.
+ */
+export function tokenF1(prediction: string, gold: string): number {
+  const predTokens = answerTokens(prediction)
+  const goldTokens = answerTokens(gold)
+  if (predTokens.length === 0 || goldTokens.length === 0) {
+    return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0
+  }
+  const goldCounts = new Map<string, number>()
+  for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1)
+  let common = 0
+  for (const t of predTokens) {
+    const left = goldCounts.get(t)
+    if (left !== undefined && left > 0) {
+      common += 1
+      goldCounts.set(t, left - 1)
+    }
+  }
+  if (common === 0) return 0
+  const precision = common / predTokens.length
+  const recall = common / goldTokens.length
+  return (2 * precision * recall) / (precision + recall)
+}
+
+/** Read the configured F1 pass threshold; fail loud on a malformed override. */
+function f1PassThreshold(): number {
+  const raw = process.env.HOTPOTQA_F1_PASS
+  if (raw === undefined || raw.length === 0) return DEFAULT_F1_PASS
+  const v = Number(raw)
+  if (!Number.isFinite(v) || v < 0 || v > 1) {
+    throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`)
+  }
+  return v
+}
+
+/**
+ * Parse the worker artifact into the final answer string.
+ * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+ * fall back to the trimmed last non-empty line. Returns '' when nothing is
+ * parseable (fail-closed — never guess), which judge() counts as resolved=false.
+ */
+export function parseFinalAnswer(artifact: string): string {
+  const lines = artifact.split(/\r?\n/)
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const line = lines[i] ?? ''
+    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)
+    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()
+  }
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const t = (lines[i] ?? '').trim()
+    if (t.length > 0) return t
+  }
+  return ''
+}
+
+function rowToTask(row: HotpotRow): BenchTask {
+  const meta: HotpotMeta = {
+    gold: row.answer,
+    supportingFacts: row.supporting_facts ?? { title: [], sent_id: [] },
+    type: row.type ?? '',
+    level: row.level ?? '',
+    rawQuestion: row.question,
+  }
+  return {
+    id: `hotpotqa-${row.id}`,
+    split: DATASET_SPLIT,
+    prompt: row.question + WORKER_CONTRACT,
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): HotpotMeta {
+  const md = task.metadata
+  if (!md || typeof md.gold !== 'string') {
+    throw new Error(`HotpotQA task ${task.id} missing metadata.gold — loadTasks did not populate it`)
+  }
+  return md as unknown as HotpotMeta
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as HotpotRow[]
+  console.warn(
+    `[hotpotqa] HOTPOTQA_FIXTURES=1 — loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`,
+  )
+  let tasks = rows.map(rowToTask)
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((t) => want.has(t.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  return tasks
+}
+
+export function createHotpotqaAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.HOTPOTQA_FIXTURES === '1'
+  // Validate the threshold at construction so a malformed env fails before any run.
+  f1PassThreshold()
+
+  return {
+    name: 'hotpotqa',
+
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, 'utf8').catch((err) => {
+          throw new Error(
+            `HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,
+          )
+        })
+        return
+      }
+      try {
+        await py(
+          `from datasets import load_dataset
+load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
+print('ok')`,
+        )
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        throw new Error(
+          `hotpotqa preflight failed: ${msg}\n` +
+            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +
+            `(2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; ` +
+            `or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`,
+        )
+      }
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const limit = opts.limit ?? 10
+      const script = `
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for r in ds:
+    rid = f"hotpotqa-{r['id']}"
+    if ids is not None and rid not in ids:
+        continue
+    sf = r.get('supporting_facts', {}) or {}
+    out.append({
+        "id": r["id"],
+        "question": r.get("question", ""),
+        "answer": r.get("answer", ""),
+        "type": str(r.get("type", "")),
+        "level": str(r.get("level", "")),
+        "supporting_facts": {
+            "title": list(sf.get("title", [])),
+            "sent_id": [int(x) for x in sf.get("sent_id", [])],
+        },
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`
+      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])
+      const rows = JSON.parse(stdout) as HotpotRow[]
+      return rows.map(rowToTask)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      // Gold artifact = the worker-contract serialization of the gold answer, so
+      // verify-judge proves gold→resolved through the SAME parse path the real
+      // artifact takes.
+      const meta = readMeta(task)
+      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const finalAnswer = parseFinalAnswer(artifact)
+
+      if (finalAnswer.length === 0) {
+        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({
+            reason: 'no parseable answer',
+            em: false,
+            f1: 0,
+            normalizedGold: normalizeAnswer(meta.gold),
+          }),
+        }
+      }
+
+      const em = exactMatch(finalAnswer, meta.gold)
+      const f1 = tokenF1(finalAnswer, meta.gold)
+      const threshold = f1PassThreshold()
+      const resolved = em || f1 >= threshold
+      return {
+        resolved,
+        score: f1,
+        detail: JSON.stringify({
+          em,
+          f1,
+          threshold,
+          normalizedAnswer: normalizeAnswer(finalAnswer),
+          normalizedGold: normalizeAnswer(meta.gold),
+          type: meta.type,
+          level: meta.level,
+          supportingFacts: meta.supportingFacts,
+        }),
+      }
+    },
+  }
+}

package/src/benchmarks/humaneval.ts

@@ -0,0 +1,251 @@
+/**
+ * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the
+ * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout
+ * through the `Supervisor` that self-corrects across rounds, vs blind
+ * random@k. This is the experiment `humaneval-gate.mts` names as "the next one" —
+ * the gate measures SELECTION over stateless single completions; this measures
+ * whether observe→steer (self-correction) beats blind compute at equal k.
+ *
+ * Worker artifact = the model's reply (a Python function, fenced or raw). The
+ * DETERMINISTIC judge runs the candidate against the task's own `test` in an
+ * isolated `--network=none` python container — exit 0 = pass. No gold
+ * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only
+ * to self-verify the judge before spending tokens.
+ *
+ * The primitives (loader / extractor / Docker checker) live here and are reused by
+ * `humaneval-gate.mts` — one home, no duplication.
+ */
+
+import { execFile } from 'node:child_process'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { gunzipSync } from 'node:zlib'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const humanevalUrl = 'https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz'
+const dockerImage = 'python:3.12-slim'
+const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)
+
+export interface HumanEvalTask {
+  taskId: string
+  prompt: string
+  test: string
+  entryPoint: string
+  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */
+  canonicalSolution?: string
+}
+
+/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch
+ *  or a malformed line — a silently-short task set would poison the gate. `offset`
+ *  selects a deeper slice (the later tasks are harder) so the worker has a
+ *  correctable middle band rather than a saturated easy prefix. */
+export async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {
+  const res = await fetch(humanevalUrl)
+  if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
+  const gz = Buffer.from(await res.arrayBuffer())
+  const text = gunzipSync(gz).toString('utf8')
+  const tasks: HumanEvalTask[] = []
+  for (const line of text.split('\n')) {
+    if (line.trim() === '') continue
+    const d = JSON.parse(line) as {
+      task_id?: string
+      prompt?: string
+      test?: string
+      entry_point?: string
+      canonical_solution?: string
+    }
+    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {
+      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`)
+    }
+    tasks.push({
+      taskId: d.task_id,
+      prompt: d.prompt,
+      test: d.test,
+      entryPoint: d.entry_point,
+      ...(d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}),
+    })
+  }
+  if (tasks.length === 0) throw new Error('HumanEval parsed to 0 tasks')
+  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`)
+  return tasks.slice(offset, offset + limit)
+}
+
+const solveInstruction =
+  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
+
+export function basePrompt(task: HumanEvalTask): string {
+  return `${solveInstruction}\n\n\`\`\`python\n${task.prompt}\`\`\``
+}
+
+/** Extract the function source from a model reply: prefer a fenced ```python (or
+ *  bare ```) block, else fall back to the raw text. The deployable program adds the
+ *  prompt header (imports + signature context), so a candidate that returns only a
+ *  body still runs; a candidate that re-defines the function shadows the header. */
+export function extractCode(reply: string): string {
+  const fenced = reply.match(/```(?:python|py)?\s*\n([\s\S]*?)```/i)
+  if (fenced && typeof fenced[1] === 'string') return fenced[1].trim()
+  return reply.trim()
+}
+
+/** The deployable test program: the prompt header (imports + signature/docstring the
+ *  model was given), then the candidate (its def shadows the header's stub), then the
+ *  task's own check() suite and the call. No gold solution anywhere. */
+function buildProgram(task: HumanEvalTask, candidate: string): string {
+  return `${task.prompt}\n${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\n`
+}
+
+export interface CheckResult {
+  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */
+  pass: number
+  /** On failure: the interpreter stderr tail (traceback / failing assertion). The
+   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */
+  detail?: string
+}
+
+/** Run one candidate's deployable test program in an isolated container:
+ *  `docker run --rm --network=none -v <tmp>:/w -w /w <img> python /w/p.py`.
+ *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image
+ *  pull failure) is NOT a test failure — it throws so the harness fails loud rather
+ *  than scoring every candidate 0 from a broken checker. */
+let dockerRunSeq = 0
+
+export function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {
+  const dir = mkdtempSync(join(tmpdir(), 'hev-'))
+  writeFileSync(join(dir, 'p.py'), buildProgram(task, candidate))
+  // Unique container name so we can force-reap it regardless of the docker client's state.
+  const name = `hev-${process.pid}-${dockerRunSeq++}`
+  return new Promise<CheckResult>((resolvePromise, reject) => {
+    let settled = false
+    const cleanup = () => {
+      rmSync(dir, { recursive: true, force: true })
+      // `execFile`'s `timeout` kills the docker CLIENT, not the container — a hung
+      // `python` would otherwise pin a CPU forever. Force-reap by name (fire-and-forget;
+      // the name is unique, so no reuse race).
+      execFile('docker', ['rm', '-f', name], () => {})
+    }
+    const finish = (res: CheckResult) => {
+      if (settled) return
+      settled = true
+      clearTimeout(backstop)
+      cleanup()
+      resolvePromise(res)
+    }
+    const fail = (e: Error) => {
+      if (settled) return
+      settled = true
+      clearTimeout(backstop)
+      cleanup()
+      reject(e)
+    }
+    // A hung container can leave the docker client stuck forwarding SIGTERM, so the
+    // execFile callback never fires. This guarantees resolution (and reap) after the
+    // timeout, independent of the callback.
+    const backstop = setTimeout(() => finish({ pass: 0 }), dockerTimeoutMs + 3000)
+    execFile(
+      'docker',
+      [
+        'run',
+        '--rm',
+        '--name',
+        name,
+        '--network=none',
+        '--cpus=1',
+        '--memory=512m',
+        '-v',
+        `${dir}:/w:ro`,
+        '-w',
+        '/w',
+        dockerImage,
+        'python',
+        '/w/p.py',
+      ],
+      { timeout: dockerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },
+      (err, _stdout, stderr) => {
+        if (err) {
+          const e = err as NodeJS.ErrnoException & { killed?: boolean; code?: number | string }
+          if (e.code === 'ENOENT') {
+            fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))
+            return
+          }
+          if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {
+            fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))
+            return
+          }
+          if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(stderr)) {
+            fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))
+            return
+          }
+          // killed-by-timeout or a non-zero exit (assert failure / error) are genuine
+          // test FAILURES — score 0, do not throw. Carry the stderr tail as the
+          // execution-grounded failure detail (empty ⇒ timeout/SIGKILL left no output).
+          finish({ pass: 0, detail: (stderr || '').slice(-600) || 'timed out (no output)' })
+          return
+        }
+        finish({ pass: 1 })
+      },
+    )
+  })
+}
+
+/** A HumanEval task carries its checker inputs in metadata so the deterministic
+ *  judge can rebuild the deployable program from a `BenchTask` alone. */
+interface HumanEvalMeta extends Record<string, unknown> {
+  promptHeader: string
+  test: string
+  entryPoint: string
+  canonicalSolution?: string
+}
+
+function toBenchTask(t: HumanEvalTask): BenchTask {
+  const metadata: HumanEvalMeta = {
+    promptHeader: t.prompt,
+    test: t.test,
+    entryPoint: t.entryPoint,
+    ...(t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}),
+  }
+  return { id: t.taskId, prompt: basePrompt(t), metadata }
+}
+
+function taskFromMeta(task: BenchTask): HumanEvalTask {
+  const m = task.metadata as HumanEvalMeta | undefined
+  if (!m?.promptHeader || !m.test || !m.entryPoint) {
+    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`)
+  }
+  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint }
+}
+
+/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle
+ *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */
+export function createHumanEvalAdapter(): BenchmarkAdapter {
+  return {
+    name: 'humaneval',
+    async preflight() {
+      // The judge is the only hard dependency; it fails loud on a missing/broken
+      // docker, so a cheap presence check here gives an earlier, clearer signal.
+      await new Promise<void>((resolve, reject) => {
+        execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {
+          if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))
+          else resolve()
+        })
+      })
+    },
+    async loadTasks(opts?: LoadOptions) {
+      const offset = Number(process.env.OFFSET ?? 0)
+      // Pull a generous window when filtering by id, else exactly `limit` from offset.
+      const all = await loadHumanEval(opts?.ids ? 164 : (opts?.limit ?? 8), offset)
+      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all
+      return picked.map(toBenchTask)
+    },
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact))
+      return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }
+    },
+    async goldArtifact(task: BenchTask) {
+      const sol = (task.metadata as HumanEvalMeta | undefined)?.canonicalSolution
+      // The canonical solution is the function BODY; the program header supplies the
+      // signature, so the body alone runs against check().
+      return sol ? sol : undefined
+    },
+  }
+}