@tangle-network/agent-bench 0.1.0

package/src/benchmarks/mind2web.ts

@@ -0,0 +1,311 @@
+/**
+ * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).
+ * Web-agent ACTION PREDICTION: each task is one step of a real web task — given
+ * the natural-language goal and the page's candidate elements, the worker picks
+ * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).
+ *
+ * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:
+ *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids
+ *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches
+ *   resolved (Step-SR) ⇔ element correct AND operation correct
+ *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right
+ *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)
+ * This is the low-noise reward a certifiable directive-lift needs: the number is a
+ * programmatic match against human-verified ground truth, not a judge's opinion.
+ *
+ * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the
+ * judge extracts deterministically; the prompt presents the candidate set as a
+ * choice over backend_node_id ordered by id (position uncorrelated with the answer).
+ *
+ * Each step carries the dataset's OWN page screenshot (written to a temp file by
+ * the loader); the worker drops it into a browser.<op> span so run-capsule's screen
+ * capsule turns the run into a film — the real page, not a re-rendered DOM.
+ *
+ * Requires for a live run: a python with `datasets` + `pillow` and network to
+ * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).
+ */
+
+import { execFile } from 'node:child_process'
+import { mkdir } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const execFileAsync = promisify(execFile)
+const BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, '.venv', 'bin', 'python')
+
+const DATASET = 'osunlp/Multimodal-Mind2Web'
+const DEFAULT_SPLIT = process.env.M2W_SPLIT ?? 'test_domain'
+const SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), 'm2w-shots')
+/** Candidate-set cap presented to the worker (always keeps all pos candidates). */
+const CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30)
+
+/** The worker contract appended to every task prompt; the judge keys off these. */
+const WORKER_CONTRACT = [
+  '',
+  'Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:',
+  'ELEMENT: <the [id] number of the chosen element>',
+  'ACTION: <CLICK | TYPE | SELECT>',
+  'VALUE: <text to type or option to select; leave empty for CLICK>',
+].join('\n')
+
+interface RawCandidate {
+  id: string
+  label: string
+}
+interface Mind2WebRow {
+  id: string
+  task: string
+  website: string
+  domain: string
+  subdomain: string
+  op: string
+  value: string
+  goldIds: string[]
+  candidates: RawCandidate[]
+  screenshotPath: string
+  targetRepr: string
+}
+interface Mind2WebMeta {
+  task: string
+  website: string
+  domain: string
+  op: string
+  value: string
+  goldIds: string[]
+  candidateIds: string[]
+  screenshotPath: string
+  targetRepr: string
+}
+
+/** Run the bench python with a script on stdin; return stdout (throws on nonzero). */
+async function py(script: string, args: string[] = []): Promise<string> {
+  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], { maxBuffer: 1024 * 1024 * 256 })
+  return stdout
+}
+
+/** The loader: stream the split, skip steps with no positive candidate (fail-loud,
+ *  never a silent score-0), cap the candidate set, save each screenshot to a file. */
+const LOADER = `
+import json, sys, os
+from datasets import load_dataset
+cfg = json.loads(sys.argv[1])
+split = cfg["split"]; limit = cfg.get("limit"); cap = cfg.get("cap", 30)
+shots = cfg["shotsDir"]; ids = set(cfg["ids"]) if cfg.get("ids") else None
+os.makedirs(shots, exist_ok=True)
+KEEP = ("aria_label","aria-label","role","type","name","placeholder","title","alt","value","text","id","class","href")
+def label(tag, attr):
+    try:
+        a = json.loads(attr) if isinstance(attr, str) else (attr or {})
+    except Exception:
+        a = {}
+    parts = []
+    for k in KEEP:
+        v = a.get(k)
+        if v:
+            sv = str(v).replace("\\n", " ").strip()
+            if k == "class": sv = sv[:40]
+            if k == "href": sv = sv[:50]
+            if sv: parts.append(k + "=" + sv[:60])
+    return "<" + str(tag) + "> " + " ".join(parts[:6])
+def cands(raw):
+    out = []
+    for c in raw or []:
+        try:
+            d = json.loads(c) if isinstance(c, str) else c
+        except Exception:
+            continue
+        bid = str(d.get("backend_node_id", ""))
+        if not bid: continue
+        out.append({"id": bid, "label": label(d.get("tag", "?"), d.get("attributes"))})
+    return out
+ds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)
+emitted = []; skipped = 0; scanned = 0
+for r in ds:
+    scanned += 1
+    if ids is not None and r["action_uid"] not in ids:
+        if scanned > 8000: break
+        continue
+    pos = cands(r.get("pos_candidates"))
+    if not pos:
+        skipped += 1
+        continue
+    neg = cands(r.get("neg_candidates"))
+    seen = set(p["id"] for p in pos); merged = list(pos)
+    for n in neg:
+        if len(merged) >= cap: break
+        if n["id"] in seen: continue
+        seen.add(n["id"]); merged.append(n)
+    merged.sort(key=lambda x: int(x["id"]) if x["id"].isdigit() else 0)
+    try:
+        op = json.loads(r["operation"]) if isinstance(r["operation"], str) else r["operation"]
+    except Exception:
+        op = {}
+    sp = os.path.join(shots, str(r["action_uid"]) + ".jpg")
+    try:
+        r["screenshot"].convert("RGB").save(sp, "JPEG", quality=70)
+    except Exception:
+        sp = ""
+    emitted.append({
+        "id": r["action_uid"], "task": r.get("confirmed_task", ""),
+        "website": r.get("website", ""), "domain": r.get("domain", ""), "subdomain": r.get("subdomain", ""),
+        "op": str(op.get("op", "")).upper(), "value": str(op.get("value", "")),
+        "goldIds": [p["id"] for p in pos], "candidates": merged,
+        "screenshotPath": sp, "targetRepr": r.get("target_action_reprs", ""),
+    })
+    if ids is None and limit is not None and len(emitted) >= limit: break
+    if ids is not None and len(emitted) >= len(ids): break
+sys.stderr.write("[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\n" % (len(emitted), skipped, scanned)); sys.stderr.flush()
+print(json.dumps(emitted)); sys.stdout.flush()
+os._exit(0)
+`
+
+function buildPrompt(row: Mind2WebRow): string {
+  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join('\n')
+  return [
+    `Web task: ${row.task}`,
+    '',
+    'You are taking the NEXT single action on the current web page. Choose the one element',
+    'to act on from the candidates below (each line is "[id] <tag> attributes"):',
+    choices,
+    WORKER_CONTRACT,
+  ].join('\n')
+}
+
+function rowToTask(row: Mind2WebRow): BenchTask {
+  const meta: Mind2WebMeta = {
+    task: row.task,
+    website: row.website,
+    domain: row.domain,
+    op: row.op,
+    value: row.value,
+    goldIds: row.goldIds,
+    candidateIds: row.candidates.map((c) => c.id),
+    screenshotPath: row.screenshotPath,
+    targetRepr: row.targetRepr,
+  }
+  return {
+    id: `mind2web-${row.id}`,
+    split: DEFAULT_SPLIT,
+    prompt: buildPrompt(row),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): Mind2WebMeta {
+  const md = task.metadata
+  if (!md || !Array.isArray((md as { goldIds?: unknown }).goldIds)) {
+    throw new Error(`mind2web task ${task.id} missing metadata.goldIds — loadTasks did not populate it`)
+  }
+  return md as unknown as Mind2WebMeta
+}
+
+/** Normalize a TYPE/SELECT value for comparison: lowercase, collapse whitespace. */
+function normValue(s: string): string {
+  return s.toLowerCase().replace(/\s+/g, ' ').trim()
+}
+
+interface ParsedAction {
+  elementId: string
+  op: string
+  value: string
+}
+
+/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.
+ *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */
+export function parseAction(artifact: string): ParsedAction | null {
+  const elem = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact)
+  const op = /ACTION:\s*(CLICK|TYPE|SELECT)/i.exec(artifact)
+  if (!elem?.[1] || !op?.[1]) return null
+  const val = /VALUE:\s*(.*)/i.exec(artifact)
+  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? '').trim() }
+}
+
+export function createMind2WebAdapter(): BenchmarkAdapter {
+  const split = DEFAULT_SPLIT
+
+  return {
+    name: 'mind2web',
+
+    async preflight() {
+      try {
+        await py(
+          `import os, sys
+from datasets import load_dataset
+import PIL
+ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)
+next(iter(ds))
+print('ok'); sys.stdout.flush(); os._exit(0)`,
+        )
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err)
+        throw new Error(
+          `mind2web preflight failed: ${msg}\n` +
+            `Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; ` +
+            `(2) network access to Hugging Face for ${DATASET} (split ${split}) ; ` +
+            `point M2W_PYTHON at the python if not bench/.venv.`,
+        )
+      }
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      const cfg = {
+        split: opts.split ?? split,
+        limit: opts.ids ? undefined : (opts.limit ?? 10),
+        cap: CANDIDATE_CAP,
+        shotsDir: SHOTS_DIR,
+        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, '')) : undefined,
+      }
+      await mkdir(SHOTS_DIR, { recursive: true })
+      const stdout = await py(LOADER, [JSON.stringify(cfg)])
+      const rows = JSON.parse(stdout) as Mind2WebRow[]
+      return rows.map(rowToTask)
+    },
+
+    async goldArtifact(task: BenchTask) {
+      // Gold = the worker-contract serialization of the ground-truth action, so
+      // verify-judge proves gold→resolved through the SAME parse path the real
+      // artifact takes. The first accepted positive id is the canonical target.
+      const meta = readMeta(task)
+      const id = meta.goldIds[0]
+      if (!id) return undefined
+      return `ELEMENT: ${id}\nACTION: ${meta.op}\nVALUE: ${meta.value}`
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const parsed = parseAction(artifact)
+      if (!parsed) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({ reason: 'no parseable ELEMENT/ACTION', goldOp: meta.op, goldIds: meta.goldIds }),
+        }
+      }
+      const elementCorrect = meta.goldIds.includes(parsed.elementId)
+      const opMatch = parsed.op === meta.op
+      const valueMatch = meta.op === 'CLICK' ? true : normValue(parsed.value) === normValue(meta.value)
+      const operationCorrect = opMatch && valueMatch
+      const resolved = elementCorrect && operationCorrect
+      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0)
+      return {
+        resolved,
+        score,
+        detail: JSON.stringify({
+          elementCorrect,
+          opMatch,
+          valueMatch,
+          predicted: parsed,
+          goldOp: meta.op,
+          goldValue: meta.value,
+          goldIds: meta.goldIds,
+          website: meta.website,
+          domain: meta.domain,
+        }),
+      }
+    },
+  }
+}

package/src/benchmarks/programbench.test.mts

@@ -0,0 +1,60 @@
+/**
+ * Offline ProgramBench adapter test. The judge needs the official `programbench`
+ * harness + Docker on linux/amd64, neither installed in CI, so this exercises the
+ * offline parts (fixtures loadTasks, the file-manifest OutputAdapter, goldArtifact)
+ * and asserts the harness-backed path FAILS LOUD with the documented fix — never a
+ * fake score. Run: PROGRAMBENCH_FIXTURES=1 npx tsx --test src/benchmarks/programbench.test.mts
+ */
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { createProgrambenchAdapter, programbenchSubmissionOutput } from './programbench'
+
+process.env.PROGRAMBENCH_FIXTURES = '1'
+
+type Events = Parameters<typeof programbenchSubmissionOutput.parse>[0]
+const stream = (text: string): Events => [{ data: { finalText: text } }] as unknown as Events
+
+test('loadTasks (fixtures) yields cleanroom tasks with instance metadata + submission contract', async () => {
+  const a = createProgrambenchAdapter()
+  const tasks = await a.loadTasks({ ids: ['abishekvashok__cmatrix.5c082c6'] })
+  assert.equal(tasks.length, 1)
+  const t = tasks[0]
+  assert.equal(t.id, 'abishekvashok__cmatrix.5c082c6')
+  assert.match(t.prompt, /compile\.sh/)
+  assert.match(t.prompt, /path:<relative\/path>/)
+  const md = t.metadata as Record<string, unknown>
+  assert.equal(md.instanceId, 'abishekvashok__cmatrix.5c082c6')
+})
+
+test('loadTasks limit slices the fixture set', async () => {
+  const a = createProgrambenchAdapter()
+  const tasks = await a.loadTasks({ limit: 2 })
+  assert.equal(tasks.length, 2)
+})
+
+test('submission OutputAdapter: ```path: blocks re-serialize into the ===FILE:=== envelope the driver splits on', () => {
+  const out = programbenchSubmissionOutput.parse(
+    stream('here\n```path:compile.sh\ncc -o executable main.c\n```\n```path:main.c\nint main(){}\n```\n'),
+  )
+  assert.match(out, /===FILE:compile\.sh===\ncc -o executable main\.c/)
+  assert.match(out, /===FILE:main\.c===\nint main\(\)\{\}/)
+})
+
+test('submission OutputAdapter: no file block → empty (fail-closed, harness scores missing compile.sh as 0)', () => {
+  assert.equal(programbenchSubmissionOutput.parse(stream('I could not solve it.')), '')
+})
+
+test('goldArtifact is undefined — oracle is stripped original source, documented, not fabricated', async () => {
+  const a = createProgrambenchAdapter()
+  const [t] = await a.loadTasks({ ids: ['abishekvashok__cmatrix.5c082c6'] })
+  assert.equal(await a.goldArtifact(t), undefined)
+})
+
+test('preflight FAILS LOUD with the install/Docker fix when the harness is absent', async () => {
+  const a = createProgrambenchAdapter()
+  await assert.rejects(a.preflight(), (e: Error) => {
+    assert.match(e.message, /pip install programbench/)
+    assert.match(e.message, /amd64/)
+    return true
+  })
+})

package/src/benchmarks/programbench.ts

@@ -0,0 +1,210 @@
+/**
+ * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom
+ * reverse-engineering: rebuild a black-box executable's behavior from scratch.
+ * The agent is given only the gold `./executable` (run-only) + stripped docs and
+ * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`
+ * with identical observable behavior. Judge = the official `programbench`
+ * harness: it extracts the submission, runs compile.sh in the per-task cleanroom
+ * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via
+ * `programbench blob sync`). Score = fraction of non-ignored tests passed
+ * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests
+ * pass. Fully deterministic — no LLM judge.
+ *
+ * OutputAdapter is stream-only, so the worker emits its codebase as fenced
+ * `path:`-prefixed file blocks (including compile.sh); the adapter materializes
+ * those into submission.tar.gz. Test execution is delegated to the real
+ * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT
+ * reimplemented here.
+ *
+ * Requires for a live run: the bench `.venv` with `programbench` installed +
+ * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for
+ * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load
+ * the committed instance ids (bench/fixtures/programbench.json); judging still
+ * needs the harness + Docker and fails loud without them — never a fabricated score.
+ */
+
+import { join } from 'node:path'
+import { readFile } from 'node:fs/promises'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')
+
+const TESTS_DATASET = 'programbench/ProgramBench-Tests'
+const TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`
+
+interface PbFixture {
+  instance_id: string
+  image_name: string
+  note?: string
+}
+
+interface PbMeta {
+  instanceId: string
+  imageName: string
+}
+
+const PROMPT = [
+  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',
+  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',
+  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',
+  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',
+].join('\n')
+
+function fixtureToTask(f: PbFixture): BenchTask {
+  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }
+  return {
+    id: f.instance_id,
+    prompt: PROMPT,
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): PbMeta {
+  const md = task.metadata
+  if (!md || typeof md.instanceId !== 'string') {
+    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)
+  }
+  return md as unknown as PbMeta
+}
+
+function selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {
+  let tasks = fixtures.map(fixtureToTask)
+  if (opts.ids) {
+    const want = new Set(opts.ids)
+    tasks = tasks.filter((t) => want.has(t.id))
+  } else if (opts.limit !== undefined) {
+    tasks = tasks.slice(0, opts.limit)
+  }
+  return tasks
+}
+
+/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */
+function imageName(instanceId: string): string {
+  return instanceId.split('.')[0].replace('__', '/')
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]
+  console.warn(
+    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,
+  )
+  return selectTasks(fixtures, opts)
+}
+
+/** Enumerate real instance ids from the HF Tests dataset tree (one dir per
+ *  instance). Throws on a non-OK response (fail loud). */
+async function fetchInstanceIds(): Promise<PbFixture[]> {
+  const res = await fetch(TREE_API)
+  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)
+  const tree = (await res.json()) as Array<{ path: string; type: string }>
+  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)
+  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)
+  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))
+}
+
+/**
+ * Run the official programbench harness for one instance over the worker's
+ * file-manifest submission. The driver builds the run-dir layout (one folder
+ * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,
+ * then parses <instance_id>.eval.json applying the tests.json ignore mask via
+ * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.
+ */
+async function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {
+  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')
+  let stdout: string
+  try {
+    // The submission manifest is piped to the driver's stdin via the shared
+    // stdin-aware runner — execFile's `input` option is not honored async and
+    // hangs the driver's sys.stdin.read() forever.
+    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })
+  } catch (err) {
+    const e = err as { message?: string }
+    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)
+  }
+  const report = JSON.parse(stdout.trim().split('\n').at(-1) ?? '{}') as {
+    passed?: number
+    total?: number
+    resolved?: boolean
+    error?: string
+  }
+  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)
+  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {
+    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)
+  }
+  const score = report.total > 0 ? report.passed / report.total : 0
+  return {
+    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),
+    score,
+    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),
+  }
+}
+
+/**
+ * Parse the worker stream into the submission text the driver materializes: the
+ * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to
+ * the python driver, which tars it. Empty when the worker emitted no file block
+ * (fail-closed → the harness scores a missing compile.sh as 0).
+ */
+export const programbenchSubmissionOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    const blocks = [...text.matchAll(/```path:([^\n`]+)\n([\s\S]*?)```/g)]
+    if (blocks.length === 0) return ''
+    // Re-serialize in the same path:<p> envelope the driver splits on.
+    return blocks.map((m) => `===FILE:${m[1].trim()}===\n${m[2]}`).join('\n')
+  },
+}
+
+export function createProgrambenchAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'
+
+  return {
+    name: 'programbench',
+    output: programbenchSubmissionOutput,
+
+    async preflight() {
+      await preflightVenvImports({
+        modules: ['programbench'],
+        requireDocker: true,
+        fix:
+          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +
+          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +
+          `(3) HF access for the hidden test blobs (the driver runs \`programbench blob sync\`). ` +
+          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,
+      })
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      let fixtures: PbFixture[]
+      try {
+        fixtures = await fetchInstanceIds()
+      } catch (err) {
+        console.warn(
+          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,
+        )
+        return loadFixtures(opts)
+      }
+      return selectTasks(fixtures, opts)
+    },
+
+    async goldArtifact() {
+      // The oracle is the original repo's source (stripped from the task image),
+      // not redistributable as a portable submission string. Judge correctness is
+      // proven by running the real harness on a real solve, not a synthetic gold.
+      return undefined
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      return runHarness(meta, artifact)
+    },
+  }
+}