@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/david-goliath.mts

@@ -0,0 +1,149 @@
+/**
+ * DAVID vs GOLIATH — the program's northstar, tested head-on: does a CHEAP model
+ * with a self-verification harness beat a FRONTIER model running solo, at EQUAL OR
+ * LOWER dollar cost, on held-out code?
+ *
+ *   GOLIATH — a strong model, ONE shot. The "just use the big model" baseline.
+ *   DAVID   — a cheap/weak model + test-time compute: generate N candidate
+ *             solutions AND M of its own unit tests, EXECUTE every candidate
+ *             against the generated tests, and submit the candidate that passes
+ *             the most (CodeT-style execution self-selection). No ground-truth
+ *             test is ever used to select — only the model's own generated tests.
+ *
+ * Both are graded by the HIDDEN HumanEval test (never shown). Cost is the real
+ * token spend × the router-reported/priced rate per arm. The win condition is a
+ * Pareto beat: David's held-out pass rate >= Goliath's AND David's $ <= Goliath's.
+ * Mechanism under test: EXECUTION-BASED VERIFICATION is the lever that lets a weak
+ * generator punch above its solo weight — the standing "verification is live" claim
+ * at its most dramatic. Paired McNemar on per-task discordant pairs for significance.
+ *
+ * Run from cwd=bench:  env DAVID=groq/llama-3.1-8b-instant GOLIATH=anthropic/claude-haiku-4-5-20251001 \
+ *   N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts
+ */
+import { execFile } from 'node:child_process'
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+
+const KEY = process.env.TANGLE_API_KEY
+if (!KEY) throw new Error('TANGLE_API_KEY required')
+const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
+const GOLIATH = process.env.GOLIATH ?? 'anthropic/claude-haiku-4-5-20251001'
+const N = Number(process.env.N ?? 8) // David candidate solutions
+const T = Number(process.env.T ?? 5) // David generated tests
+const NTASKS = Number(process.env.NTASKS ?? 164)
+const REPS = Number(process.env.REPS ?? 2)
+const CONC = Number(process.env.CONCURRENCY ?? 6)
+const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000)
+
+// Approx $/1M tokens (in,out) for cost accounting — the router does not price
+// every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the
+// point, and the ratio is robust to small rate errors. Override via PRICES env.
+const PRICES: Record<string, [number, number]> = {
+  'groq/llama-3.1-8b-instant': [0.05, 0.08],
+  'google/gemini-2.5-flash-lite': [0.10, 0.40],
+  'openai/gpt-4o-mini': [0.15, 0.60],
+  'anthropic/claude-haiku-4-5-20251001': [1.0, 5.0],
+  'glm-5.2': [0.60, 2.20],
+}
+const priceOf = (m: string) => PRICES[m] ?? [0.5, 1.5]
+
+interface Usage { in: number; out: number }
+const zero = (): Usage => ({ in: 0, out: 0 })
+const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out }
+const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 }
+
+async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise<string> {
+  for (let a = 0; ; a++) {
+    try {
+      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model, messages, temperature, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
+      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
+      if (!r.ok) return ''
+      const j = (await r.json()) as { choices?: { message?: { content?: string } }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } }
+      addU(usage, { in: j.usage?.prompt_tokens ?? 0, out: j.usage?.completion_tokens ?? 0 })
+      return j.choices?.[0]?.message?.content ?? ''
+    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  }
+}
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) })))
+async function runPy(program: string): Promise<{ ok: boolean }> {
+  const d = mkdtempSync(join(tmpdir(), 'dg-'))
+  try { writeFileSync(join(d, 'p.py'), program); const r = await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: EXEC_TIMEOUT }); return { ok: r.code === 0 } } finally { rmSync(d, { recursive: true, force: true }) }
+}
+
+const SOLVE = 'You are an expert Python programmer. Output the COMPLETE function (signature + body + imports) in a single ```python block. No prose, no tests.'
+async function genSolution(model: string, t: HumanEvalTask, temp: number, u: Usage): Promise<string> {
+  return extractCode(await chat(model, [{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp, u))
+}
+// David writes its OWN tests (never sees the hidden test). Parse assert lines.
+async function genTests(model: string, t: HumanEvalTask, u: Usage): Promise<string[]> {
+  const reply = await chat(model, [
+    { role: 'system', content: 'Write Python assert-based unit tests for the described function. Output ONLY a ```python block of standalone `assert <entry>(...) == ...` lines (at least a few, covering normal + edge cases). No function definition, no prose.' },
+    { role: 'user', content: `Function to test (entry point: ${t.entryPoint}):\n\n\`\`\`python\n${t.prompt}\`\`\`` },
+  ], 0.4, u)
+  const block = extractCode(reply) || reply
+  return block.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
+}
+// Hidden held-out judge — the truth. Never used for selection.
+async function judge(t: HumanEvalTask, code: string): Promise<boolean> {
+  if (!code.trim()) return false
+  return (await runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`)).ok
+}
+// David's self-selection: score each candidate by how many of ITS OWN tests it passes.
+async function scoreOnTests(code: string, tests: string[]): Promise<number> {
+  if (!code.trim() || tests.length === 0) return 0
+  let pass = 0
+  // one program per test keeps a crash on one test from voiding the rest
+  for (const a of tests) if ((await runPy(`${code}\n\n${a}\n`)).ok) pass++
+  return pass
+}
+
+async function davidArm(t: HumanEvalTask, u: Usage): Promise<string> {
+  const cands = (await Promise.all(Array.from({ length: N }, () => genSolution(DAVID, t, 0.7, u)))).filter((c) => c.trim())
+  if (cands.length === 0) return ''
+  const tests = await genTests(DAVID, t, u)
+  if (tests.length === 0) return cands[0]! // no verifier signal → first sample
+  const scored = await Promise.all(cands.map(async (c) => ({ c, s: await scoreOnTests(c, tests) })))
+  scored.sort((a, b) => b.s - a.s || b.c.length - a.c.length)
+  return scored[0]!.c
+}
+const goliathArm = (t: HumanEvalTask, u: Usage) => genSolution(GOLIATH, t, 0.2, u)
+
+async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2, i: number) => Promise<R>): Promise<R[]> {
+  const o = new Array<R>(xs.length); let i = 0
+  await Promise.all(Array.from({ length: Math.min(n, xs.length) }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!, k) } }))
+  return o
+}
+function mcnemar(b: number, c: number): number { const n = b + c; if (n === 0) return 1; const k = Math.min(b, c); const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }; let tl = 0; for (let i = 0; i <= k; i++) tl += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2)); return Math.min(1, 2 * tl) }
+
+async function main(): Promise<void> {
+  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: david-goliath loaded'); return }
+  const tasks = await loadHumanEval(NTASKS, 0)
+  console.error(`=== DAVID(${DAVID}, N=${N} sols + ${T} self-tests) vs GOLIATH(${GOLIATH}, 1 shot) · HumanEval n=${tasks.length} · reps=${REPS} ===`)
+  const dU = zero(), gU = zero()
+  const units = tasks.flatMap((task) => Array.from({ length: REPS }, () => task))
+  let done = 0
+  const res = await pool(units, CONC, async (task) => {
+    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
+    const [dCode, gCode] = await Promise.all([safe(() => davidArm(task, dU)), safe(() => goliathArm(task, gU))])
+    const [d, g] = await Promise.all([judge(task, dCode), judge(task, gCode)])
+    if (++done % 20 === 0) console.error(`  ${done}/${units.length} units`)
+    return { d, g }
+  })
+  const n = res.length, dPass = res.filter((r) => r.d).length, gPass = res.filter((r) => r.g).length
+  const b = res.filter((r) => r.d && !r.g).length, c = res.filter((r) => !r.d && r.g).length
+  const p = mcnemar(b, c)
+  const dCost = usd(DAVID, dU), gCost = usd(GOLIATH, gU)
+  console.log('\n=== RESULT (held-out HumanEval) ===')
+  console.log(`  GOLIATH ${GOLIATH} solo : ${gPass}/${n} = ${(gPass / n * 100).toFixed(1)}%   $${gCost.toFixed(4)}`)
+  console.log(`  DAVID   ${DAVID} + verify: ${dPass}/${n} = ${(dPass / n * 100).toFixed(1)}%   $${dCost.toFixed(4)}`)
+  console.log(`  accuracy: David ${dPass >= gPass ? '>=' : '<'} Goliath (${(dPass / n * 100).toFixed(1)} vs ${(gPass / n * 100).toFixed(1)}); paired McNemar David-only=${b} Goliath-only=${c} p=${p.toFixed(4)}`)
+  console.log(`  cost: David is ${(gCost / Math.max(dCost, 1e-9)).toFixed(1)}x CHEAPER ($${dCost.toFixed(4)} vs $${gCost.toFixed(4)})`)
+  const paretoBeat = dPass >= gPass && dCost <= gCost
+  const sigBeat = dPass > gPass && p < 0.05
+  console.log(`  VERDICT: ${sigBeat ? 'DAVID SIGNIFICANTLY BEATS GOLIATH' : paretoBeat ? 'DAVID PARETO-DOMINATES (>= accuracy, <= cost)' : dPass >= gPass ? 'David matches accuracy (check cost)' : 'Goliath wins accuracy'}`)
+}
+main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })

package/src/hev-improve.mts

@@ -65,8 +65,11 @@ async function main(): Promise<void> {
   const trainN = Number(process.env.TRAIN_N ?? 12)
   const holdoutN = Number(process.env.HOLDOUT_N ?? 12)
   const offset = Number(process.env.OFFSET ?? 80)
-  const generations = Number(process.env.GENERATIONS ?? 1)
-  const population = Number(process.env.POPULATION ?? 2)
+  // generations=1 never exercises the GEPA Pareto/combine path (the frontier
+  // needs >=1 completed generation before combine can fire) — default to a
+  // multi-generation budget so the default run measures the full loop.
+  const generations = Number(process.env.GENERATIONS ?? 6)
+  const population = Number(process.env.POPULATION ?? 4)
   const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
   const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
   const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
@@ -115,9 +118,21 @@ async function main(): Promise<void> {
         console.log(`  [judge] ${scenario.id} pass=0 (empty)`)
         return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' }
       }
-      const { pass } = await runChecker(t, code)
+      const { pass, detail } = await runChecker(t, code)
       console.log(`  [judge] ${scenario.id} pass=${pass}`)
-      return { dimensions: { pass }, composite: pass, notes: pass === 1 ? 'passed' : 'failed' }
+      if (pass === 1) return { dimensions: { pass }, composite: pass, notes: 'passed' }
+      // Trajectory-grounded failure note: the checker's traceback/assertion tail
+      // plus the model's own emitted code, so GEPA reflection sees WHAT failed and
+      // WHAT the model wrote — not just the word 'failed'. The candidate's full
+      // raw reply additionally reaches the proposer via the campaign breakdown's
+      // `emitted` field (carried automatically from the string artifact).
+      const traceback = (detail ?? 'checker produced no output (timeout or silent non-zero exit)').slice(-800)
+      const excerpt = code.slice(0, 700)
+      return {
+        dimensions: { pass },
+        composite: pass,
+        notes: `${traceback}\n--- emitted code (first 700 chars) ---\n${excerpt}`,
+      }
     },
   }
 
@@ -146,12 +161,17 @@ async function main(): Promise<void> {
     judge,
     agent,
     expectUsage: 'warn',
+    // rawTraceContext stays OFF deliberately: it swaps the distilled findings for
+    // filesystem paths + grep/cat instructions (rawTraceDistiller), which only a
+    // coding harness can execute. This run's proposer is prompt-tier (gepaProposer
+    // — a single LLM call that cannot run grep), so the trace evidence arrives via
+    // the judge's traceback notes + the breakdown's `emitted` excerpt instead.
     budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
     llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel },
   })
 
   console.log('\n═══ RESULT ═══')
-  console.log(`decision=${out.decision}  lift=${out.lift}`)
+  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
   console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`)
   console.log(`winner   holdout pass-rate = ${out.raw.winner.compositeMean}`)
   console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
@@ -160,7 +180,6 @@ async function main(): Promise<void> {
   if ((out.raw.winner as { surface?: unknown }).surface) {
     console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`)
   }
-  console.log(`live instruction unchanged: ${profile.prompt?.systemPrompt}`)
 }
 
 main().catch((e) => {

package/src/humaneval-object-ablation.mts

@@ -0,0 +1,201 @@
+/**
+ * OBJECT-OF-IMPROVEMENT ablation on HumanEval — the head-on test of the claim our
+ * whole self-improvement line rests on: at EQUAL budget, does adding CAPABILITY
+ * (a real code-execution tool) beat adding IMPROVER-cleverness (blind prompt
+ * self-refinement)? Both arms get the same model, the same K rounds per task, the
+ * same held-out tasks. The ONLY difference is what the budget buys:
+ *
+ *   IMPROVER arm  — K rounds of "critique your own function and rewrite it",
+ *                   with NO execution. The classic reflect-without-a-tool loop.
+ *   CAPABILITY arm — K rounds WITH a `run_python` tool: the agent executes its
+ *                   function on inputs it chooses, sees real output/errors, fixes.
+ *
+ * Grading is a DETERMINISTIC hidden test (never shown): the task's own `check`.
+ * If the capability arm wins the held-out pass rate, the object of improvement
+ * (what you can DO) dominates the improver (how cleverly you rewrite) — the
+ * finding the prompt-only self-improvement runs kept nulling on.
+ *
+ * Fast by construction: HumanEval tasks are tiny, graded in an isolated Python
+ * container with a hard timeout — seconds per task. Paired
+ * McNemar over the per-task pass/fail difference gives the significance.
+ *
+ * Run from cwd=bench:  env WORKER_MODEL=google/gemini-2.5-flash-lite N=60 K=3 \
+ *   REPS=2 node_modules/.bin/tsx src/humaneval-object-ablation.mts
+ */
+import {
+  loadHumanEval,
+  extractCode,
+  runPythonProgram,
+  type HumanEvalTask,
+} from './benchmarks/humaneval'
+
+const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+const KEY = process.env.TANGLE_API_KEY
+if (!KEY) throw new Error('TANGLE_API_KEY required')
+const MODEL = process.env.WORKER_MODEL ?? 'google/gemini-2.5-flash-lite'
+const N = Number(process.env.N ?? 60)
+const OFFSET = Number(process.env.OFFSET ?? 0)
+const K = Number(process.env.K ?? 3) // rounds/budget per task (equal for both arms)
+const REPS = Number(process.env.REPS ?? 2)
+const CONC = Number(process.env.CONCURRENCY ?? 6)
+const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 8000)
+
+interface ChatMsg { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string }
+interface Tool { type: 'function'; function: { name: string; description: string; parameters: unknown } }
+
+async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: string; toolCalls: { id: string; name: string; args: Record<string, unknown> }[] }> {
+  const body: Record<string, unknown> = { model: MODEL, messages, temperature: 0.4 }
+  if (tools) { body.tools = tools; body.tool_choice = 'auto' }
+  for (let attempt = 0; ; attempt++) {
+    let res: Response
+    try {
+      res = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(60_000) })
+    } catch (e) { if (attempt >= 5) throw e; await sleep(800 * 2 ** attempt); continue }
+    if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(res.status)) { if (attempt >= 5) throw new Error(`router ${res.status} exhausted`); await sleep(800 * 2 ** attempt); continue }
+    if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 200)}`)
+    const j = (await res.json()) as { choices?: { message?: { content?: string; tool_calls?: { id: string; function: { name: string; arguments: string } }[] } }[] }
+    const m = j.choices?.[0]?.message
+    const toolCalls = (m?.tool_calls ?? []).map((t) => { let args: Record<string, unknown> = {}; try { args = JSON.parse(t.function.arguments) } catch { /* keep {} */ } return { id: t.id, name: t.function.name, args } })
+    return { content: m?.content ?? '', toolCalls }
+  }
+}
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+/** Run model-written Python in the shared networkless, resource-capped container. */
+async function runPython(program: string): Promise<{ stdout: string; stderr: string; ok: boolean }> {
+  const result = await runPythonProgram(program, EXEC_TIMEOUT)
+  return {
+    stdout: result.stdout.slice(0, 2000),
+    stderr: result.stderr.slice(0, 2000),
+    ok: result.exitCode === 0,
+  }
+}
+
+/** The HIDDEN judge: candidate full function + the task's own check. Never shown. */
+async function judge(task: HumanEvalTask, candidate: string): Promise<boolean> {
+  if (!candidate.trim()) return false
+  const program = `${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\nprint("PASS")\n`
+  const r = await runPython(program)
+  return r.ok && r.stdout.includes('PASS')
+}
+
+const SYSTEM = 'You are an expert Python programmer. Output the COMPLETE function definition (signature + body, plus any imports) in a single ```python block. No tests, no prose outside the block.'
+const userPrompt = (t: HumanEvalTask) => `Complete this function:\n\n\`\`\`python\n${t.prompt}\`\`\``
+
+/** IMPROVER arm: K rounds of blind self-refinement — no execution, just "review and rewrite". */
+async function improverArm(task: HumanEvalTask): Promise<string> {
+  const messages: ChatMsg[] = [{ role: 'system', content: SYSTEM }, { role: 'user', content: userPrompt(task) }]
+  let code = ''
+  for (let round = 0; round < K; round++) {
+    const { content } = await router(messages)
+    code = extractCode(content) || content
+    if (round < K - 1) {
+      messages.push({ role: 'assistant', content })
+      messages.push({ role: 'user', content: 'Carefully review your function for correctness bugs and edge cases. If it can be improved, output the corrected COMPLETE function in a python block; if it is already correct, output it again unchanged.' })
+    }
+  }
+  return code
+}
+
+const RUN_TOOL: Tool = { type: 'function', function: { name: 'run_python', description: 'Execute a Python snippet and return its stdout/stderr. Use it to test your function on example inputs from the docstring before finalizing.', parameters: { type: 'object', properties: { code: { type: 'string', description: 'python source to run' } }, required: ['code'] } } }
+
+/** CAPABILITY arm: K rounds WITH a real code-execution tool — write, run, see real output, fix. */
+async function capabilityArm(task: HumanEvalTask): Promise<string> {
+  const messages: ChatMsg[] = [
+    { role: 'system', content: `${SYSTEM}\nYou have a run_python tool: test your function on the docstring's example inputs before giving your final answer. Fix any failures you observe.` },
+    { role: 'user', content: userPrompt(task) },
+  ]
+  let code = ''
+  // (K-1) tool-exploration rounds + 1 forced final answer = K calls total, matching
+  // the improver arm's K blind-refine rounds (equal budget).
+  for (let round = 0; round < Math.max(1, K - 1); round++) {
+    const { content, toolCalls } = await router(messages, [RUN_TOOL])
+    if (content && extractCode(content)) code = extractCode(content)
+    messages.push({ role: 'assistant', content: content || '', ...(toolCalls.length ? { tool_calls: toolCalls.map((t) => ({ id: t.id, type: 'function', function: { name: t.name, arguments: JSON.stringify(t.args) } })) } : {}) })
+    if (toolCalls.length) {
+      for (const tc of toolCalls) {
+        const snippet = String(tc.args.code ?? '')
+        const r = await runPython(snippet)
+        messages.push({ role: 'tool', tool_call_id: tc.id, name: tc.name, content: `stdout:\n${r.stdout}\nstderr:\n${r.stderr}\nexit_ok=${r.ok}` })
+      }
+    } else {
+      messages.push({ role: 'user', content: 'Test your function with run_python on the docstring examples before finalizing.' })
+    }
+  }
+  // FAIR FINAL ANSWER: the tool rounds are exploration; force one no-tool call to
+  // emit the complete function. Without this, an agent that ends mid-tool-call
+  // yields empty code and is unfairly scored 0 (an extraction artifact, not a
+  // real "the tool hurt" signal). Only override if it produces a real block.
+  {
+    messages.push({ role: 'user', content: 'Now output your FINAL complete function in a single ```python block, no tools, no prose.' })
+    const { content } = await router(messages)
+    const finalCode = extractCode(content)
+    if (finalCode.trim()) code = finalCode
+  }
+  return code
+}
+
+async function pool<T, R>(items: T[], limit: number, fn: (t: T, i: number) => Promise<R>): Promise<R[]> {
+  const out = new Array<R>(items.length)
+  let next = 0
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (next < items.length) { const i = next++; out[i] = await fn(items[i]!, i) }
+  }))
+  return out
+}
+
+// McNemar exact (paired): b = capability-only wins, c = improver-only wins.
+function mcnemarP(b: number, c: number): number {
+  const n = b + c; if (n === 0) return 1
+  const k = Math.min(b, c)
+  const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }
+  let tail = 0; for (let i = 0; i <= k; i++) tail += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2))
+  return Math.min(1, 2 * tail)
+}
+
+async function main(): Promise<void> {
+  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: humaneval-object-ablation loaded'); return }
+  const tasks = await loadHumanEval(N, OFFSET)
+  console.error(`=== OBJECT-OF-IMPROVEMENT ablation · HumanEval n=${tasks.length} (offset ${OFFSET}) · model=${MODEL} · K=${K} rounds · reps=${REPS} · equal budget ===`)
+  // self-check the local grader on the gold solution of task 0 (never shown to the model)
+  if (tasks[0]?.canonicalSolution) {
+    const gold = `${tasks[0].prompt}${tasks[0].canonicalSolution}`
+    console.error(`  grader self-check (gold passes): ${await judge(tasks[0], gold)}`)
+  }
+
+  // per (task, rep): run both arms; record pass/fail.
+  const units = tasks.flatMap((task) => Array.from({ length: REPS }, (_, rep) => ({ task, rep })))
+  let done = 0
+  let capEmpty = 0
+  const results = await pool(units, CONC, async ({ task }) => {
+    // Per-unit resilience: a transient model failure (e.g. a weak model emitting a
+    // malformed tool call → router 400) scores that arm 0 for this unit, never
+    // crashes the whole run. Both arms wrapped identically so neither is favored.
+    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
+    const [impCode, capCode] = await Promise.all([safe(() => improverArm(task)), safe(() => capabilityArm(task))])
+    const [imp, cap] = await Promise.all([judge(task, impCode), judge(task, capCode)])
+    done++
+    // AUTOPSY: a capability-arm failure with EMPTY final code is an extraction
+    // artifact (ended mid-tool-call, never emitted a final function), NOT a real
+    // "the tool hurt" signal. Count it so the effect can be separated.
+    if (!cap && !capCode.trim()) { capEmpty++; if (imp) console.error(`  [cap-empty] ${task.taskId} (improver passed)`) }
+    if (done % 20 === 0) console.error(`  ${done}/${units.length} units`)
+    return { id: task.taskId, imp, cap }
+  })
+  console.error(`  capability-arm empty-final-code (artifact) count: ${capEmpty}/${results.length}`)
+
+  const impPass = results.filter((r) => r.imp).length
+  const capPass = results.filter((r) => r.cap).length
+  const b = results.filter((r) => r.cap && !r.imp).length // capability-only wins
+  const c = results.filter((r) => !r.cap && r.imp).length // improver-only wins
+  const p = mcnemarP(b, c)
+  const n = results.length
+  const liftPp = ((capPass - impPass) / n) * 100
+
+  console.log('')
+  console.log('=== RESULT (held-out HumanEval, equal budget) ===')
+  console.log(`  IMPROVER   (blind self-refine, no tool): ${impPass}/${n} = ${((impPass / n) * 100).toFixed(1)}%`)
+  console.log(`  CAPABILITY (self-built execution tool)  : ${capPass}/${n} = ${((capPass / n) * 100).toFixed(1)}%`)
+  console.log(`  lift = ${liftPp >= 0 ? '+' : ''}${liftPp.toFixed(1)}pp   paired McNemar: capability-only=${b} improver-only=${c} p=${p.toFixed(4)}`)
+  console.log(`  verdict: ${p < 0.05 && capPass > impPass ? 'CAPABILITY > IMPROVER (significant)' : p < 0.05 && impPass > capPass ? 'IMPROVER > CAPABILITY (significant)' : 'no significant difference'}`)
+}
+main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })