@tangle-network/agent-bench 0.3.5 → 0.3.7

package/src/benchmarks/humaneval.test.mts

@@ -0,0 +1,122 @@
+import assert from 'node:assert/strict'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it } from 'node:test'
+import { runPythonProgram } from './humaneval'
+
+describe('HumanEval Python isolation', () => {
+  it('runs the exact program through the resource-capped networkless container', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'humaneval-docker-test-'))
+    const fakeDocker = join(dir, 'docker')
+    const capture = join(dir, 'capture.jsonl')
+    writeFileSync(
+      fakeDocker,
+      `#!/usr/bin/env node
+const fs = require('node:fs')
+const path = require('node:path')
+const args = process.argv.slice(2)
+if (args[0] === 'rm') process.exit(0)
+if (process.env.FAKE_DOCKER_MISSING === '1') {
+  process.stderr.write('docker: Error response from daemon: No such image: python:3.12-slim')
+  process.exit(125)
+}
+const mount = args[args.indexOf('-v') + 1]
+const hostDir = mount.slice(0, -':/w:ro'.length)
+fs.appendFileSync(process.env.FAKE_DOCKER_CAPTURE, JSON.stringify({
+  args,
+  program: fs.readFileSync(path.join(hostDir, 'p.py'), 'utf8'),
+}) + '\\n')
+if (process.env.FAKE_DOCKER_NO_START === '1') {
+  process.stderr.write('docker: Error response from daemon: unable to start container process')
+  process.exit(125)
+}
+process.stdout.write(args[args.length - 2] + '\\n')
+if (process.env.FAKE_DOCKER_CANDIDATE === 'daemon-text') {
+  process.stderr.write('Cannot connect to the Docker daemon')
+  process.exit(1)
+}
+if (process.env.FAKE_DOCKER_CANDIDATE === 'exit-125') process.exit(125)
+process.stdout.write('CONTAINER_OK\\n')
+`,
+      { mode: 0o755 },
+    )
+    const originalPath = process.env.PATH
+    const originalCapture = process.env.FAKE_DOCKER_CAPTURE
+    const originalMissing = process.env.FAKE_DOCKER_MISSING
+    const originalCandidate = process.env.FAKE_DOCKER_CANDIDATE
+    const originalNoStart = process.env.FAKE_DOCKER_NO_START
+    process.env.PATH = `${dir}:${originalPath ?? ''}`
+    process.env.FAKE_DOCKER_CAPTURE = capture
+    try {
+      const program = 'print("exact bytes")\n'
+      const result = await runPythonProgram(program, 2_000)
+      assert.equal(result.exitCode, 0)
+      assert.match(result.stdout, /CONTAINER_OK/)
+
+      const firstCall = JSON.parse(readFileSync(capture, 'utf8').trim().split('\n')[0]!) as {
+        args: string[]
+        program: string
+      }
+      assert.equal(firstCall.program, program)
+      assert.deepEqual(firstCall.args.slice(0, 2), ['run', '--rm'])
+      assert.ok(firstCall.args.includes('--network=none'))
+      assert.ok(firstCall.args.includes('--cpus=1'))
+      assert.ok(firstCall.args.includes('--memory=512m'))
+      assert.ok(firstCall.args.includes('--pids-limit=64'))
+      assert.ok(firstCall.args.includes('--cap-drop=ALL'))
+      assert.ok(firstCall.args.includes('--security-opt=no-new-privileges'))
+      assert.ok(firstCall.args.includes('--pull=never'))
+      assert.ok(firstCall.args.includes('--read-only'))
+      assert.equal(
+        firstCall.args[firstCall.args.indexOf('--tmpfs') + 1],
+        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',
+      )
+      assert.equal(firstCall.args[firstCall.args.indexOf('--user') + 1], '65534:65534')
+      assert.match(firstCall.args[firstCall.args.indexOf('-v') + 1] ?? '', /:\/w:ro$/)
+      const command = firstCall.args.slice(firstCall.args.indexOf('python:3.12-slim'))
+      assert.equal(command[0], 'python:3.12-slim')
+      assert.equal(command[1], 'sh')
+      assert.equal(command[2], '-c')
+      assert.match(command[3] ?? '', /command -v timeout/)
+      assert.equal(command[4], 'agent-runtime-checker')
+      assert.match(command[5] ?? '', /^__AGENT_RUNTIME_CANDIDATE_STARTED_hev-/)
+      assert.equal(command[6], '2s')
+
+      process.env.FAKE_DOCKER_MISSING = '1'
+      await assert.rejects(
+        runPythonProgram('print("must not score")\n', 2_000),
+        /docker image python:3\.12-slim unavailable/,
+      )
+
+      delete process.env.FAKE_DOCKER_MISSING
+      process.env.FAKE_DOCKER_CANDIDATE = 'daemon-text'
+      const daemonText = await runPythonProgram('raise SystemExit(1)\n', 2_000)
+      assert.equal(daemonText.exitCode, 1)
+      assert.match(daemonText.stderr, /Cannot connect to the Docker daemon/)
+
+      process.env.FAKE_DOCKER_CANDIDATE = 'exit-125'
+      const exit125 = await runPythonProgram('raise SystemExit(125)\n', 2_000)
+      assert.equal(exit125.exitCode, 125)
+
+      delete process.env.FAKE_DOCKER_CANDIDATE
+      process.env.FAKE_DOCKER_NO_START = '1'
+      await assert.rejects(
+        runPythonProgram('print("never started")\n', 2_000),
+        /did not start the candidate/,
+      )
+    } finally {
+      if (originalPath === undefined) delete process.env.PATH
+      else process.env.PATH = originalPath
+      if (originalCapture === undefined) delete process.env.FAKE_DOCKER_CAPTURE
+      else process.env.FAKE_DOCKER_CAPTURE = originalCapture
+      if (originalMissing === undefined) delete process.env.FAKE_DOCKER_MISSING
+      else process.env.FAKE_DOCKER_MISSING = originalMissing
+      if (originalCandidate === undefined) delete process.env.FAKE_DOCKER_CANDIDATE
+      else process.env.FAKE_DOCKER_CANDIDATE = originalCandidate
+      if (originalNoStart === undefined) delete process.env.FAKE_DOCKER_NO_START
+      else process.env.FAKE_DOCKER_NO_START = originalNoStart
+      rmSync(dir, { recursive: true, force: true })
+    }
+  })
+})

package/src/benchmarks/humaneval.ts

@@ -17,7 +17,7 @@
  */
 
 import { execFile } from 'node:child_process'
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { gunzipSync } from 'node:zlib'
@@ -111,19 +111,36 @@ export interface CheckResult {
   detail?: string
 }
 
+export interface PythonProgramResult {
+  exitCode: number
+  stdout: string
+  stderr: string
+}
+
 /** Run one candidate's deployable test program in an isolated container:
- *  `docker run --rm --network=none -v <tmp>:/w -w /w <img> python /w/p.py`.
+ *  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.
  *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image
- *  pull failure) is NOT a test failure — it throws so the harness fails loud rather
+ *  unavailable) is NOT a test failure — it throws so the harness fails loud rather
  *  than scoring every candidate 0 from a broken checker. */
 let dockerRunSeq = 0
 
-export function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {
+/** Run arbitrary Python in the same isolated container used by the HumanEval
+ * checker. This is also the execution-tool primitive for experiments that let
+ * a model test its own snippets; model-written code must never run on the host. */
+export function runPythonProgram(
+  program: string,
+  timeoutMs = dockerTimeoutMs,
+): Promise<PythonProgramResult> {
   const dir = mkdtempSync(join(tmpdir(), 'hev-'))
-  writeFileSync(join(dir, 'p.py'), buildProgram(task, candidate))
+  // Rootless Docker maps container root to a subordinate host uid that cannot
+  // traverse mkdtemp's default 0700 directory. The directory contains only the
+  // disposable candidate script and is mounted read-only into the container.
+  chmodSync(dir, 0o755)
+  writeFileSync(join(dir, 'p.py'), program, { mode: 0o644 })
   // Unique container name so we can force-reap it regardless of the docker client's state.
   const name = `hev-${process.pid}-${dockerRunSeq++}`
-  return new Promise<CheckResult>((resolvePromise, reject) => {
+  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`
+  return new Promise<PythonProgramResult>((resolvePromise, reject) => {
     let settled = false
     const cleanup = () => {
       rmSync(dir, { recursive: true, force: true })
@@ -132,7 +149,7 @@ export function runChecker(task: HumanEvalTask, candidate: string): Promise<Chec
       // the name is unique, so no reuse race).
       execFile('docker', ['rm', '-f', name], () => {})
     }
-    const finish = (res: CheckResult) => {
+    const finish = (res: PythonProgramResult) => {
       if (settled) return
       settled = true
       clearTimeout(backstop)
@@ -146,56 +163,106 @@ export function runChecker(task: HumanEvalTask, candidate: string): Promise<Chec
       cleanup()
       reject(e)
     }
-    // A hung container can leave the docker client stuck forwarding SIGTERM, so the
-    // execFile callback never fires. This guarantees resolution (and reap) after the
-    // timeout, independent of the callback.
-    const backstop = setTimeout(() => finish({ pass: 0 }), dockerTimeoutMs + 3000)
+    // Candidate timeouts are enforced inside the container. If the outer Docker
+    // client or daemon misses this larger deadline, that is infrastructure failure,
+    // not a wrong answer.
+    const outerTimeoutMs = timeoutMs + 3_000
+    const backstop = setTimeout(
+      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3_000}ms`)),
+      outerTimeoutMs + 3_000,
+    )
+    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1_000))
     execFile(
       'docker',
       [
         'run',
         '--rm',
+        '--pull=never',
         '--name',
         name,
         '--network=none',
         '--cpus=1',
         '--memory=512m',
+        '--pids-limit=64',
+        '--cap-drop=ALL',
+        '--security-opt=no-new-privileges',
+        '--read-only',
+        '--tmpfs',
+        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',
+        '--user',
+        '65534:65534',
         '-v',
         `${dir}:/w:ro`,
         '-w',
         '/w',
         dockerImage,
-        'python',
-        '/w/p.py',
+        'sh',
+        '-c',
+        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf "%s\\n" "$1"; exec timeout -s KILL "$2" python /w/p.py',
+        'agent-runtime-checker',
+        startMarker,
+        `${inContainerSeconds}s`,
       ],
-      { timeout: dockerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },
-      (err, _stdout, stderr) => {
+      { timeout: outerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },
+      (err, stdout, stderr) => {
+        const markerLine = `${startMarker}\n`
+        const candidateStarted = stdout?.startsWith(markerLine) === true
+        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : (stdout ?? '')
         if (err) {
-          const e = err as NodeJS.ErrnoException & { killed?: boolean; code?: number | string }
+          const e = err as Error & { killed?: boolean; code?: number | string }
           if (e.code === 'ENOENT') {
             fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))
             return
           }
-          if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {
-            fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))
+          if (e.killed) {
+            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`))
             return
           }
-          if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(stderr)) {
-            fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))
+          if (!candidateStarted) {
+            if (
+              /cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(
+                stderr,
+              )
+            ) {
+              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))
+              return
+            }
+            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {
+              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))
+              return
+            }
+            fail(
+              new Error(
+                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`,
+              ),
+            )
             return
           }
-          // killed-by-timeout or a non-zero exit (assert failure / error) are genuine
-          // test FAILURES — score 0, do not throw. Carry the stderr tail as the
-          // execution-grounded failure detail (empty ⇒ timeout/SIGKILL left no output).
-          finish({ pass: 0, detail: (stderr || '').slice(-600) || 'timed out (no output)' })
+          const exitCode = typeof e.code === 'number' ? e.code : 1
+          finish({
+            exitCode,
+            stdout: candidateStdout,
+            stderr: stderr ?? '',
+          })
           return
         }
-        finish({ pass: 1 })
+        if (!candidateStarted) {
+          fail(new Error('docker checker exited without starting the candidate'))
+          return
+        }
+        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? '' })
       },
     )
   })
 }
 
+export async function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {
+  const result = await runPythonProgram(buildProgram(task, candidate))
+  return result.exitCode === 0
+    ? { pass: 1 }
+    : { pass: 0, detail: result.stderr.slice(-600) || 'timed out (no output)' }
+}
+
 /** A HumanEval task carries its checker inputs in metadata so the deterministic
  *  judge can rebuild the deployable program from a `BenchTask` alone. */
 interface HumanEvalMeta extends Record<string, unknown> {
@@ -229,14 +296,20 @@ export function createHumanEvalAdapter(): BenchmarkAdapter {
   return {
     name: 'humaneval',
     async preflight() {
-      // The judge is the only hard dependency; it fails loud on a missing/broken
-      // docker, so a cheap presence check here gives an earlier, clearer signal.
+      // Prove the daemon and exact local image before any model call. Scoring uses
+      // --pull=never so a registry or cold pull cannot become a candidate outcome.
       await new Promise<void>((resolve, reject) => {
         execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {
           if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))
           else resolve()
         })
       })
+      await new Promise<void>((resolve, reject) => {
+        execFile('docker', ['image', 'inspect', dockerImage], (err) => {
+          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`))
+          else resolve()
+        })
+      })
     },
     async loadTasks(opts?: LoadOptions) {
       const offset = Number(process.env.OFFSET ?? 0)

package/src/david-attribution.mts

@@ -0,0 +1,78 @@
+/**
+ * DAVID mechanism attribution — decompose the cheap-model harness's held-out
+ * accuracy into what SAMPLING buys vs what VERIFICATION-SELECTION buys, so a
+ * David-Goliath win is credited to the right lever (not just best-of-N luck).
+ *
+ * For each task, generate N candidate solutions + the model's own tests, then
+ * report four numbers on the HIDDEN test:
+ *   pass@1        — first candidate (no harness).
+ *   mean-cand     — expected accuracy of a RANDOM candidate (sampling floor).
+ *   oracle@N      — a correct candidate exists among the N (ceiling of selection).
+ *   verify-select — the candidate the self-tests picked (the actual David).
+ * verify-select − mean-cand = what VERIFICATION adds over blind sampling;
+ * oracle@N − verify-select = the selection gap left on the table.
+ *
+ * Run from cwd=bench: env DAVID=groq/llama-3.1-8b-instant N=8 T=5 NTASKS=60 \
+ *   node_modules/.bin/tsx src/david-attribution.mts
+ */
+import { execFile } from 'node:child_process'
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+
+const KEY = process.env.TANGLE_API_KEY!
+const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
+const N = Number(process.env.N ?? 8)
+const T = Number(process.env.T ?? 5)
+const NTASKS = Number(process.env.NTASKS ?? 60)
+const CONC = Number(process.env.CONCURRENCY ?? 6)
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+async function chat(messages: { role: string; content: string }[], temp: number): Promise<string> {
+  for (let a = 0; ; a++) {
+    try {
+      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model: DAVID, messages, temperature: temp, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
+      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
+      if (!r.ok) return ''
+      return (((await r.json()) as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content) ?? ''
+    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  }
+}
+const exec = (f: string, a: string[], o: object) => new Promise<number>((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0))))
+async function runPy(p: string): Promise<boolean> { const d = mkdtempSync(join(tmpdir(), 'da-')); try { writeFileSync(join(d, 'p.py'), p); return (await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: 6000 })) === 0 } finally { rmSync(d, { recursive: true, force: true }) } }
+const SOLVE = 'Expert Python. Output the COMPLETE function in one ```python block, no prose, no tests.'
+const genSol = async (t: HumanEvalTask, temp: number) => extractCode(await chat([{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp))
+async function genTests(t: HumanEvalTask): Promise<string[]> {
+  const b = extractCode(await chat([{ role: 'system', content: 'Write Python assert unit tests. Output ONLY a ```python block of `assert <entry>(...) == ...` lines. No function, no prose.' }, { role: 'user', content: `entry: ${t.entryPoint}\n\n\`\`\`python\n${t.prompt}\`\`\`` }], 0.4))
+  return b.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
+}
+const judge = async (t: HumanEvalTask, code: string) => code.trim() ? runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`) : false
+async function scoreTests(code: string, tests: string[]): Promise<number> { if (!code.trim() || !tests.length) return 0; let p = 0; for (const a of tests) if (await runPy(`${code}\n\n${a}\n`)) p++; return p }
+async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2) => Promise<R>): Promise<R[]> { const o = new Array<R>(xs.length); let i = 0; await Promise.all(Array.from({ length: n }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!) } })); return o }
+
+async function main(): Promise<void> {
+  const tasks = await loadHumanEval(NTASKS, 0)
+  console.error(`=== ATTRIBUTION · ${DAVID} · N=${N} sols + ${T} tests · n=${tasks.length} ===`)
+  let done = 0
+  const rows = await pool(tasks, CONC, async (t) => {
+    const cands = (await Promise.all(Array.from({ length: N }, () => genSol(t, 0.7)))).filter((c) => c.trim())
+    if (!cands.length) return { p1: 0, mean: 0, oracle: 0, sel: 0 }
+    const tests = await genTests(t)
+    const hidden = await Promise.all(cands.map((c) => judge(t, c))) // hidden-test pass per candidate (for attribution only)
+    const selScores = tests.length ? await Promise.all(cands.map((c) => scoreTests(c, tests))) : cands.map(() => 0)
+    let bi = 0; for (let i = 1; i < cands.length; i++) if (selScores[i]! > selScores[bi]! || (selScores[i]! === selScores[bi]! && cands[i]!.length > cands[bi]!.length)) bi = i
+    if (++done % 15 === 0) console.error(`  ${done}/${tasks.length}`)
+    return { p1: hidden[0] ? 1 : 0, mean: hidden.filter(Boolean).length / cands.length, oracle: hidden.some(Boolean) ? 1 : 0, sel: hidden[bi] ? 1 : 0 }
+  })
+  const n = rows.length, avg = (f: (r: typeof rows[number]) => number) => (rows.reduce((s, r) => s + f(r), 0) / n) * 100
+  console.log('\n=== ATTRIBUTION (held-out) ===')
+  console.log(`  pass@1 (no harness)        : ${avg((r) => r.p1).toFixed(1)}%`)
+  console.log(`  mean random candidate      : ${avg((r) => r.mean).toFixed(1)}%   (sampling floor)`)
+  console.log(`  verify-select (DAVID)      : ${avg((r) => r.sel).toFixed(1)}%`)
+  console.log(`  oracle@N (a correct exists): ${avg((r) => r.oracle).toFixed(1)}%   (selection ceiling)`)
+  console.log(`  --> verification adds over random sampling: +${(avg((r) => r.sel) - avg((r) => r.mean)).toFixed(1)}pp`)
+  console.log(`  --> selection gap still on table (oracle-select): ${(avg((r) => r.oracle) - avg((r) => r.sel)).toFixed(1)}pp`)
+}
+main().catch((e) => { console.error('MAIN:', e instanceof Error ? e.stack : e); process.exit(1) })

package/src/david-goliath.mts

@@ -0,0 +1,149 @@
+/**
+ * DAVID vs GOLIATH — the program's northstar, tested head-on: does a CHEAP model
+ * with a self-verification harness beat a FRONTIER model running solo, at EQUAL OR
+ * LOWER dollar cost, on held-out code?
+ *
+ *   GOLIATH — a strong model, ONE shot. The "just use the big model" baseline.
+ *   DAVID   — a cheap/weak model + test-time compute: generate N candidate
+ *             solutions AND M of its own unit tests, EXECUTE every candidate
+ *             against the generated tests, and submit the candidate that passes
+ *             the most (CodeT-style execution self-selection). No ground-truth
+ *             test is ever used to select — only the model's own generated tests.
+ *
+ * Both are graded by the HIDDEN HumanEval test (never shown). Cost is the real
+ * token spend × the router-reported/priced rate per arm. The win condition is a
+ * Pareto beat: David's held-out pass rate >= Goliath's AND David's $ <= Goliath's.
+ * Mechanism under test: EXECUTION-BASED VERIFICATION is the lever that lets a weak
+ * generator punch above its solo weight — the standing "verification is live" claim
+ * at its most dramatic. Paired McNemar on per-task discordant pairs for significance.
+ *
+ * Run from cwd=bench:  env DAVID=groq/llama-3.1-8b-instant GOLIATH=anthropic/claude-haiku-4-5-20251001 \
+ *   N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts
+ */
+import { execFile } from 'node:child_process'
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+
+const KEY = process.env.TANGLE_API_KEY
+if (!KEY) throw new Error('TANGLE_API_KEY required')
+const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
+const GOLIATH = process.env.GOLIATH ?? 'anthropic/claude-haiku-4-5-20251001'
+const N = Number(process.env.N ?? 8) // David candidate solutions
+const T = Number(process.env.T ?? 5) // David generated tests
+const NTASKS = Number(process.env.NTASKS ?? 164)
+const REPS = Number(process.env.REPS ?? 2)
+const CONC = Number(process.env.CONCURRENCY ?? 6)
+const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000)
+
+// Approx $/1M tokens (in,out) for cost accounting — the router does not price
+// every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the
+// point, and the ratio is robust to small rate errors. Override via PRICES env.
+const PRICES: Record<string, [number, number]> = {
+  'groq/llama-3.1-8b-instant': [0.05, 0.08],
+  'google/gemini-2.5-flash-lite': [0.10, 0.40],
+  'openai/gpt-4o-mini': [0.15, 0.60],
+  'anthropic/claude-haiku-4-5-20251001': [1.0, 5.0],
+  'glm-5.2': [0.60, 2.20],
+}
+const priceOf = (m: string) => PRICES[m] ?? [0.5, 1.5]
+
+interface Usage { in: number; out: number }
+const zero = (): Usage => ({ in: 0, out: 0 })
+const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out }
+const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 }
+
+async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise<string> {
+  for (let a = 0; ; a++) {
+    try {
+      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model, messages, temperature, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
+      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
+      if (!r.ok) return ''
+      const j = (await r.json()) as { choices?: { message?: { content?: string } }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } }
+      addU(usage, { in: j.usage?.prompt_tokens ?? 0, out: j.usage?.completion_tokens ?? 0 })
+      return j.choices?.[0]?.message?.content ?? ''
+    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  }
+}
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) })))
+async function runPy(program: string): Promise<{ ok: boolean }> {
+  const d = mkdtempSync(join(tmpdir(), 'dg-'))
+  try { writeFileSync(join(d, 'p.py'), program); const r = await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: EXEC_TIMEOUT }); return { ok: r.code === 0 } } finally { rmSync(d, { recursive: true, force: true }) }
+}
+
+const SOLVE = 'You are an expert Python programmer. Output the COMPLETE function (signature + body + imports) in a single ```python block. No prose, no tests.'
+async function genSolution(model: string, t: HumanEvalTask, temp: number, u: Usage): Promise<string> {
+  return extractCode(await chat(model, [{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp, u))
+}
+// David writes its OWN tests (never sees the hidden test). Parse assert lines.
+async function genTests(model: string, t: HumanEvalTask, u: Usage): Promise<string[]> {
+  const reply = await chat(model, [
+    { role: 'system', content: 'Write Python assert-based unit tests for the described function. Output ONLY a ```python block of standalone `assert <entry>(...) == ...` lines (at least a few, covering normal + edge cases). No function definition, no prose.' },
+    { role: 'user', content: `Function to test (entry point: ${t.entryPoint}):\n\n\`\`\`python\n${t.prompt}\`\`\`` },
+  ], 0.4, u)
+  const block = extractCode(reply) || reply
+  return block.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
+}
+// Hidden held-out judge — the truth. Never used for selection.
+async function judge(t: HumanEvalTask, code: string): Promise<boolean> {
+  if (!code.trim()) return false
+  return (await runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`)).ok
+}
+// David's self-selection: score each candidate by how many of ITS OWN tests it passes.
+async function scoreOnTests(code: string, tests: string[]): Promise<number> {
+  if (!code.trim() || tests.length === 0) return 0
+  let pass = 0
+  // one program per test keeps a crash on one test from voiding the rest
+  for (const a of tests) if ((await runPy(`${code}\n\n${a}\n`)).ok) pass++
+  return pass
+}
+
+async function davidArm(t: HumanEvalTask, u: Usage): Promise<string> {
+  const cands = (await Promise.all(Array.from({ length: N }, () => genSolution(DAVID, t, 0.7, u)))).filter((c) => c.trim())
+  if (cands.length === 0) return ''
+  const tests = await genTests(DAVID, t, u)
+  if (tests.length === 0) return cands[0]! // no verifier signal → first sample
+  const scored = await Promise.all(cands.map(async (c) => ({ c, s: await scoreOnTests(c, tests) })))
+  scored.sort((a, b) => b.s - a.s || b.c.length - a.c.length)
+  return scored[0]!.c
+}
+const goliathArm = (t: HumanEvalTask, u: Usage) => genSolution(GOLIATH, t, 0.2, u)
+
+async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2, i: number) => Promise<R>): Promise<R[]> {
+  const o = new Array<R>(xs.length); let i = 0
+  await Promise.all(Array.from({ length: Math.min(n, xs.length) }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!, k) } }))
+  return o
+}
+function mcnemar(b: number, c: number): number { const n = b + c; if (n === 0) return 1; const k = Math.min(b, c); const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }; let tl = 0; for (let i = 0; i <= k; i++) tl += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2)); return Math.min(1, 2 * tl) }
+
+async function main(): Promise<void> {
+  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: david-goliath loaded'); return }
+  const tasks = await loadHumanEval(NTASKS, 0)
+  console.error(`=== DAVID(${DAVID}, N=${N} sols + ${T} self-tests) vs GOLIATH(${GOLIATH}, 1 shot) · HumanEval n=${tasks.length} · reps=${REPS} ===`)
+  const dU = zero(), gU = zero()
+  const units = tasks.flatMap((task) => Array.from({ length: REPS }, () => task))
+  let done = 0
+  const res = await pool(units, CONC, async (task) => {
+    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
+    const [dCode, gCode] = await Promise.all([safe(() => davidArm(task, dU)), safe(() => goliathArm(task, gU))])
+    const [d, g] = await Promise.all([judge(task, dCode), judge(task, gCode)])
+    if (++done % 20 === 0) console.error(`  ${done}/${units.length} units`)
+    return { d, g }
+  })
+  const n = res.length, dPass = res.filter((r) => r.d).length, gPass = res.filter((r) => r.g).length
+  const b = res.filter((r) => r.d && !r.g).length, c = res.filter((r) => !r.d && r.g).length
+  const p = mcnemar(b, c)
+  const dCost = usd(DAVID, dU), gCost = usd(GOLIATH, gU)
+  console.log('\n=== RESULT (held-out HumanEval) ===')
+  console.log(`  GOLIATH ${GOLIATH} solo : ${gPass}/${n} = ${(gPass / n * 100).toFixed(1)}%   $${gCost.toFixed(4)}`)
+  console.log(`  DAVID   ${DAVID} + verify: ${dPass}/${n} = ${(dPass / n * 100).toFixed(1)}%   $${dCost.toFixed(4)}`)
+  console.log(`  accuracy: David ${dPass >= gPass ? '>=' : '<'} Goliath (${(dPass / n * 100).toFixed(1)} vs ${(gPass / n * 100).toFixed(1)}); paired McNemar David-only=${b} Goliath-only=${c} p=${p.toFixed(4)}`)
+  console.log(`  cost: David is ${(gCost / Math.max(dCost, 1e-9)).toFixed(1)}x CHEAPER ($${dCost.toFixed(4)} vs $${gCost.toFixed(4)})`)
+  const paretoBeat = dPass >= gPass && dCost <= gCost
+  const sigBeat = dPass > gPass && p < 0.05
+  console.log(`  VERDICT: ${sigBeat ? 'DAVID SIGNIFICANTLY BEATS GOLIATH' : paretoBeat ? 'DAVID PARETO-DOMINATES (>= accuracy, <= cost)' : dPass >= gPass ? 'David matches accuracy (check cost)' : 'Goliath wins accuracy'}`)
+}
+main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })

package/src/hev-improve.mts

@@ -65,8 +65,11 @@ async function main(): Promise<void> {
   const trainN = Number(process.env.TRAIN_N ?? 12)
   const holdoutN = Number(process.env.HOLDOUT_N ?? 12)
   const offset = Number(process.env.OFFSET ?? 80)
-  const generations = Number(process.env.GENERATIONS ?? 1)
-  const population = Number(process.env.POPULATION ?? 2)
+  // generations=1 never exercises the GEPA Pareto/combine path (the frontier
+  // needs >=1 completed generation before combine can fire) — default to a
+  // multi-generation budget so the default run measures the full loop.
+  const generations = Number(process.env.GENERATIONS ?? 6)
+  const population = Number(process.env.POPULATION ?? 4)
   const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
   const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
   const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
@@ -115,9 +118,21 @@ async function main(): Promise<void> {
         console.log(`  [judge] ${scenario.id} pass=0 (empty)`)
         return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' }
       }
-      const { pass } = await runChecker(t, code)
+      const { pass, detail } = await runChecker(t, code)
       console.log(`  [judge] ${scenario.id} pass=${pass}`)
-      return { dimensions: { pass }, composite: pass, notes: pass === 1 ? 'passed' : 'failed' }
+      if (pass === 1) return { dimensions: { pass }, composite: pass, notes: 'passed' }
+      // Trajectory-grounded failure note: the checker's traceback/assertion tail
+      // plus the model's own emitted code, so GEPA reflection sees WHAT failed and
+      // WHAT the model wrote — not just the word 'failed'. The candidate's full
+      // raw reply additionally reaches the proposer via the campaign breakdown's
+      // `emitted` field (carried automatically from the string artifact).
+      const traceback = (detail ?? 'checker produced no output (timeout or silent non-zero exit)').slice(-800)
+      const excerpt = code.slice(0, 700)
+      return {
+        dimensions: { pass },
+        composite: pass,
+        notes: `${traceback}\n--- emitted code (first 700 chars) ---\n${excerpt}`,
+      }
     },
   }
 
@@ -146,12 +161,17 @@ async function main(): Promise<void> {
     judge,
     agent,
     expectUsage: 'warn',
+    // rawTraceContext stays OFF deliberately: it swaps the distilled findings for
+    // filesystem paths + grep/cat instructions (rawTraceDistiller), which only a
+    // coding harness can execute. This run's proposer is prompt-tier (gepaProposer
+    // — a single LLM call that cannot run grep), so the trace evidence arrives via
+    // the judge's traceback notes + the breakdown's `emitted` excerpt instead.
     budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
     llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel },
   })
 
   console.log('\n═══ RESULT ═══')
-  console.log(`decision=${out.decision}  lift=${out.lift}`)
+  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
   console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`)
   console.log(`winner   holdout pass-rate = ${out.raw.winner.compositeMean}`)
   console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
@@ -160,7 +180,6 @@ async function main(): Promise<void> {
   if ((out.raw.winner as { surface?: unknown }).surface) {
     console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`)
   }
-  console.log(`live instruction unchanged: ${profile.prompt?.systemPrompt}`)
 }
 
 main().catch((e) => {