@tangle-network/agent-bench 0.3.6 → 0.3.8

package/dist/index.js

@@ -1,47 +1,47 @@
 import {
   ADAPTERS,
   resolveAdapter
-} from "./chunk-5SBJCB6W.js";
-import "./chunk-2PVVP7GN.js";
+} from "./chunk-V7AEBY6U.js";
+import "./chunk-XKEFIFIC.js";
 import "./chunk-VQRS7VUC.js";
-import "./chunk-Z7ML6L77.js";
+import "./chunk-HWST3SED.js";
 import {
   createRagBenchAdapter
-} from "./chunk-R67DFVLO.js";
+} from "./chunk-IFVINJ4B.js";
 import "./chunk-SEVJPLZC.js";
 import {
   createSweBenchAdapter,
   scoreSweReport,
   sweEvaluationArgv,
   swePatchOutput
-} from "./chunk-CKUVRZ2T.js";
+} from "./chunk-3U5TXJZS.js";
 import {
   createT2RagBenchAdapter
-} from "./chunk-Y6O2OCUO.js";
-import "./chunk-HBSWHQNJ.js";
-import "./chunk-UAIOHCUK.js";
-import "./chunk-R36V2VP7.js";
-import "./chunk-JRWWGMK7.js";
+} from "./chunk-SHM6MRRF.js";
+import "./chunk-RCYQEFNX.js";
+import "./chunk-SFLA7OH3.js";
+import "./chunk-IZ5M6OAC.js";
+import "./chunk-XYA4XSNU.js";
 import "./chunk-INNOYXCP.js";
 import "./chunk-PA2ZKHJC.js";
-import "./chunk-PPYSEKFM.js";
+import "./chunk-5H5XV76F.js";
 import "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter
-} from "./chunk-TSWPNOYM.js";
+} from "./chunk-NQG5XDSB.js";
 import {
   createOpenRagBenchAdapter
-} from "./chunk-KDIKRJGB.js";
-import "./chunk-ZEWMTR5M.js";
+} from "./chunk-SHYIRB7I.js";
+import "./chunk-MQMRLGOG.js";
 import "./chunk-PUIRNYI7.js";
 import "./chunk-C7T7WEK2.js";
 import {
   runRefineLoop
 } from "./chunk-JTHWEDEW.js";
-import "./chunk-2XU6OGEN.js";
+import "./chunk-Z4TZ76N7.js";
 import {
   createCragAdapter
-} from "./chunk-HHXFIHXC.js";
+} from "./chunk-SVR2LKYI.js";
 import {
   FINAL_ANSWER_SENTINEL,
   answerScoreToBenchScore,
@@ -54,18 +54,18 @@ import {
   scoreAnswerArtifact,
   tokenF1
 } from "./chunk-X3BTXCJ4.js";
-import "./chunk-J3KDJNX2.js";
-import "./chunk-X5YKXC6V.js";
-import "./chunk-YCGY7UIZ.js";
+import "./chunk-RH5F53JT.js";
+import "./chunk-YSMEKBTD.js";
+import "./chunk-7GRVHU22.js";
 import "./chunk-UPAMRDX4.js";
-import "./chunk-ODT47UAY.js";
-import "./chunk-7WSD27QQ.js";
-import "./chunk-SYDW647C.js";
-import "./chunk-IFAV6KEM.js";
+import "./chunk-K3BQGZCT.js";
+import "./chunk-PB64GYIG.js";
+import "./chunk-IA2FBTWC.js";
+import "./chunk-KP5KD6EN.js";
 import {
   StagedJudgeError,
   runStagedJudge
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 import "./chunk-53UPUNBZ.js";
 
 // src/run-benchmarks.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "repository": {
@@ -25,10 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.122.8",
-    "@tangle-network/agent-interface": "0.31.0",
-    "@tangle-network/sandbox": "^0.11.1",
-    "@tangle-network/agent-runtime": "0.97.0"
+    "@tangle-network/agent-eval": "0.125.0",
+    "@tangle-network/agent-interface": "0.32.0",
+    "@tangle-network/agent-knowledge": "^4.1.0",
+    "@tangle-network/sandbox": "^0.12.0",
+    "@tangle-network/agent-runtime": "0.104.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/scripts/run-package-tests.mjs

@@ -1,5 +1,5 @@
 import { execFile } from 'node:child_process'
-import { access, readdir } from 'node:fs/promises'
+import { access, readdir, readFile } from 'node:fs/promises'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
@@ -43,14 +43,36 @@ try {
 }
 
 const tests = await collectTests(sourceDir)
-const relativeTests = tests.map((file) => path.relative(benchDir, file))
-if (relativeTests.length === 0) throw new Error('no package tests found under src/')
+if (tests.length === 0) throw new Error('no package tests found under src/')
 
-await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], {
-  ...process.env,
-  TSX_TSCONFIG_PATH: 'tsconfig.public.json',
-})
+// Two test runtimes coexist under src/: node:test files run under `node --test`;
+// vitest files (the swe-arena suite) crash there (`vitest` APIs need the vitest
+// worker), so partition by the framework each file actually imports.
+const nodeTests = []
+const vitestTests = []
+for (const file of tests) {
+  const body = await readFile(file, 'utf8')
+  if (/from\s+['"]vitest['"]/.test(body)) vitestTests.push(file)
+  else nodeTests.push(file)
+}
+
+if (nodeTests.length > 0) {
+  await run(
+    process.execPath,
+    ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))],
+    {
+      ...process.env,
+      TSX_TSCONFIG_PATH: 'tsconfig.public.json',
+    },
+  )
+}
+
+if (vitestTests.length > 0) {
+  await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
+}
 
 await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
 
-console.log(`package tests passed: ${tests.length}/${tests.length} TypeScript files + Pier bridge`)
+console.log(
+  `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,
+)

package/scripts/verify-packed-consumer.mjs

@@ -120,7 +120,7 @@ try {
   )
   await writeFile(
     path.join(consumerDir, 'index.mjs'),
-    "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\n",
+    "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
   )
   await writeFile(
     path.join(consumerDir, 'tsconfig.json'),

package/scripts/verify-pier-agent.mts

@@ -370,6 +370,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
       },
       instructionDelivery: { kind: 'argv-append' as const },
       cwd: { workspace: 'task' as const, path: '.' },
+      env: { PATH: { kind: 'public' as const, value: '/usr/local/bin:/usr/bin:/bin' } },
       environment: { kind: 'evaluator-task-container' as const },
       workspace: candidateWorkspace,
       isolation: {

package/src/benchmarks/_harness.ts

@@ -18,6 +18,7 @@
 
 import { execFile, spawn } from 'node:child_process'
 import { createHash } from 'node:crypto'
+import { existsSync, readFileSync } from 'node:fs'
 import {
   cp,
   lstat,
@@ -42,8 +43,25 @@ import type {
 
 const execFileAsync = promisify(execFile)
 
-/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
-export const benchRoot = fileURLToPath(new URL('../..', import.meta.url))
+/** Locate the package by identity because source files and compiled chunks have different depths. */
+function resolveBenchRoot(moduleUrl: string): string {
+  let current = dirname(fileURLToPath(moduleUrl))
+  while (true) {
+    const manifestPath = join(current, 'package.json')
+    if (existsSync(manifestPath)) {
+      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }
+      if (manifest.name === '@tangle-network/agent-bench') return current
+    }
+    const parent = dirname(current)
+    if (parent === current) {
+      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)
+    }
+    current = parent
+  }
+}
+
+/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */
+export const benchRoot = resolveBenchRoot(import.meta.url)
 
 /** Resolve the shared interpreter without requiring an installed package to contain a venv. */
 export function resolveBenchPython(

package/src/benchmarks/humaneval.test.mts

@@ -0,0 +1,122 @@
+import assert from 'node:assert/strict'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it } from 'node:test'
+import { runPythonProgram } from './humaneval'
+
+describe('HumanEval Python isolation', () => {
+  it('runs the exact program through the resource-capped networkless container', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'humaneval-docker-test-'))
+    const fakeDocker = join(dir, 'docker')
+    const capture = join(dir, 'capture.jsonl')
+    writeFileSync(
+      fakeDocker,
+      `#!/usr/bin/env node
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+const args = process.argv.slice(2)
+if (args[0] === 'rm') process.exit(0)
+if (process.env.FAKE_DOCKER_MISSING === '1') {
+  process.stderr.write('docker: Error response from daemon: No such image: python:3.12-slim')
+  process.exit(125)
+}
+const mount = args[args.indexOf('-v') + 1]
+const hostDir = mount.slice(0, -':/w:ro'.length)
+fs.appendFileSync(process.env.FAKE_DOCKER_CAPTURE, JSON.stringify({
+  args,
+  program: fs.readFileSync(path.join(hostDir, 'p.py'), 'utf8'),
+}) + '\\n')
+if (process.env.FAKE_DOCKER_NO_START === '1') {
+  process.stderr.write('docker: Error response from daemon: unable to start container process')
+  process.exit(125)
+}
+process.stdout.write(args[args.length - 2] + '\\n')
+if (process.env.FAKE_DOCKER_CANDIDATE === 'daemon-text') {
+  process.stderr.write('Cannot connect to the Docker daemon')
+  process.exit(1)
+}
+if (process.env.FAKE_DOCKER_CANDIDATE === 'exit-125') process.exit(125)
+process.stdout.write('CONTAINER_OK\\n')
+`,
+      { mode: 0o755 },
+    )
+    const originalPath = process.env.PATH
+    const originalCapture = process.env.FAKE_DOCKER_CAPTURE
+    const originalMissing = process.env.FAKE_DOCKER_MISSING
+    const originalCandidate = process.env.FAKE_DOCKER_CANDIDATE
+    const originalNoStart = process.env.FAKE_DOCKER_NO_START
+    process.env.PATH = `${dir}:${originalPath ?? ''}`
+    process.env.FAKE_DOCKER_CAPTURE = capture
+    try {
+      const program = 'print("exact bytes")\n'
+      const result = await runPythonProgram(program, 2_000)
+      assert.equal(result.exitCode, 0)
+      assert.match(result.stdout, /CONTAINER_OK/)
+
+      const firstCall = JSON.parse(readFileSync(capture, 'utf8').trim().split('\n')[0]!) as {
+        args: string[]
+        program: string
+      }
+      assert.equal(firstCall.program, program)
+      assert.deepEqual(firstCall.args.slice(0, 2), ['run', '--rm'])
+      assert.ok(firstCall.args.includes('--network=none'))
+      assert.ok(firstCall.args.includes('--cpus=1'))
+      assert.ok(firstCall.args.includes('--memory=512m'))
+      assert.ok(firstCall.args.includes('--pids-limit=64'))
+      assert.ok(firstCall.args.includes('--cap-drop=ALL'))
+      assert.ok(firstCall.args.includes('--security-opt=no-new-privileges'))
+      assert.ok(firstCall.args.includes('--pull=never'))
+      assert.ok(firstCall.args.includes('--read-only'))
+      assert.equal(
+        firstCall.args[firstCall.args.indexOf('--tmpfs') + 1],
+        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',
+      )
+      assert.equal(firstCall.args[firstCall.args.indexOf('--user') + 1], '65534:65534')
+      assert.match(firstCall.args[firstCall.args.indexOf('-v') + 1] ?? '', /:\/w:ro$/)
+      const command = firstCall.args.slice(firstCall.args.indexOf('python:3.12-slim'))
+      assert.equal(command[0], 'python:3.12-slim')
+      assert.equal(command[1], 'sh')
+      assert.equal(command[2], '-c')
+      assert.match(command[3] ?? '', /command -v timeout/)
+      assert.equal(command[4], 'agent-runtime-checker')
+      assert.match(command[5] ?? '', /^__AGENT_RUNTIME_CANDIDATE_STARTED_hev-/)
+      assert.equal(command[6], '2s')
+
+      process.env.FAKE_DOCKER_MISSING = '1'
+      await assert.rejects(
+        runPythonProgram('print("must not score")\n', 2_000),
+        /docker image python:3\.12-slim unavailable/,
+      )
+
+      delete process.env.FAKE_DOCKER_MISSING
+      process.env.FAKE_DOCKER_CANDIDATE = 'daemon-text'
+      const daemonText = await runPythonProgram('raise SystemExit(1)\n', 2_000)
+      assert.equal(daemonText.exitCode, 1)
+      assert.match(daemonText.stderr, /Cannot connect to the Docker daemon/)
+
+      process.env.FAKE_DOCKER_CANDIDATE = 'exit-125'
+      const exit125 = await runPythonProgram('raise SystemExit(125)\n', 2_000)
+      assert.equal(exit125.exitCode, 125)
+
+      delete process.env.FAKE_DOCKER_CANDIDATE
+      process.env.FAKE_DOCKER_NO_START = '1'
+      await assert.rejects(
+        runPythonProgram('print("never started")\n', 2_000),
+        /did not start the candidate/,
+      )
+    } finally {
+      if (originalPath === undefined) delete process.env.PATH
+      else process.env.PATH = originalPath
+      if (originalCapture === undefined) delete process.env.FAKE_DOCKER_CAPTURE
+      else process.env.FAKE_DOCKER_CAPTURE = originalCapture
+      if (originalMissing === undefined) delete process.env.FAKE_DOCKER_MISSING
+      else process.env.FAKE_DOCKER_MISSING = originalMissing
+      if (originalCandidate === undefined) delete process.env.FAKE_DOCKER_CANDIDATE
+      else process.env.FAKE_DOCKER_CANDIDATE = originalCandidate
+      if (originalNoStart === undefined) delete process.env.FAKE_DOCKER_NO_START
+      else process.env.FAKE_DOCKER_NO_START = originalNoStart
+      rmSync(dir, { recursive: true, force: true })
+    }
+  })
+})

package/src/benchmarks/humaneval.ts

@@ -17,7 +17,7 @@
  */
 
 import { execFile } from 'node:child_process'
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { gunzipSync } from 'node:zlib'
@@ -111,19 +111,36 @@ export interface CheckResult {
   detail?: string
 }
 
+export interface PythonProgramResult {
+  exitCode: number
+  stdout: string
+  stderr: string
+}
+
 /** Run one candidate's deployable test program in an isolated container:
- *  `docker run --rm --network=none -v <tmp>:/w -w /w <img> python /w/p.py`.
+ *  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.
  *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image
- *  pull failure) is NOT a test failure — it throws so the harness fails loud rather
+ *  unavailable) is NOT a test failure — it throws so the harness fails loud rather
  *  than scoring every candidate 0 from a broken checker. */
 let dockerRunSeq = 0
 
-export function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {
+/** Run arbitrary Python in the same isolated container used by the HumanEval
+ * checker. This is also the execution-tool primitive for experiments that let
+ * a model test its own snippets; model-written code must never run on the host. */
+export function runPythonProgram(
+  program: string,
+  timeoutMs = dockerTimeoutMs,
+): Promise<PythonProgramResult> {
   const dir = mkdtempSync(join(tmpdir(), 'hev-'))
-  writeFileSync(join(dir, 'p.py'), buildProgram(task, candidate))
+  // Rootless Docker maps container root to a subordinate host uid that cannot
+  // traverse mkdtemp's default 0700 directory. The directory contains only the
+  // disposable candidate script and is mounted read-only into the container.
+  chmodSync(dir, 0o755)
+  writeFileSync(join(dir, 'p.py'), program, { mode: 0o644 })
   // Unique container name so we can force-reap it regardless of the docker client's state.
   const name = `hev-${process.pid}-${dockerRunSeq++}`
-  return new Promise<CheckResult>((resolvePromise, reject) => {
+  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`
+  return new Promise<PythonProgramResult>((resolvePromise, reject) => {
     let settled = false
     const cleanup = () => {
       rmSync(dir, { recursive: true, force: true })
@@ -132,7 +149,7 @@ export function runChecker(task: HumanEvalTask, candidate: string): Promise<Chec
       // the name is unique, so no reuse race).
       execFile('docker', ['rm', '-f', name], () => {})
     }
-    const finish = (res: CheckResult) => {
+    const finish = (res: PythonProgramResult) => {
       if (settled) return
       settled = true
       clearTimeout(backstop)
@@ -146,56 +163,106 @@ export function runChecker(task: HumanEvalTask, candidate: string): Promise<Chec
       cleanup()
       reject(e)
     }
-    // A hung container can leave the docker client stuck forwarding SIGTERM, so the
-    // execFile callback never fires. This guarantees resolution (and reap) after the
-    // timeout, independent of the callback.
-    const backstop = setTimeout(() => finish({ pass: 0 }), dockerTimeoutMs + 3000)
+    // Candidate timeouts are enforced inside the container. If the outer Docker
+    // client or daemon misses this larger deadline, that is infrastructure failure,
+    // not a wrong answer.
+    const outerTimeoutMs = timeoutMs + 3_000
+    const backstop = setTimeout(
+      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3_000}ms`)),
+      outerTimeoutMs + 3_000,
+    )
+    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1_000))
     execFile(
       'docker',
       [
         'run',
         '--rm',
+        '--pull=never',
         '--name',
         name,
         '--network=none',
         '--cpus=1',
         '--memory=512m',
+        '--pids-limit=64',
+        '--cap-drop=ALL',
+        '--security-opt=no-new-privileges',
+        '--read-only',
+        '--tmpfs',
+        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',
+        '--user',
+        '65534:65534',
         '-v',
         `${dir}:/w:ro`,
         '-w',
         '/w',
         dockerImage,
-        'python',
-        '/w/p.py',
+        'sh',
+        '-c',
+        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf "%s\\n" "$1"; exec timeout -s KILL "$2" python /w/p.py',
+        'agent-runtime-checker',
+        startMarker,
+        `${inContainerSeconds}s`,
       ],
-      { timeout: dockerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },
-      (err, _stdout, stderr) => {
+      { timeout: outerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },
+      (err, stdout, stderr) => {
+        const markerLine = `${startMarker}\n`
+        const candidateStarted = stdout?.startsWith(markerLine) === true
+        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : (stdout ?? '')
         if (err) {
-          const e = err as NodeJS.ErrnoException & { killed?: boolean; code?: number | string }
+          const e = err as Error & { killed?: boolean; code?: number | string }
           if (e.code === 'ENOENT') {
             fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))
             return
           }
-          if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {
-            fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))
+          if (e.killed) {
+            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`))
             return
           }
-          if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(stderr)) {
-            fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))
+          if (!candidateStarted) {
+            if (
+              /cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(
+                stderr,
+              )
+            ) {
+              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))
+              return
+            }
+            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {
+              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))
+              return
+            }
+            fail(
+              new Error(
+                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`,
+              ),
+            )
             return
           }
-          // killed-by-timeout or a non-zero exit (assert failure / error) are genuine
-          // test FAILURES — score 0, do not throw. Carry the stderr tail as the
-          // execution-grounded failure detail (empty ⇒ timeout/SIGKILL left no output).
-          finish({ pass: 0, detail: (stderr || '').slice(-600) || 'timed out (no output)' })
+          const exitCode = typeof e.code === 'number' ? e.code : 1
+          finish({
+            exitCode,
+            stdout: candidateStdout,
+            stderr: stderr ?? '',
+          })
           return
         }
-        finish({ pass: 1 })
+        if (!candidateStarted) {
+          fail(new Error('docker checker exited without starting the candidate'))
+          return
+        }
+        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? '' })
       },
     )
   })
 }
 
+export async function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {
+  const result = await runPythonProgram(buildProgram(task, candidate))
+  return result.exitCode === 0
+    ? { pass: 1 }
+    : { pass: 0, detail: result.stderr.slice(-600) || 'timed out (no output)' }
+}
+
 /** A HumanEval task carries its checker inputs in metadata so the deterministic
  *  judge can rebuild the deployable program from a `BenchTask` alone. */
 interface HumanEvalMeta extends Record<string, unknown> {
@@ -229,14 +296,20 @@ export function createHumanEvalAdapter(): BenchmarkAdapter {
   return {
     name: 'humaneval',
     async preflight() {
-      // The judge is the only hard dependency; it fails loud on a missing/broken
-      // docker, so a cheap presence check here gives an earlier, clearer signal.
+      // Prove the daemon and exact local image before any model call. Scoring uses
+      // --pull=never so a registry or cold pull cannot become a candidate outcome.
       await new Promise<void>((resolve, reject) => {
         execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {
           if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))
           else resolve()
         })
       })
+      await new Promise<void>((resolve, reject) => {
+        execFile('docker', ['image', 'inspect', dockerImage], (err) => {
+          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`))
+          else resolve()
+        })
+      })
     },
     async loadTasks(opts?: LoadOptions) {
       const offset = Number(process.env.OFFSET ?? 0)

package/src/david-attribution.mts

@@ -0,0 +1,78 @@
+/**
+ * DAVID mechanism attribution — decompose the cheap-model harness's held-out
+ * accuracy into what SAMPLING buys vs what VERIFICATION-SELECTION buys, so a
+ * David-Goliath win is credited to the right lever (not just best-of-N luck).
+ *
+ * For each task, generate N candidate solutions + the model's own tests, then
+ * report four numbers on the HIDDEN test:
+ *   pass@1        — first candidate (no harness).
+ *   mean-cand     — expected accuracy of a RANDOM candidate (sampling floor).
+ *   oracle@N      — a correct candidate exists among the N (ceiling of selection).
+ *   verify-select — the candidate the self-tests picked (the actual David).
+ * verify-select − mean-cand = what VERIFICATION adds over blind sampling;
+ * oracle@N − verify-select = the selection gap left on the table.
+ *
+ * Run from cwd=bench: env DAVID=groq/llama-3.1-8b-instant N=8 T=5 NTASKS=60 \
+ *   node_modules/.bin/tsx src/david-attribution.mts
+ */
+import { execFile } from 'node:child_process'
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
+
+const KEY = process.env.TANGLE_API_KEY!
+const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
+const N = Number(process.env.N ?? 8)
+const T = Number(process.env.T ?? 5)
+const NTASKS = Number(process.env.NTASKS ?? 60)
+const CONC = Number(process.env.CONCURRENCY ?? 6)
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+async function chat(messages: { role: string; content: string }[], temp: number): Promise<string> {
+  for (let a = 0; ; a++) {
+    try {
+      const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model: DAVID, messages, temperature: temp, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) })
+      if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue }
+      if (!r.ok) return ''
+      return (((await r.json()) as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content) ?? ''
+    } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) }
+  }
+}
+const exec = (f: string, a: string[], o: object) => new Promise<number>((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0))))
+async function runPy(p: string): Promise<boolean> { const d = mkdtempSync(join(tmpdir(), 'da-')); try { writeFileSync(join(d, 'p.py'), p); return (await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: 6000 })) === 0 } finally { rmSync(d, { recursive: true, force: true }) } }
+const SOLVE = 'Expert Python. Output the COMPLETE function in one ```python block, no prose, no tests.'
+const genSol = async (t: HumanEvalTask, temp: number) => extractCode(await chat([{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp))
+async function genTests(t: HumanEvalTask): Promise<string[]> {
+  const b = extractCode(await chat([{ role: 'system', content: 'Write Python assert unit tests. Output ONLY a ```python block of `assert <entry>(...) == ...` lines. No function, no prose.' }, { role: 'user', content: `entry: ${t.entryPoint}\n\n\`\`\`python\n${t.prompt}\`\`\`` }], 0.4))
+  return b.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
+}
+const judge = async (t: HumanEvalTask, code: string) => code.trim() ? runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`) : false
+async function scoreTests(code: string, tests: string[]): Promise<number> { if (!code.trim() || !tests.length) return 0; let p = 0; for (const a of tests) if (await runPy(`${code}\n\n${a}\n`)) p++; return p }
+async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2) => Promise<R>): Promise<R[]> { const o = new Array<R>(xs.length); let i = 0; await Promise.all(Array.from({ length: n }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!) } })); return o }
+
+async function main(): Promise<void> {
+  const tasks = await loadHumanEval(NTASKS, 0)
+  console.error(`=== ATTRIBUTION · ${DAVID} · N=${N} sols + ${T} tests · n=${tasks.length} ===`)
+  let done = 0
+  const rows = await pool(tasks, CONC, async (t) => {
+    const cands = (await Promise.all(Array.from({ length: N }, () => genSol(t, 0.7)))).filter((c) => c.trim())
+    if (!cands.length) return { p1: 0, mean: 0, oracle: 0, sel: 0 }
+    const tests = await genTests(t)
+    const hidden = await Promise.all(cands.map((c) => judge(t, c))) // hidden-test pass per candidate (for attribution only)
+    const selScores = tests.length ? await Promise.all(cands.map((c) => scoreTests(c, tests))) : cands.map(() => 0)
+    let bi = 0; for (let i = 1; i < cands.length; i++) if (selScores[i]! > selScores[bi]! || (selScores[i]! === selScores[bi]! && cands[i]!.length > cands[bi]!.length)) bi = i
+    if (++done % 15 === 0) console.error(`  ${done}/${tasks.length}`)
+    return { p1: hidden[0] ? 1 : 0, mean: hidden.filter(Boolean).length / cands.length, oracle: hidden.some(Boolean) ? 1 : 0, sel: hidden[bi] ? 1 : 0 }
+  })
+  const n = rows.length, avg = (f: (r: typeof rows[number]) => number) => (rows.reduce((s, r) => s + f(r), 0) / n) * 100
+  console.log('\n=== ATTRIBUTION (held-out) ===')
+  console.log(`  pass@1 (no harness)        : ${avg((r) => r.p1).toFixed(1)}%`)
+  console.log(`  mean random candidate      : ${avg((r) => r.mean).toFixed(1)}%   (sampling floor)`)
+  console.log(`  verify-select (DAVID)      : ${avg((r) => r.sel).toFixed(1)}%`)
+  console.log(`  oracle@N (a correct exists): ${avg((r) => r.oracle).toFixed(1)}%   (selection ceiling)`)
+  console.log(`  --> verification adds over random sampling: +${(avg((r) => r.sel) - avg((r) => r.mean)).toFixed(1)}pp`)
+  console.log(`  --> selection gap still on table (oracle-select): ${(avg((r) => r.oracle) - avg((r) => r.sel)).toFixed(1)}pp`)
+}
+main().catch((e) => { console.error('MAIN:', e instanceof Error ? e.stack : e); process.exit(1) })