@tangle-network/agent-bench 0.8.10 → 0.8.12

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.8.12
+
+### Changed
+
+- The first-party cohort this benchmark resolves through the workspace catalog moves with the runtime: `agent-interface` 0.53.0 to 1.0.0, `agent-eval` 0.145.15 to 0.145.21, `agent-knowledge` 8.0.1 to 8.0.5, and `sandbox` 0.27.0 to 0.27.1. Interface 1.0.0 publishes the surface of 0.56.0 unchanged and states a compatibility promise, so one interface copy resolves for the whole tree.
+
+## 0.8.11
+
+- Consume Runtime 0.135.3, Eval 0.145.15, Interface 0.53.0, Knowledge 8.0.1, and Sandbox 0.26.2 as one compatible set.
+
 ## 0.8.10
 
 - Consume Runtime 0.134.9 and Knowledge 8.0.0 as one compatible dependency set.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.8.10",
+  "version": "0.8.12",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.145.11",
-    "@tangle-network/agent-interface": "0.52.0",
-    "@tangle-network/agent-knowledge": "8.0.0",
-    "@tangle-network/sandbox": "0.26.1",
-    "@tangle-network/agent-runtime": "0.134.9"
+    "@tangle-network/agent-eval": "0.145.21",
+    "@tangle-network/agent-interface": "1.0.0",
+    "@tangle-network/agent-knowledge": "8.0.5",
+    "@tangle-network/sandbox": "0.27.1",
+    "@tangle-network/agent-runtime": "0.137.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/aec-gate.mts

@@ -1,238 +0,0 @@
-/**
- * Router-based gate runner for aec-bench — the fix for the null-score sandbox path.
- *
- * aec-bench is closed-form reasoning + a deterministic local verify.py judge, so a
- * sandbox is unnecessary: solve each task with one direct router chat call, then
- * judge the raw response locally (verify.py extracts the last fenced ```json block
- * itself). The prior sandbox path emitted the JSON in-stream but never fed it to
- * the judge, so every verdict.score came out null — the bug this runner fixes by
- * passing the model's full response straight to adapter.judge().
- *
- * Two paired arms over the SAME task set (loadTasks once):
- *   random@K  — K identical-base-prompt shots/task (the compute control)
- *   diverse@K — K shots, the i-th prefixed with composeStrategies(base, K)[i]
- *
- * Each attempt carries a REAL numeric verdict.score (from verify.py) + the output,
- * written as a corpus RunRecord (condition random@K / diverse@K) the existing
- * corpus-replay --selector + corpus-report consume unchanged. Fail loud on a router
- * error — never a fabricated score.
- */
-
-import { resolveAdapter } from './adapters'
-import type { BenchmarkAdapter, BenchTask } from './benchmarks/types'
-import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
-import { composeStrategies } from './directives'
-import {
-  benchProfileModel,
-  benchRouterProfile,
-  type BenchRouterTarget,
-  runBenchRouterTurn,
-  withBenchProfile,
-} from './router-turn'
-import { pool } from './stats.mts'
-
-function must(name: string): string {
-  const v = process.env[name]
-  if (!v) throw new Error(`env ${name} is required`)
-  return v
-}
-
-interface ArmSpec {
-  /** Corpus condition label the selector/report filter on (e.g. random@4). */
-  condition: string
-  /** Per-attempt prompt builder: the i-th of K shots for a task. */
-  promptFor(task: BenchTask, i: number, k: number): string
-}
-
-interface AttemptOutcome {
-  prompt: string
-  output: string
-  score: number
-  resolved: boolean
-  costUsd?: number
-  tokensIn?: number
-  tokensOut?: number
-  wallMs: number
-  /** the router/judge call failed after retries — EXCLUDED from stats, never scored 0. */
-  infraError?: boolean
-}
-
-async function runAttempt(
-  cfg: BenchRouterTarget,
-  adapter: BenchmarkAdapter,
-  task: BenchTask,
-  prompt: string,
-): Promise<AttemptOutcome> {
-  const startedAt = Date.now()
-  // Retry transient router/judge failures (rate limits, stream drops, 5xx) with
-  // backoff; a genuine empty completion still scores a real 0 (verify.py fail-closes).
-  // Only after retries are exhausted do we record an EXCLUDED infraError — never a
-  // fabricated score, and never a throw that aborts the whole multi-model run.
-  let lastErr: unknown
-  for (let attempt = 0; attempt < 3; attempt += 1) {
-    try {
-      const res = await runBenchRouterTurn(
-        {
-          routerBaseUrl: cfg.routerBaseUrl,
-          routerKey: cfg.routerKey,
-          profile: withBenchProfile(cfg.profile, { name: 'aec-worker' }),
-        },
-        prompt,
-      )
-      const content = res.finalText
-      const verdict = await adapter.judge(task, content)
-      return {
-        prompt,
-        output: content,
-        score: verdict.score,
-        resolved: verdict.resolved,
-        wallMs: Date.now() - startedAt,
-        ...(res.usage.costUsd !== undefined ? { costUsd: res.usage.costUsd } : {}),
-        ...(res.usage.tokensKnown === false
-          ? {}
-          : { tokensIn: res.usage.input, tokensOut: res.usage.output }),
-      }
-    } catch (err) {
-      lastErr = err
-      if (attempt < 2) await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt))
-    }
-  }
-  console.warn(`[aec-gate] ${task.id}: infra error after 3 tries — excluded: ${(lastErr instanceof Error ? lastErr.message : String(lastErr)).slice(0, 160)}`)
-  return { prompt, output: '', score: 0, resolved: false, wallMs: Date.now() - startedAt, infraError: true }
-}
-
-function toAttemptRecord(o: AttemptOutcome, round: number): AttemptRecord {
-  return {
-    round,
-    prompt: o.prompt,
-    output: o.output,
-    // infra-errored attempts carry NO score/valid → corpus-replay skips them.
-    ...(o.infraError ? {} : { valid: o.resolved, score: o.score }),
-    wallMs: o.wallMs,
-    eventCount: 1,
-    eventTypes: o.infraError ? { 'router.error': 1 } : { 'router.chat': 1 },
-    traceTail: o.output.slice(-600),
-    ...(o.costUsd !== undefined ? { costUsd: o.costUsd } : {}),
-    ...(o.tokensIn !== undefined ? { tokensIn: o.tokensIn } : {}),
-    ...(o.tokensOut !== undefined ? { tokensOut: o.tokensOut } : {}),
-  }
-}
-
-interface ArmResult {
-  /** mean graded score across SCORED (non-infra) attempts */
-  meanScore: number
-  /** fraction of scored attempts at full credit (score >= 1) */
-  fullCreditRate: number
-  attemptCount: number
-  /** attempts excluded as infra errors (router/judge failed after retries) */
-  infraCount: number
-}
-
-async function runArm(
-  arm: ArmSpec,
-  cfg: BenchRouterTarget,
-  adapter: BenchmarkAdapter,
-  tasks: BenchTask[],
-  k: number,
-  concurrency: number,
-  corpusPath: string,
-): Promise<ArmResult> {
-  // Flatten (task, shot) into one unit of work so the pool bounds TOTAL in-flight
-  // router calls across all tasks, not per-task.
-  const units = tasks.flatMap((task) => Array.from({ length: k }, (_, i) => ({ task, i })))
-  const outcomes = await pool(units, concurrency, (u) => runAttempt(cfg, adapter, u.task, arm.promptFor(u.task, u.i, k)))
-
-  const scored = outcomes.filter((o) => !o.infraError)
-  let scoreSum = 0
-  let fullCredit = 0
-  for (const o of scored) {
-    scoreSum += o.score
-    if (o.score >= 1) fullCredit += 1
-  }
-
-  // Group K outcomes back per task → one RunRecord/task (the controller-run shape).
-  for (let t = 0; t < tasks.length; t += 1) {
-    const task = tasks[t] as BenchTask
-    const taskOutcomes = outcomes.slice(t * k, t * k + k)
-    const attempts = taskOutcomes.map((o, i) => toAttemptRecord(o, i))
-    const record = buildRunRecordFromAttempts(attempts, {
-      benchmark: adapter.name,
-      instanceId: task.id,
-      condition: arm.condition,
-      model: benchProfileModel(cfg.profile),
-      // k-attempt outcome = any usable attempt resolved (the oracle@k ceiling for
-      // this run; the deployable selector is scored separately by corpus-replay).
-      resolved: taskOutcomes.some((o) => o.resolved),
-      // a task whose every attempt infra-errored is itself infra-errored.
-      infraError: taskOutcomes.length > 0 && taskOutcomes.every((o) => o.infraError),
-    })
-    await appendRunRecord(corpusPath, record)
-  }
-
-  return {
-    meanScore: scored.length > 0 ? scoreSum / scored.length : 0,
-    fullCreditRate: scored.length > 0 ? fullCredit / scored.length : 0,
-    infraCount: outcomes.length - scored.length,
-    attemptCount: outcomes.length,
-  }
-}
-
-async function main(): Promise<void> {
-  const n = Number(process.env.N ?? 5)
-  const k = Number(process.env.K ?? 4)
-  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
-  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-  const routerKey = must('TANGLE_API_KEY')
-  const concurrency = Number(process.env.CONCURRENCY ?? 6)
-  const randomCorpus = process.env.RANDOM_CORPUS ?? '/tmp/aec-r.jsonl'
-  const diverseCorpus = process.env.DIVERSE_CORPUS ?? '/tmp/aec-d.jsonl'
-
-  if (!Number.isFinite(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
-  if (!Number.isFinite(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
-
-  const cfg: BenchRouterTarget = {
-    routerBaseUrl,
-    routerKey,
-    profile: benchRouterProfile('aec-worker', model, {
-      retry: { maxAttempts: Number(process.env.MAX_ATTEMPTS ?? 3) },
-    }),
-  }
-  const bench = process.env.BENCH ?? 'aec-bench'
-  const adapter = resolveAdapter(bench)
-
-  console.log(`=== ${bench} router gate · N=${n} K=${k} model=${model} conc=${concurrency} ===`)
-  await adapter.preflight()
-  const tasks = await adapter.loadTasks({ limit: n })
-  console.log(`loaded ${tasks.length} task(s): ${tasks.map((t) => t.id).join(', ')}`)
-
-  // random arm: the task prompt verbatim, K times (the compute control).
-  const randomArm: ArmSpec = {
-    condition: `random@${k}`,
-    promptFor: (task) => task.prompt,
-  }
-  // diverse arm: the i-th shot prefixed with the i-th distinct strategy lens.
-  const diverseArm: ArmSpec = {
-    condition: `diverse@${k}`,
-    promptFor: (task, i, kk) => composeStrategies(task.prompt, kk)[i] as string,
-  }
-
-  console.log(`\n▶ random@${k} (control — identical base prompt) → ${randomCorpus}`)
-  const r = await runArm(randomArm, cfg, adapter, tasks, k, concurrency, randomCorpus)
-  console.log(`  random@${k}:  mean score ${(r.meanScore * 100).toFixed(1)}%  full-credit ${(r.fullCreditRate * 100).toFixed(1)}%  (n=${r.attemptCount} attempts${r.infraCount ? `, ${r.infraCount} infra-excluded` : ''})`)
-
-  console.log(`\n▶ diverse@${k} (K distinct strategy lenses) → ${diverseCorpus}`)
-  const d = await runArm(diverseArm, cfg, adapter, tasks, k, concurrency, diverseCorpus)
-  console.log(`  diverse@${k}: mean score ${(d.meanScore * 100).toFixed(1)}%  full-credit ${(d.fullCreditRate * 100).toFixed(1)}%  (n=${d.attemptCount} attempts${d.infraCount ? `, ${d.infraCount} infra-excluded` : ''})`)
-
-  console.log(
-    `\n=== next: read the gate ===\n` +
-      `  npx tsx src/corpus-replay.mts ${randomCorpus} --selector\n` +
-      `  npx tsx src/corpus-replay.mts ${diverseCorpus} --selector --condition=diverse\n` +
-      `  npx tsx src/corpus-report.mts ${randomCorpus} ${diverseCorpus}`,
-  )
-}
-
-main().catch((err) => {
-  console.error(`aec-gate: ${err instanceof Error ? err.message : String(err)}`)
-  process.exit(1)
-})

package/src/atom-humaneval.mts

@@ -1,218 +0,0 @@
-/**
- * The "useful or BS" verdict: agents-driving-agents on a REAL deployable-checked domain.
- *
- * A `driverAgent` with a REAL router-LLM brain drives, per HumanEval task: it spawns
- * worker agents (each a router LLM that writes the function), every worker GATED by the
- * deterministic local Docker checker (the deliverable — a worker settles `valid` ⟺ its tests
- * pass), and the completion-oracle keeps-best a DELIVERED worker. The supervisor returns a winner
- * ONLY when a worker actually passed the tests (no self-declared done). We measure the driver's
- * delivered rate against a BLIND best-of-K baseline (K independent workers, no orchestration) at
- * the same K — the honest "does the recursion+oracle beat blind compute, or is it BS" question.
- *
- * Run (creds via dotenvx; Docker daemon must be up):
- *   DOTENV_PRIVATE_KEY_FILE=~/company/devops/secrets/.env.keys \
- *   dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
- *   N=5 K=3 WORKER_MODEL=deepseek-v4-flash DRIVER_MODEL=deepseek-v4-flash \
- *   npx tsx bench/src/atom-humaneval.mts
- */
-
-import {
-  type Agent,
-  type AgentProfile,
-  type AgentSpec,
-  contentAddress,
-  createExecutor,
-  createExecutorRegistry,
-  createSupervisor,
-  gateOnDeliverable,
-  InMemoryResultBlobStore,
-  InMemorySpawnJournal,
-  mapExecutorResult,
-  supervisorAgent,
-} from '../../src/runtime/index'
-import { basePrompt, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
-import {
-  benchProfileModel,
-  benchRouterProfile,
-  type BenchRouterTarget,
-  runBenchRouterTurn,
-  withBenchProfile,
-} from './router-turn'
-
-function must(k: string): string {
-  const v = process.env[k]
-  if (!v) throw new Error(`missing required env ${k}`)
-  return v
-}
-
-const N = Number(process.env.N ?? 5)
-const K = Number(process.env.K ?? 3)
-const OFFSET = Number(process.env.OFFSET ?? 0)
-const WORKER_TEMP = Number(process.env.WORKER_TEMP ?? 0.7)
-
-const cfg: BenchRouterTarget = {
-  routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
-  routerKey: must('TANGLE_API_KEY'),
-  profile: benchRouterProfile(
-    'humaneval-worker',
-    process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-    { temperature: WORKER_TEMP },
-  ),
-}
-const driverCfg: BenchRouterTarget = {
-  ...cfg,
-  profile: benchRouterProfile(
-    'humaneval-driver',
-    process.env.DRIVER_MODEL ?? benchProfileModel(cfg.profile),
-    { maxTurns: K + 4 },
-  ),
-}
-
-// ── A gated router worker: one router call → candidate code, settled valid ⟺ the tests pass ──
-function humanEvalWorker(task: HumanEvalTask, label: string): Agent<unknown, unknown> {
-  const profile: AgentProfile = withBenchProfile(cfg.profile, {
-    name: label,
-    systemPrompt: basePrompt(task),
-  })
-  const routerFactory = createExecutor({
-    backend: 'router',
-    routerBaseUrl: cfg.routerBaseUrl,
-    routerKey: cfg.routerKey,
-  })
-  const executorFactory = (spec: AgentSpec, ctx: Parameters<typeof routerFactory>[1]) => {
-    const inner = routerFactory(spec, ctx)
-    const mapped = mapExecutorResult(inner, (result) => {
-      const raw = result.out as { content?: unknown }
-      const code = extractCode(typeof raw?.content === 'string' ? raw.content : '')
-      return { outRef: contentAddress(code), out: code }
-    })
-    return gateOnDeliverable(mapped, {
-      check: async (out) => (await runChecker(task, String(out))).pass === 1,
-      describe: `${task.taskId}: the provided test suite passes`,
-    })
-  }
-  const spec: AgentSpec = { profile, harness: null, executorFactory }
-  return { name: label, act: async () => '', executorSpec: spec } as Agent<unknown, unknown> & {
-    executorSpec: AgentSpec
-  }
-}
-
-const driverSystem = `You are an orchestrator driving worker agents to solve a Python coding task. You do NOT write code yourself. Each worker independently attempts the task and is graded by a deterministic, hidden test suite. Tools: spawn_worker (dispatch one attempt; the "profile" argument may be {} and "task" a short note), await_event (collect the next settled worker — its result tells you valid:true if its tests PASSED, valid:false if they failed), and stopping (reply with NO tool call) once a worker has DELIVERED. Spawn one worker, await it; if it delivered, stop; if not, spawn another, up to ${K} workers total. You cannot declare success yourself — only a delivered (valid:true) worker counts.`
-
-interface TaskOutcome {
-  taskId: string
-  driverDelivered: boolean
-  blindDelivered: boolean
-  driverSpawns: number
-  driverWorkerTokens: number
-}
-
-// ── Driver arm: the orchestrated atom ────────────────────────────────────────────────────────
-async function driveTask(
-  task: HumanEvalTask,
-): Promise<{ delivered: boolean; spawns: number; tokens: number }> {
-  const blobs = new InMemoryResultBlobStore()
-  const journal = new InMemorySpawnJournal()
-  let spawns = 0
-  const makeWorker = (): Agent<unknown, unknown> => {
-    const w = humanEvalWorker(task, `w-${spawns}`)
-    spawns += 1
-    return w
-  }
-  const root = supervisorAgent(
-    withBenchProfile(driverCfg.profile, {
-      name: `drv-${task.taskId}`,
-      systemPrompt: driverSystem,
-    }),
-    {
-      router: {
-        routerBaseUrl: driverCfg.routerBaseUrl,
-        routerKey: driverCfg.routerKey,
-      },
-      blobs,
-      makeWorkerAgent: makeWorker,
-      perWorker: { maxIterations: 2, maxTokens: 4000 },
-    },
-  )
-  const runId = `he-${task.taskId.replace('/', '-')}`
-  const result = await createSupervisor<unknown, unknown>().run(root, basePrompt(task), {
-    budget: { maxIterations: 100, maxTokens: 400_000 },
-    runId,
-    journal,
-    blobs,
-    executors: createExecutorRegistry(),
-    maxDepth: 4,
-    now: () => Date.now(),
-  })
-  const tree = await journal.loadTree(runId)
-  const tokens = (tree ?? [])
-    .filter((e): e is Extract<NonNullable<typeof tree>[number], { kind: 'settled' }> => e.kind === 'settled')
-    .reduce((s, e) => s + e.spent.tokens.input + e.spent.tokens.output, 0)
-  return { delivered: result.kind === 'winner', spawns, tokens }
-}
-
-// ── Blind arm: K independent workers, best-of-K by the checker (no orchestration) ─────────────
-async function blindTask(task: HumanEvalTask): Promise<boolean> {
-  for (let i = 0; i < K; i += 1) {
-    // A transient router error is a FAILED attempt, not a crash — the driver arm already types
-    // an executor throw into a `down` settlement, so the blind arm must match (fair comparison).
-    let content = ''
-    try {
-      const res = await runBenchRouterTurn(
-        {
-          routerBaseUrl: cfg.routerBaseUrl,
-          routerKey: cfg.routerKey,
-          profile: withBenchProfile(cfg.profile, {
-            name: 'humaneval-blind-atom-worker',
-          }),
-        },
-        basePrompt(task),
-      )
-      content = res.finalText
-    } catch {
-      continue
-    }
-    if ((await runChecker(task, extractCode(content))).pass === 1) return true
-  }
-  return false
-}
-
-async function main(): Promise<void> {
-  console.log(
-    `atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${benchProfileModel(cfg.profile)} driver=${benchProfileModel(driverCfg.profile)}`,
-  )
-  const tasks = await loadHumanEval(N, OFFSET)
-  const outcomes: TaskOutcome[] = []
-  for (const task of tasks) {
-    const drv = await driveTask(task)
-    const blind = await blindTask(task)
-    outcomes.push({
-      taskId: task.taskId,
-      driverDelivered: drv.delivered,
-      blindDelivered: blind,
-      driverSpawns: drv.spawns,
-      driverWorkerTokens: drv.tokens,
-    })
-    console.log(
-      `  ${task.taskId.padEnd(14)} driver=${drv.delivered ? 'PASS' : 'fail'} (spawns=${drv.spawns}, tok=${drv.tokens})  blind@${K}=${blind ? 'PASS' : 'fail'}`,
-    )
-  }
-  const driverPass = outcomes.filter((o) => o.driverDelivered).length
-  const blindPass = outcomes.filter((o) => o.blindDelivered).length
-  const avgSpawns = outcomes.reduce((s, o) => s + o.driverSpawns, 0) / Math.max(1, outcomes.length)
-  console.log('\n── verdict ──')
-  console.log(`driver-orchestrated delivered: ${driverPass}/${outcomes.length}  (avg spawns ${avgSpawns.toFixed(1)} of ${K} allowed)`)
-  console.log(`blind best-of-${K} delivered:    ${blindPass}/${outcomes.length}`)
-  console.log(
-    driverPass > blindPass
-      ? `→ orchestration BEAT blind by +${driverPass - blindPass} tasks`
-      : driverPass === blindPass
-        ? `→ orchestration TIED blind (the atom delivers, but adds no lift here at this N)`
-        : `→ orchestration LOST to blind by ${blindPass - driverPass} tasks`,
-  )
-}
-
-main().catch((e) => {
-  console.error(e)
-  process.exit(1)
-})

package/src/david-attribution.mts

@@ -1,97 +0,0 @@
-/**
- * DAVID mechanism attribution — decompose the cheap-model harness's held-out
- * accuracy into what SAMPLING buys vs what VERIFICATION-SELECTION buys, so a
- * David-Goliath win is credited to the right lever (not just best-of-N luck).
- *
- * For each task, generate N candidate solutions + the model's own tests, then
- * report four numbers on the HIDDEN test:
- *   pass@1        — first candidate (no harness).
- *   mean-cand     — expected accuracy of a RANDOM candidate (sampling floor).
- *   oracle@N      — a correct candidate exists among the N (ceiling of selection).
- *   verify-select — the candidate the self-tests picked (the actual David).
- * verify-select − mean-cand = what VERIFICATION adds over blind sampling;
- * oracle@N − verify-select = the selection gap left on the table.
- *
- * Run from cwd=bench: env DAVID=groq/llama-3.1-8b-instant N=8 T=5 NTASKS=60 \
- *   node_modules/.bin/tsx src/david-attribution.mts
- */
-import { execFile } from 'node:child_process'
-import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
-import { benchRouterProfile, runBenchRouterTurn } from './router-turn'
-
-function requiredEnv(name: string): string {
-  const value = process.env[name]
-  if (!value) throw new Error(`${name} required`)
-  return value
-}
-
-const KEY = requiredEnv('TANGLE_API_KEY')
-const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
-const N = Number(process.env.N ?? 8)
-const T = Number(process.env.T ?? 5)
-const NTASKS = Number(process.env.NTASKS ?? 60)
-const CONC = Number(process.env.CONCURRENCY ?? 6)
-const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
-const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
-
-async function chat(messages: { role: string; content: string }[], temp: number): Promise<string> {
-  try {
-    const system = messages.find((message) => message.role === 'system')?.content
-    const turn = await runBenchRouterTurn(
-      {
-        routerBaseUrl: ROUTER,
-        routerKey: KEY,
-        profile: benchRouterProfile('david-attribution-worker', DAVID, {
-          ...(system ? { systemPrompt: system } : {}),
-          temperature: temp,
-          maxTokens: MAX_TOKENS,
-        }),
-        timeoutMs: LLM_TIMEOUT_MS,
-      },
-      { messages: messages.filter((message) => message.role !== 'system') },
-    )
-    return turn.finalText
-  } catch {
-    return ''
-  }
-}
-const exec = (f: string, a: string[], o: object) => new Promise<number>((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0))))
-async function runPy(p: string): Promise<boolean> { const d = mkdtempSync(join(tmpdir(), 'da-')); try { writeFileSync(join(d, 'p.py'), p); return (await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: 6000 })) === 0 } finally { rmSync(d, { recursive: true, force: true }) } }
-const SOLVE = 'Expert Python. Output the COMPLETE function in one ```python block, no prose, no tests.'
-const genSol = async (t: HumanEvalTask, temp: number) => extractCode(await chat([{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp))
-async function genTests(t: HumanEvalTask): Promise<string[]> {
-  const b = extractCode(await chat([{ role: 'system', content: 'Write Python assert unit tests. Output ONLY a ```python block of `assert <entry>(...) == ...` lines. No function, no prose.' }, { role: 'user', content: `entry: ${t.entryPoint}\n\n\`\`\`python\n${t.prompt}\`\`\`` }], 0.4))
-  return b.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
-}
-const judge = async (t: HumanEvalTask, code: string) => code.trim() ? runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`) : false
-async function scoreTests(code: string, tests: string[]): Promise<number> { if (!code.trim() || !tests.length) return 0; let p = 0; for (const a of tests) if (await runPy(`${code}\n\n${a}\n`)) p++; return p }
-async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2) => Promise<R>): Promise<R[]> { const o = new Array<R>(xs.length); let i = 0; await Promise.all(Array.from({ length: n }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!) } })); return o }
-
-async function main(): Promise<void> {
-  const tasks = await loadHumanEval(NTASKS, 0)
-  console.error(`=== ATTRIBUTION · ${DAVID} · N=${N} sols + ${T} tests · n=${tasks.length} ===`)
-  let done = 0
-  const rows = await pool(tasks, CONC, async (t) => {
-    const cands = (await Promise.all(Array.from({ length: N }, () => genSol(t, 0.7)))).filter((c) => c.trim())
-    if (!cands.length) return { p1: 0, mean: 0, oracle: 0, sel: 0 }
-    const tests = await genTests(t)
-    const hidden = await Promise.all(cands.map((c) => judge(t, c))) // hidden-test pass per candidate (for attribution only)
-    const selScores = tests.length ? await Promise.all(cands.map((c) => scoreTests(c, tests))) : cands.map(() => 0)
-    let bi = 0; for (let i = 1; i < cands.length; i++) if (selScores[i]! > selScores[bi]! || (selScores[i]! === selScores[bi]! && cands[i]!.length > cands[bi]!.length)) bi = i
-    if (++done % 15 === 0) console.error(`  ${done}/${tasks.length}`)
-    return { p1: hidden[0] ? 1 : 0, mean: hidden.filter(Boolean).length / cands.length, oracle: hidden.some(Boolean) ? 1 : 0, sel: hidden[bi] ? 1 : 0 }
-  })
-  const n = rows.length, avg = (f: (r: typeof rows[number]) => number) => (rows.reduce((s, r) => s + f(r), 0) / n) * 100
-  console.log('\n=== ATTRIBUTION (held-out) ===')
-  console.log(`  pass@1 (no harness)        : ${avg((r) => r.p1).toFixed(1)}%`)
-  console.log(`  mean random candidate      : ${avg((r) => r.mean).toFixed(1)}%   (sampling floor)`)
-  console.log(`  verify-select (DAVID)      : ${avg((r) => r.sel).toFixed(1)}%`)
-  console.log(`  oracle@N (a correct exists): ${avg((r) => r.oracle).toFixed(1)}%   (selection ceiling)`)
-  console.log(`  --> verification adds over random sampling: +${(avg((r) => r.sel) - avg((r) => r.mean)).toFixed(1)}pp`)
-  console.log(`  --> selection gap still on table (oracle-select): ${(avg((r) => r.oracle) - avg((r) => r.sel)).toFixed(1)}pp`)
-}
-main().catch((e) => { console.error('MAIN:', e instanceof Error ? e.stack : e); process.exit(1) })