@tangle-network/agent-bench 0.8.10 → 0.8.15

package/src/atom-humaneval.mts

@@ -1,218 +0,0 @@
-/**
- * The "useful or BS" verdict: agents-driving-agents on a REAL deployable-checked domain.
- *
- * A `driverAgent` with a REAL router-LLM brain drives, per HumanEval task: it spawns
- * worker agents (each a router LLM that writes the function), every worker GATED by the
- * deterministic local Docker checker (the deliverable — a worker settles `valid` ⟺ its tests
- * pass), and the completion-oracle keeps-best a DELIVERED worker. The supervisor returns a winner
- * ONLY when a worker actually passed the tests (no self-declared done). We measure the driver's
- * delivered rate against a BLIND best-of-K baseline (K independent workers, no orchestration) at
- * the same K — the honest "does the recursion+oracle beat blind compute, or is it BS" question.
- *
- * Run (creds via dotenvx; Docker daemon must be up):
- *   DOTENV_PRIVATE_KEY_FILE=~/company/devops/secrets/.env.keys \
- *   dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
- *   N=5 K=3 WORKER_MODEL=deepseek-v4-flash DRIVER_MODEL=deepseek-v4-flash \
- *   npx tsx bench/src/atom-humaneval.mts
- */
-
-import {
-  type Agent,
-  type AgentProfile,
-  type AgentSpec,
-  contentAddress,
-  createExecutor,
-  createExecutorRegistry,
-  createSupervisor,
-  gateOnDeliverable,
-  InMemoryResultBlobStore,
-  InMemorySpawnJournal,
-  mapExecutorResult,
-  supervisorAgent,
-} from '../../src/runtime/index'
-import { basePrompt, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
-import {
-  benchProfileModel,
-  benchRouterProfile,
-  type BenchRouterTarget,
-  runBenchRouterTurn,
-  withBenchProfile,
-} from './router-turn'
-
-function must(k: string): string {
-  const v = process.env[k]
-  if (!v) throw new Error(`missing required env ${k}`)
-  return v
-}
-
-const N = Number(process.env.N ?? 5)
-const K = Number(process.env.K ?? 3)
-const OFFSET = Number(process.env.OFFSET ?? 0)
-const WORKER_TEMP = Number(process.env.WORKER_TEMP ?? 0.7)
-
-const cfg: BenchRouterTarget = {
-  routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
-  routerKey: must('TANGLE_API_KEY'),
-  profile: benchRouterProfile(
-    'humaneval-worker',
-    process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-    { temperature: WORKER_TEMP },
-  ),
-}
-const driverCfg: BenchRouterTarget = {
-  ...cfg,
-  profile: benchRouterProfile(
-    'humaneval-driver',
-    process.env.DRIVER_MODEL ?? benchProfileModel(cfg.profile),
-    { maxTurns: K + 4 },
-  ),
-}
-
-// ── A gated router worker: one router call → candidate code, settled valid ⟺ the tests pass ──
-function humanEvalWorker(task: HumanEvalTask, label: string): Agent<unknown, unknown> {
-  const profile: AgentProfile = withBenchProfile(cfg.profile, {
-    name: label,
-    systemPrompt: basePrompt(task),
-  })
-  const routerFactory = createExecutor({
-    backend: 'router',
-    routerBaseUrl: cfg.routerBaseUrl,
-    routerKey: cfg.routerKey,
-  })
-  const executorFactory = (spec: AgentSpec, ctx: Parameters<typeof routerFactory>[1]) => {
-    const inner = routerFactory(spec, ctx)
-    const mapped = mapExecutorResult(inner, (result) => {
-      const raw = result.out as { content?: unknown }
-      const code = extractCode(typeof raw?.content === 'string' ? raw.content : '')
-      return { outRef: contentAddress(code), out: code }
-    })
-    return gateOnDeliverable(mapped, {
-      check: async (out) => (await runChecker(task, String(out))).pass === 1,
-      describe: `${task.taskId}: the provided test suite passes`,
-    })
-  }
-  const spec: AgentSpec = { profile, harness: null, executorFactory }
-  return { name: label, act: async () => '', executorSpec: spec } as Agent<unknown, unknown> & {
-    executorSpec: AgentSpec
-  }
-}
-
-const driverSystem = `You are an orchestrator driving worker agents to solve a Python coding task. You do NOT write code yourself. Each worker independently attempts the task and is graded by a deterministic, hidden test suite. Tools: spawn_worker (dispatch one attempt; the "profile" argument may be {} and "task" a short note), await_event (collect the next settled worker — its result tells you valid:true if its tests PASSED, valid:false if they failed), and stopping (reply with NO tool call) once a worker has DELIVERED. Spawn one worker, await it; if it delivered, stop; if not, spawn another, up to ${K} workers total. You cannot declare success yourself — only a delivered (valid:true) worker counts.`
-
-interface TaskOutcome {
-  taskId: string
-  driverDelivered: boolean
-  blindDelivered: boolean
-  driverSpawns: number
-  driverWorkerTokens: number
-}
-
-// ── Driver arm: the orchestrated atom ────────────────────────────────────────────────────────
-async function driveTask(
-  task: HumanEvalTask,
-): Promise<{ delivered: boolean; spawns: number; tokens: number }> {
-  const blobs = new InMemoryResultBlobStore()
-  const journal = new InMemorySpawnJournal()
-  let spawns = 0
-  const makeWorker = (): Agent<unknown, unknown> => {
-    const w = humanEvalWorker(task, `w-${spawns}`)
-    spawns += 1
-    return w
-  }
-  const root = supervisorAgent(
-    withBenchProfile(driverCfg.profile, {
-      name: `drv-${task.taskId}`,
-      systemPrompt: driverSystem,
-    }),
-    {
-      router: {
-        routerBaseUrl: driverCfg.routerBaseUrl,
-        routerKey: driverCfg.routerKey,
-      },
-      blobs,
-      makeWorkerAgent: makeWorker,
-      perWorker: { maxIterations: 2, maxTokens: 4000 },
-    },
-  )
-  const runId = `he-${task.taskId.replace('/', '-')}`
-  const result = await createSupervisor<unknown, unknown>().run(root, basePrompt(task), {
-    budget: { maxIterations: 100, maxTokens: 400_000 },
-    runId,
-    journal,
-    blobs,
-    executors: createExecutorRegistry(),
-    maxDepth: 4,
-    now: () => Date.now(),
-  })
-  const tree = await journal.loadTree(runId)
-  const tokens = (tree ?? [])
-    .filter((e): e is Extract<NonNullable<typeof tree>[number], { kind: 'settled' }> => e.kind === 'settled')
-    .reduce((s, e) => s + e.spent.tokens.input + e.spent.tokens.output, 0)
-  return { delivered: result.kind === 'winner', spawns, tokens }
-}
-
-// ── Blind arm: K independent workers, best-of-K by the checker (no orchestration) ─────────────
-async function blindTask(task: HumanEvalTask): Promise<boolean> {
-  for (let i = 0; i < K; i += 1) {
-    // A transient router error is a FAILED attempt, not a crash — the driver arm already types
-    // an executor throw into a `down` settlement, so the blind arm must match (fair comparison).
-    let content = ''
-    try {
-      const res = await runBenchRouterTurn(
-        {
-          routerBaseUrl: cfg.routerBaseUrl,
-          routerKey: cfg.routerKey,
-          profile: withBenchProfile(cfg.profile, {
-            name: 'humaneval-blind-atom-worker',
-          }),
-        },
-        basePrompt(task),
-      )
-      content = res.finalText
-    } catch {
-      continue
-    }
-    if ((await runChecker(task, extractCode(content))).pass === 1) return true
-  }
-  return false
-}
-
-async function main(): Promise<void> {
-  console.log(
-    `atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${benchProfileModel(cfg.profile)} driver=${benchProfileModel(driverCfg.profile)}`,
-  )
-  const tasks = await loadHumanEval(N, OFFSET)
-  const outcomes: TaskOutcome[] = []
-  for (const task of tasks) {
-    const drv = await driveTask(task)
-    const blind = await blindTask(task)
-    outcomes.push({
-      taskId: task.taskId,
-      driverDelivered: drv.delivered,
-      blindDelivered: blind,
-      driverSpawns: drv.spawns,
-      driverWorkerTokens: drv.tokens,
-    })
-    console.log(
-      `  ${task.taskId.padEnd(14)} driver=${drv.delivered ? 'PASS' : 'fail'} (spawns=${drv.spawns}, tok=${drv.tokens})  blind@${K}=${blind ? 'PASS' : 'fail'}`,
-    )
-  }
-  const driverPass = outcomes.filter((o) => o.driverDelivered).length
-  const blindPass = outcomes.filter((o) => o.blindDelivered).length
-  const avgSpawns = outcomes.reduce((s, o) => s + o.driverSpawns, 0) / Math.max(1, outcomes.length)
-  console.log('\n── verdict ──')
-  console.log(`driver-orchestrated delivered: ${driverPass}/${outcomes.length}  (avg spawns ${avgSpawns.toFixed(1)} of ${K} allowed)`)
-  console.log(`blind best-of-${K} delivered:    ${blindPass}/${outcomes.length}`)
-  console.log(
-    driverPass > blindPass
-      ? `→ orchestration BEAT blind by +${driverPass - blindPass} tasks`
-      : driverPass === blindPass
-        ? `→ orchestration TIED blind (the atom delivers, but adds no lift here at this N)`
-        : `→ orchestration LOST to blind by ${blindPass - driverPass} tasks`,
-  )
-}
-
-main().catch((e) => {
-  console.error(e)
-  process.exit(1)
-})

package/src/david-attribution.mts

@@ -1,97 +0,0 @@
-/**
- * DAVID mechanism attribution — decompose the cheap-model harness's held-out
- * accuracy into what SAMPLING buys vs what VERIFICATION-SELECTION buys, so a
- * David-Goliath win is credited to the right lever (not just best-of-N luck).
- *
- * For each task, generate N candidate solutions + the model's own tests, then
- * report four numbers on the HIDDEN test:
- *   pass@1        — first candidate (no harness).
- *   mean-cand     — expected accuracy of a RANDOM candidate (sampling floor).
- *   oracle@N      — a correct candidate exists among the N (ceiling of selection).
- *   verify-select — the candidate the self-tests picked (the actual David).
- * verify-select − mean-cand = what VERIFICATION adds over blind sampling;
- * oracle@N − verify-select = the selection gap left on the table.
- *
- * Run from cwd=bench: env DAVID=groq/llama-3.1-8b-instant N=8 T=5 NTASKS=60 \
- *   node_modules/.bin/tsx src/david-attribution.mts
- */
-import { execFile } from 'node:child_process'
-import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
-import { benchRouterProfile, runBenchRouterTurn } from './router-turn'
-
-function requiredEnv(name: string): string {
-  const value = process.env[name]
-  if (!value) throw new Error(`${name} required`)
-  return value
-}
-
-const KEY = requiredEnv('TANGLE_API_KEY')
-const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
-const N = Number(process.env.N ?? 8)
-const T = Number(process.env.T ?? 5)
-const NTASKS = Number(process.env.NTASKS ?? 60)
-const CONC = Number(process.env.CONCURRENCY ?? 6)
-const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
-const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
-
-async function chat(messages: { role: string; content: string }[], temp: number): Promise<string> {
-  try {
-    const system = messages.find((message) => message.role === 'system')?.content
-    const turn = await runBenchRouterTurn(
-      {
-        routerBaseUrl: ROUTER,
-        routerKey: KEY,
-        profile: benchRouterProfile('david-attribution-worker', DAVID, {
-          ...(system ? { systemPrompt: system } : {}),
-          temperature: temp,
-          maxTokens: MAX_TOKENS,
-        }),
-        timeoutMs: LLM_TIMEOUT_MS,
-      },
-      { messages: messages.filter((message) => message.role !== 'system') },
-    )
-    return turn.finalText
-  } catch {
-    return ''
-  }
-}
-const exec = (f: string, a: string[], o: object) => new Promise<number>((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0))))
-async function runPy(p: string): Promise<boolean> { const d = mkdtempSync(join(tmpdir(), 'da-')); try { writeFileSync(join(d, 'p.py'), p); return (await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: 6000 })) === 0 } finally { rmSync(d, { recursive: true, force: true }) } }
-const SOLVE = 'Expert Python. Output the COMPLETE function in one ```python block, no prose, no tests.'
-const genSol = async (t: HumanEvalTask, temp: number) => extractCode(await chat([{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp))
-async function genTests(t: HumanEvalTask): Promise<string[]> {
-  const b = extractCode(await chat([{ role: 'system', content: 'Write Python assert unit tests. Output ONLY a ```python block of `assert <entry>(...) == ...` lines. No function, no prose.' }, { role: 'user', content: `entry: ${t.entryPoint}\n\n\`\`\`python\n${t.prompt}\`\`\`` }], 0.4))
-  return b.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
-}
-const judge = async (t: HumanEvalTask, code: string) => code.trim() ? runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`) : false
-async function scoreTests(code: string, tests: string[]): Promise<number> { if (!code.trim() || !tests.length) return 0; let p = 0; for (const a of tests) if (await runPy(`${code}\n\n${a}\n`)) p++; return p }
-async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2) => Promise<R>): Promise<R[]> { const o = new Array<R>(xs.length); let i = 0; await Promise.all(Array.from({ length: n }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!) } })); return o }
-
-async function main(): Promise<void> {
-  const tasks = await loadHumanEval(NTASKS, 0)
-  console.error(`=== ATTRIBUTION · ${DAVID} · N=${N} sols + ${T} tests · n=${tasks.length} ===`)
-  let done = 0
-  const rows = await pool(tasks, CONC, async (t) => {
-    const cands = (await Promise.all(Array.from({ length: N }, () => genSol(t, 0.7)))).filter((c) => c.trim())
-    if (!cands.length) return { p1: 0, mean: 0, oracle: 0, sel: 0 }
-    const tests = await genTests(t)
-    const hidden = await Promise.all(cands.map((c) => judge(t, c))) // hidden-test pass per candidate (for attribution only)
-    const selScores = tests.length ? await Promise.all(cands.map((c) => scoreTests(c, tests))) : cands.map(() => 0)
-    let bi = 0; for (let i = 1; i < cands.length; i++) if (selScores[i]! > selScores[bi]! || (selScores[i]! === selScores[bi]! && cands[i]!.length > cands[bi]!.length)) bi = i
-    if (++done % 15 === 0) console.error(`  ${done}/${tasks.length}`)
-    return { p1: hidden[0] ? 1 : 0, mean: hidden.filter(Boolean).length / cands.length, oracle: hidden.some(Boolean) ? 1 : 0, sel: hidden[bi] ? 1 : 0 }
-  })
-  const n = rows.length, avg = (f: (r: typeof rows[number]) => number) => (rows.reduce((s, r) => s + f(r), 0) / n) * 100
-  console.log('\n=== ATTRIBUTION (held-out) ===')
-  console.log(`  pass@1 (no harness)        : ${avg((r) => r.p1).toFixed(1)}%`)
-  console.log(`  mean random candidate      : ${avg((r) => r.mean).toFixed(1)}%   (sampling floor)`)
-  console.log(`  verify-select (DAVID)      : ${avg((r) => r.sel).toFixed(1)}%`)
-  console.log(`  oracle@N (a correct exists): ${avg((r) => r.oracle).toFixed(1)}%   (selection ceiling)`)
-  console.log(`  --> verification adds over random sampling: +${(avg((r) => r.sel) - avg((r) => r.mean)).toFixed(1)}pp`)
-  console.log(`  --> selection gap still on table (oracle-select): ${(avg((r) => r.oracle) - avg((r) => r.sel)).toFixed(1)}pp`)
-}
-main().catch((e) => { console.error('MAIN:', e instanceof Error ? e.stack : e); process.exit(1) })

package/src/david-goliath.mts

@@ -1,168 +0,0 @@
-/**
- * DAVID vs GOLIATH — the program's northstar, tested head-on: does a CHEAP model
- * with a self-verification harness beat a FRONTIER model running solo, at EQUAL OR
- * LOWER dollar cost, on held-out code?
- *
- *   GOLIATH — a strong model, ONE shot. The "just use the big model" baseline.
- *   DAVID   — a cheap/weak model + test-time compute: generate N candidate
- *             solutions AND M of its own unit tests, EXECUTE every candidate
- *             against the generated tests, and submit the candidate that passes
- *             the most (CodeT-style execution self-selection). No ground-truth
- *             test is ever used to select — only the model's own generated tests.
- *
- * Both are graded by the HIDDEN HumanEval test (never shown). Cost is the real
- * token spend × the router-reported/priced rate per arm. The win condition is a
- * Pareto beat: David's held-out pass rate >= Goliath's AND David's $ <= Goliath's.
- * Mechanism under test: EXECUTION-BASED VERIFICATION is the lever that lets a weak
- * generator punch above its solo weight — the standing "verification is live" claim
- * at its most dramatic. Paired McNemar on per-task discordant pairs for significance.
- *
- * Run from cwd=bench:  env DAVID=glm-5.2 GOLIATH=deepseek-v4-flash \
- *   N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts
- */
-import { execFile } from 'node:child_process'
-import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
-import { runBenchRouterTurn } from './router-turn'
-
-const KEY = process.env.TANGLE_API_KEY
-if (!KEY) throw new Error('TANGLE_API_KEY required')
-const ROUTER_KEY = KEY
-const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
-const GOLIATH = process.env.GOLIATH ?? 'deepseek-v4-flash'
-const N = Number(process.env.N ?? 8) // David candidate solutions
-const T = Number(process.env.T ?? 5) // David generated tests
-const NTASKS = Number(process.env.NTASKS ?? 164)
-const REPS = Number(process.env.REPS ?? 2)
-const CONC = Number(process.env.CONCURRENCY ?? 6)
-const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000)
-const LLM_TIMEOUT = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
-const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
-
-// Approx $/1M tokens (in,out) for cost accounting — the router does not price
-// every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the
-// point, and the ratio is robust to small rate errors. Override via PRICES env.
-const PRICES: Record<string, [number, number]> = {
-  'groq/llama-3.1-8b-instant': [0.05, 0.08],
-  'google/gemini-2.5-flash-lite': [0.10, 0.40],
-  'openai/gpt-4o-mini': [0.15, 0.60],
-  'anthropic/claude-haiku-4-5-20251001': [1.0, 5.0],
-  'glm-5.2': [0.60, 2.20],
-}
-const priceOf = (m: string) => PRICES[m] ?? [0.5, 1.5]
-
-interface Usage { in: number; out: number }
-const zero = (): Usage => ({ in: 0, out: 0 })
-const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out }
-const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 }
-
-async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise<string> {
-  try {
-    const system = messages.find((message) => message.role === 'system')?.content
-    const result = await runBenchRouterTurn(
-      {
-        routerBaseUrl: ROUTER,
-        routerKey: ROUTER_KEY,
-        profile: {
-          name: 'david-goliath-worker',
-          harness: 'cli-base',
-          model: {
-            provider: 'tangle-router',
-            default: model,
-            metadata: { temperature, maxTokens: MAX_TOKENS },
-          },
-          ...(system ? { prompt: { systemPrompt: system } } : {}),
-        },
-        timeoutMs: LLM_TIMEOUT,
-      },
-      { messages: messages.filter((message) => message.role !== 'system') },
-    )
-    if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage')
-    addU(usage, { in: result.usage.input, out: result.usage.output })
-    return result.finalText
-  } catch {
-    return ''
-  }
-}
-const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) })))
-async function runPy(program: string): Promise<{ ok: boolean }> {
-  const d = mkdtempSync(join(tmpdir(), 'dg-'))
-  try { writeFileSync(join(d, 'p.py'), program); const r = await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: EXEC_TIMEOUT }); return { ok: r.code === 0 } } finally { rmSync(d, { recursive: true, force: true }) }
-}
-
-const SOLVE = 'You are an expert Python programmer. Output the COMPLETE function (signature + body + imports) in a single ```python block. No prose, no tests.'
-async function genSolution(model: string, t: HumanEvalTask, temp: number, u: Usage): Promise<string> {
-  return extractCode(await chat(model, [{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp, u))
-}
-// David writes its OWN tests (never sees the hidden test). Parse assert lines.
-async function genTests(model: string, t: HumanEvalTask, u: Usage): Promise<string[]> {
-  const reply = await chat(model, [
-    { role: 'system', content: 'Write Python assert-based unit tests for the described function. Output ONLY a ```python block of standalone `assert <entry>(...) == ...` lines (at least a few, covering normal + edge cases). No function definition, no prose.' },
-    { role: 'user', content: `Function to test (entry point: ${t.entryPoint}):\n\n\`\`\`python\n${t.prompt}\`\`\`` },
-  ], 0.4, u)
-  const block = extractCode(reply) || reply
-  return block.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
-}
-// Hidden held-out judge — the truth. Never used for selection.
-async function judge(t: HumanEvalTask, code: string): Promise<boolean> {
-  if (!code.trim()) return false
-  return (await runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`)).ok
-}
-// David's self-selection: score each candidate by how many of ITS OWN tests it passes.
-async function scoreOnTests(code: string, tests: string[]): Promise<number> {
-  if (!code.trim() || tests.length === 0) return 0
-  let pass = 0
-  // one program per test keeps a crash on one test from voiding the rest
-  for (const a of tests) if ((await runPy(`${code}\n\n${a}\n`)).ok) pass++
-  return pass
-}
-
-async function davidArm(t: HumanEvalTask, u: Usage): Promise<string> {
-  const cands = (await Promise.all(Array.from({ length: N }, () => genSolution(DAVID, t, 0.7, u)))).filter((c) => c.trim())
-  if (cands.length === 0) return ''
-  const tests = await genTests(DAVID, t, u)
-  if (tests.length === 0) return cands[0]! // no verifier signal → first sample
-  const scored = await Promise.all(cands.map(async (c) => ({ c, s: await scoreOnTests(c, tests) })))
-  scored.sort((a, b) => b.s - a.s || b.c.length - a.c.length)
-  return scored[0]!.c
-}
-const goliathArm = (t: HumanEvalTask, u: Usage) => genSolution(GOLIATH, t, 0.2, u)
-
-async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2, i: number) => Promise<R>): Promise<R[]> {
-  const o = new Array<R>(xs.length); let i = 0
-  await Promise.all(Array.from({ length: Math.min(n, xs.length) }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!, k) } }))
-  return o
-}
-function mcnemar(b: number, c: number): number { const n = b + c; if (n === 0) return 1; const k = Math.min(b, c); const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }; let tl = 0; for (let i = 0; i <= k; i++) tl += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2)); return Math.min(1, 2 * tl) }
-
-async function main(): Promise<void> {
-  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: david-goliath loaded'); return }
-  const tasks = await loadHumanEval(NTASKS, 0)
-  console.error(`=== DAVID(${DAVID}, N=${N} sols + ${T} self-tests) vs GOLIATH(${GOLIATH}, 1 shot) · HumanEval n=${tasks.length} · reps=${REPS} ===`)
-  const dU = zero(), gU = zero()
-  const units = tasks.flatMap((task) => Array.from({ length: REPS }, () => task))
-  let done = 0
-  const res = await pool(units, CONC, async (task) => {
-    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
-    const [dCode, gCode] = await Promise.all([safe(() => davidArm(task, dU)), safe(() => goliathArm(task, gU))])
-    const [d, g] = await Promise.all([judge(task, dCode), judge(task, gCode)])
-    if (++done % 20 === 0) console.error(`  ${done}/${units.length} units`)
-    return { d, g }
-  })
-  const n = res.length, dPass = res.filter((r) => r.d).length, gPass = res.filter((r) => r.g).length
-  const b = res.filter((r) => r.d && !r.g).length, c = res.filter((r) => !r.d && r.g).length
-  const p = mcnemar(b, c)
-  const dCost = usd(DAVID, dU), gCost = usd(GOLIATH, gU)
-  console.log('\n=== RESULT (held-out HumanEval) ===')
-  console.log(`  GOLIATH ${GOLIATH} solo : ${gPass}/${n} = ${(gPass / n * 100).toFixed(1)}%   $${gCost.toFixed(4)}`)
-  console.log(`  DAVID   ${DAVID} + verify: ${dPass}/${n} = ${(dPass / n * 100).toFixed(1)}%   $${dCost.toFixed(4)}`)
-  console.log(`  accuracy: David ${dPass >= gPass ? '>=' : '<'} Goliath (${(dPass / n * 100).toFixed(1)} vs ${(gPass / n * 100).toFixed(1)}); paired McNemar David-only=${b} Goliath-only=${c} p=${p.toFixed(4)}`)
-  console.log(`  cost: David is ${(gCost / Math.max(dCost, 1e-9)).toFixed(1)}x CHEAPER ($${dCost.toFixed(4)} vs $${gCost.toFixed(4)})`)
-  const paretoBeat = dPass >= gPass && dCost <= gCost
-  const sigBeat = dPass > gPass && p < 0.05
-  console.log(`  VERDICT: ${sigBeat ? 'DAVID SIGNIFICANTLY BEATS GOLIATH' : paretoBeat ? 'DAVID PARETO-DOMINATES (>= accuracy, <= cost)' : dPass >= gPass ? 'David matches accuracy (check cost)' : 'Goliath wins accuracy'}`)
-}
-main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })

package/src/decoder-live.mts

@@ -1,133 +0,0 @@
-/**
- * LIVE validation of the per-harness tool-part decoders against a REAL sandbox box.
- *
- * Spin a real box for HARNESS, make the harness call tools, capture the actual session part shapes,
- * and prove the harness's decoder extracts the tool calls (with error status where the harness carries
- * it). No mock. Exits non-zero on a decode miss so it can gate CI. The box is ALWAYS deleted.
- *
- * Run: HARNESS=opencode dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
- *        pnpm exec tsx bench/src/decoder-live.mts        (also: claude-code, codex, kimi-code, …)
- */
-import { Sandbox } from '@tangle-network/sandbox'
-import { decodeToolPart } from '../../src/runtime/supervise/trace-source'
-
-function must(k: string): string {
-  const v = process.env[k]
-  if (!v) throw new Error(`missing env ${k}`)
-  return v
-}
-
-function* candidateParts(node: unknown, depth = 0): Generator<unknown> {
-  if (!node || typeof node !== 'object' || depth > 6) return
-  const o = node as Record<string, unknown>
-  yield o
-  if (o.part) yield o.part
-  if (Array.isArray(o.parts)) for (const p of o.parts) yield p
-  if (o.data) yield* candidateParts(o.data, depth + 1)
-  if (o.message) yield* candidateParts(o.message, depth + 1)
-}
-
-const HARNESS = process.env.HARNESS ?? 'opencode'
-
-async function main(): Promise<number> {
-  const client = new Sandbox({
-    baseUrl: process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools',
-    apiKey: must('TANGLE_API_KEY'),
-    timeoutMs: 600_000,
-  } as never)
-
-  console.error(`[live] creating ${HARNESS} box (${process.env.WORKER_MODEL ?? 'deepseek-v4-flash'})…`)
-  const box = (await client.create({
-    backend: {
-      type: HARNESS,
-      model: {
-        provider: process.env.WORKER_PROVIDER ?? 'openai',
-        model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-        baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
-      },
-      profile: { name: 'decoder-live' },
-    },
-  } as never)) as unknown as Record<string, (...a: never[]) => unknown> & { id?: string }
-
-  try {
-    console.error('[live] box', box.id, '— waiting for running…')
-    await box.waitFor('running' as never, { timeoutMs: 180_000 } as never)
-
-    // Force several distinct tool calls AND at least one tool error (to exercise error-status decode).
-    const prompt =
-      'Use your tools, one tool call at a time: (1) list the files in the current directory; ' +
-      '(2) run `echo hello-from-tool`; (3) run `false` (this command fails on purpose); ' +
-      '(4) create notes.txt containing "hi". A separate tool call for each. Then reply "done".'
-
-    const rawEvents: unknown[] = []
-    const ac = new AbortController()
-    const timer = setTimeout(() => ac.abort(), 240_000)
-    try {
-      for await (const ev of box.streamPrompt(prompt as never, {
-        signal: ac.signal,
-      } as never) as AsyncGenerator<unknown>) {
-        rawEvents.push(ev)
-      }
-    } finally {
-      clearTimeout(timer)
-    }
-
-    // Vocabulary: every distinct part `type` the harness emitted (so we learn its real schema).
-    const types = new Map<string, number>()
-    for (const ev of rawEvents)
-      for (const part of candidateParts(ev)) {
-        const t = (part as Record<string, unknown>)?.type
-        if (typeof t === 'string') types.set(t, (types.get(t) ?? 0) + 1)
-      }
-
-    // Decode tool calls via the HARNESS adapter + de-dup by callId.
-    const decoded: Array<{ toolName: string; status?: string; callId?: string }> = []
-    const seen = new Set<string>()
-    for (const ev of rawEvents)
-      for (const part of candidateParts(ev)) {
-        const step = decodeToolPart(part, HARNESS)
-        if (!step) continue
-        if (step.callId && seen.has(step.callId)) continue
-        if (step.callId) seen.add(step.callId)
-        decoded.push({ toolName: step.toolName, ...(step.status ? { status: step.status } : {}), ...(step.callId ? { callId: step.callId } : {}) })
-      }
-
-    // Surface any harness/model error events (so a no-tools run isn't mistaken for a decoder miss).
-    const errs = rawEvents.filter((e) => JSON.stringify(e).match(/"error"/i)).slice(0, 2)
-    if (errs.length) console.error(`[live] harness error events:`, errs.map((e) => JSON.stringify(e).slice(0, 300)))
-
-    console.error(`\n========== LIVE DECODER RESULT — harness=${HARNESS} ==========`)
-    console.error(`raw stream events: ${rawEvents.length}`)
-    console.error(`part-type vocabulary:`, JSON.stringify(Object.fromEntries(types)))
-    console.error(`decoded ${decoded.length} tool calls:`, JSON.stringify(decoded))
-    console.error(`with error status: ${decoded.filter((d) => d.status === 'error').length}`)
-    // Sample raw shapes the decoder did NOT match but that mention a tool (the schema to learn from).
-    console.error(`\n--- raw toolish parts NOT decoded (first 4) ---`)
-    let shown = 0
-    for (const ev of rawEvents) {
-      if (shown >= 4) break
-      for (const part of candidateParts(ev)) {
-        if (shown >= 4) break
-        const s = JSON.stringify(part)
-        if (s.match(/tool/i) && !decodeToolPart(part, HARNESS)) {
-          console.error(s.slice(0, 380))
-          shown++
-        }
-      }
-    }
-    return decoded.length > 0 ? 0 : 2
-  } finally {
-    await (box.delete as (...a: never[]) => Promise<void>)().catch(() => {})
-    console.error('[live] box deleted.')
-  }
-}
-
-main()
-  .then((code) => {
-    console.error(code === 0 ? `\n✅ ${HARNESS}: decoder extracts real tool calls.` : `\n❌ ${HARNESS}: decoder extracted ZERO — see raw shapes above.`)
-    process.exit(code)
-  })
-  .catch((e) => {
-    console.error('[live] FAILED:', e)
-    process.exit(1)
-  })