@tangle-network/agent-bench 0.8.9 → 0.8.12

package/src/david-goliath.mts

@@ -1,168 +0,0 @@
-/**
- * DAVID vs GOLIATH — the program's northstar, tested head-on: does a CHEAP model
- * with a self-verification harness beat a FRONTIER model running solo, at EQUAL OR
- * LOWER dollar cost, on held-out code?
- *
- *   GOLIATH — a strong model, ONE shot. The "just use the big model" baseline.
- *   DAVID   — a cheap/weak model + test-time compute: generate N candidate
- *             solutions AND M of its own unit tests, EXECUTE every candidate
- *             against the generated tests, and submit the candidate that passes
- *             the most (CodeT-style execution self-selection). No ground-truth
- *             test is ever used to select — only the model's own generated tests.
- *
- * Both are graded by the HIDDEN HumanEval test (never shown). Cost is the real
- * token spend × the router-reported/priced rate per arm. The win condition is a
- * Pareto beat: David's held-out pass rate >= Goliath's AND David's $ <= Goliath's.
- * Mechanism under test: EXECUTION-BASED VERIFICATION is the lever that lets a weak
- * generator punch above its solo weight — the standing "verification is live" claim
- * at its most dramatic. Paired McNemar on per-task discordant pairs for significance.
- *
- * Run from cwd=bench:  env DAVID=glm-5.2 GOLIATH=deepseek-v4-flash \
- *   N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts
- */
-import { execFile } from 'node:child_process'
-import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval'
-import { runBenchRouterTurn } from './router-turn'
-
-const KEY = process.env.TANGLE_API_KEY
-if (!KEY) throw new Error('TANGLE_API_KEY required')
-const ROUTER_KEY = KEY
-const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant'
-const GOLIATH = process.env.GOLIATH ?? 'deepseek-v4-flash'
-const N = Number(process.env.N ?? 8) // David candidate solutions
-const T = Number(process.env.T ?? 5) // David generated tests
-const NTASKS = Number(process.env.NTASKS ?? 164)
-const REPS = Number(process.env.REPS ?? 2)
-const CONC = Number(process.env.CONCURRENCY ?? 6)
-const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000)
-const LLM_TIMEOUT = Number(process.env.LLM_TIMEOUT_MS ?? 60_000)
-const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000)
-
-// Approx $/1M tokens (in,out) for cost accounting — the router does not price
-// every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the
-// point, and the ratio is robust to small rate errors. Override via PRICES env.
-const PRICES: Record<string, [number, number]> = {
-  'groq/llama-3.1-8b-instant': [0.05, 0.08],
-  'google/gemini-2.5-flash-lite': [0.10, 0.40],
-  'openai/gpt-4o-mini': [0.15, 0.60],
-  'anthropic/claude-haiku-4-5-20251001': [1.0, 5.0],
-  'glm-5.2': [0.60, 2.20],
-}
-const priceOf = (m: string) => PRICES[m] ?? [0.5, 1.5]
-
-interface Usage { in: number; out: number }
-const zero = (): Usage => ({ in: 0, out: 0 })
-const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out }
-const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 }
-
-async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise<string> {
-  try {
-    const system = messages.find((message) => message.role === 'system')?.content
-    const result = await runBenchRouterTurn(
-      {
-        routerBaseUrl: ROUTER,
-        routerKey: ROUTER_KEY,
-        profile: {
-          name: 'david-goliath-worker',
-          harness: 'cli-base',
-          model: {
-            provider: 'tangle-router',
-            default: model,
-            metadata: { temperature, maxTokens: MAX_TOKENS },
-          },
-          ...(system ? { prompt: { systemPrompt: system } } : {}),
-        },
-        timeoutMs: LLM_TIMEOUT,
-      },
-      { messages: messages.filter((message) => message.role !== 'system') },
-    )
-    if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage')
-    addU(usage, { in: result.usage.input, out: result.usage.output })
-    return result.finalText
-  } catch {
-    return ''
-  }
-}
-const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) })))
-async function runPy(program: string): Promise<{ ok: boolean }> {
-  const d = mkdtempSync(join(tmpdir(), 'dg-'))
-  try { writeFileSync(join(d, 'p.py'), program); const r = await exec('python3', [join(d, 'p.py')], { cwd: d, timeout: EXEC_TIMEOUT }); return { ok: r.code === 0 } } finally { rmSync(d, { recursive: true, force: true }) }
-}
-
-const SOLVE = 'You are an expert Python programmer. Output the COMPLETE function (signature + body + imports) in a single ```python block. No prose, no tests.'
-async function genSolution(model: string, t: HumanEvalTask, temp: number, u: Usage): Promise<string> {
-  return extractCode(await chat(model, [{ role: 'system', content: SOLVE }, { role: 'user', content: `Complete:\n\n\`\`\`python\n${t.prompt}\`\`\`` }], temp, u))
-}
-// David writes its OWN tests (never sees the hidden test). Parse assert lines.
-async function genTests(model: string, t: HumanEvalTask, u: Usage): Promise<string[]> {
-  const reply = await chat(model, [
-    { role: 'system', content: 'Write Python assert-based unit tests for the described function. Output ONLY a ```python block of standalone `assert <entry>(...) == ...` lines (at least a few, covering normal + edge cases). No function definition, no prose.' },
-    { role: 'user', content: `Function to test (entry point: ${t.entryPoint}):\n\n\`\`\`python\n${t.prompt}\`\`\`` },
-  ], 0.4, u)
-  const block = extractCode(reply) || reply
-  return block.split('\n').map((l) => l.trim()).filter((l) => l.startsWith('assert ') && l.includes(t.entryPoint)).slice(0, T + 3)
-}
-// Hidden held-out judge — the truth. Never used for selection.
-async function judge(t: HumanEvalTask, code: string): Promise<boolean> {
-  if (!code.trim()) return false
-  return (await runPy(`${code}\n\n${t.test}\n\ncheck(${t.entryPoint})\n`)).ok
-}
-// David's self-selection: score each candidate by how many of ITS OWN tests it passes.
-async function scoreOnTests(code: string, tests: string[]): Promise<number> {
-  if (!code.trim() || tests.length === 0) return 0
-  let pass = 0
-  // one program per test keeps a crash on one test from voiding the rest
-  for (const a of tests) if ((await runPy(`${code}\n\n${a}\n`)).ok) pass++
-  return pass
-}
-
-async function davidArm(t: HumanEvalTask, u: Usage): Promise<string> {
-  const cands = (await Promise.all(Array.from({ length: N }, () => genSolution(DAVID, t, 0.7, u)))).filter((c) => c.trim())
-  if (cands.length === 0) return ''
-  const tests = await genTests(DAVID, t, u)
-  if (tests.length === 0) return cands[0]! // no verifier signal → first sample
-  const scored = await Promise.all(cands.map(async (c) => ({ c, s: await scoreOnTests(c, tests) })))
-  scored.sort((a, b) => b.s - a.s || b.c.length - a.c.length)
-  return scored[0]!.c
-}
-const goliathArm = (t: HumanEvalTask, u: Usage) => genSolution(GOLIATH, t, 0.2, u)
-
-async function pool<T2, R>(xs: T2[], n: number, fn: (x: T2, i: number) => Promise<R>): Promise<R[]> {
-  const o = new Array<R>(xs.length); let i = 0
-  await Promise.all(Array.from({ length: Math.min(n, xs.length) }, async () => { while (i < xs.length) { const k = i++; o[k] = await fn(xs[k]!, k) } }))
-  return o
-}
-function mcnemar(b: number, c: number): number { const n = b + c; if (n === 0) return 1; const k = Math.min(b, c); const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }; let tl = 0; for (let i = 0; i <= k; i++) tl += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2)); return Math.min(1, 2 * tl) }
-
-async function main(): Promise<void> {
-  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: david-goliath loaded'); return }
-  const tasks = await loadHumanEval(NTASKS, 0)
-  console.error(`=== DAVID(${DAVID}, N=${N} sols + ${T} self-tests) vs GOLIATH(${GOLIATH}, 1 shot) · HumanEval n=${tasks.length} · reps=${REPS} ===`)
-  const dU = zero(), gU = zero()
-  const units = tasks.flatMap((task) => Array.from({ length: REPS }, () => task))
-  let done = 0
-  const res = await pool(units, CONC, async (task) => {
-    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
-    const [dCode, gCode] = await Promise.all([safe(() => davidArm(task, dU)), safe(() => goliathArm(task, gU))])
-    const [d, g] = await Promise.all([judge(task, dCode), judge(task, gCode)])
-    if (++done % 20 === 0) console.error(`  ${done}/${units.length} units`)
-    return { d, g }
-  })
-  const n = res.length, dPass = res.filter((r) => r.d).length, gPass = res.filter((r) => r.g).length
-  const b = res.filter((r) => r.d && !r.g).length, c = res.filter((r) => !r.d && r.g).length
-  const p = mcnemar(b, c)
-  const dCost = usd(DAVID, dU), gCost = usd(GOLIATH, gU)
-  console.log('\n=== RESULT (held-out HumanEval) ===')
-  console.log(`  GOLIATH ${GOLIATH} solo : ${gPass}/${n} = ${(gPass / n * 100).toFixed(1)}%   $${gCost.toFixed(4)}`)
-  console.log(`  DAVID   ${DAVID} + verify: ${dPass}/${n} = ${(dPass / n * 100).toFixed(1)}%   $${dCost.toFixed(4)}`)
-  console.log(`  accuracy: David ${dPass >= gPass ? '>=' : '<'} Goliath (${(dPass / n * 100).toFixed(1)} vs ${(gPass / n * 100).toFixed(1)}); paired McNemar David-only=${b} Goliath-only=${c} p=${p.toFixed(4)}`)
-  console.log(`  cost: David is ${(gCost / Math.max(dCost, 1e-9)).toFixed(1)}x CHEAPER ($${dCost.toFixed(4)} vs $${gCost.toFixed(4)})`)
-  const paretoBeat = dPass >= gPass && dCost <= gCost
-  const sigBeat = dPass > gPass && p < 0.05
-  console.log(`  VERDICT: ${sigBeat ? 'DAVID SIGNIFICANTLY BEATS GOLIATH' : paretoBeat ? 'DAVID PARETO-DOMINATES (>= accuracy, <= cost)' : dPass >= gPass ? 'David matches accuracy (check cost)' : 'Goliath wins accuracy'}`)
-}
-main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })

package/src/decoder-live.mts

@@ -1,133 +0,0 @@
-/**
- * LIVE validation of the per-harness tool-part decoders against a REAL sandbox box.
- *
- * Spin a real box for HARNESS, make the harness call tools, capture the actual session part shapes,
- * and prove the harness's decoder extracts the tool calls (with error status where the harness carries
- * it). No mock. Exits non-zero on a decode miss so it can gate CI. The box is ALWAYS deleted.
- *
- * Run: HARNESS=opencode dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
- *        pnpm exec tsx bench/src/decoder-live.mts        (also: claude-code, codex, kimi-code, …)
- */
-import { Sandbox } from '@tangle-network/sandbox'
-import { decodeToolPart } from '../../src/runtime/supervise/trace-source'
-
-function must(k: string): string {
-  const v = process.env[k]
-  if (!v) throw new Error(`missing env ${k}`)
-  return v
-}
-
-function* candidateParts(node: unknown, depth = 0): Generator<unknown> {
-  if (!node || typeof node !== 'object' || depth > 6) return
-  const o = node as Record<string, unknown>
-  yield o
-  if (o.part) yield o.part
-  if (Array.isArray(o.parts)) for (const p of o.parts) yield p
-  if (o.data) yield* candidateParts(o.data, depth + 1)
-  if (o.message) yield* candidateParts(o.message, depth + 1)
-}
-
-const HARNESS = process.env.HARNESS ?? 'opencode'
-
-async function main(): Promise<number> {
-  const client = new Sandbox({
-    baseUrl: process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools',
-    apiKey: must('TANGLE_API_KEY'),
-    timeoutMs: 600_000,
-  } as never)
-
-  console.error(`[live] creating ${HARNESS} box (${process.env.WORKER_MODEL ?? 'deepseek-v4-flash'})…`)
-  const box = (await client.create({
-    backend: {
-      type: HARNESS,
-      model: {
-        provider: process.env.WORKER_PROVIDER ?? 'openai',
-        model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
-        baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
-      },
-      profile: { name: 'decoder-live' },
-    },
-  } as never)) as unknown as Record<string, (...a: never[]) => unknown> & { id?: string }
-
-  try {
-    console.error('[live] box', box.id, '— waiting for running…')
-    await box.waitFor('running' as never, { timeoutMs: 180_000 } as never)
-
-    // Force several distinct tool calls AND at least one tool error (to exercise error-status decode).
-    const prompt =
-      'Use your tools, one tool call at a time: (1) list the files in the current directory; ' +
-      '(2) run `echo hello-from-tool`; (3) run `false` (this command fails on purpose); ' +
-      '(4) create notes.txt containing "hi". A separate tool call for each. Then reply "done".'
-
-    const rawEvents: unknown[] = []
-    const ac = new AbortController()
-    const timer = setTimeout(() => ac.abort(), 240_000)
-    try {
-      for await (const ev of box.streamPrompt(prompt as never, {
-        signal: ac.signal,
-      } as never) as AsyncGenerator<unknown>) {
-        rawEvents.push(ev)
-      }
-    } finally {
-      clearTimeout(timer)
-    }
-
-    // Vocabulary: every distinct part `type` the harness emitted (so we learn its real schema).
-    const types = new Map<string, number>()
-    for (const ev of rawEvents)
-      for (const part of candidateParts(ev)) {
-        const t = (part as Record<string, unknown>)?.type
-        if (typeof t === 'string') types.set(t, (types.get(t) ?? 0) + 1)
-      }
-
-    // Decode tool calls via the HARNESS adapter + de-dup by callId.
-    const decoded: Array<{ toolName: string; status?: string; callId?: string }> = []
-    const seen = new Set<string>()
-    for (const ev of rawEvents)
-      for (const part of candidateParts(ev)) {
-        const step = decodeToolPart(part, HARNESS)
-        if (!step) continue
-        if (step.callId && seen.has(step.callId)) continue
-        if (step.callId) seen.add(step.callId)
-        decoded.push({ toolName: step.toolName, ...(step.status ? { status: step.status } : {}), ...(step.callId ? { callId: step.callId } : {}) })
-      }
-
-    // Surface any harness/model error events (so a no-tools run isn't mistaken for a decoder miss).
-    const errs = rawEvents.filter((e) => JSON.stringify(e).match(/"error"/i)).slice(0, 2)
-    if (errs.length) console.error(`[live] harness error events:`, errs.map((e) => JSON.stringify(e).slice(0, 300)))
-
-    console.error(`\n========== LIVE DECODER RESULT — harness=${HARNESS} ==========`)
-    console.error(`raw stream events: ${rawEvents.length}`)
-    console.error(`part-type vocabulary:`, JSON.stringify(Object.fromEntries(types)))
-    console.error(`decoded ${decoded.length} tool calls:`, JSON.stringify(decoded))
-    console.error(`with error status: ${decoded.filter((d) => d.status === 'error').length}`)
-    // Sample raw shapes the decoder did NOT match but that mention a tool (the schema to learn from).
-    console.error(`\n--- raw toolish parts NOT decoded (first 4) ---`)
-    let shown = 0
-    for (const ev of rawEvents) {
-      if (shown >= 4) break
-      for (const part of candidateParts(ev)) {
-        if (shown >= 4) break
-        const s = JSON.stringify(part)
-        if (s.match(/tool/i) && !decodeToolPart(part, HARNESS)) {
-          console.error(s.slice(0, 380))
-          shown++
-        }
-      }
-    }
-    return decoded.length > 0 ? 0 : 2
-  } finally {
-    await (box.delete as (...a: never[]) => Promise<void>)().catch(() => {})
-    console.error('[live] box deleted.')
-  }
-}
-
-main()
-  .then((code) => {
-    console.error(code === 0 ? `\n✅ ${HARNESS}: decoder extracts real tool calls.` : `\n❌ ${HARNESS}: decoder extracted ZERO — see raw shapes above.`)
-    process.exit(code)
-  })
-  .catch((e) => {
-    console.error('[live] FAILED:', e)
-    process.exit(1)
-  })

package/src/diverse-gate.mjs

@@ -1,112 +0,0 @@
-// The beat-blind gate, as ONE command — the experiment that decides whether the
-// diversification/selection surface (the one PR #145 defers to as "tracked separately")
-// actually beats compute-matched best-of-N. Composes only LANDED pieces:
-//   1. random@k corpus  — k identical-directive shots/instance (the compute control)
-//   2. diverse@k corpus — k DIFFERENT strategy lenses/instance (DIVERSE=1; the bet)
-//   3. selector replay  — selfConsistencySelect@k over each (corpus-replay --selector)
-//   4. paired report    — bootstrap CI + Benjamini-Hochberg over both (corpus-report)
-//
-// The decomposition it yields:
-//   random@k           = more-compute, no picking          (control)
-//   selector@k (homog) = picking over IDENTICAL attempts   (#143: −8.2pp on the committed corpus)
-//   diverse-selector@k = picking over DIVERSE attempts      (THE bet: does approach-diversity
-//                        give self-consistency the signal identical attempts don't?)
-// Beat-blind iff diverse-selector@k > random@k at significant n.
-//
-//   node diverse-gate.mjs            run it (generates corpora — a real worker run)
-//   node diverse-gate.mjs --dry      print the plan only (no run; safe while another
-//                                    sandbox run is live — zero router/sandbox contention)
-//
-// Knobs (env): BENCH (default hotpotqa) · N (default 30) · K (default 4) ·
-//   RESEARCH=1 (local opencode, default) | SANDBOX=1 (prod sandbox web worker) · MODELS ·
-//   DIVERSE_BASE (compose #145's GEPA-learned directive as the lens base — follow-on).
-
-import { spawn } from 'node:child_process'
-
-const DRY = process.argv.includes('--dry')
-const BENCH = process.env.BENCH ?? 'hotpotqa'
-const N = process.env.N ?? '30'
-const K = process.env.K ?? '4'
-const RANDOM_CORPUS = process.env.RANDOM_CORPUS ?? '/tmp/dg-random.jsonl'
-const DIVERSE_CORPUS = process.env.DIVERSE_CORPUS ?? '/tmp/dg-diverse.jsonl'
-// Worker-mode env passes through to batch-oracle unchanged (RESEARCH=1 / SANDBOX=1 / MODELS / TANGLE_API_KEY).
-const passEnv = { ...process.env, BENCH, N, K }
-
-const steps = [
-  {
-    label: 'random@k corpus (control — identical directive)',
-    cmd: 'npx',
-    args: ['tsx', 'src/run.ts', 'batch-oracle', N],
-    env: { ...passEnv, CORPUS: RANDOM_CORPUS },
-  },
-  {
-    label: 'diverse@k corpus (the bet — k distinct strategy lenses)',
-    cmd: 'npx',
-    args: ['tsx', 'src/run.ts', 'batch-oracle', N],
-    env: { ...passEnv, CORPUS: DIVERSE_CORPUS, DIVERSE: '1' },
-  },
-  {
-    label: 'selector@k over the CONTROL corpus (homogeneous)',
-    cmd: 'npx',
-    args: ['tsx', 'src/corpus-replay.mts', RANDOM_CORPUS, '--selector'],
-    env: passEnv,
-  },
-  {
-    label: 'selector@k over the DIVERSE corpus (the beat-blind number)',
-    cmd: 'npx',
-    // The diverse corpus records carry condition="diverse@4"; corpus-replay's
-    // selector filter defaults to "random", so match the diverse condition here.
-    args: ['tsx', 'src/corpus-replay.mts', DIVERSE_CORPUS, '--selector', '--condition=diverse'],
-    env: passEnv,
-  },
-  {
-    label: 'paired bootstrap CI + Benjamini-Hochberg over both corpora',
-    cmd: 'npx',
-    args: ['tsx', 'src/corpus-report.mts', RANDOM_CORPUS, DIVERSE_CORPUS],
-    env: passEnv,
-  },
-]
-
-const shellPreview = (s) => {
-  const envStr = Object.entries(s.env)
-    .filter(([k]) => ['BENCH', 'N', 'K', 'CORPUS', 'DIVERSE', 'RESEARCH', 'SANDBOX', 'MODELS'].includes(k))
-    .map(([k, v]) => `${k}=${v}`)
-    .join(' ')
-  return `  ${envStr} ${s.cmd} ${s.args.join(' ')}`.replace(/\s+/g, ' ')
-}
-
-function runStep(s) {
-  return new Promise((resolve, reject) => {
-    console.log(`\n▶ ${s.label}`)
-    const child = spawn(s.cmd, s.args, { cwd: process.cwd(), env: s.env, stdio: 'inherit' })
-    child.on('error', reject)
-    // Fail loud: a non-zero step aborts the gate (no silent partial result).
-    child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`${s.label} exited ${code}`))))
-  })
-}
-
-async function main() {
-  console.log(`=== beat-blind gate · BENCH=${BENCH} N=${N} K=${K} ${DRY ? '(DRY — plan only)' : ''} ===`)
-  if (DRY) {
-    console.log('plan (no run — zero sandbox/router contention):')
-    for (const s of steps) console.log(shellPreview(s))
-    console.log(
-      '\nbeat-blind iff diverse-selector@k > random@k at significant n.' +
-        '\nDIVERSE_BASE=<file> composes #145\'s GEPA-learned directive as the lens base (follow-on).',
-    )
-    return
-  }
-  for (const s of steps) await runStep(s)
-  console.log(
-    '\n=== read the gate ===\n' +
-      `  random@k (control)        — from ${RANDOM_CORPUS} replay\n` +
-      `  selector@k (homogeneous)  — same corpus, the pick (#143: −8.2pp on committed finsearch)\n` +
-      `  diverse-selector@k        — ${DIVERSE_CORPUS} replay: THE bet\n` +
-      '  beat-blind iff diverse-selector@k > random@k, significant per the BH report above.',
-  )
-}
-
-main().catch((err) => {
-  console.error(`diverse-gate: ${err instanceof Error ? err.message : String(err)}`)
-  process.exit(1)
-})

package/src/hev-eval.mts

@@ -1,101 +0,0 @@
-/**
- * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the
- * worker model on each task and print the pass rate + the per-task result. Used to
- * measure a baseline instruction vs a proposer-supplied instruction on the SAME
- * held-out set (the proposer proposes; this grades — kept separate for honesty).
- *
- *   INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \
- *     HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts
- */
-import { readFileSync } from 'node:fs'
-import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
-import { runBenchRouterTurn } from './router-turn'
-
-const SEED_INSTRUCTION =
-  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
-
-async function complete(
-  base: string,
-  key: string,
-  model: string,
-  instruction: string,
-  prompt: string,
-  maxTokens: number,
-): Promise<string> {
-  try {
-    const turn = await runBenchRouterTurn(
-      {
-        routerBaseUrl: base,
-        routerKey: key,
-        profile: {
-          name: 'humaneval-worker',
-          harness: 'cli-base',
-          model: {
-            provider: 'tangle-router',
-            default: model,
-            metadata: { temperature: 0.2, maxTokens },
-          },
-          prompt: { systemPrompt: instruction },
-        },
-      },
-      prompt,
-    )
-    return turn.finalText
-  } catch {
-    return ''
-  }
-}
-
-async function main(): Promise<void> {
-  const key = process.env.TANGLE_API_KEY
-  if (!key) throw new Error('TANGLE_API_KEY required')
-  const apiKey: string = key
-  const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
-  const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
-  const instruction = process.env.INSTRUCTION_FILE
-    ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8')
-    : (process.env.INSTRUCTION ?? SEED_INSTRUCTION)
-  const maxTokens = Number(process.env.MAX_TOKENS ?? 2500)
-  const conc = Number(process.env.CONC ?? 6)
-  const offset = Number(process.env.OFFSET ?? 55)
-  const n = Number(process.env.N ?? 40)
-  const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
-
-  const all = await loadHumanEval(164, 0)
-  const byId = new Map(all.map((t) => [t.taskId, t]))
-  const tasks: HumanEvalTask[] = idsEnv.length
-    ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t)
-    : all.slice(offset, offset + n)
-
-  console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`)
-  let pass = 0
-  const fails: string[] = []
-  // simple concurrency pool
-  let i = 0
-  async function worker(): Promise<void> {
-    while (i < tasks.length) {
-      const t = tasks[i]
-      i += 1
-      if (!t) continue
-      const reply = await complete(
-        base,
-        apiKey,
-        model,
-        instruction,
-        `\`\`\`python\n${t.prompt}\`\`\``,
-        maxTokens,
-      )
-      const { pass: p } = await runChecker(t, extractCode(reply))
-      if (p === 1) pass += 1
-      else fails.push(t.taskId)
-    }
-  }
-  await Promise.all(Array.from({ length: conc }, () => worker()))
-  console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`)
-  console.log(`FAILED: ${fails.sort().join(', ')}`)
-}
-
-main().catch((e) => {
-  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
-  process.exit(1)
-})