@tangle-network/agent-bench 0.1.0

package/README.md

@@ -0,0 +1,15 @@
+# agent-runtime-bench
+
+Private experiment workspace nested in agent-runtime; decoupled from its build/lint/release (the package builds `src/`, lints `src tests examples` — `bench/` is none of those).
+
+**Read [`bench/HARNESS.md`](./HARNESS.md) FIRST.** It is the one maintained map: the commands, the `rollout → corpus → selector → CI → gate` data flow, the canonical-suite table, the wired/needs-creds/scaffolded matrix, and the gate one-liners — kept verified against source.
+
+## SWE-bench judge setup (the one block not in HARNESS.md)
+
+```bash
+python3 -m venv .venv && .venv/bin/pip install swebench   # SWE-bench harness
+pnpm install                                              # tsx + link parent
+# Docker daemon must be running (judges build/run per-instance images)
+```
+
+The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API_KEY`, or a direct provider).

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@tangle-network/agent-bench",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "The unified benchmark suite for agent-runtime agents: 18 adapters (commit0, enterpriseops-gym, trata-hedge, finsearchcomp, swe-bench, humaneval, …) behind one resolveAdapter registry, each with a real deterministic judge. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts",
+    "./adapters": "./src/adapters.ts",
+    "./benchmarks/*": "./src/benchmarks/*.ts"
+  },
+  "scripts": {
+    "gate": "tsx src/corpus-replay.mts corpus/finsearch.jsonl --selector",
+    "gate-cli": "tsx src/gate-cli.mts",
+    "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl",
+    "terminal-compare": "tsx src/terminal-compare.ts"
+  },
+  "dependencies": {
+    "@tangle-network/agent-eval": "^0.100.0",
+    "@tangle-network/agent-runtime": "^0.78.0",
+    "@tangle-network/sandbox": "^0.9.3"
+  },
+  "devDependencies": {
+    "tsx": "^4.19.0",
+    "typescript": "^6.0.3"
+  },
+  "files": [
+    "src",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/adapters.ts

@@ -0,0 +1,60 @@
+/**
+ * The benchmark registry — the single source of truth for every wired benchmark.
+ * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`
+ * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it
+ * here rather than each keeping their own copy.
+ */
+
+import { createAecBenchAdapter } from './benchmarks/aec-bench'
+import { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'
+import { createCadBenchAdapter } from './benchmarks/cadbench'
+import { createCadDesignAdapter } from './benchmarks/cad-design'
+import { createCadGenBenchAdapter } from './benchmarks/cadgenbench'
+import { createCommit0Adapter } from './benchmarks/commit0'
+import { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'
+import { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'
+import { createFramesAdapter } from './benchmarks/frames'
+import { createHotpotqaAdapter } from './benchmarks/hotpotqa'
+import { createHumanEvalAdapter } from './benchmarks/humaneval'
+import { createMind2WebAdapter } from './benchmarks/mind2web'
+import { createProgrambenchAdapter } from './benchmarks/programbench'
+import { createSimpleQaAdapter } from './benchmarks/simpleqa'
+import { createSweBenchAdapter } from './benchmarks/swe-bench'
+import { createTerminalBenchAdapter } from './benchmarks/terminal-bench'
+import { createTrataHedgeAdapter } from './benchmarks/trata-hedge'
+import type { BenchmarkAdapter } from './benchmarks/types'
+
+export const ADAPTERS: Record<string, () => BenchmarkAdapter> = {
+  'swe-bench': createSweBenchAdapter,
+  'terminal-bench': createTerminalBenchAdapter,
+  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).
+  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge
+  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.
+  'aec-bench': createAecBenchAdapter,
+  commit0: createCommit0Adapter,
+  programbench: createProgrambenchAdapter,
+  appworld: createAppWorldAdapter,
+  // AppWorld's native interactive protocol — the worker is the in-engine ReAct
+  // episode (execution feedback every turn), the mode published baselines use.
+  'appworld-react': createAppWorldReactAdapter,
+  'enterpriseops-gym': createEnterpriseOpsGymAdapter,
+  'cad-design': createCadDesignAdapter,
+  cadbench: createCadBenchAdapter,
+  cadgenbench: createCadGenBenchAdapter,
+  frames: createFramesAdapter,
+  finsearchcomp: createFinsearchcompAdapter,
+  simpleqa: createSimpleQaAdapter,
+  hotpotqa: createHotpotqaAdapter,
+  // Deployable-checker code domain: worker = router completion, judge = Docker test
+  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).
+  humaneval: createHumanEvalAdapter,
+  mind2web: createMind2WebAdapter,
+  'trata-hedge': createTrataHedgeAdapter,
+}
+
+/** Resolve a benchmark key to its adapter, failing loud with the known keys. */
+export function resolveAdapter(key: string): BenchmarkAdapter {
+  const make = ADAPTERS[key]
+  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(', ')})`)
+  return make()
+}

package/src/aec-gate.mts

@@ -0,0 +1,217 @@
+/**
+ * Router-based gate runner for aec-bench — the fix for the null-score sandbox path.
+ *
+ * aec-bench is closed-form reasoning + a deterministic local verify.py judge, so a
+ * sandbox is unnecessary: solve each task with one direct router chat call, then
+ * judge the raw response locally (verify.py extracts the last fenced ```json block
+ * itself). The prior sandbox path emitted the JSON in-stream but never fed it to
+ * the judge, so every verdict.score came out null — the bug this runner fixes by
+ * passing the model's full response straight to adapter.judge().
+ *
+ * Two paired arms over the SAME task set (loadTasks once):
+ *   random@K  — K identical-base-prompt shots/task (the compute control)
+ *   diverse@K — K shots, the i-th prefixed with composeStrategies(base, K)[i]
+ *
+ * Each attempt carries a REAL numeric verdict.score (from verify.py) + the output,
+ * written as a corpus RunRecord (condition random@K / diverse@K) the existing
+ * corpus-replay --selector + corpus-report consume unchanged. Fail loud on a router
+ * error — never a fabricated score.
+ */
+
+import { resolveAdapter } from './adapters'
+import type { BenchmarkAdapter, BenchTask } from './benchmarks/types'
+import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
+import { composeStrategies } from './directives'
+import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/loops'
+import { pool } from './stats.mts'
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+interface ArmSpec {
+  /** Corpus condition label the selector/report filter on (e.g. random@4). */
+  condition: string
+  /** Per-attempt prompt builder: the i-th of K shots for a task. */
+  promptFor(task: BenchTask, i: number, k: number): string
+}
+
+interface AttemptOutcome {
+  prompt: string
+  output: string
+  score: number
+  resolved: boolean
+  costUsd?: number
+  tokensIn?: number
+  tokensOut?: number
+  wallMs: number
+  /** the router/judge call failed after retries — EXCLUDED from stats, never scored 0. */
+  infraError?: boolean
+}
+
+async function runAttempt(
+  cfg: RouterConfig,
+  adapter: BenchmarkAdapter,
+  task: BenchTask,
+  prompt: string,
+): Promise<AttemptOutcome> {
+  const startedAt = Date.now()
+  // Retry transient router/judge failures (rate limits, stream drops, 5xx) with
+  // backoff; a genuine empty completion still scores a real 0 (verify.py fail-closes).
+  // Only after retries are exhausted do we record an EXCLUDED infraError — never a
+  // fabricated score, and never a throw that aborts the whole multi-model run.
+  let lastErr: unknown
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    try {
+      const res = await routerChatWithUsage(cfg, [{ role: 'user', content: prompt }])
+      const content = typeof res.content === 'string' ? res.content : ''
+      const verdict = await adapter.judge(task, content)
+      return {
+        prompt,
+        output: content,
+        score: verdict.score,
+        resolved: verdict.resolved,
+        wallMs: Date.now() - startedAt,
+        ...(res.costUsd !== undefined ? { costUsd: res.costUsd } : {}),
+        ...(res.usage ? { tokensIn: res.usage.input, tokensOut: res.usage.output } : {}),
+      }
+    } catch (err) {
+      lastErr = err
+      if (attempt < 2) await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt))
+    }
+  }
+  console.warn(`[aec-gate] ${task.id}: infra error after 3 tries — excluded: ${(lastErr instanceof Error ? lastErr.message : String(lastErr)).slice(0, 160)}`)
+  return { prompt, output: '', score: 0, resolved: false, wallMs: Date.now() - startedAt, infraError: true }
+}
+
+function toAttemptRecord(o: AttemptOutcome, round: number): AttemptRecord {
+  return {
+    round,
+    prompt: o.prompt,
+    output: o.output,
+    // infra-errored attempts carry NO score/valid → corpus-replay skips them.
+    ...(o.infraError ? {} : { valid: o.resolved, score: o.score }),
+    wallMs: o.wallMs,
+    eventCount: 1,
+    eventTypes: o.infraError ? { 'router.error': 1 } : { 'router.chat': 1 },
+    traceTail: o.output.slice(-600),
+    ...(o.costUsd !== undefined ? { costUsd: o.costUsd } : {}),
+    ...(o.tokensIn !== undefined ? { tokensIn: o.tokensIn } : {}),
+    ...(o.tokensOut !== undefined ? { tokensOut: o.tokensOut } : {}),
+  }
+}
+
+interface ArmResult {
+  /** mean graded score across SCORED (non-infra) attempts */
+  meanScore: number
+  /** fraction of scored attempts at full credit (score >= 1) */
+  fullCreditRate: number
+  attemptCount: number
+  /** attempts excluded as infra errors (router/judge failed after retries) */
+  infraCount: number
+}
+
+async function runArm(
+  arm: ArmSpec,
+  cfg: RouterConfig,
+  adapter: BenchmarkAdapter,
+  tasks: BenchTask[],
+  k: number,
+  concurrency: number,
+  corpusPath: string,
+): Promise<ArmResult> {
+  // Flatten (task, shot) into one unit of work so the pool bounds TOTAL in-flight
+  // router calls across all tasks, not per-task.
+  const units = tasks.flatMap((task) => Array.from({ length: k }, (_, i) => ({ task, i })))
+  const outcomes = await pool(units, concurrency, (u) => runAttempt(cfg, adapter, u.task, arm.promptFor(u.task, u.i, k)))
+
+  const scored = outcomes.filter((o) => !o.infraError)
+  let scoreSum = 0
+  let fullCredit = 0
+  for (const o of scored) {
+    scoreSum += o.score
+    if (o.score >= 1) fullCredit += 1
+  }
+
+  // Group K outcomes back per task → one RunRecord/task (the controller-run shape).
+  for (let t = 0; t < tasks.length; t += 1) {
+    const task = tasks[t] as BenchTask
+    const taskOutcomes = outcomes.slice(t * k, t * k + k)
+    const attempts = taskOutcomes.map((o, i) => toAttemptRecord(o, i))
+    const record = buildRunRecordFromAttempts(attempts, {
+      benchmark: adapter.name,
+      instanceId: task.id,
+      condition: arm.condition,
+      model: cfg.model,
+      // k-attempt outcome = any usable attempt resolved (the oracle@k ceiling for
+      // this run; the deployable selector is scored separately by corpus-replay).
+      resolved: taskOutcomes.some((o) => o.resolved),
+      // a task whose every attempt infra-errored is itself infra-errored.
+      infraError: taskOutcomes.length > 0 && taskOutcomes.every((o) => o.infraError),
+    })
+    await appendRunRecord(corpusPath, record)
+  }
+
+  return {
+    meanScore: scored.length > 0 ? scoreSum / scored.length : 0,
+    fullCreditRate: scored.length > 0 ? fullCredit / scored.length : 0,
+    infraCount: outcomes.length - scored.length,
+    attemptCount: outcomes.length,
+  }
+}
+
+async function main(): Promise<void> {
+  const n = Number(process.env.N ?? 5)
+  const k = Number(process.env.K ?? 4)
+  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const routerKey = must('TANGLE_API_KEY')
+  const concurrency = Number(process.env.CONCURRENCY ?? 6)
+  const randomCorpus = process.env.RANDOM_CORPUS ?? '/tmp/aec-r.jsonl'
+  const diverseCorpus = process.env.DIVERSE_CORPUS ?? '/tmp/aec-d.jsonl'
+
+  if (!Number.isFinite(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
+  if (!Number.isFinite(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
+
+  const cfg: RouterConfig = { routerBaseUrl, routerKey, model }
+  const bench = process.env.BENCH ?? 'aec-bench'
+  const adapter = resolveAdapter(bench)
+
+  console.log(`=== ${bench} router gate · N=${n} K=${k} model=${model} conc=${concurrency} ===`)
+  await adapter.preflight()
+  const tasks = await adapter.loadTasks({ limit: n })
+  console.log(`loaded ${tasks.length} task(s): ${tasks.map((t) => t.id).join(', ')}`)
+
+  // random arm: the task prompt verbatim, K times (the compute control).
+  const randomArm: ArmSpec = {
+    condition: `random@${k}`,
+    promptFor: (task) => task.prompt,
+  }
+  // diverse arm: the i-th shot prefixed with the i-th distinct strategy lens.
+  const diverseArm: ArmSpec = {
+    condition: `diverse@${k}`,
+    promptFor: (task, i, kk) => composeStrategies(task.prompt, kk)[i] as string,
+  }
+
+  console.log(`\n▶ random@${k} (control — identical base prompt) → ${randomCorpus}`)
+  const r = await runArm(randomArm, cfg, adapter, tasks, k, concurrency, randomCorpus)
+  console.log(`  random@${k}:  mean score ${(r.meanScore * 100).toFixed(1)}%  full-credit ${(r.fullCreditRate * 100).toFixed(1)}%  (n=${r.attemptCount} attempts${r.infraCount ? `, ${r.infraCount} infra-excluded` : ''})`)
+
+  console.log(`\n▶ diverse@${k} (K distinct strategy lenses) → ${diverseCorpus}`)
+  const d = await runArm(diverseArm, cfg, adapter, tasks, k, concurrency, diverseCorpus)
+  console.log(`  diverse@${k}: mean score ${(d.meanScore * 100).toFixed(1)}%  full-credit ${(d.fullCreditRate * 100).toFixed(1)}%  (n=${d.attemptCount} attempts${d.infraCount ? `, ${d.infraCount} infra-excluded` : ''})`)
+
+  console.log(
+    `\n=== next: read the gate ===\n` +
+      `  npx tsx src/corpus-replay.mts ${randomCorpus} --selector\n` +
+      `  npx tsx src/corpus-replay.mts ${diverseCorpus} --selector --condition=diverse\n` +
+      `  npx tsx src/corpus-report.mts ${randomCorpus} ${diverseCorpus}`,
+  )
+}
+
+main().catch((err) => {
+  console.error(`aec-gate: ${err instanceof Error ? err.message : String(err)}`)
+  process.exit(1)
+})

package/src/atom-humaneval.mts

@@ -0,0 +1,197 @@
+/**
+ * The "useful or BS" verdict: agents-driving-agents on a REAL deployable-checked domain.
+ *
+ * A `driverAgent` with a REAL router-LLM brain drives, per HumanEval task: it spawns
+ * worker agents (each a router LLM that writes the function), every worker GATED by the
+ * deterministic local Docker checker (the deliverable — a worker settles `valid` ⟺ its tests
+ * pass), and the completion-oracle keeps-best a DELIVERED worker. The supervisor returns a winner
+ * ONLY when a worker actually passed the tests (no self-declared done). We measure the driver's
+ * delivered rate against a BLIND best-of-K baseline (K independent workers, no orchestration) at
+ * the same K — the honest "does the recursion+oracle beat blind compute, or is it BS" question.
+ *
+ * Run (creds via dotenvx; Docker daemon must be up):
+ *   DOTENV_PRIVATE_KEY_FILE=~/company/devops/secrets/.env.keys \
+ *   dotenvx run -f ~/company/devops/secrets/agent-state.env -- \
+ *   N=5 K=3 WORKER_MODEL=deepseek-v4-flash DRIVER_MODEL=deepseek-v4-flash \
+ *   npx tsx bench/src/atom-humaneval.mts
+ */
+
+import {
+  type Agent,
+  type AgentProfile,
+  type AgentSpec,
+  contentAddress,
+  type DriverAgentOptions,
+  driverAgent,
+  createExecutorRegistry,
+  createSupervisor,
+  type Executor,
+  type ExecutorResult,
+  gateOnDeliverable,
+  InMemoryResultBlobStore,
+  InMemorySpawnJournal,
+  type RouterConfig,
+  routerBrain,
+  routerChatWithUsage,
+} from '../../src/runtime/index'
+import { basePrompt, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
+
+function must(k: string): string {
+  const v = process.env[k]
+  if (!v) throw new Error(`missing required env ${k}`)
+  return v
+}
+
+const N = Number(process.env.N ?? 5)
+const K = Number(process.env.K ?? 3)
+const OFFSET = Number(process.env.OFFSET ?? 0)
+const WORKER_TEMP = Number(process.env.WORKER_TEMP ?? 0.7)
+
+const cfg: RouterConfig = {
+  routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+  routerKey: must('TANGLE_API_KEY'),
+  model: process.env.WORKER_MODEL ?? 'deepseek-v4-flash',
+}
+const driverCfg: RouterConfig = { ...cfg, model: process.env.DRIVER_MODEL ?? cfg.model }
+
+// The driver-LLM brain is the SHARED `routerBrain` (the canonical ToolLoopChat seam) — it forwards
+// usage/costUsd, so this bench's driver arms meter their own inference into the conserved pool.
+
+// ── A gated router worker: one router call → candidate code, settled valid ⟺ the tests pass ──
+function humanEvalWorker(task: HumanEvalTask, label: string): Agent<unknown, unknown> {
+  let artifact: ExecutorResult<unknown> | undefined
+  const inner: Executor<unknown> = {
+    runtime: 'router',
+    async execute(_t, signal) {
+      const res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], {
+        temperature: WORKER_TEMP,
+        ...(signal ? { signal } : {}),
+      })
+      const code = extractCode(res.content)
+      artifact = {
+        outRef: contentAddress(code),
+        out: code,
+        spent: { iterations: 1, tokens: res.usage ?? { input: 0, output: 0 }, usd: res.costUsd ?? 0, ms: 0 },
+      }
+      return artifact
+    },
+    teardown: () => Promise.resolve({ destroyed: true }),
+    resultArtifact: () => {
+      if (!artifact) throw new Error('resultArtifact read before execute')
+      return artifact
+    },
+  }
+  const gated = gateOnDeliverable(inner, {
+    check: async (out) => (await runChecker(task, String(out))).pass === 1,
+    describe: `${task.taskId}: the provided test suite passes`,
+  })
+  const spec: AgentSpec = { profile: { name: label } as AgentProfile, harness: null, executor: gated }
+  return { name: label, act: async () => '', executorSpec: spec } as Agent<unknown, unknown> & {
+    executorSpec: AgentSpec
+  }
+}
+
+const driverSystem = `You are an orchestrator driving worker agents to solve a Python coding task. You do NOT write code yourself. Each worker independently attempts the task and is graded by a deterministic, hidden test suite. Tools: spawn_worker (dispatch one attempt; the "profile" argument may be {} and "task" a short note), await_event (collect the next settled worker — its result tells you valid:true if its tests PASSED, valid:false if they failed), and stopping (reply with NO tool call) once a worker has DELIVERED. Spawn one worker, await it; if it delivered, stop; if not, spawn another, up to ${K} workers total. You cannot declare success yourself — only a delivered (valid:true) worker counts.`
+
+interface TaskOutcome {
+  taskId: string
+  driverDelivered: boolean
+  blindDelivered: boolean
+  driverSpawns: number
+  driverWorkerTokens: number
+}
+
+// ── Driver arm: the orchestrated atom ────────────────────────────────────────────────────────
+async function driveTask(
+  task: HumanEvalTask,
+): Promise<{ delivered: boolean; spawns: number; tokens: number }> {
+  const blobs = new InMemoryResultBlobStore()
+  const journal = new InMemorySpawnJournal()
+  let spawns = 0
+  const makeWorker = (): Agent<unknown, unknown> => {
+    const w = humanEvalWorker(task, `w-${spawns}`)
+    spawns += 1
+    return w
+  }
+  const opts: DriverAgentOptions = {
+    name: `drv-${task.taskId}`,
+    brain: routerBrain(driverCfg),
+    blobs,
+    makeWorkerAgent: makeWorker,
+    perWorker: { maxIterations: 2, maxTokens: 4000 },
+    systemPrompt: driverSystem,
+    maxTurns: K + 4,
+  }
+  const root = driverAgent(opts)
+  const runId = `he-${task.taskId.replace('/', '-')}`
+  const result = await createSupervisor<unknown, unknown>().run(root, basePrompt(task), {
+    budget: { maxIterations: 100, maxTokens: 400_000 },
+    runId,
+    journal,
+    blobs,
+    executors: createExecutorRegistry(),
+    maxDepth: 4,
+    now: () => Date.now(),
+  })
+  const tree = await journal.loadTree(runId)
+  const tokens = (tree ?? [])
+    .filter((e): e is Extract<NonNullable<typeof tree>[number], { kind: 'settled' }> => e.kind === 'settled')
+    .reduce((s, e) => s + e.spent.tokens.input + e.spent.tokens.output, 0)
+  return { delivered: result.kind === 'winner', spawns, tokens }
+}
+
+// ── Blind arm: K independent workers, best-of-K by the checker (no orchestration) ─────────────
+async function blindTask(task: HumanEvalTask): Promise<boolean> {
+  for (let i = 0; i < K; i += 1) {
+    // A transient router error is a FAILED attempt, not a crash — the driver arm already types
+    // an executor throw into a `down` settlement, so the blind arm must match (fair comparison).
+    let res: { content: string }
+    try {
+      res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], {
+        temperature: WORKER_TEMP,
+      })
+    } catch {
+      continue
+    }
+    if ((await runChecker(task, extractCode(res.content))).pass === 1) return true
+  }
+  return false
+}
+
+async function main(): Promise<void> {
+  console.log(`atom-humaneval: N=${N} K=${K} offset=${OFFSET} worker=${cfg.model} driver=${driverCfg.model}`)
+  const tasks = await loadHumanEval(N, OFFSET)
+  const outcomes: TaskOutcome[] = []
+  for (const task of tasks) {
+    const drv = await driveTask(task)
+    const blind = await blindTask(task)
+    outcomes.push({
+      taskId: task.taskId,
+      driverDelivered: drv.delivered,
+      blindDelivered: blind,
+      driverSpawns: drv.spawns,
+      driverWorkerTokens: drv.tokens,
+    })
+    console.log(
+      `  ${task.taskId.padEnd(14)} driver=${drv.delivered ? 'PASS' : 'fail'} (spawns=${drv.spawns}, tok=${drv.tokens})  blind@${K}=${blind ? 'PASS' : 'fail'}`,
+    )
+  }
+  const driverPass = outcomes.filter((o) => o.driverDelivered).length
+  const blindPass = outcomes.filter((o) => o.blindDelivered).length
+  const avgSpawns = outcomes.reduce((s, o) => s + o.driverSpawns, 0) / Math.max(1, outcomes.length)
+  console.log('\n── verdict ──')
+  console.log(`driver-orchestrated delivered: ${driverPass}/${outcomes.length}  (avg spawns ${avgSpawns.toFixed(1)} of ${K} allowed)`)
+  console.log(`blind best-of-${K} delivered:    ${blindPass}/${outcomes.length}`)
+  console.log(
+    driverPass > blindPass
+      ? `→ orchestration BEAT blind by +${driverPass - blindPass} tasks`
+      : driverPass === blindPass
+        ? `→ orchestration TIED blind (the atom delivers, but adds no lift here at this N)`
+        : `→ orchestration LOST to blind by ${blindPass - driverPass} tasks`,
+  )
+}
+
+main().catch((e) => {
+  console.error(e)
+  process.exit(1)
+})