@tangle-network/agent-bench 0.1.0

package/src/generate-eval/schema.ts

@@ -0,0 +1,78 @@
+/**
+ * The generated-eval interchange contract.
+ *
+ * This schema — not our code — is the integration surface: authoring agents
+ * emit it, the certifier validates it, runners consume it, the leaderboard
+ * aggregates it. It extends the bench `SearchTask` (so admitted evals drop
+ * straight into the existing search-bench runner) with the two things
+ * generation needs: a reproducible environment (`setup`) and an executable
+ * reference (`reference`) the certifier can re-run independently.
+ *
+ * Soundness contract (enforced by certify.ts, never by the authoring model):
+ *   - grounding: the reference must execute and pass against the real pinned
+ *     target named in `setup`.
+ *   - discrimination: a no-tools parametric baseline must FAIL the oracle.
+ * An eval without a passed `certification` block is a CANDIDATE, not an eval.
+ */
+import type { SearchTask } from '../search-bench/tasks'
+
+export const generatedEvalSchemaVersion = 1
+
+export interface EvalReference {
+  /** Files written into the certification workspace, path → content. */
+  files: Record<string, string>
+  /** Command that exercises the reference against the pinned target.
+   *  Exit 0 ⇒ grounding gate passes. Run from the workspace root. */
+  cmd: string
+  /** Optional stdout requirement — every string must appear in the command's
+   *  stdout. Guards against a reference that "passes" by doing nothing. */
+  stdoutContains?: string[]
+}
+
+export interface EvalCertification {
+  schemaVersion: number
+  /** Reference executed + passed against the pinned target. */
+  groundingPassed: boolean
+  /** No-tools baseline failed the oracle (the task discriminates). */
+  parametricFailed: boolean
+  /** Model used for the parametric gate. */
+  parametricModel: string
+  /** Exact target the reference ran against (e.g. hono@4.6.3). */
+  resolvedTarget: string
+  certifiedAt: string
+  /** Tool that performed certification (name@version). */
+  certifier: string
+}
+
+export interface GeneratedEval extends SearchTask {
+  schemaVersion: number
+  /** Shell commands that provision a CLEAN workspace with the pinned target
+   *  (e.g. ["npm init -y", "npm i hono@4.6.3"]). Re-run by the certifier from
+   *  scratch — the authoring agent's own workspace is never trusted. */
+  setup: string[]
+  reference: EvalReference
+  /** Present + both-gates-true ⇔ admitted. */
+  certification?: EvalCertification
+}
+
+/** Parse + structurally validate a candidate emitted by an authoring agent.
+ *  Fail-loud: a malformed candidate is a generation failure, not a skip. */
+export function parseCandidate(raw: string): GeneratedEval {
+  const v = JSON.parse(raw) as Partial<GeneratedEval>
+  const problems: string[] = []
+  if (v.schemaVersion !== generatedEvalSchemaVersion)
+    problems.push(`schemaVersion must be ${generatedEvalSchemaVersion}`)
+  if (!v.id || !/^[a-z0-9][a-z0-9-]*$/.test(v.id)) problems.push('id must be kebab-case')
+  if (!v.domain) problems.push('missing domain')
+  if (!v.prompt || v.prompt.length < 40) problems.push('prompt missing or too short')
+  if (!v.needsFreshDocs) problems.push('missing needsFreshDocs')
+  const o = v.oracle
+  if (!o || !(o.containsAll?.length || o.containsAny?.length || o.regex?.length))
+    problems.push('oracle needs containsAll, containsAny, or regex')
+  if (!v.setup?.length) problems.push('setup must pin the target')
+  if (!v.reference?.cmd || !v.reference.files || Object.keys(v.reference.files).length === 0)
+    problems.push('reference needs files + cmd')
+  if (!v.sourceUrl) problems.push('missing sourceUrl (primary-doc provenance)')
+  if (problems.length) throw new Error(`invalid candidate: ${problems.join('; ')}`)
+  return v as GeneratedEval
+}

package/src/humaneval-gate.mts

@@ -0,0 +1,204 @@
+/**
+ * HumanEval deployable-verifier gate — the thesis test the answer-oracle benches
+ * (aec-bench, finsearch) could not reach.
+ *
+ * The repo's deployable self-consistency selector LOSES on both aec-bench (−9.4pp)
+ * and finsearch (−8.2pp): diversity opens a large ORACLE ceiling, but the selector
+ * cannot capture it because those benches verify by RECOMPUTING the gold answer (an
+ * oracle, not a deployable check). Code+tests is the deployable-checker regime — the
+ * agent runs the task's provided tests (which it legitimately has in production) and
+ * keeps a passer. This file asks: at EQUAL k, does diverse@k + a deployable
+ * verifier-grounded pick beat random@k + the same pick, and beat blind@1?
+ *
+ * SCOPE — read the numbers as a LOWER BOUND. Here a "shot" is a single STATELESS
+ * completion (one router call, `maxTurns=0`, NO `AgentProfile` / sandbox / keystone —
+ * it calls the router directly). That is the *degenerate* rollout (HARNESS.md's
+ * "Terminology"): it isolates the SELECTOR with the generator unable to self-correct,
+ * so it measures the selector's value at its MAXIMUM. A real rollout (an `AgentProfile`
+ * through `runLoop`, `maxTurns>0` over a persistent workspace) self-verifies by
+ * iterating, which shrinks the external selector's job — that is the next experiment,
+ * not this one. A positive result here is the science (the selector works in a
+ * deployable-checker regime), not the product.
+ *
+ * Two paired arms over the SAME tasks (each "shot" = one stateless completion):
+ *   random@K  — K identical-base-prompt completions/task (the compute control)
+ *   diverse@K — K completions, the i-th prefixed with composeStrategies(base, K)[i]
+ *
+ * The DEPLOYABLE CHECKER runs each candidate against the task's own `test` in an
+ * isolated `--network=none` python:3.12-slim container (hard timeout) — exit 0 = pass.
+ * No gold `canonical_solution` is ever shown to the model or the selector.
+ *
+ * Metrics (paired across the same tasks):
+ *   blind pass@1       — first attempt passes
+ *   random-pass@k      — verifierGroundedSelect over the K random shots passes
+ *   diverse-pass@k     — verifierGroundedSelect over the K diverse shots passes
+ *   oracle@k           — any of the K passes (the ceiling)
+ *   self-consistency@k — selfConsistencySelect (answer-clustering, NOT the checker)
+ *                        over the diverse shots — the direct contrast with the
+ *                        −8/−9pp answer-oracle selector.
+ * Each delta carries a 95% paired-bootstrap CI.
+ *
+ *   N=20 K=4 npx tsx src/humaneval-gate.mts
+ */
+
+import { composeStrategies } from './directives'
+import { basePrompt, type CheckResult, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval'
+import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/loops'
+import { selfConsistencySelect, verifierGroundedSelect } from './selector'
+import { type PairedLift, pairedLift, pool } from './stats.mts'
+
+const dockerImage = 'python:3.12-slim'
+const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+interface Attempt {
+  code: string
+  pass: number
+}
+
+interface TaskOutcome {
+  taskId: string
+  randomAttempts: Attempt[]
+  diverseAttempts: Attempt[]
+}
+
+const pct = (x: number) => `${(x * 100).toFixed(1)}%`
+const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp`
+
+async function main(): Promise<void> {
+  const n = Number(process.env.N ?? 20)
+  const k = Number(process.env.K ?? 4)
+  const offset = Number(process.env.OFFSET ?? 0)
+  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const routerKey = must('TANGLE_API_KEY')
+  const solveConcurrency = Number(process.env.CONCURRENCY ?? 8)
+  const dockerConcurrency = Number(process.env.DOCKER_CONCURRENCY ?? 6)
+
+  if (!Number.isInteger(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
+  if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
+  if (!Number.isInteger(offset) || offset < 0) throw new Error(`OFFSET must be a non-negative integer, got ${process.env.OFFSET}`)
+
+  const cfg: RouterConfig = { routerBaseUrl, routerKey, model }
+
+  console.log(`=== HumanEval deployable-verifier gate · N=${n} K=${k} offset=${offset} model=${model} ===`)
+  console.log(`  router=${routerBaseUrl}  docker=${dockerImage} (--network=none, timeout ${dockerTimeoutMs}ms)`)
+  console.log(
+    '  regime: STATELESS single completions (maxTurns=0, no AgentProfile/sandbox) — the selector no-self-correction LOWER BOUND, not a rollout/product number',
+  )
+
+  const tasks = await loadHumanEval(n, offset)
+  console.log(`loaded ${tasks.length} HumanEval task(s): ${tasks.map((t) => t.taskId).join(', ')}`)
+
+  // Build the full work set: per task, K random + K diverse solve calls.
+  type Unit = { taskIdx: number; arm: 'random' | 'diverse'; shot: number; prompt: string }
+  const units: Unit[] = []
+  for (let ti = 0; ti < tasks.length; ti += 1) {
+    const task = tasks[ti] as HumanEvalTask
+    const base = basePrompt(task)
+    const diverse = composeStrategies(base, k)
+    for (let s = 0; s < k; s += 1) {
+      units.push({ taskIdx: ti, arm: 'random', shot: s, prompt: base })
+      units.push({ taskIdx: ti, arm: 'diverse', shot: s, prompt: diverse[s] as string })
+    }
+  }
+  console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`)
+
+  const codes = await pool(units, solveConcurrency, async (u) => {
+    const res = await routerChatWithUsage(cfg, [{ role: 'user', content: u.prompt }], {
+      temperature: Number(process.env.TEMPERATURE ?? '0.8'),
+    })
+    return extractCode(typeof res.content === 'string' ? res.content : '')
+  })
+
+  console.log(`▶ running ${codes.length} candidates through the Docker deployable checker, conc=${dockerConcurrency}`)
+  const passes = await pool(units, dockerConcurrency, (u, i) => runChecker(tasks[u.taskIdx] as HumanEvalTask, codes[i] as string))
+
+  // Regroup into per-task arms, preserving shot order.
+  const outcomes: TaskOutcome[] = tasks.map((t) => ({ taskId: t.taskId, randomAttempts: [], diverseAttempts: [] }))
+  units.forEach((u, i) => {
+    const att: Attempt = { code: codes[i] as string, pass: (passes[i] as CheckResult).pass }
+    const o = outcomes[u.taskIdx] as TaskOutcome
+    if (u.arm === 'random') o.randomAttempts[u.shot] = att
+    else o.diverseAttempts[u.shot] = att
+  })
+
+  // Per-task {0,1} outcomes for each metric, aligned across the same tasks.
+  const blind: number[] = []
+  const randomAtK: number[] = []
+  const diverseAtK: number[] = []
+  const oracleAtK: number[] = []
+  const selfConsistencyAtK: number[] = []
+
+  for (const o of outcomes) {
+    const rPasses = o.randomAttempts.map((a) => a.pass)
+    const dPasses = o.diverseAttempts.map((a) => a.pass)
+    // blind = first random shot (one-shot baseline)
+    blind.push((o.randomAttempts[0] as Attempt).pass)
+    // random@k = verifier-grounded pick over the K random shots
+    randomAtK.push((o.randomAttempts[verifierGroundedSelect(rPasses)] as Attempt).pass)
+    // diverse@k = verifier-grounded pick over the K diverse shots
+    diverseAtK.push((o.diverseAttempts[verifierGroundedSelect(dPasses)] as Attempt).pass)
+    // oracle@k = any diverse shot passes (the ceiling the diverse arm opens)
+    oracleAtK.push(dPasses.some((p) => p > 0) ? 1 : 0)
+    // self-consistency@k = answer-clustering pick over the diverse shots (the
+    // −8/−9pp answer-oracle selector; uses code text, NOT the checker).
+    const scIdx = selfConsistencySelect(o.diverseAttempts.map((a) => a.code))
+    selfConsistencyAtK.push((o.diverseAttempts[scIdx] as Attempt).pass)
+  }
+
+  const rate = (xs: number[]) => xs.reduce((s, x) => s + x, 0) / xs.length
+
+  const blindRate = rate(blind)
+  const randomRate = rate(randomAtK)
+  const diverseRate = rate(diverseAtK)
+  const oracleRate = rate(oracleAtK)
+  const scRate = rate(selfConsistencyAtK)
+  // oracle ceiling of the RANDOM arm — for the diverse-vs-random ceiling contrast.
+  const randomOracleRate = rate(outcomes.map((o) => (o.randomAttempts.some((a) => a.pass > 0) ? 1 : 0)))
+
+  console.log(`\n${'='.repeat(78)}`)
+  console.log(`RESULTS · HumanEval · n=${tasks.length} tasks · k=${k} · model=${model}`)
+  console.log('='.repeat(78))
+  console.log(`  blind pass@1               ${pct(blindRate)}`)
+  console.log(`  random-pass@k (verifier)   ${pct(randomRate)}`)
+  console.log(`  diverse-pass@k (verifier)  ${pct(diverseRate)}`)
+  console.log(`  oracle@k (diverse, any)    ${pct(oracleRate)}    [random-arm oracle ${pct(randomOracleRate)}]`)
+  console.log(`  self-consistency@k         ${pct(scRate)}    (answer-clustering selector over the diverse set)`)
+
+  const liftDiverseVsRandom = pairedLift(randomAtK, diverseAtK)
+  const liftRandomVsBlind = pairedLift(blind, randomAtK)
+  const liftDiverseVsBlind = pairedLift(blind, diverseAtK)
+  const liftScVsRandom = pairedLift(randomAtK, selfConsistencyAtK)
+  const liftDiverseVsScVerifier = pairedLift(selfConsistencyAtK, diverseAtK)
+
+  const row = (label: string, l: PairedLift) =>
+    console.log(
+      `  ${label.padEnd(34)} ${pp(l.point).padStart(7)}   CI [${pp(l.low)}, ${pp(l.high)}]   (paired ${l.pairs}, discordant ${l.discordant})`,
+    )
+
+  console.log(`\n  PAIRED LIFTS (95% bootstrap CI, B=10000):`)
+  row('diverse@k − random@k (verifier)', liftDiverseVsRandom)
+  row('random@k − blind (compute)', liftRandomVsBlind)
+  row('diverse@k − blind (total)', liftDiverseVsBlind)
+  row('self-consistency@k − random@k', liftScVsRandom)
+  row('verifier-pick − sc-pick (diverse)', liftDiverseVsScVerifier)
+
+  const sig = (l: PairedLift) => (l.low > 0 ? 'POSITIVE (CI excludes 0)' : l.high < 0 ? 'NEGATIVE (CI excludes 0)' : 'n.s. (CI spans 0)')
+  console.log(`\n  VERDICT:`)
+  console.log(`    diverse@k beats blind@1?       ${liftDiverseVsBlind.point > 0 ? 'yes' : 'no'} (${pp(liftDiverseVsBlind.point)}, ${sig(liftDiverseVsBlind)})`)
+  console.log(`    diverse@k beats random@k @k?   ${liftDiverseVsRandom.point > 0 ? 'yes' : 'no'} (${pp(liftDiverseVsRandom.point)}, ${sig(liftDiverseVsRandom)})`)
+  console.log(`    is the diversity ceiling capturable with a deployable checker?`)
+  console.log(`      diverse oracle ${pct(oracleRate)} → verifier-pick ${pct(diverseRate)} (gap ${pp(oracleRate - diverseRate)});`)
+  console.log(`      contrast: the SAME diverse set under the answer-clustering selector resolves ${pct(scRate)} (verifier−sc ${pp(diverseRate - scRate)}).`)
+}
+
+main().catch((err) => {
+  console.error(`humaneval-gate: ${err instanceof Error ? err.message : String(err)}`)
+  process.exit(1)
+})

package/src/humaneval-repair-gate.mts

@@ -0,0 +1,143 @@
+/**
+ * HumanEval self-repair gate — the TOOL-USING router backend vs blind resampling
+ * at equal compute, the strongest form of the steering question.
+ *
+ * The earlier steering gate (the rsi analyst arm) used an LLM that AUDITED the
+ * prior code WITHOUT running it — and was a null (−1.2pp, n.s.). This removes that
+ * weakness: the worker gets a `run_tests` tool (the deployable Docker checker), so
+ * it actually RUNS the tests, sees the real failure, and fixes — execution-grounded
+ * self-repair, off-box over the Tangle router's tool-calling (no sandbox). If
+ * steering ever beats compute on a deployable checker, this is where it should.
+ *
+ *   blind@K   — K independent completions, verifier-grounded pick           (breadth/resample)
+ *   repair@K  — ONE worker, up to K tool-turns: write → run_tests → fix → …  (depth/tool-grounded)
+ *
+ * Equal budget: one inference turn = one router completion, so both arms spend ≤K
+ * completions. Both finals are judged by the SAME check() suite. Per-task {0,1}
+ * outcomes, paired 95% bootstrap CI (discordant pairs = the power).
+ *
+ *   TANGLE_API_KEY=… N=82 K=3 OFFSET=82 WORKER_MODEL=gpt-3.5-turbo \
+ *     tsx src/humaneval-repair-gate.mts
+ */
+import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval, runChecker } from './benchmarks/humaneval'
+import { type RouterConfig, type ToolSpec, routerChatWithUsage, routerToolLoop } from '@tangle-network/agent-runtime/loops'
+import { verifierGroundedSelect } from './selector'
+import { type PairedLift, pairedLift, pool } from './stats.mts'
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+const runTestsTool: ToolSpec = {
+  type: 'function',
+  function: {
+    name: 'run_tests',
+    description:
+      "Run the task's test suite against your candidate function and return PASS or the real failure output. Verify with this before giving your final answer.",
+    parameters: {
+      type: 'object',
+      properties: { code: { type: 'string', description: 'The COMPLETE Python function definition to test (signature + body, plus any imports).' } },
+      required: ['code'],
+    },
+  },
+}
+
+const repairSystem = [
+  'You complete a Python function. You have a run_tests tool that runs the REAL test suite against your code.',
+  'Workflow: write the function, call run_tests to check it, and if it fails read the error and fix the function, then call run_tests again.',
+  'When run_tests reports all tests passed, reply with the final function in a single ```python block and do NOT call the tool again.',
+].join(' ')
+
+/** repair@K: one worker, up to K inference turns, steering on real test failures. */
+async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise<number> {
+  let lastTested = ''
+  const r = await routerToolLoop(
+    cfg,
+    repairSystem,
+    basePrompt(task),
+    [runTestsTool],
+    async (name, args) => {
+      if (name !== 'run_tests') return `error: unknown tool ${name}`
+      const code = extractCode(String(args.code ?? ''))
+      lastTested = code
+      const res = await runChecker(task, code)
+      return res.pass === 1
+        ? 'ALL TESTS PASSED. Reply with the final function now; do not call run_tests again.'
+        : `TESTS FAILED:\n${res.detail ?? 'no output'}\n\nFix the function and call run_tests again.`
+    },
+    { maxTurns: k, temperature: 0.3 },
+  )
+  // Judge the model's final answer; fall back to the last code it tested (it may
+  // report "done" without re-pasting the passing function).
+  const finalCode = extractCode(r.final) || lastTested
+  if (!finalCode) return 0
+  return (await runChecker(task, finalCode)).pass
+}
+
+/** blind@K: K independent completions, verifier-grounded pick (the resample control). */
+async function blindAttempts(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise<number[]> {
+  const base = basePrompt(task)
+  const passes: number[] = []
+  for (let i = 0; i < k; i += 1) {
+    const res = await routerChatWithUsage(cfg, [{ role: 'user', content: base }], { temperature: 0.8 })
+    passes.push((await runChecker(task, extractCode(res.content))).pass)
+  }
+  return passes
+}
+
+const pct = (x: number) => `${(x * 100).toFixed(1)}%`
+const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp`
+
+async function main(): Promise<void> {
+  const n = Number(process.env.N ?? 82)
+  const k = Number(process.env.K ?? 3)
+  const offset = Number(process.env.OFFSET ?? 82)
+  const model = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
+  const cfg: RouterConfig = { routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: must('TANGLE_API_KEY'), model }
+  const concurrency = Number(process.env.CONCURRENCY ?? 6)
+  if (k < 2) throw new Error('K must be >= 2 (repair needs at least write + one fix)')
+
+  console.log(`=== HumanEval self-repair gate · tool-using router worker · N=${n} K=${k} offset=${offset} model=${model} ===`)
+  const tasks = await loadHumanEval(n, offset)
+  console.log(`loaded ${tasks.length} task(s); running blind@${k} (resample) vs repair@${k} (run_tests-grounded), conc=${concurrency}\n`)
+
+  const rows = await pool(tasks, concurrency, async (task, i) => {
+    const blind = await blindAttempts(cfg, task, k)
+    const repair = await repairAttempt(cfg, task, k)
+    const blind1 = blind[0] ?? 0
+    const blindK = blind[verifierGroundedSelect(blind)] ?? 0
+    process.stderr.write(`  [${i + 1}/${tasks.length}] ${task.taskId}: blind@1=${blind1} blind@${k}=${blindK} repair@${k}=${repair}\n`)
+    return { blind1, blindK, repair }
+  })
+
+  const blind1 = rows.map((r) => r.blind1)
+  const blindK = rows.map((r) => r.blindK)
+  const repairK = rows.map((r) => r.repair)
+  const rate = (xs: number[]) => xs.reduce((s, x) => s + x, 0) / xs.length
+
+  console.log(`\n${'='.repeat(74)}`)
+  console.log(`RESULTS · HumanEval self-repair · n=${tasks.length} · k=${k} · ${model}`)
+  console.log('='.repeat(74))
+  console.log(`  blind pass@1        ${pct(rate(blind1))}`)
+  console.log(`  blind@${k} (resample) ${pct(rate(blindK))}`)
+  console.log(`  repair@${k} (tools)   ${pct(rate(repairK))}`)
+
+  const row = (label: string, l: PairedLift) =>
+    console.log(`  ${label.padEnd(34)} ${pp(l.point).padStart(7)}   CI [${pp(l.low)}, ${pp(l.high)}]   (paired ${l.pairs}, discordant ${l.discordant})`)
+  const sig = (l: PairedLift) => (l.low > 0 ? 'SIGNIF +' : l.high < 0 ? 'SIGNIF -' : 'n.s. (CI spans 0)')
+
+  const repairVsBlind = pairedLift(blindK, repairK)
+  const computeVsBlind1 = pairedLift(blind1, blindK)
+  console.log(`\n  PAIRED LIFTS (95% bootstrap CI, B=10000):`)
+  row(`repair@${k} − blind@${k} (steering)`, repairVsBlind)
+  row(`blind@${k} − blind@1 (more-compute)`, computeVsBlind1)
+  console.log(`\n  VERDICT:`)
+  console.log(`    execution-grounded self-repair beats blind resampling @ equal k?  ${repairVsBlind.point > 0 ? 'yes' : 'no'} (${pp(repairVsBlind.point)}, ${sig(repairVsBlind)})`)
+}
+
+main().catch((err) => {
+  console.error(`humaneval-repair-gate: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`)
+  process.exit(1)
+})

package/src/index.ts

@@ -0,0 +1,19 @@
+/**
+ * `@tangle-network/agent-bench` — the unified benchmark suite for agent-runtime agents.
+ *
+ * Any consumer (a product, a supervisor harness, a profile/skill/prompt change) imports the registry
+ * and scores its agent's artifact against a real, deterministic judge — without owning the benchmark:
+ *
+ *   import { resolveAdapter } from '@tangle-network/agent-bench'
+ *   const verdict = await resolveAdapter('commit0').judge(task, myAgentsDiff)
+ *
+ * Each adapter fails loud in `preflight()` when its harness/deps (Docker, a venv, a dataset) are
+ * absent, so importing the registry is cheap; running a specific benchmark pulls only its deps.
+ */
+export { ADAPTERS, resolveAdapter } from './adapters'
+export type {
+  BenchmarkAdapter,
+  BenchScore,
+  BenchTask,
+  LoadOptions,
+} from './benchmarks/types'

package/src/mcp-mount-probe.mts

@@ -0,0 +1,126 @@
+/**
+ * The critical de-risk for the real e2e: does an in-box opencode harness (via the cli-bridge)
+ * actually MOUNT my coordination MCP and CALL spawn_worker — landing on a real Scope.spawn?
+ *
+ * Serves the coordination MCP over a live Scope, then asks the bridge's opencode (with that MCP in
+ * its config) to call spawn_worker + await_event. If the Scope spawned+settled, the in-box driving
+ * path is real. No mock.
+ *
+ *   ROUTER_BASE=http://127.0.0.1:3355/v1 TANGLE_API_KEY=<bridge-bearer> \
+ *   WORKER_MODEL=opencode/zai-coding-plan/glm-5-turbo npx tsx bench/src/mcp-mount-probe.mts
+ */
+
+import {
+  type Agent,
+  type AgentProfile,
+  type AgentSpec,
+  createExecutorRegistry,
+  createSupervisor,
+  type Executor,
+  type ExecutorResult,
+  InMemoryResultBlobStore,
+  InMemorySpawnJournal,
+  type Scope,
+  type UsageEvent,
+} from '../../src/runtime/index'
+import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp'
+
+const BRIDGE = process.env.ROUTER_BASE ?? 'http://127.0.0.1:3355/v1'
+const BEARER = process.env.TANGLE_API_KEY ?? ''
+const MODEL = process.env.WORKER_MODEL ?? 'opencode/zai-coding-plan/glm-5-turbo'
+
+function deliveringLeaf(name: string, out: unknown): Agent<unknown, unknown> {
+  const ex: Executor<unknown> = {
+    runtime: 'router',
+    execute() {
+      return (async function* () {
+        yield { kind: 'iteration' } as UsageEvent
+        yield { kind: 'tokens', input: 5, output: 5 } as UsageEvent
+      })()
+    },
+    teardown: () => Promise.resolve({ destroyed: true }),
+    resultArtifact: (): ExecutorResult<unknown> => ({
+      outRef: `w:${name}`,
+      out,
+      verdict: { valid: true, score: 1 },
+      spent: { iterations: 1, tokens: { input: 5, output: 5 }, usd: 0, ms: 0 },
+    }),
+  }
+  const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex }
+  return { name, act: async () => out, executorSpec: spec } as Agent<unknown, unknown> & {
+    executorSpec: AgentSpec
+  }
+}
+
+async function bridgeChat(messages: Array<{ role: string; content: string }>, mcpUrl: string): Promise<string> {
+  const r = await fetch(`${BRIDGE.replace(/\/$/, '')}/chat/completions`, {
+    method: 'POST',
+    headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' },
+    body: JSON.stringify({
+      model: MODEL,
+      messages,
+      mcp: { mcpServers: { coordination: { type: 'http', url: mcpUrl } } },
+    }),
+  })
+  if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})`
+  const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  return j.choices?.[0]?.message?.content ?? ''
+}
+
+async function main(): Promise<void> {
+  const blobs = new InMemoryResultBlobStore()
+  let mounted = false
+  const root: Agent<unknown, unknown> = {
+    name: 'mcp-mount-probe',
+    async act(_t, scope: Scope<unknown>) {
+      const mcp = await serveCoordinationMcp({
+        scope,
+        blobs,
+        makeWorkerAgent: () => deliveringLeaf('w', { ok: true }),
+        perWorker: { maxIterations: 4, maxTokens: 2000 },
+      })
+      console.error(`[probe] coordination MCP live at ${mcp.url}`)
+      try {
+        const content = await bridgeChat(
+          [
+            {
+              role: 'user',
+              content:
+                'You have an MCP server named "coordination" with tools: spawn_worker, await_event, stop. ' +
+                'Call spawn_worker with arguments {"profile":{},"task":"hello"}. Then call await_event. ' +
+                'Then reply with exactly what await_event returned.',
+            },
+          ],
+          mcp.url,
+        )
+        const settled = mcp.settled()
+        mounted = settled.length > 0
+        console.error(`[probe] opencode replied: ${content.slice(0, 400)}`)
+        console.error(`[probe] Scope spawned+settled = ${settled.length}: ${JSON.stringify(settled)}`)
+        return mounted ? { mounted: true } : undefined
+      } finally {
+        await mcp.close()
+      }
+    },
+  }
+
+  const result = await createSupervisor<unknown, unknown>().run(root, 'probe', {
+    budget: { maxIterations: 100, maxTokens: 400_000 },
+    runId: 'mcp-probe',
+    journal: new InMemorySpawnJournal(),
+    blobs,
+    executors: createExecutorRegistry(),
+    maxDepth: 4,
+    now: () => Date.now(),
+  })
+  console.log(
+    mounted
+      ? '✅ MCP MOUNT WORKS — the in-box opencode harness called spawn_worker → real Scope.spawn'
+      : `❌ opencode did NOT call spawn_worker (result=${result.kind}) — protocol/mount needs work`,
+  )
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})