@tangle-network/agent-bench 0.8.10 → 0.8.12

package/src/hev-improve.mts

@@ -1,245 +0,0 @@
-/**
- * Official GEPA prompt optimization on HumanEval. The worker is a single chat
- * completion and the judge is the deterministic Docker checker.
- *
- * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test
- * is withheld — the worker cannot verify, so prompt wording cannot move resolve.
- * HumanEval hands the worker a well-specified function to complete and grades by
- * running tests, so the instruction prompt DOES move pass-rate. This run measures
- * whether self-improvement lifts a CHEAP model when the task is prompt-sensitive.
- *
- * Worker + reflect models call the zai coding endpoint directly (no tangle router,
- * no WAF, no 503):  TANGLE_API_KEY=$ZAI_API_KEY  ROUTER_BASE=https://api.z.ai/api/coding/paas/v4
- */
-import {
-  improve,
-  officialGepa,
-  type ReadonlyAgentProfile,
-} from '@tangle-network/agent-runtime'
-import {
-  canonicalCandidateDigest,
-  type AgentProfile,
-  agentProfileSchema,
-} from '@tangle-network/agent-interface'
-import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
-import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
-import {
-  assertCompleteCost,
-  officialOptimizerModel,
-  requiredTokenPricing,
-} from './official-optimizer-config.mjs'
-import { runBenchRouterTurn, withBenchProfile } from './router-turn'
-
-// The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
-// solveInstruction so the baseline arm reproduces the plain-prompt denominator.
-const SEED_INSTRUCTION =
-  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
-
-interface Completion {
-  text: string
-  tokIn: number
-  tokOut: number
-}
-
-async function complete(
-  base: string,
-  key: string,
-  profile: AgentProfile,
-  prompt: string,
-  maxTokens: number,
-): Promise<Completion> {
-  const result = await runBenchRouterTurn(
-    {
-      routerBaseUrl: base,
-      routerKey: key,
-      profile: withBenchProfile(profile, { temperature: 0.2, maxTokens }),
-    },
-    prompt,
-  )
-  return {
-    text: result.finalText,
-    tokIn: result.usage.input,
-    tokOut: result.usage.output,
-  }
-}
-
-async function main(): Promise<void> {
-  const key = process.env.TANGLE_API_KEY
-  if (!key) throw new Error('TANGLE_API_KEY required (worker + reflect completions)')
-  const base = process.env.ROUTER_BASE ?? 'https://api.z.ai/api/coding/paas/v4'
-  const workerModel = process.env.WORKER_MODEL ?? 'glm-4.5-air'
-  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
-  // The GEPA reflector may live on a DIFFERENT endpoint than the (cheap) worker —
-  // e.g. a small worker on Together + a strong optimizer on zai. Defaults to the
-  // worker endpoint when unset.
-  const reflectBase = process.env.REFLECT_BASE ?? base
-  const reflectKey = process.env.REFLECT_KEY ?? key
-  const trainN = Number(process.env.TRAIN_N ?? 12)
-  const selectionN = Number(process.env.SELECTION_N ?? 12)
-  const testN = Number(process.env.TEST_N ?? 12)
-  const offset = Number(process.env.OFFSET ?? 80)
-  const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 24)
-  const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 5)
-  const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
-  const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
-  const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
-  const runDir = process.env.RUN_DIR ?? '.runs/humaneval-official-gepa'
-  if (process.env.DRYRUN) {
-    console.log(
-      `DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa})`,
-    )
-    return
-  }
-  const workerPricing = requiredTokenPricing(process.env, 'WORKER')
-  const optimizer = officialOptimizerModel({
-    env: process.env,
-    model: reflectModel,
-    baseUrl: reflectBase,
-    apiKey: reflectKey,
-    maxCostUsd: maxProposerCostUsd,
-    maxOutputTokensPerRequest: reflectMaxTokens,
-  })
-
-  // All three partitions are disjoint slices of the harder middle band.
-  const train = await loadHumanEval(trainN, offset)
-  const selection = await loadHumanEval(selectionN, offset + trainN)
-  const testCases = await loadHumanEval(testN, offset + trainN + selectionN)
-  const byId = new Map<string, HumanEvalTask>(
-    [...train, ...selection, ...testCases].map((t) => [t.taskId, t]),
-  )
-
-  console.log('=== HumanEval prompt optimization with official GEPA ===')
-  console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`)
-  console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`)
-  console.log(`selection=[${selection.map((t) => t.taskId).join(', ')}]`)
-  console.log(`test=[${testCases.map((t) => t.taskId).join(', ')}]`)
-  console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} offset=${offset} maxTokens=${workerMaxTokens}`)
-  console.log(`runDir=${runDir}\n`)
-
-  const stats = { n: 0 }
-  const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
-    const instr = candidate.prompt?.systemPrompt
-    if (instr === undefined) throw new Error('agent: candidate profile has no system prompt')
-    const t = byId.get(scenario.id)
-    if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
-    const prompt = `\`\`\`python\n${t.prompt}\`\`\``
-    const executionProfile: AgentProfile = agentProfileSchema.parse({
-      ...candidate,
-      name: candidate.name ?? 'humaneval-improvement-worker',
-      model: { ...candidate.model, provider: 'tangle-router', default: workerModel },
-    })
-    const t0 = Date.now()
-    const paid = await ctx.cost.runPaidCall({
-      channel: 'agent',
-      actor: 'humaneval-worker',
-      model: workerModel,
-      execute: () => complete(base, key, executionProfile, prompt, workerMaxTokens),
-      receipt: (result) => {
-        const usageUnknown = result.tokIn === 0 && result.tokOut === 0
-        return {
-          model: workerModel,
-          inputTokens: result.tokIn,
-          outputTokens: result.tokOut,
-          customTokenPricing: workerPricing,
-          ...(usageUnknown ? { usageUnknown: true } : {}),
-        }
-      },
-    })
-    if (!paid.succeeded) throw paid.error
-    const r = paid.value
-    const hasText = r.text.trim().length > 0
-    stats.n += 1
-    const codeLen = extractCode(r.text).length
-    console.log(`  [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`)
-    return hasText ? r.text : null
-  }
-
-  const judge: JudgeConfig<string | null, Scenario> = {
-    name: 'humaneval-docker',
-    dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }],
-    async score({ artifact, scenario }) {
-      const t = byId.get(scenario.id)
-      if (!t) throw new Error(`judge: unknown scenario ${scenario.id}`)
-      const code = extractCode(String(artifact ?? ''))
-      if (!code.trim()) {
-        console.log(`  [judge] ${scenario.id} pass=0 (empty)`)
-        return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' }
-      }
-      const { pass, detail } = await runChecker(t, code)
-      console.log(`  [judge] ${scenario.id} pass=${pass}`)
-      if (pass === 1) return { dimensions: { pass }, composite: pass, notes: 'passed' }
-      // Trajectory-grounded failure note: the checker's traceback/assertion tail
-      // plus the model's own emitted code, so GEPA reflection sees WHAT failed and
-      // WHAT the model wrote — not just the word 'failed'. The candidate's full
-      // raw reply additionally reaches the proposer via the campaign breakdown's
-      // `emitted` field (carried automatically from the string artifact).
-      const traceback = (detail ?? 'checker produced no output (timeout or silent non-zero exit)').slice(-800)
-      const excerpt = code.slice(0, 700)
-      return {
-        dimensions: { pass },
-        composite: pass,
-        notes: `${traceback}\n--- emitted code (first 700 chars) ---\n${excerpt}`,
-      }
-    },
-  }
-
-  const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } }
-  const scenario = (task: HumanEvalTask): Scenario => ({ id: task.taskId, kind: 'humaneval' })
-
-  const out = await improve(profile, {
-    surface: 'prompt',
-    executionRef: canonicalCandidateDigest({
-      callback: 'bench/hev-improve',
-      model: workerModel,
-      endpoint: new URL(base).origin,
-      maxTokens: workerMaxTokens,
-      checker: 'local-python',
-    }),
-    method: officialGepa<Scenario, string | null>({
-      objective:
-        'Improve the complete instruction for a small model that writes Python functions which pass hidden unit tests.',
-      background:
-        'Prefer behavioral strategies over wording changes. Address algorithm choice, edge cases, boundary values, type behavior, and self-checking. Return only the complete instruction.',
-      recipe: {
-        kind: 'engine',
-        run: {
-          engine: 'gepa',
-          maxEvaluations,
-          maxProposerCostUsd,
-        },
-      },
-      optimizer,
-      resume: 'if-compatible',
-      trustResumeState: true,
-      describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }),
-    }),
-    trainScenarios: train.map(scenario),
-    selectionScenarios: selection.map(scenario),
-    testScenarios: testCases.map(scenario),
-    judges: [judge],
-    agent,
-    expectUsage: 'warn',
-    maxConcurrency,
-    reps: 1,
-    runDir,
-    optimizationRunOptions: {
-      expectUsage: 'warn',
-      maxConcurrency,
-      reps: 1,
-    },
-  })
-
-  assertCompleteCost('humaneval official GEPA run', out.cost)
-  console.log('\n=== RESULT ===')
-  console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`)
-  console.log(`baseline test pass-rate=${out.raw.best.baselineComposite}`)
-  console.log(`winner test pass-rate=${out.raw.best.winnerComposite}`)
-  console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`)
-  console.log(`cost=${JSON.stringify(out.cost)}`)
-  console.log(`winner instruction:\n${String(out.candidate.value).slice(0, 2000)}`)
-}
-
-main().catch((e) => {
-  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
-  process.exit(1)
-})

package/src/humaneval-object-ablation.mts

@@ -1,239 +0,0 @@
-/**
- * OBJECT-OF-IMPROVEMENT ablation on HumanEval — the head-on test of the claim our
- * whole self-improvement line rests on: at EQUAL budget, does adding CAPABILITY
- * (a real code-execution tool) beat adding IMPROVER-cleverness (blind prompt
- * self-refinement)? Both arms get the same model, the same K rounds per task, the
- * same held-out tasks. The ONLY difference is what the budget buys:
- *
- *   IMPROVER arm  — K rounds of "critique your own function and rewrite it",
- *                   with NO execution. The classic reflect-without-a-tool loop.
- *   CAPABILITY arm — K rounds WITH a `run_python` tool: the agent executes its
- *                   function on inputs it chooses, sees real output/errors, fixes.
- *
- * Grading is a DETERMINISTIC hidden test (never shown): the task's own `check`.
- * If the capability arm wins the held-out pass rate, the object of improvement
- * (what you can DO) dominates the improver (how cleverly you rewrite) — the
- * finding the prompt-only self-improvement runs kept nulling on.
- *
- * Fast by construction: HumanEval tasks are tiny, graded in an isolated Python
- * container with a hard timeout — seconds per task. Paired
- * McNemar over the per-task pass/fail difference gives the significance.
- *
- * Run from cwd=bench:  env WORKER_MODEL=deepseek-v4-flash N=60 K=3 \
- *   REPS=2 node_modules/.bin/tsx src/humaneval-object-ablation.mts
- */
-import {
-  loadHumanEval,
-  extractCode,
-  runPythonProgram,
-  type HumanEvalTask,
-} from './benchmarks/humaneval'
-import { runBenchRouterTurn } from './router-turn'
-
-const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
-const KEY = process.env.TANGLE_API_KEY
-if (!KEY) throw new Error('TANGLE_API_KEY required')
-const ROUTER_KEY = KEY
-const MODEL = process.env.WORKER_MODEL ?? 'deepseek-v4-flash'
-const N = Number(process.env.N ?? 60)
-const OFFSET = Number(process.env.OFFSET ?? 0)
-const K = Number(process.env.K ?? 3) // rounds/budget per task (equal for both arms)
-const REPS = Number(process.env.REPS ?? 2)
-const CONC = Number(process.env.CONCURRENCY ?? 6)
-const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 8000)
-
-interface ChatMsg extends Record<string, unknown> { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string }
-interface Tool { type: 'function'; function: { name: string; description: string; parameters: unknown } }
-
-async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: string; toolCalls: { id: string; name: string; args: Record<string, unknown> }[] }> {
-  for (let attempt = 0; ; attempt++) {
-    try {
-      const system = messages.find((message) => message.role === 'system')?.content
-      const result = await runBenchRouterTurn(
-        {
-          routerBaseUrl: ROUTER,
-          routerKey: ROUTER_KEY,
-          profile: {
-            name: 'humaneval-object-ablation-worker',
-            harness: 'cli-base',
-            model: {
-              provider: 'tangle-router',
-              default: MODEL,
-              metadata: {
-                temperature: 0.4,
-                ...(tools ? { toolChoice: 'auto' } : {}),
-              },
-            },
-            ...(system ? { prompt: { systemPrompt: system } } : {}),
-            ...(tools
-              ? { tools: Object.fromEntries(tools.map((tool) => [tool.function.name, true])) }
-              : {}),
-          },
-          ...(tools ? { tools } : {}),
-          timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 60_000),
-        },
-        { messages: messages.filter((message) => message.role !== 'system') },
-      )
-      const toolCalls = result.toolCalls.map((call) => {
-        if (call.id === undefined) {
-          throw new Error(`router tool call '${call.name}' omitted its required id`)
-        }
-        let args: Record<string, unknown> = {}
-        try {
-          args = JSON.parse(call.arguments) as Record<string, unknown>
-        } catch {
-          // Keep the empty argument object; the tool returns a useful error.
-        }
-        return { id: call.id, name: call.name, args }
-      })
-      return { content: result.finalText, toolCalls }
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error)
-      const status = Number(/router (\d+)/.exec(message)?.[1])
-      const transient =
-        !Number.isFinite(status) || [408, 429, 500, 502, 503, 504, 520, 522, 524].includes(status)
-      if (!transient || attempt >= 5) throw error
-      await sleep(800 * 2 ** attempt)
-    }
-  }
-}
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
-/** Run model-written Python in the shared networkless, resource-capped container. */
-async function runPython(program: string): Promise<{ stdout: string; stderr: string; ok: boolean }> {
-  const result = await runPythonProgram(program, EXEC_TIMEOUT)
-  return {
-    stdout: result.stdout.slice(0, 2000),
-    stderr: result.stderr.slice(0, 2000),
-    ok: result.exitCode === 0,
-  }
-}
-
-/** The HIDDEN judge: candidate full function + the task's own check. Never shown. */
-async function judge(task: HumanEvalTask, candidate: string): Promise<boolean> {
-  if (!candidate.trim()) return false
-  const program = `${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\nprint("PASS")\n`
-  const r = await runPython(program)
-  return r.ok && r.stdout.includes('PASS')
-}
-
-const SYSTEM = 'You are an expert Python programmer. Output the COMPLETE function definition (signature + body, plus any imports) in a single ```python block. No tests, no prose outside the block.'
-const userPrompt = (t: HumanEvalTask) => `Complete this function:\n\n\`\`\`python\n${t.prompt}\`\`\``
-
-/** IMPROVER arm: K rounds of blind self-refinement — no execution, just "review and rewrite". */
-async function improverArm(task: HumanEvalTask): Promise<string> {
-  const messages: ChatMsg[] = [{ role: 'system', content: SYSTEM }, { role: 'user', content: userPrompt(task) }]
-  let code = ''
-  for (let round = 0; round < K; round++) {
-    const { content } = await router(messages)
-    code = extractCode(content) || content
-    if (round < K - 1) {
-      messages.push({ role: 'assistant', content })
-      messages.push({ role: 'user', content: 'Carefully review your function for correctness bugs and edge cases. If it can be improved, output the corrected COMPLETE function in a python block; if it is already correct, output it again unchanged.' })
-    }
-  }
-  return code
-}
-
-const RUN_TOOL: Tool = { type: 'function', function: { name: 'run_python', description: 'Execute a Python snippet and return its stdout/stderr. Use it to test your function on example inputs from the docstring before finalizing.', parameters: { type: 'object', properties: { code: { type: 'string', description: 'python source to run' } }, required: ['code'] } } }
-
-/** CAPABILITY arm: K rounds WITH a real code-execution tool — write, run, see real output, fix. */
-async function capabilityArm(task: HumanEvalTask): Promise<string> {
-  const messages: ChatMsg[] = [
-    { role: 'system', content: `${SYSTEM}\nYou have a run_python tool: test your function on the docstring's example inputs before giving your final answer. Fix any failures you observe.` },
-    { role: 'user', content: userPrompt(task) },
-  ]
-  let code = ''
-  // (K-1) tool-exploration rounds + 1 forced final answer = K calls total, matching
-  // the improver arm's K blind-refine rounds (equal budget).
-  for (let round = 0; round < Math.max(1, K - 1); round++) {
-    const { content, toolCalls } = await router(messages, [RUN_TOOL])
-    if (content && extractCode(content)) code = extractCode(content)
-    messages.push({ role: 'assistant', content: content || '', ...(toolCalls.length ? { tool_calls: toolCalls.map((t) => ({ id: t.id, type: 'function', function: { name: t.name, arguments: JSON.stringify(t.args) } })) } : {}) })
-    if (toolCalls.length) {
-      for (const tc of toolCalls) {
-        const snippet = String(tc.args.code ?? '')
-        const r = await runPython(snippet)
-        messages.push({ role: 'tool', tool_call_id: tc.id, name: tc.name, content: `stdout:\n${r.stdout}\nstderr:\n${r.stderr}\nexit_ok=${r.ok}` })
-      }
-    } else {
-      messages.push({ role: 'user', content: 'Test your function with run_python on the docstring examples before finalizing.' })
-    }
-  }
-  // FAIR FINAL ANSWER: the tool rounds are exploration; force one no-tool call to
-  // emit the complete function. Without this, an agent that ends mid-tool-call
-  // yields empty code and is unfairly scored 0 (an extraction artifact, not a
-  // real "the tool hurt" signal). Only override if it produces a real block.
-  {
-    messages.push({ role: 'user', content: 'Now output your FINAL complete function in a single ```python block, no tools, no prose.' })
-    const { content } = await router(messages)
-    const finalCode = extractCode(content)
-    if (finalCode.trim()) code = finalCode
-  }
-  return code
-}
-
-async function pool<T, R>(items: T[], limit: number, fn: (t: T, i: number) => Promise<R>): Promise<R[]> {
-  const out = new Array<R>(items.length)
-  let next = 0
-  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
-    while (next < items.length) { const i = next++; out[i] = await fn(items[i]!, i) }
-  }))
-  return out
-}
-
-// McNemar exact (paired): b = capability-only wins, c = improver-only wins.
-function mcnemarP(b: number, c: number): number {
-  const n = b + c; if (n === 0) return 1
-  const k = Math.min(b, c)
-  const lf = (x: number) => { let s = 0; for (let i = 2; i <= x; i++) s += Math.log(i); return s }
-  let tail = 0; for (let i = 0; i <= k; i++) tail += Math.exp(lf(n) - lf(i) - lf(n - i) - n * Math.log(2))
-  return Math.min(1, 2 * tail)
-}
-
-async function main(): Promise<void> {
-  if (['1', 'true'].includes((process.env.SMOKE ?? '').toLowerCase())) { console.error('SMOKE ok: humaneval-object-ablation loaded'); return }
-  const tasks = await loadHumanEval(N, OFFSET)
-  console.error(`=== OBJECT-OF-IMPROVEMENT ablation · HumanEval n=${tasks.length} (offset ${OFFSET}) · model=${MODEL} · K=${K} rounds · reps=${REPS} · equal budget ===`)
-  // self-check the local grader on the gold solution of task 0 (never shown to the model)
-  if (tasks[0]?.canonicalSolution) {
-    const gold = `${tasks[0].prompt}${tasks[0].canonicalSolution}`
-    console.error(`  grader self-check (gold passes): ${await judge(tasks[0], gold)}`)
-  }
-
-  // per (task, rep): run both arms; record pass/fail.
-  const units = tasks.flatMap((task) => Array.from({ length: REPS }, (_, rep) => ({ task, rep })))
-  let done = 0
-  let capEmpty = 0
-  const results = await pool(units, CONC, async ({ task }) => {
-    // Per-unit resilience: a transient model failure (e.g. a weak model emitting a
-    // malformed tool call → router 400) scores that arm 0 for this unit, never
-    // crashes the whole run. Both arms wrapped identically so neither is favored.
-    const safe = async (fn: () => Promise<string>) => { try { return await fn() } catch { return '' } }
-    const [impCode, capCode] = await Promise.all([safe(() => improverArm(task)), safe(() => capabilityArm(task))])
-    const [imp, cap] = await Promise.all([judge(task, impCode), judge(task, capCode)])
-    done++
-    // AUTOPSY: a capability-arm failure with EMPTY final code is an extraction
-    // artifact (ended mid-tool-call, never emitted a final function), NOT a real
-    // "the tool hurt" signal. Count it so the effect can be separated.
-    if (!cap && !capCode.trim()) { capEmpty++; if (imp) console.error(`  [cap-empty] ${task.taskId} (improver passed)`) }
-    if (done % 20 === 0) console.error(`  ${done}/${units.length} units`)
-    return { id: task.taskId, imp, cap }
-  })
-  console.error(`  capability-arm empty-final-code (artifact) count: ${capEmpty}/${results.length}`)
-
-  const impPass = results.filter((r) => r.imp).length
-  const capPass = results.filter((r) => r.cap).length
-  const b = results.filter((r) => r.cap && !r.imp).length // capability-only wins
-  const c = results.filter((r) => !r.cap && r.imp).length // improver-only wins
-  const p = mcnemarP(b, c)
-  const n = results.length
-  const liftPp = ((capPass - impPass) / n) * 100
-
-  console.log('')
-  console.log('=== RESULT (held-out HumanEval, equal budget) ===')
-  console.log(`  IMPROVER   (blind self-refine, no tool): ${impPass}/${n} = ${((impPass / n) * 100).toFixed(1)}%`)
-  console.log(`  CAPABILITY (self-built execution tool)  : ${capPass}/${n} = ${((capPass / n) * 100).toFixed(1)}%`)
-  console.log(`  lift = ${liftPp >= 0 ? '+' : ''}${liftPp.toFixed(1)}pp   paired McNemar: capability-only=${b} improver-only=${c} p=${p.toFixed(4)}`)
-  console.log(`  verdict: ${p < 0.05 && capPass > impPass ? 'CAPABILITY > IMPROVER (significant)' : p < 0.05 && impPass > capPass ? 'IMPROVER > CAPABILITY (significant)' : 'no significant difference'}`)
-}
-main().catch((e) => { console.error('MAIN:', e instanceof Error ? (e.stack ?? e.message) : e); process.exit(1) })