@tangle-network/agent-bench 0.1.0 → 0.3.5

package/src/examples/lean-proof-gate.mts

@@ -0,0 +1,162 @@
+/**
+ * Lean prover-verifier gate — a real "prover-verifier loop" for MATH, the pipeline-math pattern
+ * on this substrate. It is `math-demo.mts` with the `check` swapped from a numeric compare to the
+ * REAL Lean 4 compiler: a prover model proposes a proof, Lean verifies it (ground truth, cannot be
+ * fooled), the error feeds back, and it retries. There is NO new "loop" primitive — the loop is the
+ * existing `refine` strategy over `createVerifierEnvironment`; the only new thing is the verifier.
+ *
+ * Two loops stack: within a turn the model calls the `lean_check` tool (real compiler) to fix its
+ * proof (the humaneval-repair pattern), and across shots `refine` carries the failure forward.
+ *
+ * Run the DETERMINISTIC VERIFIER right now, no model, no key (proves the judge is real):
+ *   tsx src/examples/lean-proof-gate.mts --verify-only
+ *
+ * Run the FULL prover-verifier loop (real agent iterating against Lean):
+ *   TANGLE_API_KEY=… WORKER_MODEL=gpt-4.1 BUDGET=3 tsx src/examples/lean-proof-gate.mts
+ */
+import {
+  type AgenticTask,
+  createVerifierEnvironment,
+  printBenchmarkReport,
+  refine,
+  runBenchmark,
+  sample,
+  sampleThenRefine,
+} from '@tangle-network/agent-runtime/loops'
+import { ensureLeanImage, leanCheck } from './lean-verify.js'
+
+// Real, mathlib-free Lean 4 theorems (compile in core Lean). `header` is everything up to `:=`;
+// the prover supplies the proof term / tactic block. `reference` is a known-good proof used only
+// by --verify-only to prove the verifier — the graded run never sees it.
+interface ProofTask {
+  id: string
+  header: string
+  statement: string
+  reference: string
+}
+
+const PROBLEMS: ProofTask[] = [
+  {
+    id: 'and-swap',
+    header: 'theorem and_swap (p q : Prop) (h : p ∧ q) : q ∧ p',
+    statement: 'From a proof of `p ∧ q`, produce a proof of `q ∧ p`.',
+    reference: '⟨h.2, h.1⟩',
+  },
+  {
+    id: 'or-swap',
+    header: 'theorem or_swap (p q : Prop) (h : p ∨ q) : q ∨ p',
+    statement: 'From a proof of `p ∨ q`, produce a proof of `q ∨ p`.',
+    reference: 'h.symm',
+  },
+  {
+    id: 'add-comm',
+    header: 'theorem add_comm_nat (a b : Nat) : a + b = b + a',
+    statement: 'Addition of natural numbers is commutative.',
+    reference: 'by omega',
+  },
+  {
+    id: 'mul-one',
+    header: 'theorem mul_one_nat (n : Nat) : n * 1 = n',
+    statement: 'Multiplying a natural number by one gives the number.',
+    reference: 'by simp',
+  },
+  {
+    id: 'reverse-reverse',
+    header: 'theorem reverse_reverse {α : Type} (l : List α) : l.reverse.reverse = l',
+    statement: 'Reversing a list twice returns the original list.',
+    reference: 'by simp',
+  },
+]
+
+const SYSTEM =
+  'You are a Lean 4 theorem prover. Prove the theorem. You have a `lean_check` tool that runs the ' +
+  'REAL Lean compiler on a candidate proof and returns the exact error — call it to iterate until ' +
+  'it compiles. Then submit ONLY the proof (everything that goes after `:=`, e.g. `⟨h.2, h.1⟩` or ' +
+  '`by omega`) with `submit_answer`. Never use `sorry`, `admit`, or `native_decide`.'
+
+const tasks: AgenticTask[] = PROBLEMS.map((p) => ({
+  id: p.id,
+  systemPrompt: SYSTEM,
+  userPrompt: `Prove this Lean 4 theorem.\n\n${p.header} := ?\n\nStatement: ${p.statement}`,
+  meta: { header: p.header },
+}))
+
+const leanEnv = createVerifierEnvironment({
+  name: 'lean-proofs',
+  // THE VERIFIER — real Lean, deterministic. settled.valid ⟺ Lean accepts a real (non-sorry) proof.
+  check: async (task, answer) => {
+    const header = (task.meta as { header: string }).header
+    const r = await leanCheck(header, answer)
+    return { passes: r.passed ? 1 : 0, total: 1, errored: 0 }
+  },
+  // The prover's mid-turn verifier tool — it compiles a candidate against real Lean and returns the error.
+  extraTools: [
+    {
+      type: 'function',
+      function: {
+        name: 'lean_check',
+        description:
+          'Compile a candidate proof with the real Lean 4 compiler. Returns ACCEPTED, or the Lean error to fix. Pass only the proof (what follows `:=`).',
+        parameters: {
+          type: 'object',
+          properties: { proof: { type: 'string', description: 'The proof term or `by ...` tactic block.' } },
+          required: ['proof'],
+        },
+      },
+    },
+  ],
+  callExtra: async (task, name, args) => {
+    if (name !== 'lean_check') return `ERROR: unknown tool ${name}`
+    const header = (task.meta as { header: string }).header
+    const r = await leanCheck(header, String(args.proof ?? ''))
+    return r.passed
+      ? 'ACCEPTED by Lean. Submit this exact proof with submit_answer.'
+      : `LEAN REJECTED:\n${r.error}\n\nFix the proof and call lean_check again.`
+  },
+})
+
+/** Prove the verifier itself — no model, no key. Every reference proof must PASS; a mangled one must FAIL. */
+async function verifyOnly(): Promise<void> {
+  await ensureLeanImage()
+  let ok = 0
+  for (const p of PROBLEMS) {
+    const good = await leanCheck(p.header, p.reference)
+    console.log(`${good.passed ? 'PASS' : 'FAIL'}  ${p.id}  (reference: ${p.reference})`)
+    if (good.passed) ok++
+    else console.log(`      ${good.error.split('\n')[0]}`)
+  }
+  // Negative control: break the first proof, expect a FAIL — proves the judge rejects wrong proofs.
+  const bad = await leanCheck(PROBLEMS[0].header, '⟨h.1, h.2⟩')
+  console.log(`\nnegative control (deliberately wrong): ${bad.passed ? 'PASS (BUG!)' : 'FAIL ✓ rejected'}`)
+  console.log(`      ${bad.error.split('\n').slice(0, 2).join(' ')}`)
+  console.log(`\nverifier: ${ok}/${PROBLEMS.length} reference proofs accepted, wrong proof rejected=${!bad.passed}`)
+  if (ok !== PROBLEMS.length || bad.passed) process.exit(1)
+}
+
+async function main(): Promise<void> {
+  if (process.argv.includes('--verify-only')) return verifyOnly()
+
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('set TANGLE_API_KEY to run the live prover-verifier loop (or pass --verify-only to prove just the verifier)')
+  await ensureLeanImage() // fail loud before spending model budget if Lean can't run
+  const report = await runBenchmark({
+    environment: leanEnv,
+    tasks,
+    worker: {
+      routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',
+      routerKey,
+      model: process.env.WORKER_MODEL ?? 'gpt-4.1',
+      innerTurns: 8, // room to call lean_check and fix
+      temperature: 0.4,
+    },
+    strategies: [sample, refine, sampleThenRefine],
+    budget: Number(process.env.BUDGET ?? 3),
+    concurrency: 2,
+  })
+  printBenchmarkReport(report)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/examples/lean-verify.ts

@@ -0,0 +1,95 @@
+/**
+ * The deterministic Lean 4 verifier — the ground-truth judge for the prover-verifier gate.
+ *
+ * It assembles `${header} := ${proof}`, compiles it with the real Lean compiler in a cached,
+ * network-isolated Docker image, and passes ONLY when Lean accepts it AND the proof uses no
+ * trust-escape (`sorry`/`admit`/`native_decide`). There is no LLM in this path: a wrong proof
+ * fails with Lean's own type error, which the loop feeds back verbatim for the next attempt.
+ *
+ * Proven real (Lean 4.31.0): `⟨h.2, h.1⟩` compiles (PASS); `⟨h.1, h.2⟩` fails with
+ * "argument h.left has type p but is expected to have type q" (FAIL). No mocks anywhere.
+ */
+
+import { execFile } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+
+const pexec = promisify(execFile)
+const HERE = dirname(fileURLToPath(import.meta.url))
+const IMAGE = process.env.LEAN_IMAGE ?? 'agent-runtime-lean-verify:local'
+
+/** A proof that reaches acceptance via an unchecked escape is not a proof — reject it. */
+const TRUST_ESCAPE = /\b(sorry|admit|sorryAx|native_decide)\b/
+
+export interface LeanResult {
+  /** Lean accepted the proof AND it used no trust-escape. */
+  readonly passed: boolean
+  /** Lean's compiler output on failure — feed this back to the prover verbatim. */
+  readonly error: string
+}
+
+let imagePromise: Promise<void> | undefined
+
+/** Build the Lean toolchain image once (idempotent), or fail loud with the fix. Never fabricates. */
+export function ensureLeanImage(): Promise<void> {
+  if (!imagePromise) {
+    imagePromise = (async () => {
+      try {
+        await pexec('docker', ['image', 'inspect', IMAGE])
+        return // already built
+      } catch {
+        // not present — build it
+      }
+      try {
+        await pexec('docker', ['--version'])
+      } catch {
+        throw new Error(
+          'Lean verifier needs Docker. Install/start Docker, or set LEAN_IMAGE to a prebuilt Lean 4 image. (This gate never fabricates a pass.)',
+        )
+      }
+      await pexec(
+        'docker',
+        ['build', '-t', IMAGE, '-f', join(HERE, 'lean.Dockerfile'), HERE],
+        { maxBuffer: 1 << 26 },
+      )
+    })()
+  }
+  return imagePromise
+}
+
+/**
+ * Compile `${header} := ${proof}` with real Lean. Returns `{ passed, error }` — `error` is the
+ * Lean output on failure (empty on pass), suitable to feed straight back to the prover.
+ */
+export async function leanCheck(header: string, proof: string): Promise<LeanResult> {
+  const body = proof.trim()
+  if (TRUST_ESCAPE.test(body)) {
+    return { passed: false, error: `rejected: the proof uses a trust-escape (sorry/admit/native_decide) — prove it for real` }
+  }
+  await ensureLeanImage()
+  const dir = await mkdtemp(join(tmpdir(), 'leanchk-'))
+  try {
+    await writeFile(join(dir, 'T.lean'), `${header} := ${body}\n`, 'utf8')
+    try {
+      const { stdout, stderr } = await pexec(
+        'docker',
+        ['run', '--rm', '--network', 'none', '-v', `${dir}:/w`, '-w', '/w', IMAGE, 'lean', 'T.lean'],
+        { timeout: 90_000, maxBuffer: 1 << 26 },
+      )
+      // Lean emits a WARNING (exit 0) for `sorry`; treat that as not-proven too.
+      if (/declaration uses 'sorry'/.test(stdout + stderr)) {
+        return { passed: false, error: "rejected: Lean reports the proof still uses 'sorry'" }
+      }
+      return { passed: true, error: '' }
+    } catch (e) {
+      const r = e as { stdout?: string; stderr?: string; killed?: boolean }
+      if (r.killed) return { passed: false, error: 'Lean timed out (90s) — the proof did not compile in time' }
+      return { passed: false, error: (r.stderr || r.stdout || String(e)).slice(0, 2500) }
+    }
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+}

package/src/examples/lean.Dockerfile

@@ -0,0 +1,12 @@
+# A minimal Lean 4 toolchain image — the deterministic verifier for the Lean proof gate.
+# Built ONCE by lean-verify.ts (docker build) and cached; each check runs `lean` in it
+# (~seconds). This is the ground-truth judge: it compiles the proposed proof and cannot be
+# fooled — a wrong proof fails with a real type error that feeds the next attempt.
+FROM ubuntu:24.04
+RUN apt-get update -qq \
+ && apt-get install -y -qq curl ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+# elan is Lean's toolchain installer; `stable` pins the current Lean 4 release.
+RUN curl -sSf https://elan.lean-lang.org/elan-init.sh | sh -s -- -y --default-toolchain stable
+ENV PATH="/root/.elan/bin:${PATH}"
+RUN lean --version

package/src/examples/math-demo.mts

@@ -1,13 +1,15 @@
 /**
- * Math through the suite — the ANY-DOMAIN generalization proof. No tool surface, no gym,
- * no sandbox: the domain is one `check` function (numeric answer match), built with
- * `createVerifierEnvironment`. The same strategies (sample / refine / your own) compete
- * on word problems exactly as they do on EOPS tickets or commit0 repos.
+ * Grade an AI's answers, then let it retry until they're right.
  *
- * This is the template for every answer-shaped product domain: tax (a computed return
- * value), legal (a clause checklist), creative/gtm (a rubric scorer) — swap `check`.
+ * The "task" is three word problems; the "check" is one function that scores the final
+ * number by exact match. `createVerifierEnvironment` turns that single check into a full
+ * benchmark, and the toolkit compares ways of spending the compute budget: sample (N blind
+ * attempts, keep the best), refine (a critic reads the failure and steers the retry), and
+ * sampleThenRefine (both). Swap the `check` and this same file grades a tax return value,
+ * a rubric score, anything answer-shaped.
  *
- *   WORKER_MODEL=gpt-4o-mini BUDGET=3 tsx src/examples/math-demo.mts
+ * Run from bench/ (needs a router key; default model deepseek-v4-flash):
+ *   TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini BUDGET=3 tsx src/examples/math-demo.mts
  */
 import {
   type AgenticTask,

package/src/examples/strategy-demo.mts

@@ -1,18 +1,16 @@
 /**
- * Strategy demo — the optimization suite in three layers, on a toy Environment (no gym).
+ * The whole toolkit in one small file, on a toy task (drive a counter to 5).
  *
- * The whole idea in one file: you implement an `Environment` (5 hooks: open/tools/call/
- * score/close), and you get optimization STRATEGIES — sample (best-of-N), refine
- * (iterate-with-feedback), and any you author — compared and scored by your own check,
- * for free. This uses a trivial "counter" environment so it runs with just a router key
- * (no benchmark dataset, no sandbox).
+ * You write one small adapter (open a task, expose tools, score the result) and get back
+ * automatic comparison of ways to spend a compute budget, each scored by your own check.
+ * Three levels, shown below:
+ *   1. just run it     — runBenchmark(env, …) compares the built-in tactics for you.
+ *   2. pick tactics    — sample (N blind attempts, keep best), refine (critic steers the
+ *                        retry), adaptiveRefine (refine, restart a stalled line).
+ *   3. author your own — defineStrategy(name, body) in ~10 lines from shot() + critique().
  *
- *   dotenvx run -f …/.env.keys -- env WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts
- *
- * The three layers shown below:
- *   1. just run it     — runBenchmark(env, …) compares the default strategies, free.
- *   2. pick strategies — pass [sample, refine, adaptiveRefine].
- *   3. author your own — defineStrategy(name, body) in ~10 lines, no Supervisor ceremony.
+ * Toy task = only a router key needed (no dataset, no sandbox). Run from bench/:
+ *   TANGLE_API_KEY=... WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts
  */
 import { adaptiveRefine, type AgenticTask, type ArtifactHandle, defineStrategy, type Environment, printBenchmarkReport, refine, runBenchmark, sample } from '@tangle-network/agent-runtime/loops'
 

package/src/gate.ts

@@ -1,8 +1,9 @@
 /**
  * The gate — run the open binding question THROUGH the recursive runtime.
  *
- * The bench unifier (`run-benchmarks.ts`) drives `runLoop`. This module drives the recursive atom
- * instead: a `Persona` + the generic `fanout` combinator over the budget-conserving `Supervisor`,
+ * The bench unifier (`run-benchmarks.ts`) ranks a matrix of cells via per-cell `openSandboxRun`.
+ * This module drives the recursive atom instead: a `Persona` + the generic `fanout` combinator over
+ * the budget-conserving `Supervisor`,
  * so the diverse-strategy-vs-blind gate is measured through the same recursive atom every
  * personified loop uses — not a bespoke harness.
  *

package/src/hev-eval.mts

@@ -0,0 +1,69 @@
+/**
+ * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the
+ * worker model on each task and print the pass rate + the per-task result. Used to
+ * measure a baseline instruction vs a proposer-supplied instruction on the SAME
+ * held-out set (the proposer proposes; this grades — kept separate for honesty).
+ *
+ *   INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \
+ *     HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts
+ */
+import { readFileSync } from 'node:fs'
+import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+
+const SEED_INSTRUCTION =
+  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
+
+async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<string> {
+  const res = await fetch(`${base}/chat/completions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
+  })
+  if (!res.ok) return ''
+  const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  return d.choices?.[0]?.message?.content ?? ''
+}
+
+async function main(): Promise<void> {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY required')
+  const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
+  const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
+  const instruction = process.env.INSTRUCTION_FILE
+    ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8')
+    : (process.env.INSTRUCTION ?? SEED_INSTRUCTION)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 2500)
+  const conc = Number(process.env.CONC ?? 6)
+  const offset = Number(process.env.OFFSET ?? 55)
+  const n = Number(process.env.N ?? 40)
+  const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+
+  const all = await loadHumanEval(164, 0)
+  const byId = new Map(all.map((t) => [t.taskId, t]))
+  const tasks: HumanEvalTask[] = idsEnv.length
+    ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t)
+    : all.slice(offset, offset + n)
+
+  console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`)
+  let pass = 0
+  const fails: string[] = []
+  // simple concurrency pool
+  let i = 0
+  async function worker(): Promise<void> {
+    while (i < tasks.length) {
+      const t = tasks[i++]
+      const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const { pass: p } = await runChecker(t, extractCode(reply))
+      if (p === 1) pass += 1
+      else fails.push(t.taskId)
+    }
+  }
+  await Promise.all(Array.from({ length: conc }, () => worker()))
+  console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`)
+  console.log(`FAILED: ${fails.sort().join(', ')}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/hev-improve.mts

@@ -0,0 +1,169 @@
+/**
+ * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart
+ * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer +
+ * held-out gate), but the worker is a single chat completion and the judge is the
+ * deterministic Docker checker (run the function against its own hidden unit tests).
+ *
+ * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test
+ * is withheld — the worker cannot verify, so prompt wording cannot move resolve.
+ * HumanEval hands the worker a well-specified function to complete and grades by
+ * running tests, so the instruction prompt DOES move pass-rate. This run measures
+ * whether self-improvement lifts a CHEAP model when the task is prompt-sensitive.
+ *
+ * Worker + reflect models call the zai coding endpoint directly (no tangle router,
+ * no WAF, no 503):  TANGLE_API_KEY=$ZAI_API_KEY  ROUTER_BASE=https://api.z.ai/api/coding/paas/v4
+ */
+import { improve } from '@tangle-network/agent-runtime'
+import type { AgentProfile } from '@tangle-network/agent-interface'
+import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
+import { gepaProposer } from '@tangle-network/agent-eval/campaign'
+import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+
+// The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
+// solveInstruction so the baseline arm reproduces the plain-prompt denominator.
+const SEED_INSTRUCTION =
+  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
+
+interface Completion {
+  text: string
+  usd: number
+  tokIn: number
+  tokOut: number
+}
+
+async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<Completion> {
+  const res = await fetch(`${base}/chat/completions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
+  })
+  if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`)
+  const d = (await res.json()) as {
+    choices?: Array<{ message?: { content?: string } }>
+    usage?: { prompt_tokens?: number; completion_tokens?: number }
+  }
+  const text = d.choices?.[0]?.message?.content ?? ''
+  const tokIn = d.usage?.prompt_tokens ?? 0
+  const tokOut = d.usage?.completion_tokens ?? 0
+  // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the
+  // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is).
+  const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000
+  return { text, usd, tokIn, tokOut }
+}
+
+async function main(): Promise<void> {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY required (worker + reflect completions)')
+  const base = process.env.ROUTER_BASE ?? 'https://api.z.ai/api/coding/paas/v4'
+  const workerModel = process.env.WORKER_MODEL ?? 'glm-4.5-air'
+  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
+  // The GEPA reflector may live on a DIFFERENT endpoint than the (cheap) worker —
+  // e.g. a small worker on Together + a strong optimizer on zai. Defaults to the
+  // worker endpoint when unset.
+  const reflectBase = process.env.REFLECT_BASE ?? base
+  const reflectKey = process.env.REFLECT_KEY ?? key
+  const trainN = Number(process.env.TRAIN_N ?? 12)
+  const holdoutN = Number(process.env.HOLDOUT_N ?? 12)
+  const offset = Number(process.env.OFFSET ?? 80)
+  const generations = Number(process.env.GENERATIONS ?? 1)
+  const population = Number(process.env.POPULATION ?? 2)
+  const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
+  const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
+  const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
+
+  // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset).
+  const train = await loadHumanEval(trainN, offset)
+  const holdout = await loadHumanEval(holdoutN, offset + trainN)
+  const byId = new Map<string, HumanEvalTask>([...train, ...holdout].map((t) => [t.taskId, t]))
+  const allIds = [...byId.keys()]
+
+  console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══')
+  console.log(`worker=${workerModel}  reflect=${reflectModel}  base=${base}`)
+  console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`)
+  console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`)
+  console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`)
+  console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`)
+
+  const stats = { n: 0 }
+  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const instr = String(surface)
+    const t = byId.get(scenario.id)
+    if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
+    const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\``
+    const t0 = Date.now()
+    const r = await complete(base, key, workerModel, prompt, workerMaxTokens)
+    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
+    const hasText = r.text.trim().length > 0
+    ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
+    ctx.cost.observeTokens(
+      zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
+    )
+    stats.n += 1
+    const codeLen = extractCode(r.text).length
+    console.log(`  [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`)
+    return hasText ? r.text : null
+  }
+
+  const judge: JudgeConfig<string, Scenario> = {
+    name: 'humaneval-docker',
+    dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }],
+    async score({ artifact, scenario }) {
+      const t = byId.get(scenario.id)
+      if (!t) throw new Error(`judge: unknown scenario ${scenario.id}`)
+      const code = extractCode(String(artifact ?? ''))
+      if (!code.trim()) {
+        console.log(`  [judge] ${scenario.id} pass=0 (empty)`)
+        return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' }
+      }
+      const { pass } = await runChecker(t, code)
+      console.log(`  [judge] ${scenario.id} pass=${pass}`)
+      return { dimensions: { pass }, composite: pass, notes: pass === 1 ? 'passed' : 'failed' }
+    },
+  }
+
+  const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } }
+  const proposer = gepaProposer({
+    llm: { baseUrl: reflectBase, apiKey: reflectKey },
+    model: reflectModel,
+    target:
+      'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' +
+      'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' +
+      'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' +
+      'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' +
+      'Bold rewrites that change model BEHAVIOR beat cosmetic edits.',
+    maxTokens: reflectMaxTokens,
+    temperature: 0.7,
+  })
+
+  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' }))
+  const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' }))
+
+  const out = await improve(profile, [], {
+    surface: 'prompt',
+    gate: 'holdout',
+    generator: proposer,
+    scenarios,
+    judge,
+    agent,
+    expectUsage: 'warn',
+    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
+    llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel },
+  })
+
+  console.log('\n═══ RESULT ═══')
+  console.log(`decision=${out.decision}  lift=${out.lift}`)
+  console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`)
+  console.log(`winner   holdout pass-rate = ${out.raw.winner.compositeMean}`)
+  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
+  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
+  if (out.raw.winner.label) console.log(`winner label   : ${out.raw.winner.label}`)
+  if ((out.raw.winner as { surface?: unknown }).surface) {
+    console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`)
+  }
+  console.log(`live instruction unchanged: ${profile.prompt?.systemPrompt}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})