@tangle-network/agent-bench 0.3.7 → 0.4.0

package/src/live-improve-campaign-mbpp.mts

@@ -1,641 +0,0 @@
-/**
- * live-improve-campaign-mbpp — the kill-flow follow-up to live-improve-campaign.mts
- * (HumanEval × Qwen2.5-7B: DEV baseline 90%, saturated, gate correctly HELD). Identical
- * loop, moved to a config WITH headroom: sanitized MBPP, where the same worker measures
- * 76.7% at the baseline recipe k5/r2/t6 against an 85.9% pass@5 bound (~9pts of room).
- * The OPEN question this run answers: does dial-tuning ship a win when there is room
- * to win? `improve()` (surface 'rollout-policy') tunes { k, repairRounds, testgen }
- * and the library's own held-out gate makes the ship/hold call — a HOLD is a valid
- * result; the gate is never loosened.
- *
- * Wiring is live-improve-campaign.mts verbatim except the dataset seams:
- *   - Tasks come from mbpp-structural.mts's loadMbpp (sanitized MBPP, MBPP_JSON env);
- *     prompt = its basePrompt (description + the shown official assert).
- *   - VISIBLE checks: the shown assert (test_list[0]) rides task.meta.visibleChecks so
- *     the strategy's default officialChecksFromMeta() source ranks it as the OFFICIAL
- *     check, lexicographically above the model-authored guesses (mbpp-structural's
- *     measured lesson: unweighted guesses flip selection negative).
- *   - HIDDEN grading: test_list[1:] (+ test_imports), script-side nonce-sentinel judge
- *     in docker --network=none, AFTER the strategy locks its artifact.
- *   - test_imports are prepended to every candidate before visible-check execution
- *     (a CheckRunner prelude wrapper) and inside the hidden program, mirroring the rig.
- *
- * Honesty split (identical to the prior run):
- *   - DEV = sanitized-MBPP usable-task index [0, DEV_N) and HELD-OUT = [DEV_N,
- *     DEV_N+HOLD_N) — fixed, disjoint slices, passed as explicit
- *     `budget.holdoutScenarios` so the library's own split machinery enforces
- *     disjointness (it throws on overlap).
- *   - Deterministic proposer; `analyzeGeneration: null` keeps the findings channel
- *     empty — the improver's context is ONLY the DEV composites the loop accumulates.
- *   - The gate decision is `result.gateDecision` from the library — never recomputed
- *     here. The recompute-from-raw block at the end cross-checks the held-out pass
- *     counts against the durable per-cell files; a mismatch voids any ship claim.
- *
- * Guards (all from the prior run): SMOKE=1 first (6 dev + 6 held-out tasks, 1
- * generation, population 2 — proves the full path completes before the burn);
- * fail-loud model preflight; MAX_CALLS hard cap (default 8000 — the run aborts loud,
- * durable provenance survives in RUN_DIR); dollars ceiling.
- *
- * Run (key via dotenvx; never in the shell history):
- *   cd ~/company/devops/secrets && dotenvx run -f agent-state.env -- bash -c ' \
- *     cd /home/drew/code/agent-runtime-swe && \
- *     MBPP_JSON=/abs/sanitized-mbpp.json npx tsx bench/src/live-improve-campaign-mbpp.mts'
- */
-
-import { execFile, execFileSync } from 'node:child_process'
-import { randomBytes } from 'node:crypto'
-import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import type {
-  DispatchContext,
-  JudgeConfig,
-  MutableSurface,
-  Scenario,
-} from '@tangle-network/agent-eval/contract'
-import type { AgentProfile } from '@tangle-network/agent-interface'
-import { improve } from '../../src/improvement/improve'
-import {
-  parseRolloutPolicy,
-  ROLLOUT_POLICY_EXTENSION,
-  serializeRolloutPolicy,
-  structuralRolloutPolicyFromProfile,
-} from '../../src/improvement/rollout-policy'
-import {
-  type AgenticRunResult,
-  type CheckExecChannel,
-  type CheckOutcome,
-  type CheckRunner,
-  createVerifierEnvironment,
-  runAgentic,
-  sandboxCheckRunner,
-  structuralRollout,
-  type StructuralRolloutResult,
-} from '../../src/runtime/index'
-import { basePrompt, loadMbpp, type MbppTask } from './mbpp-structural.mts'
-
-function must(name: string): string {
-  const v = process.env[name]
-  if (!v) throw new Error(`env ${name} is required`)
-  return v
-}
-
-const SMOKE = process.env.SMOKE === '1'
-const DEV_N = Number(process.env.DEV_N ?? (SMOKE ? 6 : 150))
-const HOLD_N = Number(process.env.HOLD_N ?? (SMOKE ? 6 : 150))
-const GENERATIONS = Number(process.env.GENERATIONS ?? (SMOKE ? 1 : 2))
-const POPULATION = Number(process.env.POPULATION ?? (SMOKE ? 2 : 4))
-const CONCURRENCY = Number(process.env.CONCURRENCY ?? 8)
-const DOCKER_CONCURRENCY = Number(process.env.DOCKER_CONCURRENCY ?? 6)
-// The worker of the measured MBPP basis: 76.7% repaired@1 at k5/r2/t6, 85.9% pass@5
-// bound — the headroom config the saturated HumanEval run lacked.
-const MODEL = process.env.MODEL ?? 'Qwen/Qwen2.5-7B-Instruct-Turbo'
-const BASE = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
-const TEMP = Number(process.env.TEMPERATURE ?? 0.8)
-const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 2500)
-const DOLLARS = Number(process.env.DOLLARS ?? 15)
-/** Hard LLM-call cap across every phase — the runaway guard. Hitting it aborts the
- *  process loud (durable provenance survives in RUN_DIR); it never silently degrades. */
-const MAX_CALLS = Number(process.env.MAX_CALLS ?? 8000)
-const RUN_DIR = process.env.RUN_DIR ?? join(tmpdir(), `live-improve-campaign-mbpp-${Date.now()}`)
-
-const systemPrompt = 'You are an expert Python programmer.'
-const dockerImage = 'python:3.12-slim'
-const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)
-
-// ── Docker: ONE semaphored --network=none exec channel for BOTH judges ───────────────
-
-let dockerInFlight = 0
-const dockerWaiters: Array<() => void> = []
-async function withDockerSlot<T>(fn: () => Promise<T>): Promise<T> {
-  if (dockerInFlight >= DOCKER_CONCURRENCY) await new Promise<void>((r) => dockerWaiters.push(r))
-  dockerInFlight += 1
-  try {
-    return await fn()
-  } finally {
-    dockerInFlight -= 1
-    dockerWaiters.shift()?.()
-  }
-}
-
-const containerPrefix = `licm-${process.pid}`
-let containerSeq = 0
-
-function reapContainers(): void {
-  try {
-    const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], {
-      timeout: 10000,
-    })
-      .toString()
-      .trim()
-    if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 })
-  } catch {
-    /* reaper is best-effort by design */
-  }
-}
-process.on('SIGINT', () => {
-  reapContainers()
-  process.exit(130)
-})
-process.on('SIGTERM', () => {
-  reapContainers()
-  process.exit(143)
-})
-
-const dockerBox: CheckExecChannel = {
-  exec(command, options) {
-    const timeoutMs = options?.timeoutMs ?? dockerTimeoutMs
-    return withDockerSlot(
-      () =>
-        new Promise((resolve, reject) => {
-          const name = `${containerPrefix}-${containerSeq++}`
-          let settled = false
-          const reap = () => execFile('docker', ['rm', '-f', name], () => {})
-          const finish = (r: { exitCode: number; stdout: string; stderr: string }) => {
-            if (settled) return
-            settled = true
-            clearTimeout(backstop)
-            reap()
-            resolve(r)
-          }
-          const fail = (e: Error) => {
-            if (settled) return
-            settled = true
-            clearTimeout(backstop)
-            reap()
-            reject(e)
-          }
-          // execFile's timeout kills the docker CLIENT; a hung container could leave
-          // the callback unfired. The backstop guarantees resolution and the named
-          // reap kills the stray container.
-          const backstop = setTimeout(
-            () => finish({ exitCode: 124, stdout: '', stderr: 'timed out (backstop)' }),
-            timeoutMs + 3000,
-          )
-          execFile(
-            'docker',
-            ['run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', dockerImage, 'sh', '-c', command],
-            { timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },
-            (err, stdout, stderr) => {
-              if (err) {
-                const e = err as NodeJS.ErrnoException & { code?: number | string }
-                if (e.code === 'ENOENT') {
-                  fail(new Error('docker binary not found on PATH'))
-                  return
-                }
-                if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr ?? '')) {
-                  fail(new Error(`docker daemon unreachable: ${(stderr ?? '').slice(0, 200)}`))
-                  return
-                }
-                finish({ exitCode: typeof e.code === 'number' ? e.code : 1, stdout: stdout ?? '', stderr: stderr ?? '' })
-                return
-              }
-              finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' })
-            },
-          )
-        }),
-    )
-  },
-}
-
-// ── The hidden nonce judge (script-side, AFTER the strategy locks its artifact) ──────
-// MBPP hidden suite = test_list[1:] with test_imports; pass requires exit 0 AND the
-// per-call nonce sentinel in stdout — a candidate printing a forged verdict cannot pass.
-
-function buildHiddenProgram(task: MbppTask, candidate: string, nonce: string): string {
-  return `${task.testImports.join('\n')}\n${candidate}\n\n${task.hiddenAsserts.join('\n')}\nprint("HIDDEN-${nonce} PASS")\n`
-}
-
-async function runHiddenJudge(
-  task: MbppTask,
-  candidate: string,
-): Promise<{ pass: number; detail?: string }> {
-  const nonce = randomBytes(8).toString('hex')
-  const b64 = Buffer.from(buildHiddenProgram(task, candidate, nonce), 'utf8').toString('base64')
-  const r = await dockerBox.exec(`printf '%s' '${b64}' | base64 -d | python3 -`, {
-    timeoutMs: dockerTimeoutMs,
-  })
-  if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 }
-  return { pass: 0, detail: (r.stderr || r.stdout).slice(-300) || 'timed out (no output)' }
-}
-
-// ── Scenarios: fixed disjoint slices of sanitized MBPP ────────────────────────────────
-
-interface MbppScenario extends Scenario {
-  kind: 'mbpp'
-}
-
-const taskById = new Map<string, MbppTask>()
-const scenarioId = (t: MbppTask) => `mbpp/${t.taskId}`
-
-// ── The real evaluator: one cell = one structuralRollout run + hidden grade ──────────
-
-interface CellArtifact {
-  taskId: string
-  policy: string
-  /** Hidden nonce-judge grade of the FINAL selected candidate: {0,1}. */
-  pass: number
-  detail?: string
-  repairStop: string
-  shots: number
-  completions: number
-  authoredChecks: number
-  officialChecks: number
-  tokens: { input: number; output: number }
-  usd: number
-  ms: number
-}
-
-interface ScoredCandidate {
-  candidate: string
-  outcome: CheckOutcome
-}
-function recordingRunner(inner: CheckRunner, log: ScoredCandidate[]): CheckRunner {
-  return {
-    async run(candidate, checks, ctx) {
-      const outcome = await inner.run(candidate, checks, ctx)
-      log.push({ candidate, outcome })
-      return outcome
-    },
-  }
-}
-
-/** MBPP's test_imports must be in scope when the visible checks run (the rig prepends
- *  them to every judged program). The recorded candidate stays RAW — the hidden judge
- *  prepends the same imports itself. */
-function testImportsPrelude(inner: CheckRunner): CheckRunner {
-  return {
-    run(candidate, checks, ctx) {
-      const raw = ctx.task.meta?.testImports
-      const imports = Array.isArray(raw) ? raw.filter((i): i is string => typeof i === 'string') : []
-      return inner.run(imports.length > 0 ? `${imports.join('\n')}\n${candidate}` : candidate, checks, ctx)
-    },
-  }
-}
-
-// Global spend meter (every cell of every phase — baseline, generations, holdout).
-const spend = { cells: 0, llmCalls: 0, tokensIn: 0, tokensOut: 0, usd: 0, hiddenPass: 0 }
-
-async function evaluateCell(
-  surface: MutableSurface,
-  scenario: MbppScenario,
-  ctx: DispatchContext,
-): Promise<CellArtifact> {
-  if (spend.llmCalls >= MAX_CALLS) {
-    console.error(
-      `\nBUDGET CAP HIT: ${spend.llmCalls} llm calls >= MAX_CALLS=${MAX_CALLS} — aborting loud; durable provenance in ${RUN_DIR}`,
-    )
-    reapContainers()
-    process.exit(1)
-  }
-  const policy = parseRolloutPolicy(surface)
-  if (!policy) {
-    throw new Error(`agent: surface carries no valid rollout policy: ${String(surface).slice(0, 120)}`)
-  }
-  const task = taskById.get(scenario.id)
-  if (!task) throw new Error(`agent: unknown scenario id ${scenario.id}`)
-
-  const scored: ScoredCandidate[] = []
-  const strategy = structuralRollout({
-    policy: { ...policy, temperature: TEMP },
-    checkRunner: recordingRunner(testImportsPrelude(sandboxCheckRunner({ box: dockerBox })), scored),
-  })
-  // INERT check: the strategy's harness-verified score channel carries no hidden
-  // signal — hidden grading happens below, after the rollout locks its artifact.
-  const inertSurface = createVerifierEnvironment({
-    name: 'mbpp-inert',
-    check: () => ({ passes: 0, total: 1, errored: 0 }),
-  })
-  const result = (await runAgentic({
-    surface: inertSurface,
-    task: {
-      id: scenario.id,
-      systemPrompt,
-      userPrompt: basePrompt(task),
-      meta: {
-        entryPoint: task.entryPoint,
-        // The OFFICIAL check: the shown assert (test_list[0], printed in the prompt)
-        // feeds the strategy's default officialChecksFromMeta() source, so it ranks
-        // above the model-authored guesses in selection and repair.
-        visibleChecks: [task.shownAssert],
-        testImports: task.testImports,
-      },
-    },
-    routerBaseUrl: BASE,
-    routerKey: must('TOGETHER_API_KEY'),
-    model: MODEL,
-    temperature: TEMP,
-    maxTokens: MAX_TOKENS,
-    innerTurns: 2,
-    strategy,
-    // The strategy's documented sizing: k samples + repair rounds + the check-author consult.
-    budget: policy.k + policy.repairRounds + 1,
-  })) as AgenticRunResult & StructuralRolloutResult
-
-  // Backend integrity: report REAL usage on every cell (expectUsage 'assert' upstream).
-  ctx.cost.observe(result.usd, 'together')
-  ctx.cost.observeTokens(result.tokens)
-
-  const winner = result.selection.find((r) => r.selected)
-  if (!winner) {
-    // Zero candidates means EVERY shot for this cell returned null — the signature of a
-    // dead worker (credits exhausted / rate-limited / outage), not a hard task. The
-    // runtime swallows exhausted-retry shots as null, so without this the run degrades
-    // silently for hundreds of cells and then dies at the holdout with a cryptic empty-
-    // gate error. Probe the API once and abort LOUD with the real HTTP status so the
-    // operator sees the actual cause (e.g. HTTP 402 credit exceeded) immediately.
-    if (result.repairStop === 'no-candidates') {
-      await abortOnDeadWorker(scenario.id)
-    }
-    throw new Error(`${scenario.id}: no receipt marked selected (repairStop=${result.repairStop})`)
-  }
-  const rec = scored[winner.candidateIndex]
-  if (!rec) {
-    throw new Error(
-      `${scenario.id}: selected receipt #${winner.candidateIndex} has no recorded candidate (${scored.length} scored)`,
-    )
-  }
-
-  const hidden = await runHiddenJudge(task, rec.candidate)
-
-  spend.cells += 1
-  spend.llmCalls += result.completions
-  spend.tokensIn += result.tokens.input
-  spend.tokensOut += result.tokens.output
-  spend.usd += result.usd
-  spend.hiddenPass += hidden.pass
-
-  const artifact: CellArtifact = {
-    taskId: scenario.id,
-    policy: serializeRolloutPolicy(policy),
-    pass: hidden.pass,
-    ...(hidden.detail ? { detail: hidden.detail } : {}),
-    repairStop: result.repairStop,
-    shots: result.shots,
-    completions: result.completions,
-    authoredChecks: result.authoredChecks,
-    officialChecks: result.officialChecks,
-    tokens: result.tokens,
-    usd: result.usd,
-    ms: result.ms,
-  }
-  appendFileSync(
-    join(RUN_DIR, 'cells.jsonl'),
-    `${JSON.stringify({ cellId: ctx.cellId, generation: ctx.generation ?? null, ...artifact, detail: undefined })}\n`,
-  )
-  console.log(
-    `  [cell ${String(spend.cells).padStart(4)}] ${scenario.id.padEnd(10)} ${artifact.policy.padEnd(38)} hidden=${hidden.pass ? 'PASS' : 'fail'} ${result.repairStop} calls=${result.completions}`,
-  )
-  return artifact
-}
-
-// The in-loop judge is a deterministic transcriber of the script-side hidden grade —
-// the grading itself never runs inside the strategy or the proposer's view.
-const hiddenJudge: JudgeConfig<CellArtifact, MbppScenario> = {
-  name: 'hidden-nonce-judge',
-  dimensions: [
-    { key: 'hidden', description: 'MBPP hidden suite test_list[1:] (docker --network=none, nonce sentinel)' },
-  ],
-  score: ({ artifact }) => ({
-    dimensions: { hidden: artifact.pass },
-    composite: artifact.pass,
-    notes: artifact.pass ? 'hidden PASS' : `hidden fail: ${(artifact.detail ?? '').slice(0, 160)}`,
-  }),
-}
-
-// ── Fail-loud model preflight (cost gate: prove the worker is live before any burn) ──
-
-/** One direct 8-token probe of the worker. Returns HTTP status + a short body slice —
- *  the shared health check for the startup preflight and the mid-run dead-worker guard. */
-async function probeWorker(): Promise<{ ok: boolean; status: number; body: string; content: string }> {
-  const res = await fetch(`${BASE}/chat/completions`, {
-    method: 'POST',
-    headers: { Authorization: `Bearer ${must('TOGETHER_API_KEY')}`, 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      model: MODEL,
-      max_tokens: 16,
-      temperature: 0,
-      messages: [{ role: 'user', content: 'Reply with the single word: ready' }],
-    }),
-  })
-  if (!res.ok) return { ok: false, status: res.status, body: (await res.text()).slice(0, 300), content: '' }
-  const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
-  return { ok: true, status: res.status, body: '', content: (d.choices?.[0]?.message?.content ?? '').trim() }
-}
-
-async function preflightModel(): Promise<void> {
-  const p = await probeWorker()
-  if (!p.ok) throw new Error(`model preflight FAILED: ${MODEL} @ ${BASE} → HTTP ${p.status}: ${p.body}`)
-  if (p.content === '') throw new Error(`model preflight FAILED: ${MODEL} returned empty content`)
-  console.log(`  preflight: ${MODEL} is live (replied ${JSON.stringify(p.content.slice(0, 40))})`)
-}
-
-/** Called when a cell produced zero candidates (every shot null). Probes the worker; if
- *  it is unhealthy (e.g. HTTP 402 credit exceeded, 429 rate limit, outage) the whole run
- *  is doomed — abort LOUD now rather than degrade through hundreds more null cells into
- *  a cryptic empty-holdout gate error. If the probe is HEALTHY the null was a one-off, so
- *  return and let the per-cell throw handle just this cell. */
-async function abortOnDeadWorker(scenarioId: string): Promise<void> {
-  const p = await probeWorker()
-  if (p.ok && p.content !== '') return
-  console.error(
-    `\nDEAD WORKER: cell ${scenarioId} produced zero candidates and a direct probe returned ` +
-      `${p.ok ? `empty content` : `HTTP ${p.status}: ${p.body}`}. Every shot is failing — aborting ` +
-      `loud (durable provenance in ${RUN_DIR}). If this is HTTP 402, add Together credits and re-run.`,
-  )
-  reapContainers()
-  process.exit(1)
-}
-
-// ── Reporting helpers (read the library's own result objects; never re-decide) ───────
-
-interface CampaignLike {
-  cells: Array<{ error?: string | null; judgeScores: Record<string, { composite: number }> }>
-}
-function passStats(campaign: CampaignLike): { passed: number; scored: number; errored: number; rate: number } {
-  let passed = 0
-  let scored = 0
-  let errored = 0
-  for (const cell of campaign.cells) {
-    if (cell.error) {
-      errored += 1
-      continue
-    }
-    scored += 1
-    const scores = Object.values(cell.judgeScores)
-    const composite = scores.length === 0 ? 0 : scores.reduce((s, j) => s + j.composite, 0) / scores.length
-    if (composite >= 0.999) passed += 1
-  }
-  return { passed, scored, errored, rate: scored > 0 ? passed / scored : 0 }
-}
-const pct = (x: number) => `${(100 * x).toFixed(1)}%`
-
-/** Recompute pass counts from the DURABLE per-cell files the loop's fs storage wrote
- *  (`<runDir>/<phase>/<scenario>_<rep>/cached-result.json`, artifact.pass = the hidden
- *  nonce-judge grade). Independent of the in-memory campaign objects — the ship-claim
- *  cross-check. */
-function recomputeFromRaw(dir: string): { passed: number; scored: number; errored: number } | null {
-  if (!existsSync(dir)) return null
-  let passed = 0
-  let scored = 0
-  let errored = 0
-  for (const entry of readdirSync(dir)) {
-    const file = join(dir, entry, 'cached-result.json')
-    if (!existsSync(file)) continue
-    const cell = JSON.parse(readFileSync(file, 'utf8')) as {
-      error?: string | null
-      artifact?: { pass?: number }
-    }
-    if (cell.error) {
-      errored += 1
-      continue
-    }
-    scored += 1
-    if ((cell.artifact?.pass ?? 0) >= 1) passed += 1
-  }
-  return { passed, scored, errored }
-}
-
-async function main(): Promise<void> {
-  must('TOGETHER_API_KEY')
-  mkdirSync(RUN_DIR, { recursive: true })
-  const started = Date.now()
-
-  const { tasks: all, droppedShort, droppedEntry } = loadMbpp(DEV_N + HOLD_N, 0)
-  if (all.length !== DEV_N + HOLD_N) {
-    throw new Error(`expected ${DEV_N + HOLD_N} usable MBPP tasks, loaded ${all.length}`)
-  }
-  const devTasks = all.slice(0, DEV_N)
-  const holdTasks = all.slice(DEV_N)
-  for (const t of all) taskById.set(scenarioId(t), t)
-
-  const toScenario = (t: MbppTask): MbppScenario => ({ id: scenarioId(t), kind: 'mbpp' })
-  const scenarios = all.map(toScenario)
-  const holdoutScenarios = holdTasks.map(toScenario)
-
-  const baselinePolicy = { k: 5, repairRounds: 2, testgen: 6 }
-  const profile: AgentProfile = {
-    name: 'mbpp-structural-worker',
-    extensions: { [ROLLOUT_POLICY_EXTENSION]: baselinePolicy },
-  }
-
-  console.log('=== LIVE self-improvement campaign · MBPP (headroom config) · improve() surface rollout-policy ===')
-  console.log(`  worker: ${MODEL} @ ${BASE} (temp=${TEMP}, maxTokens=${MAX_TOKENS}, innerTurns=2)`)
-  console.log(
-    `  dataset: sanitized MBPP (${process.env.MBPP_JSON}); dropped at load: ${droppedShort.length} (<2 asserts), ${droppedEntry.length} (entry unresolved)`,
-  )
-  console.log(
-    `  DEV slice      : usable-task index [0, ${DEV_N}) — mbpp/${devTasks[0]?.taskId} .. mbpp/${devTasks[devTasks.length - 1]?.taskId} (n=${devTasks.length})`,
-  )
-  console.log(
-    `  HELD-OUT slice : usable-task index [${DEV_N}, ${DEV_N + HOLD_N}) — mbpp/${holdTasks[0]?.taskId} .. mbpp/${holdTasks[holdTasks.length - 1]?.taskId} (n=${holdTasks.length})`,
-  )
-  console.log(`  baseline policy: ${JSON.stringify(baselinePolicy)} (measured basis: 76.7% repaired@1, 85.9% pass@5 bound)`)
-  console.log(
-    `  budget: generations=${GENERATIONS} population<=${POPULATION} reps=1 concurrency=${CONCURRENCY} docker<=${DOCKER_CONCURRENCY} ceiling=$${DOLLARS} maxCalls=${MAX_CALLS}`,
-  )
-  console.log(`  gate: library defaultProductionGate (paired bootstrap on held-out, ship iff CI.low > 0.05) — UNCHANGED`)
-  console.log(`  runDir: ${RUN_DIR}`)
-  await preflightModel()
-  console.log(`\n  profile BEFORE: ${JSON.stringify(profile)}\n`)
-
-  const result = await improve<MbppScenario, CellArtifact>(profile, [], {
-    surface: 'rollout-policy',
-    scenarios,
-    judge: hiddenJudge,
-    agent: evaluateCell,
-    budget: {
-      generations: GENERATIONS,
-      populationSize: POPULATION,
-      maxConcurrency: CONCURRENCY,
-      holdoutScenarios,
-      reps: 1,
-      dollars: DOLLARS,
-    },
-    runDir: RUN_DIR,
-    // Deterministic proposer, empty findings channel: the improver's context is ONLY
-    // the DEV composites the loop accumulates — no distilled failure text, no trace
-    // paths, and (by the loop's own structure) never a held-out cell.
-    analyzeGeneration: null,
-  })
-
-  const loop = result.raw.raw
-  const wallMin = (Date.now() - started) / 60000
-
-  console.log('\n── DEV (train) results — what the improver saw ──')
-  const baseDev = passStats(loop.baselineCampaign)
-  console.log(
-    `  gen -1 baseline  ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(profile)!).padEnd(38)} DEV ${baseDev.passed}/${baseDev.scored} = ${pct(baseDev.rate)}  (errored ${baseDev.errored})`,
-  )
-  for (const gen of loop.generations) {
-    const surfaceByHash = new Map(gen.surfaces.map((s) => [s.surfaceHash, s]))
-    const promotedHashes = new Set(gen.record.promoted)
-    for (const cand of gen.record.candidates) {
-      const s = surfaceByHash.get(cand.surfaceHash)
-      const stats = s ? passStats(s.campaign) : undefined
-      console.log(
-        `  gen ${String(gen.record.generationIndex).padStart(2)} ${String(cand.label ?? '').padEnd(16)} ${String(s?.surface ?? '?').padEnd(38)} DEV ${stats ? `${stats.passed}/${stats.scored} = ${pct(stats.rate)} (errored ${stats.errored})` : '?'}${promotedHashes.has(cand.surfaceHash) ? '  [promoted]' : ''}`,
-      )
-    }
-  }
-  console.log(`  training winner: ${String(loop.winnerSurface)}${loop.winnerLabel ? ` (${loop.winnerLabel})` : ''}`)
-
-  console.log('\n── HELD-OUT gate — the library decides ──')
-  const baseHold = passStats(loop.baselineOnHoldout)
-  const winHold = passStats(loop.winnerOnHoldout)
-  console.log(
-    `  baseline on held-out : ${baseHold.passed}/${baseHold.scored} = ${pct(baseHold.rate)}  (errored ${baseHold.errored})`,
-  )
-  console.log(
-    `  winner   on held-out : ${winHold.passed}/${winHold.scored} = ${pct(winHold.rate)}  (errored ${winHold.errored})`,
-  )
-  console.log(`  gate decision: ${result.gateDecision.toUpperCase()}  (lift ${result.lift >= 0 ? '+' : ''}${result.lift.toFixed(3)})`)
-  for (const reason of loop.gateResult.reasons) console.log(`    reason: ${reason}`)
-  for (const g of loop.gateResult.contributingGates) {
-    console.log(`    gate[${g.name}] passed=${g.passed} detail=${JSON.stringify(g.detail).slice(0, 300)}`)
-  }
-
-  console.log('\n── recompute-from-raw check (durable cell files, artifact.pass) ──')
-  const rawBase = recomputeFromRaw(join(RUN_DIR, 'holdout-baseline'))
-  const rawWin = recomputeFromRaw(join(RUN_DIR, 'holdout-winner'))
-  if (rawBase && rawWin && rawBase.scored > 0 && rawWin.scored > 0) {
-    const rawLift = rawWin.passed / rawWin.scored - rawBase.passed / rawBase.scored
-    console.log(`  holdout-baseline raw: ${rawBase.passed}/${rawBase.scored} = ${pct(rawBase.passed / rawBase.scored)}  (errored ${rawBase.errored})`)
-    console.log(`  holdout-winner   raw: ${rawWin.passed}/${rawWin.scored} = ${pct(rawWin.passed / rawWin.scored)}  (errored ${rawWin.errored})`)
-    console.log(`  raw pass-rate lift: ${rawLift >= 0 ? '+' : ''}${rawLift.toFixed(3)} (loop-reported composite lift ${result.lift >= 0 ? '+' : ''}${result.lift.toFixed(3)})`)
-    const agree = rawBase.passed === baseHold.passed && rawWin.passed === winHold.passed
-    console.log(`  agreement with library objects: ${agree ? 'EXACT' : 'MISMATCH — investigate before any ship claim'}`)
-  } else if (rawBase && rawBase.scored > 0 && !rawWin) {
-    // The library skips the winner holdout campaign when winner == baseline (empty
-    // diff) — nothing shipped, so there is nothing separate to recompute.
-    console.log(
-      `  holdout-baseline raw: ${rawBase.passed}/${rawBase.scored} = ${pct(rawBase.passed / rawBase.scored)}  (errored ${rawBase.errored})`,
-    )
-    console.log('  holdout-winner absent: winner == baseline, no separate winner campaign ran (ship impossible this run)')
-  } else {
-    console.log('  raw holdout cell files missing — cannot recompute (storage did not persist cells?)')
-  }
-
-  console.log('\n── ship/hold outcome ──')
-  console.log(`  shipped: ${result.shipped}`)
-  console.log(`  profile AFTER : ${JSON.stringify(result.profile)}`)
-  if (result.shipped) {
-    console.log(`  policy change : ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(profile)!)} → ${serializeRolloutPolicy(structuralRolloutPolicyFromProfile(result.profile)!)}`)
-  } else {
-    console.log('  policy change : none (gate held — baseline policy stays)')
-  }
-
-  console.log('\n── spend / provenance ──')
-  console.log(
-    `  cells ${spend.cells} · llm calls ${spend.llmCalls} · tokens ${spend.tokensIn} in / ${spend.tokensOut} out · router-priced $${spend.usd.toFixed(4)} · loop-reported $${result.raw.totalCostUsd.toFixed(4)}`,
-  )
-  console.log(`  wall ${wallMin.toFixed(1)} min · runDir ${RUN_DIR} (cells.jsonl + campaign cells + loop provenance)`)
-  reapContainers()
-  process.exit(0)
-}
-
-main().catch((e) => {
-  console.error(e)
-  reapContainers()
-  process.exit(1)
-})