@tangle-network/agent-bench 0.3.5 → 0.3.7

package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh

@@ -0,0 +1,51 @@
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+mkdir -p "$TMP/proj"
+cat > "$TMP/proj/conf.py" <<'PY'
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('.'))
+extensions = ['sphinx.ext.autodoc']
+# The issue: a class inheriting from a MOCKED base is documented with a
+# truncated "Bases" entry ("mocklib.nn." instead of "mocklib.nn.Module").
+autodoc_mock_imports = ['mocklib']
+PY
+cat > "$TMP/proj/mymod.py" <<'PY'
+import mocklib
+
+
+class DeepKernel(mocklib.nn.Module):
+    """A kernel inheriting a mocked torch-style base class."""
+PY
+cat > "$TMP/proj/index.rst" <<'RST'
+Test
+====
+
+.. autoclass:: mymod.DeepKernel
+   :show-inheritance:
+RST
+cat > "$TMP/repro.sh" <<'SH'
+set -u
+PROJ="$(mktemp -d)"
+cp /repro/proj/* "$PROJ"/
+OUT="$(mktemp -d)"
+PYTHONPATH=/testbed python -m sphinx -b text -q "$PROJ" "$OUT" || {
+    echo "VERIFY FAIL: sphinx build errored"
+    exit 1
+}
+cat "$OUT/index.txt"
+if grep -q 'mocklib\.nn\.Module' "$OUT/index.txt"; then
+    echo "VERIFY PASS: mocked base class documented as mocklib.nn.Module"
+    exit 0
+fi
+echo "VERIFY FAIL: Bases entry does not name mocklib.nn.Module (mocked base truncated)"
+exit 1
+SH
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && timeout -s KILL 180 bash /repro/repro.sh'

package/src/swe-arena/fixtures/worker-tokens.json

@@ -0,0 +1,42 @@
+{
+ "astropy__astropy-13033": {
+  "worker_sessions": 3,
+  "worker_tok": 180000
+ },
+ "django__django-11532": {
+  "worker_sessions": 4,
+  "worker_tok": 287038
+ },
+ "matplotlib__matplotlib-20826": {
+  "worker_sessions": 3,
+  "worker_tok": 242121
+ },
+ "pallets__flask-5014": {
+  "worker_sessions": 1,
+  "worker_tok": 46737
+ },
+ "psf__requests-1766": {
+  "worker_sessions": 2,
+  "worker_tok": 67541
+ },
+ "psf__requests-2317": {
+  "worker_sessions": 1,
+  "worker_tok": 57555
+ },
+ "psf__requests-2931": {
+  "worker_sessions": 2,
+  "worker_tok": 124545
+ },
+ "pydata__xarray-4687": {
+  "worker_sessions": 1,
+  "worker_tok": 64243
+ },
+ "sympy__sympy-14976": {
+  "worker_sessions": 1,
+  "worker_tok": 55937
+ },
+ "sympy__sympy-23534": {
+  "worker_sessions": 1,
+  "worker_tok": 36119
+ }
+}

package/src/swe-arena/fixtures.ts

@@ -0,0 +1,104 @@
+/**
+ * Hermetic fixture loaders. All artifacts live under `./fixtures/` (copied
+ * from the live experiment's scratchpad on 2026-07-15 so the replay survives
+ * scratchpad deletion). Loaders fail loud on shape drift — a silent default
+ * here would let the pinned reproduction pass against corrupted fixtures.
+ */
+
+import { readFileSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import type {
+  HoldoutEntry,
+  HoldoutRegistry,
+  LedgerRow,
+  RejudgeRow,
+  RematchRow,
+  WorkerTokens,
+} from './types'
+
+const fixturePath = (name: string): string =>
+  fileURLToPath(new URL(`./fixtures/${name}`, import.meta.url))
+
+export function readFixture(name: string): string {
+  return readFileSync(fixturePath(name), 'utf8')
+}
+
+function parseJsonl<T>(name: string): T[] {
+  return readFixture(name)
+    .split('\n')
+    .filter((l) => l.trim().length > 0)
+    .map((l, i) => {
+      try {
+        return JSON.parse(l) as T
+      } catch (err) {
+        throw new Error(`${name}:${i + 1}: invalid JSON — ${(err as Error).message}`)
+      }
+    })
+}
+
+export function loadLedger(): LedgerRow[] {
+  const rows = parseJsonl<LedgerRow>('ledger.jsonl')
+  for (const r of rows) {
+    if (typeof r.iid !== 'string' || typeof r.solo_resolved !== 'boolean') {
+      throw new Error(`ledger.jsonl: malformed row ${JSON.stringify(r).slice(0, 80)}`)
+    }
+  }
+  return rows
+}
+
+export function loadRejudge(): RejudgeRow[] {
+  const rows = parseJsonl<RejudgeRow>('rejudge.jsonl')
+  for (const r of rows) {
+    if (typeof r.iid !== 'string' || typeof r.tag !== 'string') {
+      throw new Error(`rejudge.jsonl: malformed row ${JSON.stringify(r).slice(0, 80)}`)
+    }
+  }
+  return rows
+}
+
+/** Evolution rounds in order: SUP2, SUP3, SUP4. */
+export function loadRematchRounds(): RematchRow[][] {
+  return ['rematch.jsonl', 'rematch2.jsonl', 'rematch3.jsonl'].map((f) => parseJsonl<RematchRow>(f))
+}
+
+export function loadHoldout(): HoldoutRegistry {
+  const entries = JSON.parse(readFixture('holdout.json')) as HoldoutEntry[]
+  if (!Array.isArray(entries) || entries.length === 0) {
+    throw new Error('holdout.json: expected a non-empty array')
+  }
+  const commits = new Set(entries.map((e) => e.selected_at_commit))
+  if (commits.size !== 1) {
+    throw new Error(`holdout.json: expected one selection commit, got ${[...commits].join(', ')}`)
+  }
+  return { entries, selectedAtCommit: entries[0].selected_at_commit }
+}
+
+export function loadPreregisterLog(): string[] {
+  return readFixture('holdout-preregister.log')
+    .split('\n')
+    .filter((l) => l.trim().length > 0)
+}
+
+/** Per-instance worker-session token spend (instances with 0 workers are absent). */
+export function loadWorkerTokens(): Record<string, WorkerTokens> {
+  return JSON.parse(readFixture('worker-tokens.json')) as Record<string, WorkerTokens>
+}
+
+/**
+ * Journal-true supervisor brain tokens per instance — extracted from the
+ * supervisor journals' `metered` events with the same logic as
+ * `analyze.py::_true_sup_tok`, captured into a fixture because the raw
+ * journals (runs/) are too large to commit. `null` = no journal found.
+ */
+export function loadSupJournalTrue(): Record<string, number | null> {
+  const raw = JSON.parse(readFixture('sup-journal-true.json')) as Record<string, unknown>
+  const out: Record<string, number | null> = {}
+  for (const [k, v] of Object.entries(raw)) {
+    if (k === '_provenance') continue
+    if (v !== null && typeof v !== 'number') {
+      throw new Error(`sup-journal-true.json: ${k} must be number|null`)
+    }
+    out[k] = v as number | null
+  }
+  return out
+}

package/src/swe-arena/holdout-certify.mts

@@ -0,0 +1,408 @@
+/**
+ * 2-rep holdout certification — the operator-approved run that decides whether
+ * a would-be-KEEP winner ships. Encodes the gen-2 postmortem: the winner
+ * failed certification 3/6 vs 4/6 on a ONE-REP holdout with exactly one
+ * discordant cell — a known single-rep noise class — so the protocol now
+ * pins:
+ *
+ *   - `holdoutRepsPerInstance` (default 2) replicate cells per instance,
+ *   - resolved = ALL reps resolved (fail-closed, identical to the improvement
+ *     set's AND-rule in cell-evidence.mts),
+ *   - the certification bar compares against a SAME-PROTOCOL parent
+ *     measurement: `config.holdoutBaseline` is either an explicit
+ *     `{iid -> bool}` verdict map measured under this exact protocol, or
+ *     'measure' — the incumbent runs the same 2-rep holdout FIRST in this run.
+ *
+ * Certification rule (pre-registered, gen-2 bar unchanged in kind):
+ * non-regression — the candidate's AND-resolved count must be >= the parent's,
+ * with complete conclusive coverage on the candidate side. The improvement-set
+ * gate already demanded a strict gain; the holdout certifies that the gain did
+ * not cost held-out performance.
+ *
+ * The operator-release valve is unchanged: the outer loop always runs with
+ * `holdout: 'deferred'` and only reports a would-be-KEEP brief; THIS module is
+ * the separate, operator-fired run:
+ *
+ *   tsx src/swe-arena/holdout-certify.mts <config.json> --candidate <loops-commit> [--out <dir>]
+ */
+
+import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import process from 'node:process'
+import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
+import { loadExcludes, runSupervisorArm, type SecretsEnv, type SupervisorArmSpec } from './arms.ts'
+import { gatesForArmKind, waitForCapacity } from './capacity.ts'
+import {
+  replicateCoverageComplete,
+  resolvedInstanceCount,
+  type ReplicateRun,
+} from './cell-evidence.mts'
+import {
+  acquireInstanceLock,
+  addEvalWorktree,
+  assertFrozenArm,
+  assertLaunchEnv,
+  removeEvalWorktree,
+  runWithPostGateClock,
+  type OuterLoopConfig,
+} from './outer-loop.mts'
+import { installProcessSignalAbort, runOk } from './proc.ts'
+import { loadInstanceImages } from './run-experiment.mts'
+import { createSerializedJudge, type SerializedJudge } from './serialized-judge.ts'
+
+// ---------------------------------------------------------------------------
+// Pure protocol pieces — unit-tested against synthetic replicate runs.
+// ---------------------------------------------------------------------------
+
+export const DEFAULT_HOLDOUT_REPS = 2
+
+export function holdoutReps(config: Pick<OuterLoopConfig, 'holdoutRepsPerInstance'>): number {
+  const reps = config.holdoutRepsPerInstance ?? DEFAULT_HOLDOUT_REPS
+  if (!Number.isInteger(reps) || reps < 1) {
+    throw new Error(`holdout-certify: holdoutRepsPerInstance must be a positive integer, got ${JSON.stringify(reps)}`)
+  }
+  return reps
+}
+
+/** Validate an explicit same-protocol parent verdict map: every holdout
+ *  instance must carry a boolean (a partial map cannot anchor the bar). */
+export function assertHoldoutBaselineMap(
+  baseline: Record<string, boolean>,
+  holdoutInstances: string[],
+): void {
+  const missing = holdoutInstances.filter((iid) => typeof baseline[iid] !== 'boolean')
+  if (missing.length > 0) {
+    throw new Error(
+      `holdout-certify: holdoutBaseline map is missing same-protocol verdicts for: ${missing.join(', ')} — ` +
+        "provide all instances or use holdoutBaseline: 'measure'",
+    )
+  }
+  const extras = Object.keys(baseline).filter((iid) => !holdoutInstances.includes(iid))
+  if (extras.length > 0) {
+    throw new Error(`holdout-certify: holdoutBaseline names non-holdout instances: ${extras.join(', ')}`)
+  }
+}
+
+export interface HoldoutCertification {
+  candResolved: number
+  parentResolved: number
+  coverageComplete: boolean
+  certified: boolean
+  reasons: string[]
+  perInstance: Array<{ iid: string; candidate: boolean | null; parent: boolean | null; discordant: boolean }>
+}
+
+/** The certification decision over fail-closed AND-verdicts. Pure. */
+export function decideHoldoutCertification(input: {
+  candidateRuns: ReplicateRun[]
+  /** Parent AND-verdict per instance, measured under the SAME protocol
+   *  (reps, fail-closed rule). */
+  parentVerdicts: Record<string, boolean>
+  iids: string[]
+  reps: number
+}): HoldoutCertification {
+  const { candidateRuns, parentVerdicts, iids, reps } = input
+  assertHoldoutBaselineMap(parentVerdicts, iids)
+  const candResolved = resolvedInstanceCount(candidateRuns, iids, reps)
+  const parentResolved = iids.filter((iid) => parentVerdicts[iid] === true).length
+  const coverageComplete = replicateCoverageComplete(candidateRuns, iids, reps)
+  const perInstance = iids.map((iid) => {
+    const mine = candidateRuns.filter((r) => r.iid === iid)
+    const candidate =
+      mine.length === reps && mine.every((r) => r.resolved !== null)
+        ? mine.every((r) => r.resolved === true)
+        : null
+    const parent = parentVerdicts[iid] ?? null
+    return { iid, candidate, parent, discordant: candidate !== null && parent !== null && candidate !== parent }
+  })
+  const reasons: string[] = []
+  if (!coverageComplete) reasons.push(`candidate coverage incomplete: not every instance has ${reps} conclusive reps`)
+  if (candResolved < parentResolved) {
+    reasons.push(`regression: candidate ${candResolved} < parent ${parentResolved} (same-protocol AND-verdicts)`)
+  }
+  const certified = coverageComplete && candResolved >= parentResolved
+  if (certified) reasons.push(`non-regression met: candidate ${candResolved} >= parent ${parentResolved} at ${reps} reps, fail-closed`)
+  return { candResolved, parentResolved, coverageComplete, certified, reasons, perInstance }
+}
+
+// ---------------------------------------------------------------------------
+// The runner — real arms + the official judge, resumable per (commit,iid,rep).
+// ---------------------------------------------------------------------------
+
+export interface HoldoutCellRow {
+  schema: 'swe-arena.holdout-cell.v1'
+  at: string
+  commit: string
+  side: 'candidate' | 'parent'
+  iid: string
+  rep: number
+  resolved: boolean | null
+  verifyPass: boolean | null
+  wallS: number | null
+  runDir: string
+  error?: string
+}
+
+/** Read prior cells (resume): conclusive rows are reused, inconclusive re-run. */
+export async function loadHoldoutCells(cellsPath: string): Promise<HoldoutCellRow[]> {
+  if (!existsSync(cellsPath)) return []
+  const raw = await readFile(cellsPath, 'utf8')
+  return raw
+    .split('\n')
+    .filter((l) => l.trim().length > 0)
+    .map((l) => JSON.parse(l) as HoldoutCellRow)
+    .filter((r) => r.schema === 'swe-arena.holdout-cell.v1')
+}
+
+const log = (msg: string): void => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`)
+
+export async function runHoldoutCertification(
+  config: OuterLoopConfig,
+  opts: { candidateCommit: string; outDir?: string },
+  signal?: AbortSignal,
+): Promise<HoldoutCertification> {
+  signal?.throwIfAborted()
+  assertFrozenArm(config.arm)
+  const reps = holdoutReps(config)
+  const iids = config.holdoutInstances
+  if (iids.length === 0) throw new Error('holdout-certify: empty holdout set')
+  const outDir = opts.outDir ?? join(config.outDir, 'holdout-certification')
+  await mkdir(outDir, { recursive: true })
+  const cellsPath = join(outDir, 'cells.jsonl')
+
+  const secrets: SecretsEnv = { secretsDir: config.secretsDir, envFiles: config.envFiles }
+  const excludes = await loadExcludes()
+  const images = await loadInstanceImages(config.instanceImagesPath)
+  const adapter = createSweBenchAdapter()
+  const tasks = await adapter.loadTasks({ ids: iids, split: 'test' })
+  const problemById = new Map<string, string>()
+  for (const iid of iids) {
+    const task = tasks.find((t) => t.id === iid)
+    if (!task) throw new Error(`holdout-certify: ${iid} not found in SWE-bench_Verified`)
+    const problem = String(task.metadata?.problem_statement ?? '')
+    if (!problem) throw new Error(`holdout-certify: ${iid} has an empty problem_statement`)
+    if (!images[iid]) throw new Error(`holdout-certify: ${iid} has no image mapping`)
+    if (!existsSync(join(config.verifyDir, `${iid}.sh`))) {
+      throw new Error(`holdout-certify: missing verify script ${join(config.verifyDir, `${iid}.sh`)}`)
+    }
+    problemById.set(iid, problem)
+  }
+
+  const judge: SerializedJudge = createSerializedJudge(
+    config.judgeTimeoutMs !== undefined ? { timeoutMs: config.judgeTimeoutMs } : {},
+  )
+  const prior = await loadHoldoutCells(cellsPath)
+
+  const awaitGates = async (gateSignal: AbortSignal | undefined = signal): Promise<void> => {
+    for (const gate of gatesForArmKind('supervisor', secrets, {
+      ...(config.gateWaitCeilingMs !== undefined ? { waitCeilingMs: config.gateWaitCeilingMs } : {}),
+      ...(config.capacityModel !== undefined ? { model: config.capacityModel } : {}),
+      onStatus: log,
+    })) {
+      if (!(await waitForCapacity(gate, gateSignal))) throw new Error(`no capacity on ${gate.name} within ceiling`)
+    }
+  }
+
+  const runSide = async (side: 'candidate' | 'parent', commit: string): Promise<ReplicateRun[]> => {
+    signal?.throwIfAborted()
+    const runs: ReplicateRun[] = []
+    for (const iid of iids) {
+      for (let rep = 0; rep < reps; rep++) {
+        signal?.throwIfAborted()
+        const cached = prior.find(
+          (r) => r.side === side && r.commit === commit && r.iid === iid && r.rep === rep && r.resolved !== null,
+        )
+        if (cached) {
+          log(`holdout ${side} ${iid} rep=${rep}: cached resolved=${cached.resolved}`)
+          runs.push({ iid, resolved: cached.resolved })
+          continue
+        }
+        const tag = `${side}-${commit.slice(0, 10)}`
+        const evalWt = join(outDir, 'eval-wt', `${tag}-${iid}-r${rep}`)
+        const armOutDir = join(outDir, 'arm-runs', tag, iid, `rep-${rep}`)
+        let row: HoldoutCellRow
+        try {
+          try {
+            await addEvalWorktree(config.loopsRepo, commit, evalWt, signal)
+            const spec: SupervisorArmSpec = {
+              kind: 'supervisor',
+              name: config.armName,
+              workerModel: config.arm.workerModel,
+              driverModel: config.arm.driverModel,
+              budget: config.arm.budget,
+              maxSandboxes: config.arm.maxSandboxes,
+              maxUsd: config.arm.maxUsd,
+              maxDepth: config.arm.maxDepth,
+              ...(config.arm.envKnobs ? { envKnobs: config.arm.envKnobs } : {}),
+              loopsRepo: evalWt,
+              extensionPath: join(evalWt, 'extensions', 'pi', 'loops.ts'),
+              timeoutMs: config.arm.timeoutMs,
+            }
+            log(`>>> holdout ${side} ${iid} rep=${rep} @ ${commit.slice(0, 10)}`)
+            const entry = images[iid]!
+            const { armRes, verdict } = await runWithPostGateClock({
+              awaitGates,
+              work: async (signal) => {
+                const armRes = await runSupervisorArm(spec, {
+                  instanceId: iid,
+                  image: entry.image,
+                  baseCommit: entry.base_commit,
+                  problemStatement: problemById.get(iid)!,
+                  verifyCmd: `bash ${join(config.verifyDir, `${iid}.sh`)}`,
+                  outDir: armOutDir,
+                  secrets,
+                  excludes,
+                  signal,
+                })
+                signal.throwIfAborted()
+                const verdict = await judge.judge(iid, armRes.patchPath, `holdout-${tag}`, signal)
+                return { armRes, verdict }
+              },
+              timeoutMs: config.dispatchTimeoutMs,
+              label: `holdout ${side} ${iid} r${rep}`,
+              signal,
+            })
+            row = {
+              schema: 'swe-arena.holdout-cell.v1',
+              at: new Date().toISOString(),
+              commit,
+              side,
+              iid,
+              rep,
+              resolved: verdict.resolved,
+              verifyPass: armRes.verify_pass,
+              wallS: armRes.wall_s,
+              runDir: join(armOutDir, 'runs', iid, config.armName),
+              ...(verdict.resolved === null ? { error: verdict.error ?? 'inconclusive judge verdict' } : {}),
+            }
+          } finally {
+            await removeEvalWorktree(config.loopsRepo, evalWt)
+          }
+        } catch (cause) {
+          if (signal?.aborted) throw cause
+          // An errored cell is an INCONCLUSIVE replicate (resolved: null) —
+          // fail-closed downstream, never a fabricated boolean.
+          row = {
+            schema: 'swe-arena.holdout-cell.v1',
+            at: new Date().toISOString(),
+            commit,
+            side,
+            iid,
+            rep,
+            resolved: null,
+            verifyPass: null,
+            wallS: null,
+            runDir: armOutDir,
+            error: (cause as Error).message,
+          }
+        }
+        await appendFile(cellsPath, JSON.stringify(row) + '\n')
+        log(`holdout ${side} ${iid} rep=${rep}: resolved=${row.resolved}${row.error ? ` (${row.error})` : ''}`)
+        runs.push({ iid, resolved: row.resolved })
+      }
+    }
+    return runs
+  }
+
+  // SAME-PROTOCOL parent verdicts: an explicit operator-provided map, or the
+  // incumbent measured FIRST under the identical 2-rep fail-closed protocol.
+  let parentVerdicts: Record<string, boolean>
+  let parentSource: 'provided' | 'measured'
+  let parentCommit: string | null = null
+  const baseline = config.holdoutBaseline ?? 'measure'
+  if (baseline !== 'measure') {
+    assertHoldoutBaselineMap(baseline, iids)
+    parentVerdicts = baseline
+    parentSource = 'provided'
+  } else {
+    parentCommit = (
+      await runOk('git', ['-C', config.loopsRepo, 'rev-parse', config.loopsBaseRef], { signal })
+    ).stdout.trim()
+    log(`holdout parent: measuring incumbent ${parentCommit.slice(0, 10)} under the same ${reps}-rep protocol`)
+    const parentRuns = await runSide('parent', parentCommit)
+    parentVerdicts = {}
+    for (const iid of iids) {
+      const mine = parentRuns.filter((r) => r.iid === iid)
+      // Fail-closed: an inconclusive/missing parent replicate reads as NOT
+      // resolved for the parent — it can only make the bar EASIER, never
+      // stricter, so a broken parent measurement cannot fail a good candidate.
+      parentVerdicts[iid] = mine.length === reps && mine.every((r) => r.resolved === true)
+    }
+    parentSource = 'measured'
+  }
+
+  const candidateRuns = await runSide('candidate', opts.candidateCommit)
+  signal?.throwIfAborted()
+  const decision = decideHoldoutCertification({ candidateRuns, parentVerdicts, iids, reps })
+
+  const record = {
+    schema: 'swe-arena.holdout-certification.v1',
+    at: new Date().toISOString(),
+    round: config.round,
+    reps,
+    instances: iids,
+    candidateCommit: opts.candidateCommit,
+    parent: { source: parentSource, commit: parentCommit, verdicts: parentVerdicts },
+    ...decision,
+  }
+  const recordPath = join(outDir, `certification-${opts.candidateCommit.slice(0, 10)}.json`)
+  await writeFile(recordPath, JSON.stringify(record, null, 2))
+  log(
+    `holdout certification: candidate ${decision.candResolved}/${iids.length} vs parent ` +
+      `${decision.parentResolved}/${iids.length} at ${reps} reps — ${decision.certified ? 'CERTIFIED' : 'NOT CERTIFIED'}`,
+  )
+  for (const r of decision.reasons) log(`  - ${r}`)
+  log(`certification record → ${recordPath}`)
+  signal?.throwIfAborted()
+  return decision
+}
+
+// ---------------------------------------------------------------------------
+// CLI.
+// ---------------------------------------------------------------------------
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const interrupt = installProcessSignalAbort('holdout-certify')
+  try {
+    const argv = process.argv.slice(2)
+    const configPath = argv[0]
+    const flag = (name: string): string | undefined => {
+      const i = argv.indexOf(name)
+      return i !== -1 ? argv[i + 1] : undefined
+    }
+    const candidate = flag('--candidate')
+    if (!configPath || configPath.startsWith('--') || !candidate) {
+      console.error(
+        'usage: tsx src/swe-arena/holdout-certify.mts <config.json> --candidate <loops-commit> [--out <dir>]  # SPENDS: arms + judges',
+      )
+      process.exit(2)
+    }
+    const config = JSON.parse(await readFile(configPath, 'utf8')) as OuterLoopConfig
+    assertLaunchEnv()
+    interrupt.signal.throwIfAborted()
+    const out = flag('--out')
+    const lockDir = out ?? join(config.outDir, 'holdout-certification')
+    await mkdir(lockDir, { recursive: true })
+    const lock = await acquireInstanceLock(lockDir)
+    try {
+      const decision = await runHoldoutCertification(config, {
+        candidateCommit: candidate,
+        ...(out ? { outDir: out } : {}),
+      }, interrupt.signal)
+      interrupt.signal.throwIfAborted()
+      process.exitCode = decision.certified ? 0 : 1
+    } finally {
+      await lock.release()
+    }
+  } catch (cause) {
+    if (!interrupt.signal.aborted) throw cause
+    const detail = cause instanceof Error ? cause.message : String(cause)
+    console.error(`holdout-certify stopped after cleanup: ${detail}`)
+  } finally {
+    interrupt.dispose()
+  }
+}