@tangle-network/agent-bench 0.3.5 → 0.3.7

package/src/swe-arena/replay.test.mts

@@ -0,0 +1,300 @@
+/**
+ * Proof-of-faithfulness gate for the swe-arena replay module.
+ *
+ * Every expected value below was produced by running the reference
+ * implementation (`fixtures/analyze.py`) against the committed fixtures on
+ * 2026-07-15 and captured verbatim — the typed port must reproduce it exactly
+ * before any typed execution path is built on top.
+ *
+ * KNOWN, DELIBERATE DIVERGENCE (flagged, both sides pinned): analyze.py's
+ * printed "SUP total tokens: 560,554 / 0.83x" counts the supervisor BRAIN
+ * only (journal metered events). The true SUP-arm spend adds worker-session
+ * tokens (worker-tokens.json): 560,554 + 1,161,836 = 1,722,390 → 2.55x
+ * SOLO. Both rollups are pinned; neither replaces the other.
+ */
+
+import { createHash } from 'node:crypto'
+import { describe, expect, it } from 'vitest'
+import {
+  costRollup,
+  ledgerOutcomes,
+  pairedSignTest,
+  reconciledOutcomes,
+  roundsProgression,
+  splitPairs,
+} from './analyze.ts'
+import {
+  loadHoldout,
+  loadLedger,
+  loadPreregisterLog,
+  loadRejudge,
+  loadRematchRounds,
+  loadSupJournalTrue,
+  loadWorkerTokens,
+  readFixture,
+} from './fixtures.ts'
+import { applyRejudge, reconcile } from './reconcile.ts'
+import { buildReplay, renderReplay } from './replay.mts'
+
+const DISCORDANT = [
+  'astropy__astropy-13033',
+  'django__django-11532',
+  'matplotlib__matplotlib-20826',
+]
+
+describe('fixture integrity', () => {
+  // The fixtures ARE the experiment record. Any byte drift invalidates every
+  // pinned number below, so drift must fail loudly here first.
+  const sha256 = (name: string): string =>
+    createHash('sha256').update(readFixture(name)).digest('hex')
+
+  it.each([
+    ['ledger.jsonl', '0014a1a2e7432a6551edbaf6668357cdbee8adb15b92e4f48ff530daf6ab8529'],
+    ['rejudge.jsonl', '71ecadde0925d573573a7bb0fe171d26da386149f693035d3ca47c983a5f1700'],
+    ['rematch.jsonl', 'aac48fdaa3c80f08367c220e406e1bb28b09c4f7d74fd17e4b76f50b5caaa5e4'],
+    ['rematch2.jsonl', 'fe8347eae3e3f48c838cca0aa37e835730520482b705038c19450c98edd54d54'],
+    ['rematch3.jsonl', '4eaeb9916634f3ffea7f66cb3bf747040a346524602a74a537efcdbc828ff020'],
+    ['holdout.json', '56938f7c509621bcadbd50b1d6dacbee0f63b350506795e45d785c9de37d93f2'],
+    ['worker-tokens.json', 'aa2f394120984a1c21ea7b70ed830a990477c08a1b38b6db743024660b43b129'],
+    ['sup-journal-true.json', 'a95b48bdd7ab46011355c6a2b2844d67aa5811235ed3b8a5c9a4c9342eff9784'],
+    ['analyze.py', '5512f4e1732ae4f0f92d7cb92840e5e99e54073c27f2b8ebd293b75346d9ebfc'],
+    ['holdout-preregister.log', 'e11fbf34840765d8cd452c870905acec67e5a6b9dc05c2f76c64f71c2ad70884'],
+  ])('%s is byte-identical to the captured artifact', (name, expected) => {
+    expect(sha256(name)).toBe(expected)
+  })
+
+  it('loads the expected row counts', () => {
+    expect(loadLedger()).toHaveLength(12)
+    expect(loadRejudge()).toHaveLength(15)
+    expect(loadRematchRounds().map((r) => r.length)).toEqual([3, 3, 3])
+  })
+
+  it('psf__requests-1766 carries the corrected-sup-verdict note', () => {
+    const row = loadLedger().find((r) => r.iid === 'psf__requests-1766')
+    expect(row?.sup_resolved).toBe(true)
+    expect(row?._note).toMatch(/re-judged 2x stable resolved=true/)
+  })
+})
+
+describe('reconcile: rejudge overrides + gold gate', () => {
+  const table = reconcile(loadLedger(), loadRejudge())
+
+  it('excludes exactly the gold-ungradeable instances 2317 + 2931', () => {
+    expect(table.excluded.map((e) => e.iid)).toEqual(['psf__requests-2317', 'psf__requests-2931'])
+    expect(table.excluded.map((e) => e.excludeReason)).toEqual([
+      'gold patch unresolved by judge (gold2) — instance ungradeable',
+      'gold patch unresolved by judge (gold) — instance ungradeable',
+    ])
+    expect(table.valid).toHaveLength(10)
+  })
+
+  it('gold-control (flask) grades true and stays in the denominator', () => {
+    const flask = table.valid.find((r) => r.iid === 'pallets__flask-5014')
+    expect(flask?.goldStatus).toBe('gradeable')
+    expect(flask?.excluded).toBe(false)
+  })
+
+  it('parse-error rejudge rows (resolved=null) never override; final2 retries do', () => {
+    const r2317 = applyRejudge(loadLedger(), loadRejudge()).find(
+      (r) => r.iid === 'psf__requests-2317',
+    )
+    // solo-final and sup-final on 2317 both failed to parse; the *-final2
+    // retries are the authoritative rows.
+    expect(r2317?.solo).toEqual({ resolved: false, source: 'solo-final2' })
+    expect(r2317?.sup).toEqual({ resolved: false, source: 'sup-final2' })
+    // gold parse error superseded by the conclusive gold2 retry → ungradeable.
+    expect(r2317?.goldStatus).toBe('ungradeable')
+  })
+
+  it('personal re-judges override the automated verdict for their arm only', () => {
+    const byIid = new Map(applyRejudge(loadLedger(), loadRejudge()).map((r) => [r.iid, r]))
+    for (const iid of DISCORDANT) {
+      expect(byIid.get(iid)?.solo).toEqual({ resolved: true, source: 'solo-final' })
+      expect(byIid.get(iid)?.sup.source).toBe('ledger')
+    }
+    expect(byIid.get('pydata__xarray-4687')?.solo).toEqual({
+      resolved: false,
+      source: 'solo-final',
+    })
+    expect(byIid.get('pydata__xarray-4687')?.sup).toEqual({ resolved: false, source: 'sup-final' })
+  })
+})
+
+describe('analyze: raw ledger reproduction of analyze.py', () => {
+  const outcomes = ledgerOutcomes(loadLedger())
+  const split = splitPairs(outcomes)
+  const sign = pairedSignTest(outcomes)
+
+  it('SOLO 7/12, SUP 4/12', () => {
+    expect(outcomes.filter((o) => o.solo)).toHaveLength(7)
+    expect(outcomes.filter((o) => o.sup)).toHaveLength(4)
+  })
+
+  it('discordant pairs: 3 SOLO-only, 0 SUP-only', () => {
+    expect(split.soloOnly).toEqual(DISCORDANT)
+    expect(split.supOnly).toEqual([])
+    expect(split.both).toHaveLength(4)
+    expect(split.neither).toEqual([
+      'psf__requests-2317',
+      'psf__requests-2931',
+      'pydata__xarray-4687',
+      'pytest-dev__pytest-6197',
+      'sphinx-doc__sphinx-9658',
+    ])
+  })
+
+  it('exact two-sided sign test p = 0.25 (agent-eval mcnemar)', () => {
+    expect(sign.b).toBe(0)
+    expect(sign.c).toBe(3)
+    expect(sign.nDiscordant).toBe(3)
+    expect(sign.pValue).toBeCloseTo(0.25, 10)
+    expect(sign.pValue.toFixed(4)).toBe('0.2500')
+  })
+})
+
+describe('analyze: valid-denominator verdict', () => {
+  const outcomes = reconciledOutcomes(reconcile(loadLedger(), loadRejudge()).valid)
+
+  it('SOLO 7/10, SUP 4/10 after excluding 2931 + 2317', () => {
+    expect(outcomes).toHaveLength(10)
+    expect(outcomes.filter((o) => o.solo)).toHaveLength(7)
+    expect(outcomes.filter((o) => o.sup)).toHaveLength(4)
+  })
+
+  it('same discordant set and p=0.25 (excluded pairs were concordant-neither)', () => {
+    const split = splitPairs(outcomes)
+    expect(split.soloOnly).toEqual(DISCORDANT)
+    expect(split.supOnly).toEqual([])
+    expect(pairedSignTest(outcomes).pValue).toBeCloseTo(0.25, 10)
+  })
+})
+
+describe('analyze: cost rollups', () => {
+  const cost = costRollup(loadLedger(), loadSupJournalTrue(), loadWorkerTokens())
+
+  it('SOLO total 675,412 tokens (analyze.py oracle)', () => {
+    expect(cost.soloTokens).toBe(675412)
+  })
+
+  it('SUP brain total 560,554 tokens — what analyze.py prints as "SUP total"', () => {
+    expect(cost.supBrainTokens).toBe(560554)
+    expect(cost.brainTokenRatio.toFixed(2)).toBe('0.83')
+  })
+
+  it('TRUE SUP total 1,722,390 = brain 560,554 + workers 1,161,836 → 2.55x SOLO', () => {
+    // The loudly-flagged divergence: analyze.py never printed worker tokens,
+    // so its 0.83x understates the SUP arm's true spend by the worker share.
+    expect(cost.supWorkerTokens).toBe(1161836)
+    expect(cost.supTotalTokens).toBe(1722390)
+    expect(cost.totalTokenRatio).toBeCloseTo(2.5501, 4)
+    expect(cost.totalTokenRatio.toFixed(2)).toBe('2.55')
+  })
+
+  it('USD via blended SUP rate (analyze.py oracle strings)', () => {
+    expect(cost.supUsd.toFixed(4)).toBe('0.1754')
+    expect((cost.blendedRatePerTok * 1e6).toFixed(3)).toBe('0.313')
+    expect(cost.soloUsdDerived.toFixed(4)).toBe('0.2114')
+  })
+
+  it('wall time: SOLO 3769s vs SUP 9124s → 2.42x', () => {
+    expect(cost.soloWallS).toBe(3769)
+    expect(cost.supWallS).toBe(9124)
+    expect(cost.wallRatio.toFixed(2)).toBe('2.42')
+  })
+
+  it('telemetry gaps: runtime zeroed spentTokens on the 4 no-winner/crashed runs', () => {
+    expect(cost.telemetryGaps).toEqual([
+      'astropy__astropy-13033',
+      'django__django-11532',
+      'matplotlib__matplotlib-20826',
+      'pytest-dev__pytest-6197',
+    ])
+  })
+})
+
+describe('analyze: supervisor evolution rounds', () => {
+  const rounds = roundsProgression(loadLedger(), loadRematchRounds())
+
+  it('matplotlib: 0-line unresolved at SUP round 0 → RESOLVED at SUP4', () => {
+    const states = rounds.get('matplotlib__matplotlib-20826')
+    expect(states?.map((s) => ({ round: s.round, resolved: s.resolved, patchLines: s.patchLines }))).toEqual([
+      { round: 'SUP', resolved: false, patchLines: 0 },
+      { round: 'SUP2', resolved: false, patchLines: 0 },
+      { round: 'SUP3', resolved: false, patchLines: 0 },
+      { round: 'SUP4', resolved: true, patchLines: 23 },
+    ])
+  })
+
+  it('astropy + django: never resolved across SUP..SUP4', () => {
+    for (const iid of ['astropy__astropy-13033', 'django__django-11532']) {
+      const states = rounds.get(iid)
+      expect(states).toHaveLength(4)
+      expect(states?.every((s) => !s.resolved)).toBe(true)
+    }
+  })
+})
+
+describe('holdout registry', () => {
+  const holdout = loadHoldout()
+  const log = loadPreregisterLog()
+
+  it('6 instances, all gold-verified and judge-calibrated, single selection commit', () => {
+    expect(holdout.entries).toHaveLength(6)
+    expect(holdout.selectedAtCommit).toBe('4a06fc54bd180789d1402c843a63ed245df9f8eb')
+    expect(holdout.entries.every((e) => e.gold_official_resolved && e.verify_calibrated)).toBe(true)
+  })
+
+  it('every entry was preregistered before any arm ran, then registered', () => {
+    const iids = holdout.entries.map((e) => e.iid)
+    for (const iid of iids) {
+      const pre = log.findIndex((l) => l.includes('PREREGISTER') && l.includes(iid))
+      const reg = log.findIndex((l) => l.includes('REGISTERED') && l.includes(iid))
+      expect(pre).toBeGreaterThanOrEqual(0)
+      expect(reg).toBeGreaterThan(pre)
+      expect(log[pre]).toContain('status=selected-before-any-arm-run')
+    }
+  })
+})
+
+describe('replay CLI output', () => {
+  // Lines below are verbatim from running fixtures/analyze.py on 2026-07-15
+  // (trailing whitespace trimmed — the per-instance table pads columns).
+  const lines = renderReplay(buildReplay())
+    .split('\n')
+    .map((l) => l.trimEnd())
+
+  it.each([
+    ['SOLO resolved: 7/12 = 58.3%'],
+    ['SUP  resolved: 4/12 = 33.3%'],
+    ['delta (SUP-SOLO): -3 instances (-25.0 pts)'],
+    ["  SOLO-only wins (SOLO✓ SUP✗): 3  ['astropy__astropy-13033', 'django__django-11532', 'matplotlib__matplotlib-20826']"],
+    ['  SUP-only wins (SUP✓ SOLO✗): 0  []'],
+    ['  exact two-sided sign test on discordant pairs: p=0.2500'],
+    ['COST (measured tokens; USD via shared blended rate $0.313/1M from SUP accounting):'],
+    ['  SOLO total tokens: 675,412  -> derived $0.2114'],
+    ['  SUP  total tokens: 560,554  -> runtime $0.1754'],
+    ['  SUP/SOLO token ratio: 0.83x'],
+    ['  SUP/SOLO cost ratio (token-derived): 0.83x'],
+    ["  [telemetry] instances where runtime spentTokens != journal-true (no-winner zeroing): ['astropy__astropy-13033', 'django__django-11532', 'matplotlib__matplotlib-20826', 'pytest-dev__pytest-6197']"],
+    ['  WALL: SOLO 3769s total vs SUP 9124s total -> SUP 2.42x wall'],
+    ['pallets__flask-5014              True  True  T   T   ?   30738    15731    0.0124 116   261'],
+    ['astropy__astropy-13033           True  False T   F   3   59496    0        0.0000 350   1550'],
+    ['pytest-dev__pytest-6197          False False F   F   0   133032   0        0.0000 755   248'],
+  ])('reproduces analyze.py line: %s', (expected) => {
+    expect(lines).toContain(expected)
+  })
+
+  it.each([
+    ['SOLO resolved: 7/10'],
+    ['SUP  resolved: 4/10'],
+    ['  EXCLUDED psf__requests-2317: gold patch unresolved by judge (gold2) — instance ungradeable'],
+    ['  EXCLUDED psf__requests-2931: gold patch unresolved by judge (gold) — instance ungradeable'],
+    ['exact two-sided sign test: p=0.2500'],
+    ['  brain 560,554 + workers 1,161,836 = 1,722,390 tokens'],
+    ['  SUP/SOLO true token ratio: 2.55x  (brain-only ratio: 0.83x)'],
+    ['  matplotlib__matplotlib-20826     SUP:unresolved(0L,null) -> SUP2:unresolved(0L,no-winner) -> SUP3:unresolved(0L,null) -> SUP4:RESOLVED(23L,delivered)'],
+    ['HOLDOUT REGISTRY (pre-registered at loops@4a06fc54bd, untouched):'],
+  ])('prints reconciled line: %s', (expected) => {
+    expect(lines).toContain(expected)
+  })
+})

package/src/swe-arena/run-experiment.mts

@@ -0,0 +1,361 @@
+/**
+ * Experiment runner CLI — the typed replacement for the experiment's
+ * `orchestrate.sh` + `run-instance.sh`:
+ *
+ *   tsx src/swe-arena/run-experiment.mts <config.json>
+ *   tsx src/swe-arena/run-experiment.mts --dry-run-parity [workDir]
+ *
+ * Per instance, sequentially: ledger-skip resume → endpoint capacity gates
+ * (supervisor arms gate on the ROUTER path too — probing z.ai alone was the
+ * proven blind spot) → solo arm → serialized judge → supervisor arm →
+ * serialized judge → append one typed LedgerRow (M1 schema) to the ledger.
+ *
+ * DRY-RUN PARITY (the M2 gate): `--dry-run-parity` executes NO arms and spends
+ * NO model tokens. It replays patch extraction + official judging for two
+ * committed fixture patches (pallets__flask-5014 SOLO — resolved;
+ * pydata__xarray-4687 SUP — unresolved) through materialize → apply →
+ * extractPatch → serialized-judge, and checks the verdicts against the pinned
+ * M1 fixtures. Docker time only.
+ *
+ * TODO(operator approval): full 12-instance parity re-run — re-execute both
+ * arms on the same 12 instances through this typed path and diff the resulting
+ * ledger against fixtures/ledger.jsonl. Costs ~$1 in model spend + ~4h wall;
+ * do not launch without an explicit operator go.
+ */
+
+import { appendFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { pathToFileURL, fileURLToPath } from 'node:url'
+import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
+import {
+  extractPatch,
+  loadExcludes,
+  runSoloArm,
+  runSupervisorArm,
+  type ExecutableArmSpec,
+  type SecretsEnv,
+  type SoloArmResult,
+  type SoloArmSpec,
+  type SupervisorArmResult,
+  type SupervisorArmSpec,
+} from './arms.ts'
+import { applyPatchWithFallback } from './calibrate.ts'
+import { gatesForArmKind, waitForCapacity } from './capacity.ts'
+import { materializeWorkspace } from './materialize.ts'
+import {
+  createSerializedJudge,
+  type JudgeVerdict,
+  type SerializedJudge,
+} from './serialized-judge.ts'
+import type { LedgerRow } from './types.ts'
+
+const fixturesDir = fileURLToPath(new URL('./fixtures', import.meta.url))
+
+// ---------------------------------------------------------------------------
+// Instance images (fixtures/instances.json, vendored from the experiment).
+// ---------------------------------------------------------------------------
+
+export interface InstanceImageEntry {
+  repo: string
+  base_commit: string
+  image: string
+  environment_setup_commit: string | null
+}
+
+export async function loadInstanceImages(path?: string): Promise<Record<string, InstanceImageEntry>> {
+  const raw = await readFile(path ?? join(fixturesDir, 'instances.json'), 'utf8')
+  return JSON.parse(raw) as Record<string, InstanceImageEntry>
+}
+
+// ---------------------------------------------------------------------------
+// Ledger row assembly + resume.
+// ---------------------------------------------------------------------------
+
+/**
+ * One paired LedgerRow from the two arm results + judge verdicts — the exact
+ * field mapping run-instance.sh wrote. Throws on an inconclusive verdict
+ * (resolved: null): an infra failure must abort the pair, never be written
+ * into a boolean column.
+ */
+export function buildLedgerRow(
+  solo: SoloArmResult,
+  soloVerdict: JudgeVerdict,
+  sup: SupervisorArmResult,
+  supVerdict: JudgeVerdict,
+): LedgerRow {
+  if (solo.iid !== sup.iid) throw new Error(`ledger row: arm iid mismatch ${solo.iid} vs ${sup.iid}`)
+  if (soloVerdict.resolved === null || supVerdict.resolved === null) {
+    throw new Error(
+      `ledger row ${solo.iid}: inconclusive judge verdict (solo=${soloVerdict.resolved}, sup=${supVerdict.resolved}) — not writing a fabricated boolean`,
+    )
+  }
+  const SUP_STATUSES = ['completed', 'running', 'failed', 'cancelled', null] as const
+  const SUP_VERDICTS = ['delivered', 'no-winner', 'best-effort', null] as const
+  if (!SUP_STATUSES.includes(sup.sup_status as (typeof SUP_STATUSES)[number])) {
+    throw new Error(`ledger row ${solo.iid}: unknown sup_status ${JSON.stringify(sup.sup_status)} — loops contract changed?`)
+  }
+  if (!SUP_VERDICTS.includes(sup.sup_verdict as (typeof SUP_VERDICTS)[number])) {
+    throw new Error(`ledger row ${solo.iid}: unknown sup_verdict ${JSON.stringify(sup.sup_verdict)} — loops contract changed?`)
+  }
+  return {
+    iid: solo.iid,
+    solo_resolved: soloVerdict.resolved,
+    sup_resolved: supVerdict.resolved,
+    solo_verify_pass: solo.verify_pass,
+    sup_verify_pass: sup.verify_pass,
+    solo_patch_lines: solo.patch_lines,
+    sup_patch_lines: sup.patch_lines,
+    solo_wall_s: solo.wall_s,
+    sup_wall_s: sup.wall_s,
+    solo_tokens: solo.usage.total_io,
+    solo_usage: solo.usage,
+    sup_spentTokens: sup.spentTokens,
+    sup_spentUsd: sup.spentUsd,
+    sup_spawned: sup.spawned,
+    sup_workers: sup.workers,
+    sup_settled: sup.settled,
+    sup_subtasks: sup.subtasks,
+    sup_delivered: sup.delivered,
+    sup_status: sup.sup_status as LedgerRow['sup_status'],
+    sup_verdict: sup.sup_verdict as LedgerRow['sup_verdict'],
+    solo_oc_rc: solo.oc_rc,
+    sup_driver_rc: sup.driver_rc,
+    solo_patch: solo.patchPath,
+    sup_patch: sup.patchPath,
+  }
+}
+
+/** iids already present in the ledger (resume-skip, orchestrate.sh semantics). */
+export async function ledgerIids(ledgerPath: string): Promise<Set<string>> {
+  const raw = await readFile(ledgerPath, 'utf8').catch(() => '')
+  const iids = new Set<string>()
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue
+    try {
+      const row = JSON.parse(line) as { iid?: string }
+      if (typeof row.iid === 'string') iids.add(row.iid)
+    } catch {
+      throw new Error(`corrupt ledger line in ${ledgerPath}: ${line.slice(0, 120)}`)
+    }
+  }
+  return iids
+}
+
+// ---------------------------------------------------------------------------
+// Experiment config + loop.
+// ---------------------------------------------------------------------------
+
+export interface ExperimentConfig {
+  instances: string[]
+  /** Exactly one solo and one supervisor arm (the paired-ledger contract). */
+  arms: ExecutableArmSpec[]
+  ledgerPath: string
+  outDir: string
+  secretsDir: string
+  envFiles: string[]
+  /** Per-instance self-repro verify scripts: <verifyDir>/<iid>.sh. */
+  verifyDir: string
+  /** Override fixtures/instances.json (image map). */
+  instanceImagesPath?: string
+  judgeTimeoutMs?: number
+  gateWaitCeilingMs?: number
+  /** Probe model id. Defaults per-endpoint in capacity.ts. */
+  capacityModel?: string
+  /** Pause between instances (orchestrate.sh: 15s, gentle on the shared key). */
+  cooldownMs?: number
+}
+
+function armPair(arms: ExecutableArmSpec[]): { solo: SoloArmSpec; sup: SupervisorArmSpec } {
+  const solo = arms.filter((a): a is SoloArmSpec => a.kind === 'solo')
+  const sup = arms.filter((a): a is SupervisorArmSpec => a.kind === 'supervisor')
+  if (solo.length !== 1 || sup.length !== 1) {
+    throw new Error(`expected exactly one solo + one supervisor arm, got ${arms.map((a) => a.kind).join(', ')}`)
+  }
+  return { solo: solo[0], sup: sup[0] }
+}
+
+export async function runExperiment(config: ExperimentConfig): Promise<void> {
+  const { solo, sup } = armPair(config.arms)
+  const secrets: SecretsEnv = { secretsDir: config.secretsDir, envFiles: config.envFiles }
+  const excludes = await loadExcludes()
+  const images = await loadInstanceImages(config.instanceImagesPath)
+  const judge = createSerializedJudge({
+    ...(config.judgeTimeoutMs !== undefined ? { timeoutMs: config.judgeTimeoutMs } : {}),
+  })
+  const adapter = createSweBenchAdapter()
+  const log = (msg: string): void => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`)
+
+  const done = await ledgerIids(config.ledgerPath)
+  const pending = config.instances.filter((iid) => !done.has(iid))
+  for (const iid of config.instances.filter((i) => done.has(i))) log(`SKIP ${iid} (already in ledger)`)
+  if (pending.length === 0) {
+    log('nothing to do — all instances already in ledger')
+    return
+  }
+
+  // One dataset load for all pending instances (problem statements + metadata).
+  const tasks = await adapter.loadTasks({ ids: pending, split: 'test' })
+  const taskById = new Map(tasks.map((t) => [t.id, t]))
+
+  for (const iid of pending) {
+    const task = taskById.get(iid)
+    if (!task) throw new Error(`instance ${iid} not found in SWE-bench_Verified`)
+    const entry = images[iid]
+    if (!entry) throw new Error(`instance ${iid} has no image mapping (instances.json)`)
+    const problemStatement = String(task.metadata?.problem_statement ?? '')
+    if (!problemStatement) throw new Error(`instance ${iid}: empty problem_statement`)
+
+    // Capacity gates: worker path always; router path because a supervisor arm runs.
+    const gateOpts = {
+      ...(config.gateWaitCeilingMs !== undefined ? { waitCeilingMs: config.gateWaitCeilingMs } : {}),
+      ...(config.capacityModel !== undefined ? { model: config.capacityModel } : {}),
+      onStatus: log,
+    }
+    for (const gate of gatesForArmKind('supervisor', secrets, gateOpts)) {
+      if (!(await waitForCapacity(gate))) {
+        log(`NO CAPACITY on ${gate.name} within ceiling — stopping before ${iid} (resume later)`)
+        return
+      }
+    }
+
+    const ctx = {
+      instanceId: iid,
+      image: entry.image,
+      baseCommit: entry.base_commit,
+      problemStatement,
+      verifyCmd: `bash ${join(config.verifyDir, `${iid}.sh`)}`,
+      outDir: config.outDir,
+      secrets,
+      excludes,
+    }
+
+    log(`>>> ${iid} SOLO arm`)
+    const soloResult = await runSoloArm(solo, ctx)
+    const soloVerdict = await judge.judge(iid, soloResult.patchPath, 'solo')
+    log(`${iid} SOLO judged: ${JSON.stringify(soloVerdict)}`)
+
+    log(`>>> ${iid} SUP arm`)
+    const supResult = await runSupervisorArm(sup, ctx)
+    const supVerdict = await judge.judge(iid, supResult.patchPath, 'sup')
+    log(`${iid} SUP judged: ${JSON.stringify(supVerdict)}`)
+
+    const row = buildLedgerRow(soloResult, soloVerdict, supResult, supVerdict)
+    await appendFile(config.ledgerPath, JSON.stringify(row) + '\n')
+    log(`LEDGER_ROW ${iid} solo=${row.solo_resolved} sup=${row.sup_resolved}`)
+    await new Promise((r) => setTimeout(r, config.cooldownMs ?? 15_000))
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Dry-run parity — the M2 gate. No arms, no tokens; docker only.
+// ---------------------------------------------------------------------------
+
+export interface ParityCaseSpec {
+  iid: string
+  arm: 'solo' | 'sup'
+  /** Committed patch fixture, relative to fixtures/ (e.g. patches/x.solo.patch). */
+  patchFixture: string
+}
+
+export interface ParityCaseResult {
+  iid: string
+  arm: 'solo' | 'sup'
+  applyRc: number
+  fixtureFiles: string[]
+  extractedFiles: string[]
+  extractedPatchLines: number
+  verdict: JudgeVerdict
+}
+
+/** The two pinned parity cases: one resolved SOLO patch, one unresolved SUP patch. */
+export const PARITY_CASES: ParityCaseSpec[] = [
+  { iid: 'pallets__flask-5014', arm: 'solo', patchFixture: 'patches/pallets__flask-5014.solo.patch' },
+  { iid: 'pydata__xarray-4687', arm: 'sup', patchFixture: 'patches/pydata__xarray-4687.sup.patch' },
+]
+
+/** Changed paths of a unified diff (b/ side), for extraction-parity checks. */
+export function diffChangedFiles(patch: string): string[] {
+  const files = new Set<string>()
+  for (const m of patch.matchAll(/^diff --git a\/.+ b\/(.+)$/gm)) files.add(m[1])
+  return [...files].sort()
+}
+
+/**
+ * Replay extraction + judging for committed patches WITHOUT running any arm:
+ * materialize the instance workspace from its image, apply the committed
+ * patch, re-extract it via the arms.ts extraction path, then grade the
+ * re-extracted patch with the serialized judge. Byte-identical output is not
+ * required (git normalizes); the changed-file set and the official verdict are.
+ */
+export async function replayPatchParity(
+  cases: ParityCaseSpec[],
+  opts: { workDir: string; judge?: SerializedJudge; keepWorkspaces?: boolean },
+): Promise<ParityCaseResult[]> {
+  const judge = opts.judge ?? createSerializedJudge()
+  const excludes = await loadExcludes()
+  const images = await loadInstanceImages()
+  const results: ParityCaseResult[] = []
+  for (const c of cases) {
+    const entry = images[c.iid]
+    if (!entry) throw new Error(`parity: no image mapping for ${c.iid}`)
+    const fixturePatchPath = join(fixturesDir, c.patchFixture)
+    const fixturePatch = await readFile(fixturePatchPath, 'utf8')
+    const ws = join(opts.workDir, `parity-${c.iid}-${c.arm}`)
+    try {
+      await materializeWorkspace({
+        instanceId: c.iid,
+        image: entry.image,
+        baseCommit: entry.base_commit,
+        dest: ws,
+      })
+      const applyRc = await applyPatchWithFallback(ws, fixturePatchPath)
+      if (applyRc !== 0) throw new Error(`parity ${c.iid}: committed patch failed to apply (rc=${applyRc})`)
+      const extracted = await extractPatch(ws, entry.base_commit, excludes)
+      const extractedPath = join(opts.workDir, `parity-${c.iid}.${c.arm}.extracted.patch`)
+      await writeFile(extractedPath, extracted)
+      const verdict = await judge.judge(c.iid, extractedPath, `parity-${c.arm}`)
+      results.push({
+        iid: c.iid,
+        arm: c.arm,
+        applyRc,
+        fixtureFiles: diffChangedFiles(fixturePatch),
+        extractedFiles: diffChangedFiles(extracted),
+        extractedPatchLines: extracted.length === 0 ? 0 : extracted.split('\n').length - 1,
+        verdict,
+      })
+    } finally {
+      if (!opts.keepWorkspaces) await rm(ws, { recursive: true, force: true })
+    }
+  }
+  return results
+}
+
+// ---------------------------------------------------------------------------
+// CLI.
+// ---------------------------------------------------------------------------
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const [arg, extra] = process.argv.slice(2)
+  if (arg === '--dry-run-parity') {
+    const workDir = extra ?? join(process.env.TMPDIR ?? '/tmp', 'swe-arena-parity')
+    await mkdir(workDir, { recursive: true })
+    const results = await replayPatchParity(PARITY_CASES, { workDir })
+    for (const r of results) {
+      console.log(
+        `PARITY ${r.iid} [${r.arm}] resolved=${r.verdict.resolved} score=${r.verdict.score} ` +
+          `files(fixture=${r.fixtureFiles.join(',')} extracted=${r.extractedFiles.join(',')})`,
+      )
+    }
+  } else if (arg && !arg.startsWith('--')) {
+    const config = JSON.parse(await readFile(arg, 'utf8')) as ExperimentConfig
+    await runExperiment(config)
+  } else {
+    console.error(
+      'usage: tsx src/swe-arena/run-experiment.mts <config.json>\n' +
+        '       tsx src/swe-arena/run-experiment.mts --dry-run-parity [workDir]',
+    )
+    process.exit(2)
+  }
+}