@tangle-network/agent-bench 0.11.2 → 0.13.0

package/src/swe-arena/gepa-seat.test.mts

@@ -1,1136 +0,0 @@
-import { spawnSync } from 'node:child_process'
-import { existsSync } from 'node:fs'
-import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
-import { createServer } from 'node:http'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import {
-  gepaOptimizationMethod,
-  createRunCostLedger,
-  fsCampaignStorage,
-  type DispatchContext,
-  type OptimizationMethodProvenance,
-} from '@tangle-network/agent-eval/campaign'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { ACTIVATION_PREDICATE_RELPATH, parseActivationPredicate } from './activation.mts'
-import {
-  DEFAULT_GEPA_PYTHON,
-  DEFAULT_MAX_METRIC_CALLS,
-  GEPA_INNER_RUNS_DIRNAME,
-  GEPA_PYTHON_INSTALL_HINT,
-  gepaBridgeScenarios,
-  gepaSeatEvaluationId,
-  innerSmokeComposite,
-  innerSmokeJudge,
-  isGepaSeat,
-  mechanicalActivationPredicate,
-  probeGepaRuntime,
-  recipeEvaluationBudget,
-  recipeForSeat,
-  recordGepaSeatInnerRun,
-  validateGepaSeat,
-  type GepaMethodFactory,
-  type GepaSeatInnerRun,
-  type GepaSeatSpec,
-  type ProbeExec,
-} from './gepa-seat.mts'
-import { defaultRound4Config, type OuterLoopConfig } from './outer-loop.mts'
-import { fanOutLoopsGenerator, type ProposerSpec, type SmokeRunner, type SmokeVerdict } from './proposer-fanout.mts'
-import { captureProposerProvenance } from './proposer-provenance.mts'
-import { officialOptimizerModel } from '../official-optimizer-config.mts'
-import { runOk } from './proc.ts'
-
-const SURFACE = 'extensions/pi/prompts/worker-coding-system.md'
-const IMPLEMENTATION_REFS = {
-  runnerImplementationRef: `sha256:${'a'.repeat(64)}`,
-  judgeImplementationRef: `sha256:${'b'.repeat(64)}`,
-} as const
-
-const seat = (over: Partial<ProposerSpec> = {}): ProposerSpec => ({
-  name: 'gepa-author',
-  engine: 'gepa',
-  surface: SURFACE,
-  ...over,
-})
-
-// ---------------------------------------------------------------------------
-// Spec validation.
-// ---------------------------------------------------------------------------
-
-describe('validateGepaSeat', () => {
-  it('accepts an engine seat and isGepaSeat discriminates on engine', () => {
-    expect(() => validateGepaSeat(seat())).not.toThrow()
-    expect(() => validateGepaSeat(seat({ engine: 'omni', maxMetricCalls: 8 }))).not.toThrow()
-    expect(isGepaSeat(seat())).toBe(true)
-    expect(isGepaSeat({ name: 'x', harness: 'claude-code' })).toBe(false)
-  })
-
-  it('requires a surface inside the declared change-space', () => {
-    expect(() => validateGepaSeat(seat({ surface: undefined }))).toThrow(/surface is required/)
-    expect(() => validateGepaSeat(seat({ surface: 'judge.py' }))).toThrow(/outside the declared change-space/)
-    expect(() => validateGepaSeat(seat({ surface: '../escape.md' }))).toThrow(/outside the declared change-space/)
-  })
-
-  it('rejects harness-seat fields on an engine seat instead of silently ignoring them', () => {
-    expect(() => validateGepaSeat(seat({ harness: 'claude-code' }))).toThrow(/'harness' belongs to harness-authored/)
-    expect(() => validateGepaSeat(seat({ merge: true }))).toThrow(/'merge'/)
-    expect(() => validateGepaSeat(seat({ model: 'x' }))).toThrow(/'model'/)
-    expect(() => validateGepaSeat(seat({ profile: 'p.json' }))).toThrow(/'profile'/)
-  })
-
-  it('bounds the budget: positive integer calls, omni needs >= 4, positive cost cap', () => {
-    expect(() => validateGepaSeat(seat({ maxMetricCalls: 0 }))).toThrow(/maxMetricCalls/)
-    expect(() => validateGepaSeat(seat({ maxMetricCalls: 2.5 }))).toThrow(/maxMetricCalls/)
-    expect(() => validateGepaSeat(seat({ engine: 'omni', maxMetricCalls: 3 }))).toThrow(/omni.*needs maxMetricCalls >= 4/)
-    expect(() => validateGepaSeat(seat({ maxProposerCostUsd: 0 }))).toThrow(/maxProposerCostUsd/)
-    expect(() => validateGepaSeat(seat({ engine: 'nope' as never }))).toThrow(/engine must be one of/)
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Recipe / budget cap.
-// ---------------------------------------------------------------------------
-
-describe('recipeForSeat', () => {
-  it.each(['gepa', 'omni'] as const)('transports the real %s optimizer configuration', async (engine) => {
-    const runDir = await mkdtemp(join(tmpdir(), 'gepa-seat-transport-'))
-    const requests: unknown[] = []
-    try {
-      const optimizer = officialOptimizerModel({
-        env: { OPT_INPUT_USD_PER_MILLION: '1', OPT_CACHED_INPUT_USD_PER_MILLION: '1', OPT_CACHE_WRITE_USD_PER_MILLION: '1', OPT_OUTPUT_USD_PER_MILLION: '1' },
-        envPrefix: 'OPT', model: 'fixture-model', baseUrl: 'http://127.0.0.1:1/v1', apiKey: 'fixture-key',
-        maxCostUsd: 1, maxOutputTokensPerRequest: 100, anthropicEndpoint: engine === 'omni',
-        complete: async (request) => {
-          requests.push(request)
-          return { model: 'fixture-model', choices: [{ message: { content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 2, completion_tokens: 1, cost: 0.000003 } }
-        },
-      })
-      const spec = seat({ engine })
-      validateGepaSeat(spec)
-      const runtime = {
-        python: { implementation: 'CPython', version: '3.12.0' },
-        bridge: { package: 'agent-eval-rpc', version: 'fixture', sourceUrl: 'https://github.com/tangle-network/agent-eval', revision: 'fixture', sourceSha256: 'a'.repeat(64) },
-        optimizer: { package: 'gepa', version: 'fixture', sourceUrl: 'https://github.com/gepa-ai/gepa', revision: 'fixture', sourceSha256: 'b'.repeat(64) },
-        engineModules: [],
-      }
-      // Only the child optimizer is substituted; the real Eval proxy calls Runtime.
-      const bridge = join(runDir, 'transport.mjs')
-      await writeFile(bridge, `
-import fs from 'node:fs'
-const input = JSON.parse(fs.readFileSync(process.argv[process.argv.indexOf('--input') + 1], 'utf8'))
-if (input.operation === 'inspect') {
-  fs.writeFileSync(process.argv[process.argv.indexOf('--output') + 1], JSON.stringify({ runtime: ${JSON.stringify(runtime)} }))
-} else {
-  const omni = input.recipe.kind === 'omni'
-  if (omni && input.recipe.explore.some(run => run.engine !== 'gepa' && run.engineConfig.model !== input.modelProxy.model)) throw new Error('wrong engine model')
-  const send = maxTokens => fetch(input.modelProxy.baseUrl + (omni ? '/messages' : '/chat/completions'), {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: 'Bearer ' + input.modelProxy.apiKey, 'x-api-key': input.modelProxy.apiKey, 'anthropic-version': '2023-06-01' },
-    body: JSON.stringify({ model: input.modelProxy.model, max_tokens: maxTokens, messages: [{ role: 'user', content: 'transport proof' }] }),
-  })
-  const overBudget = await send(101)
-  if (overBudget.ok) throw new Error('output ceiling was not enforced')
-  await overBudget.text()
-  const response = await send(10)
-  if (!response.ok) throw new Error('model transport failed: ' + response.status + ' ' + await response.text())
-  await response.json()
-  throw new Error('transport-handshake-complete')
-}
-`)
-      const method = gepaOptimizationMethod({
-        recipe: recipeForSeat(spec, optimizer.model), optimizer,
-        objective: 'Check transport only', evaluationId: 'bench-transport',
-        runner: { command: process.execPath, args: [bridge] },
-      })
-      await expect(method.optimize({
-        baselineSurface: 'seed', trainScenarios: [{ id: 'train', kind: 'fixture' }], selectionScenarios: [{ id: 'selection', kind: 'fixture' }],
-        dispatchWithSurface: async () => 'unused',
-        judges: [{ name: 'fixture', dimensions: [{ key: 'score', description: 'fixture' }], score: async () => ({ dimensions: { score: 1 }, composite: 1 }) }],
-        runDir, seed: 42, runOptions: {}, costLedger: createRunCostLedger({ storage: fsCampaignStorage(), runDir: join(runDir, 'cost') }),
-      })).rejects.toThrow('transport-handshake-complete')
-      expect(requests).toHaveLength(1)
-    } finally {
-      await rm(runDir, { recursive: true, force: true })
-    }
-  })
-
-  it("'gepa' is one bounded engine run carrying the full budget (default 10)", () => {
-    const recipe = recipeForSeat(seat() as GepaSeatSpec)
-    expect(recipe).toMatchObject({ kind: 'engine', run: { engine: 'gepa', maxEvaluations: DEFAULT_MAX_METRIC_CALLS } })
-    expect(recipeEvaluationBudget(recipe)).toBe(DEFAULT_MAX_METRIC_CALLS)
-  })
-
-  it("'omni' uses the official recipe and its four bounded runs preserve the budget", () => {
-    const recipe = recipeForSeat(seat({ engine: 'omni', maxMetricCalls: 10 }) as GepaSeatSpec)
-    expect(recipe.kind).toBe('omni')
-    if (recipe.kind !== 'omni') throw new Error('unreachable')
-    expect(recipe.explore.map((r) => r.engine)).toEqual(['gepa', 'autoresearch', 'meta_harness'])
-    expect(recipe.continueWith.engine).toBe('gepa')
-    expect(recipeEvaluationBudget(recipe)).toBe(10)
-    for (const run of [...recipe.explore, recipe.continueWith]) {
-      expect(run.maxEvaluations).toBeGreaterThan(0)
-      expect(run.maxProposerCostUsd).toBeGreaterThan(0)
-    }
-  })
-
-  it('every budget from 4 upward is preserved exactly by the omni split', () => {
-    for (const calls of [4, 5, 8, 12, 24]) {
-      const recipe = recipeForSeat(seat({ engine: 'omni', maxMetricCalls: calls }) as GepaSeatSpec)
-      expect(recipeEvaluationBudget(recipe)).toBe(calls)
-    }
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Public-only bridge examples.
-// ---------------------------------------------------------------------------
-
-describe('gepaBridgeScenarios (public-only invariant)', () => {
-  const split = { privateInstances: ['django__django-11532', 'sphinx-doc__sphinx-9658'] }
-
-  it('serializes ONLY the public smoke instance (train + a distinct-id selection alias)', () => {
-    const { train, selection } = gepaBridgeScenarios('astropy__astropy-13033', split)
-    expect(train).toEqual([{ id: 'astropy__astropy-13033', kind: 'swe-smoke', smokeIid: 'astropy__astropy-13033' }])
-    expect(selection[0]!.id).toBe('astropy__astropy-13033::selection')
-    expect(selection[0]!.smokeIid).toBe('astropy__astropy-13033')
-    // Disjoint ids — the adapter's scenario map requires uniqueness.
-    expect(train[0]!.id).not.toBe(selection[0]!.id)
-    const serialized = JSON.stringify([...train, ...selection])
-    for (const iid of split.privateInstances) expect(serialized).not.toContain(iid)
-  })
-
-  it('fails loud when the smoke instance is private — private ids never cross the bridge', () => {
-    expect(() => gepaBridgeScenarios('django__django-11532', split)).toThrow(/PRIVATE under the score split/)
-  })
-
-  it('passes through with no split configured (pre-gen-5 behavior)', () => {
-    expect(gepaBridgeScenarios('astropy__astropy-13033', null).train).toHaveLength(1)
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Inner score.
-// ---------------------------------------------------------------------------
-
-describe('inner smoke score', () => {
-  const verdict = (over: Partial<SmokeVerdict>): SmokeVerdict => ({
-    iid: 'astropy__astropy-13033',
-    pass: true,
-    reason: 'ok',
-    resolved: false,
-    patchLines: 3,
-    wallS: 60,
-    ...over,
-  })
-
-  it('resolve dominates; verify-pass is a bounded tiebreak that can never beat a resolve', () => {
-    expect(innerSmokeComposite(verdict({ resolved: false, verifyPass: false }))).toBe(0)
-    expect(innerSmokeComposite(verdict({ resolved: false, verifyPass: true }))).toBe(0.25)
-    expect(innerSmokeComposite(verdict({ resolved: true, verifyPass: false }))).toBe(1)
-    expect(innerSmokeComposite(verdict({ resolved: true, verifyPass: true }))).toBe(1.25)
-    // Older verdicts without the field score as no verify signal.
-    expect(innerSmokeComposite(verdict({ resolved: true }))).toBe(1)
-  })
-
-  it('the judge reports both dimensions and the composite', async () => {
-    const judge = innerSmokeJudge()
-    const score = await judge.score({
-      artifact: verdict({ resolved: true, verifyPass: true, reason: 'smoke line' }),
-      scenario: { id: 'x', kind: 'swe-smoke', smokeIid: 'x' },
-      signal: new AbortController().signal,
-    })
-    expect(score).toMatchObject({ dimensions: { resolved: 1, verifyPass: 1 }, composite: 1.25, notes: 'smoke line' })
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Optional Python runtime: loud failures with exact instructions.
-// ---------------------------------------------------------------------------
-
-describe('probeGepaRuntime', () => {
-  const execFailingOn =
-    (failFragment: string, stderr: string): ProbeExec =>
-    async (_cmd, args) => {
-      const line = args.join(' ')
-      if (line.includes(failFragment)) return { code: 1, stdout: '', stderr }
-      if (line === '--version') return { code: 0, stdout: 'Python 3.12.3', stderr: '' }
-      return { code: 0, stdout: '0.2.0', stderr: '' }
-    }
-
-  it('fails loud with pip install instructions when the bridge module is missing', async () => {
-    const exec = execFailingOn('agent_eval_rpc.gepa_bridge', "ModuleNotFoundError: No module named 'agent_eval_rpc'")
-    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(
-      GEPA_PYTHON_INSTALL_HINT,
-    )
-    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/not installed/)
-  })
-
-  it('fails loud when the installed gepa lacks the multi-engine optimize_anything API', async () => {
-    const exec = execFailingOn('OptimizeAnythingConfig', "ImportError: cannot import name 'OptimizeAnythingConfig'")
-    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/optimize_anything API/)
-  })
-
-  it('fails loud when python itself is missing', async () => {
-    const exec: ProbeExec = async () => ({ code: 127, stdout: '', stderr: 'not found' })
-    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/--version' failed/)
-  })
-
-  it('returns python + gepa versions on a complete runtime', async () => {
-    const exec: ProbeExec = async (_cmd, args) =>
-      args.join(' ') === '--version'
-        ? { code: 0, stdout: 'Python 3.12.3\n', stderr: '' }
-        : { code: 0, stdout: 'source\n', stderr: '' }
-    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).resolves.toEqual({
-      pythonVersion: 'Python 3.12.3',
-      gepaVersion: 'source',
-    })
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Provenance capture at t=0.
-// ---------------------------------------------------------------------------
-
-describe('captureProposerProvenance with a gepa seat', () => {
-  const okExec: ProbeExec = async (cmd, args) => {
-    const line = args.join(' ')
-    if (line === '--version') {
-      return { code: 0, stdout: cmd === 'python3' ? 'Python 3.12.3' : `${cmd} 1.0.0`, stderr: '' }
-    }
-    return { code: 0, stdout: 'source', stderr: '' }
-  }
-  it('records engine, surface, gepa version, bridge module, and the python runtime as harnessVersion', async () => {
-    const record = await captureProposerProvenance([{ name: 'claude-author', harness: 'claude-code' }, seat()], {
-      exec: okExec,
-      readSettingsModel: () => 'settings-model',
-    })
-    const gepa = record.proposers.find((p) => p.name === 'gepa-author')!
-    expect(gepa).toMatchObject({
-      engine: 'gepa',
-      surface: SURFACE,
-      gepaVersion: 'source',
-      bridge: 'agent_eval_rpc.gepa_bridge',
-      harnessVersion: 'Python 3.12.3',
-      pinnedModel: null,
-      merge: false,
-    })
-    expect(gepa.harness).toBeUndefined()
-    // The claude seat is untouched by the gepa capture path.
-    expect(record.proposers.find((p) => p.name === 'claude-author')).toMatchObject({
-      harness: 'claude-code',
-      settingsModel: 'settings-model',
-    })
-  })
-
-  it('fails LOUD at t=0 when the python runtime is missing — with install instructions', async () => {
-    const exec: ProbeExec = async (_cmd, args) =>
-      args.join(' ').includes('gepa_bridge')
-        ? { code: 1, stdout: '', stderr: 'ModuleNotFoundError' }
-        : { code: 0, stdout: 'Python 3.12.3', stderr: '' }
-    await expect(captureProposerProvenance([seat()], { exec })).rejects.toThrow(
-      GEPA_PYTHON_INSTALL_HINT,
-    )
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Mechanical activation predicate.
-// ---------------------------------------------------------------------------
-
-describe('mechanicalActivationPredicate', () => {
-  it('targets the longest added line and produces a parseable grep predicate', () => {
-    const seed = 'alpha\nshared line stays here\n'
-    const winner = 'alpha\nshared line stays here\nAlways run the neighboring test file before finalizing.\nshort\n'
-    const predicate = mechanicalActivationPredicate(seed, winner, SURFACE)!
-    expect(predicate.kind).toBe('grep')
-    expect(predicate.pattern).toContain('Always run the neighboring test file')
-    const parsed = parseActivationPredicate(JSON.stringify(predicate))
-    expect(parsed.ok).toBe(true)
-    // The pattern is regex-escaped: it must match its own source line.
-    expect(new RegExp(predicate.pattern!).test('Always run the neighboring test file before finalizing.')).toBe(true)
-  })
-
-  it('escapes regex metacharacters in the added line', () => {
-    const predicate = mechanicalActivationPredicate('', 'Use pattern (a|b).* with $VAR [strictly].\n', SURFACE)!
-    expect(new RegExp(predicate.pattern!).test('Use pattern (a|b).* with $VAR [strictly].')).toBe(true)
-    expect(new RegExp(predicate.pattern!).test('Use pattern axb1* with 2VAR strictly.')).toBe(false)
-  })
-
-  it('returns null when no added line is distinctive enough', () => {
-    expect(mechanicalActivationPredicate('a\nb\n', 'a\nb\nshort\n', SURFACE)).toBeNull()
-    expect(mechanicalActivationPredicate('same\n', 'same\n', SURFACE)).toBeNull()
-  })
-})
-
-describe('recordGepaSeatInnerRun', () => {
-  const sourceHash = 'a'.repeat(64)
-  const bridgeHash = 'b'.repeat(64)
-  const moduleHash = 'c'.repeat(64)
-  const provenance = (runId: string): OptimizationMethodProvenance => ({
-    source: {
-      kind: 'package',
-      evidence: 'observed',
-      package: 'gepa',
-      version: '0.1.4',
-      sourceUrl: 'https://github.com/gepa-ai/gepa.git',
-      revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f',
-      sourceSha256: sourceHash,
-    },
-    bridge: {
-      kind: 'package',
-      evidence: 'observed',
-      package: 'agent-eval-rpc',
-      version: '0.126.1',
-      sourceSha256: bridgeHash,
-    },
-    modules: [{ module: 'example.engine', sourceSha256: moduleHash }],
-    python: { implementation: 'CPython', version: '3.12.3' },
-    runId,
-    compatibleRunId: 'compatible-run',
-    resumed: false,
-    evaluationCount: 3,
-    tokenUsage: {
-      inputTokens: 120,
-      cachedInputTokens: 20,
-      outputTokens: 30,
-      reasoningTokens: 10,
-      totalTokens: 150,
-      calls: 2,
-    },
-    artifactDir: `/tmp/${runId}`,
-  })
-  const run = (runId: string, over: Partial<GepaSeatInnerRun> = {}): GepaSeatInnerRun => {
-    const result = provenance(runId)
-    if (
-      result.source.revision === undefined ||
-      result.source.sourceSha256 === undefined ||
-      result.bridge?.sourceSha256 === undefined ||
-      result.modules === undefined ||
-      result.python === undefined ||
-      result.compatibleRunId === undefined ||
-      result.tokenUsage === undefined
-    ) {
-      throw new Error('invalid test provenance')
-    }
-    return {
-      seat: 'gepa-author',
-      engine: 'gepa',
-      surface: SURFACE,
-      generation: 0,
-      budget: 10,
-      innerCallCount: 2,
-      innerScores: [],
-      bestComposite: 1,
-      source: {
-        ...result.source,
-        revision: result.source.revision,
-        sourceSha256: result.source.sourceSha256,
-      },
-      bridge: { ...result.bridge, sourceSha256: result.bridge.sourceSha256 },
-      modules: result.modules,
-      python: result.python,
-      runId: result.runId,
-      compatibleRunId: result.compatibleRunId,
-      resumed: result.resumed,
-      evaluationCount: result.evaluationCount,
-      tokenUsage: result.tokenUsage,
-      artifactDir: result.artifactDir,
-      totalCostUsd: 0.25,
-      costProvenance: { kind: 'observed', usd: 0.25 },
-      accountingComplete: true,
-      incompleteReasons: [],
-      durationMs: 5,
-      ...over,
-    }
-  }
-
-  it('writes collision-free immutable records in parallel and preserves the launch record', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'gepa-prov-'))
-    try {
-      const launchRecord = JSON.stringify({ capturedAt: '2026-07-24T00:00:00.000Z', proposers: [] }, null, 2)
-      await writeFile(join(dir, 'proposer-provenance.json'), launchRecord)
-      const paths = await Promise.all(
-        Array.from({ length: 24 }, (_, index) =>
-          recordGepaSeatInnerRun(dir, run(`run-${index}`, { generation: index })),
-        ),
-      )
-      expect(new Set(paths).size).toBe(24)
-      expect(await readFile(join(dir, 'proposer-provenance.json'), 'utf8')).toBe(launchRecord)
-
-      const names = (await readdir(join(dir, GEPA_INNER_RUNS_DIRNAME))).filter((name) => name.endsWith('.json'))
-      expect(names).toHaveLength(24)
-      const records = await Promise.all(
-        names.map(async (name) => JSON.parse(await readFile(join(dir, GEPA_INNER_RUNS_DIRNAME, name), 'utf8'))),
-      )
-      expect(new Set(records.map((record) => record.runId))).toEqual(
-        new Set(Array.from({ length: 24 }, (_, index) => `run-${index}`)),
-      )
-      expect(records.every((record) => record.source.sourceSha256 === sourceHash)).toBe(true)
-    } finally {
-      await rm(dir, { recursive: true, force: true })
-    }
-  })
-
-  it('fails on malformed existing launch or run data', async () => {
-    const launchDir = await mkdtemp(join(tmpdir(), 'gepa-prov-bad-launch-'))
-    const runDir = await mkdtemp(join(tmpdir(), 'gepa-prov-bad-run-'))
-    try {
-      await writeFile(join(launchDir, 'proposer-provenance.json'), '{not json')
-      await expect(recordGepaSeatInnerRun(launchDir, run('new-run'))).rejects.toThrow(/malformed JSON/)
-
-      await mkdir(join(runDir, GEPA_INNER_RUNS_DIRNAME), { recursive: true })
-      await writeFile(join(runDir, GEPA_INNER_RUNS_DIRNAME, 'broken.json'), '[]')
-      await expect(recordGepaSeatInnerRun(runDir, run('new-run'))).rejects.toThrow(/must contain a JSON object/)
-    } finally {
-      await rm(launchDir, { recursive: true, force: true })
-      await rm(runDir, { recursive: true, force: true })
-    }
-  })
-})
-
-// ---------------------------------------------------------------------------
-// The seat inside the fan-out generator, against a real temp git repo.
-// ---------------------------------------------------------------------------
-
-const fakeCtx = {} as unknown as DispatchContext
-const testOptimizerEnv: NodeJS.ProcessEnv = {
-  TEST_OPTIMIZER_INPUT_USD_PER_MILLION: '1',
-  TEST_OPTIMIZER_CACHED_INPUT_USD_PER_MILLION: '0.1',
-  TEST_OPTIMIZER_CACHE_WRITE_USD_PER_MILLION: '1.25',
-  TEST_OPTIMIZER_OUTPUT_USD_PER_MILLION: '5',
-  TEST_OPTIMIZER_MAX_REQUESTS: '10',
-  TEST_OPTIMIZER_MAX_REQUEST_BYTES: '100000',
-  TEST_OPTIMIZER_MAX_RESPONSE_BYTES: '100000',
-}
-
-const testOptimizer = officialOptimizerModel({
-  env: testOptimizerEnv,
-  envPrefix: 'TEST_OPTIMIZER',
-  model: 'optimizer-model',
-  baseUrl: 'http://127.0.0.1:1/v1',
-  apiKey: 'optimizer-key',
-  maxCostUsd: 1,
-  maxOutputTokensPerRequest: 2_000,
-  callRef: 'test:optimizer-model',
-})
-
-const fullProvenance = (
-  runId = 'gepa-run',
-  over: Partial<OptimizationMethodProvenance> = {},
-): OptimizationMethodProvenance => ({
-  source: {
-    kind: 'package',
-    evidence: 'observed',
-    package: 'gepa',
-    version: '0.1.4',
-    sourceUrl: 'https://github.com/gepa-ai/gepa.git',
-    revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f',
-    sourceSha256: '1'.repeat(64),
-  },
-  bridge: {
-    kind: 'package',
-    evidence: 'observed',
-    package: 'agent-eval-rpc',
-    version: '0.126.1',
-    sourceSha256: '2'.repeat(64),
-  },
-  modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }],
-  python: { implementation: 'CPython', version: '3.12.3' },
-  runId,
-  compatibleRunId: 'compatible-gepa-run',
-  resumed: false,
-  evaluationCount: 3,
-  tokenUsage: {
-    inputTokens: 100,
-    cachedInputTokens: 10,
-    cacheWriteInputTokens: 5,
-    outputTokens: 25,
-    reasoningTokens: 8,
-    totalTokens: 125,
-    calls: 2,
-  },
-  artifactDir: `/tmp/${runId}`,
-  ...over,
-})
-
-/** Mimics the adapter's loop: score the seed and each provided candidate via
- *  the seat's dispatch + judge, return the best-scoring candidate — exactly
- *  the contract gepaOptimizationMethod fulfills through the Python bridge. */
-const fakeGepaFactory =
-  (candidates: string[], observed?: { config?: unknown }): GepaMethodFactory =>
-  (config) => {
-    if (observed) observed.config = config
-    return {
-      name: config.name ?? 'fake-gepa',
-      async optimize(input) {
-        const judge = input.judges[0]!
-        const scenario = input.trainScenarios[0]!
-        let best = { surface: input.baselineSurface as string, composite: -Infinity }
-        for (const candidate of [input.baselineSurface as string, ...candidates]) {
-          const artifact = await input.dispatchWithSurface(candidate, scenario, fakeCtx)
-          const score = await judge.score({ artifact, scenario, signal: new AbortController().signal })
-          if (score.composite > best.composite) best = { surface: candidate, composite: score.composite }
-        }
-        return {
-          winnerSurface: best.surface,
-          cost: {
-            totalCostUsd: 0.125,
-            costProvenance: { kind: 'observed', usd: 0.125 },
-            accountingComplete: true,
-            incompleteReasons: [],
-          },
-          durationMs: 1,
-          provenance: fullProvenance(),
-        }
-      },
-    }
-  }
-
-describe('fanOutLoopsGenerator with the gepa seat', () => {
-  let loopsRepo: string
-  let outDir: string
-  let driverWt: string
-
-  const git = async (args: string[], cwd: string): Promise<string> =>
-    (await runOk('git', ['-C', cwd, ...args])).stdout.trim()
-
-  const SEED = '# worker coding system\nkeep tests green\n'
-
-  beforeEach(async () => {
-    loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-repo-'))
-    outDir = await mkdtemp(join(tmpdir(), 'gepa-out-'))
-    await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
-    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
-    await git(['config', 'user.email', 't@t.dev'], loopsRepo)
-    await git(['config', 'user.name', 'T'], loopsRepo)
-    await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })
-    await writeFile(join(loopsRepo, SURFACE), SEED)
-    await git(['add', '-A'], loopsRepo)
-    await git(['commit', '-q', '-m', 'init'], loopsRepo)
-    driverWt = join(outDir, 'driver-wt')
-    await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', driverWt, 'HEAD'])
-  })
-
-  afterEach(async () => {
-    await rm(outDir, { recursive: true, force: true })
-    await rm(loopsRepo, { recursive: true, force: true })
-  })
-
-  const baseConfig = (proposers: ProposerSpec[], over: Partial<OuterLoopConfig> = {}): OuterLoopConfig => ({
-    ...defaultRound4Config(),
-    loopsRepo,
-    outDir,
-    populationSize: proposers.length,
-    proposers,
-    prefilter: { enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false },
-    ...over,
-  })
-
-  const generatorArgs = (candidateIndex: number) => ({
-    worktreePath: driverWt,
-    findings: [],
-    maxShots: 1,
-    signal: new AbortController().signal,
-    generation: 0,
-    candidateIndex,
-  })
-
-  const smokeVerdict = (over: Partial<SmokeVerdict> = {}): SmokeVerdict => ({
-    iid: 'astropy__astropy-13033',
-    pass: true,
-    reason: 'smoke ok',
-    resolved: false,
-    patchLines: 3,
-    wallS: 5,
-    verifyPass: false,
-    ...over,
-  })
-
-  it('construction fails loud without the smoke runner (the inner evaluator)', () => {
-    expect(() => fanOutLoopsGenerator(baseConfig([seat()]))).toThrow(/inner evaluator/)
-    expect(() =>
-      fanOutLoopsGenerator(baseConfig([seat()]), {
-        smokeRunner: async () => smokeVerdict(),
-        smokeInstanceId: 'astropy__astropy-13033',
-      }),
-    ).toThrow(/immutable runner and judge references/)
-    expect(() => fanOutLoopsGenerator(baseConfig([{ name: 'no-seat-kind' }]))).toThrow(/neither a harness nor an engine/)
-  })
-
-  it('materializes each candidate into the scratch surface, applies the winner through the normal prefilter path, and records provenance', async () => {
-    const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n`
-    const LOSER = `${SEED}delete all tests\n`
-    const seen: Array<{
-      content: string
-      scratch: string
-      evaluationKey: string
-      hasCostLedger: boolean
-    }> = []
-    const smokeRunner: SmokeRunner = async ({
-      scratchPath,
-      evaluationKey,
-      costLedger,
-    }) => {
-      const content = await readFile(join(scratchPath, SURFACE), 'utf8')
-      seen.push({
-        content,
-        scratch: scratchPath,
-        evaluationKey,
-        hasCostLedger: costLedger !== undefined,
-      })
-      // The winner candidate resolves; the seed gets verify-pass only; the
-      // loser gets nothing — exercising resolve-dominates + tiebreak.
-      if (content === WINNER) return smokeVerdict({ resolved: true, verifyPass: true })
-      if (content === SEED) return smokeVerdict({ verifyPass: true })
-      return smokeVerdict()
-    }
-    const observed: { config?: unknown } = {}
-    const config = baseConfig([seat({ maxMetricCalls: 5 })], { activationGate: true })
-    const gen = fanOutLoopsGenerator(config, {
-      ...IMPLEMENTATION_REFS,
-      smokeRunner,
-      smokeInstanceId: 'astropy__astropy-13033',
-      scoreSplit: { privateInstances: ['django__django-11532'] },
-      gepaMethodFactory: fakeGepaFactory([LOSER, WINNER], observed),
-      gepaOptimizer: testOptimizer,
-    })
-
-    const result = await gen.generate(generatorArgs(0))
-
-    expect(result).toMatchObject({ applied: true, label: 'gepa-author' })
-    expect(result.rationale).toContain('engine gepa')
-    // 3 isolated inner calls (seed, loser, winner) + 1 stage-B prefilter smoke
-    // on the final candidate, all outside the driver worktree.
-    expect(seen).toHaveLength(4)
-    for (const call of seen) expect(call.scratch).not.toBe(driverWt)
-    expect(seen.map((c) => c.content)).toEqual([SEED, LOSER, WINNER, WINNER])
-    expect(new Set(seen.slice(0, 3).map((call) => call.scratch)).size).toBe(3)
-    expect(seen.slice(0, 3).every((call) => call.evaluationKey.startsWith('inner-'))).toBe(true)
-    expect(seen[3]!.evaluationKey).toBe('candidate-0')
-    expect(seen.slice(0, 3).every((call) => call.hasCostLedger)).toBe(true)
-    // The winner landed on the driver worktree with the mechanical predicate.
-    expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(WINNER)
-    const predicate = parseActivationPredicate(await readFile(join(driverWt, ACTIVATION_PREDICATE_RELPATH), 'utf8'))
-    expect(predicate.ok).toBe(true)
-    // Only the surface + predicate changed.
-    const changed = (await runOk('git', ['-C', driverWt, 'status', '--porcelain', '--untracked-files=all'])).stdout
-      .split('\n')
-      .map((l) => l.slice(3).trim())
-      .filter(Boolean)
-    expect(changed.sort()).toEqual([ACTIVATION_PREDICATE_RELPATH, SURFACE].sort())
-    // Budget threaded into the adapter recipe.
-    const incumbentCommit = await git(['rev-parse', 'HEAD'], driverWt)
-    expect(observed.config).toMatchObject({
-      evaluationId: gepaSeatEvaluationId({
-        smokeInstanceId: 'astropy__astropy-13033',
-        dispatchTimeoutMs: config.dispatchTimeoutMs,
-        incumbentCommit,
-        ...IMPLEMENTATION_REFS,
-      }),
-      recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } },
-      optimizer: testOptimizer,
-      resume: 'if-compatible',
-      trustResumeState: true,
-    })
-    // Inner-run provenance: the seat-local file plus one immutable shared record.
-    const inner = JSON.parse(await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'))
-    expect(inner).toMatchObject({
-      seat: 'gepa-author',
-      engine: 'gepa',
-      budget: 5,
-      innerCallCount: 3,
-      bestComposite: 1.25,
-      source: {
-        package: 'gepa',
-        version: '0.1.4',
-        revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f',
-        sourceSha256: '1'.repeat(64),
-      },
-      bridge: {
-        package: 'agent-eval-rpc',
-        version: '0.126.1',
-        sourceSha256: '2'.repeat(64),
-      },
-      modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }],
-      python: { implementation: 'CPython', version: '3.12.3' },
-      runId: 'gepa-run',
-      compatibleRunId: 'compatible-gepa-run',
-      resumed: false,
-      evaluationCount: 3,
-      tokenUsage: {
-        inputTokens: 100,
-        cachedInputTokens: 10,
-        cacheWriteInputTokens: 5,
-        outputTokens: 25,
-        reasoningTokens: 8,
-        totalTokens: 125,
-        calls: 2,
-      },
-      artifactDir: '/tmp/gepa-run',
-      totalCostUsd: 0.125,
-      accountingComplete: true,
-      incompleteReasons: [],
-    })
-    expect(inner.innerScores.map((s: { composite: number }) => s.composite)).toEqual([0.25, 0, 1.25])
-    const records = await readdir(join(outDir, GEPA_INNER_RUNS_DIRNAME))
-    expect(records.filter((name) => name.endsWith('.json'))).toHaveLength(1)
-    expect(JSON.parse(await readFile(join(outDir, GEPA_INNER_RUNS_DIRNAME, records[0]!), 'utf8'))).toEqual(inner)
-    expect(gen.drainPrefilterKills()).toEqual([])
-  })
-
-  it('isolates concurrent candidate evaluations while preserving parallel execution', async () => {
-    const candidateA = `${SEED}candidate A keeps its own workspace\n`
-    const candidateB = `${SEED}candidate B keeps its own workspace\n`
-    let releaseBoth!: () => void
-    const bothStarted = new Promise<void>((resolve) => {
-      releaseBoth = resolve
-    })
-    let started = 0
-    const innerSeen: Array<{ content: string; scratchPath: string }> = []
-    const smokeRunner: SmokeRunner = async ({
-      scratchPath,
-      evaluationKey,
-    }) => {
-      if (evaluationKey.startsWith('inner-')) {
-        started += 1
-        if (started === 2) releaseBoth()
-        await bothStarted
-      }
-      const content = await readFile(join(scratchPath, SURFACE), 'utf8')
-      if (evaluationKey.startsWith('inner-')) {
-        innerSeen.push({ content, scratchPath })
-      }
-      return smokeVerdict({ resolved: content === candidateA })
-    }
-    const parallelFactory: GepaMethodFactory = () => ({
-      name: 'parallel-gepa',
-      async optimize(input) {
-        const scenario = input.trainScenarios[0]!
-        await Promise.all([
-          input.dispatchWithSurface(candidateA, scenario, fakeCtx),
-          input.dispatchWithSurface(candidateB, scenario, fakeCtx),
-        ])
-        return {
-          winnerSurface: candidateA,
-          cost: {
-            totalCostUsd: 0,
-            costProvenance: { kind: 'observed', usd: 0 },
-            accountingComplete: true,
-            incompleteReasons: [],
-          },
-          durationMs: 1,
-          provenance: fullProvenance('parallel-gepa', { evaluationCount: 2 }),
-        }
-      },
-    })
-    const gen = fanOutLoopsGenerator(baseConfig([seat({ maxMetricCalls: 2 })]), {
-      ...IMPLEMENTATION_REFS,
-      smokeRunner,
-      smokeInstanceId: 'astropy__astropy-13033',
-      scoreSplit: null,
-      gepaMethodFactory: parallelFactory,
-    })
-
-    const result = await gen.generate(generatorArgs(0))
-
-    expect(result.applied).toBe(true)
-    expect(innerSeen.map((entry) => entry.content).sort()).toEqual(
-      [candidateA, candidateB].sort(),
-    )
-    expect(new Set(innerSeen.map((entry) => entry.scratchPath)).size).toBe(2)
-  })
-
-  it('uses explicit immutable refs so captured runner or judge behavior cannot share resume state', async () => {
-    const config = baseConfig([seat()])
-    const incumbentCommit = await git(['rev-parse', 'HEAD'], driverWt)
-    const makeRunner = (resolved: boolean): SmokeRunner =>
-      async () => smokeVerdict({ resolved })
-    const runnerV1 = makeRunner(false)
-    const runnerV2 = makeRunner(true)
-    const common = {
-      smokeInstanceId: 'astropy__astropy-13033',
-      dispatchTimeoutMs: config.dispatchTimeoutMs,
-      incumbentCommit,
-    }
-    const original = gepaSeatEvaluationId({ ...common, ...IMPLEMENTATION_REFS })
-    const changedRunner = gepaSeatEvaluationId({
-      ...common,
-      ...IMPLEMENTATION_REFS,
-      runnerImplementationRef: `sha256:${'c'.repeat(64)}`,
-    })
-    const changedJudge = gepaSeatEvaluationId({
-      ...common,
-      ...IMPLEMENTATION_REFS,
-      judgeImplementationRef: `sha256:${'d'.repeat(64)}`,
-    })
-    const changedIncumbent = gepaSeatEvaluationId({
-      ...common,
-      ...IMPLEMENTATION_REFS,
-      incumbentCommit: 'e'.repeat(40),
-    })
-    const smokeArgs: Parameters<SmokeRunner>[0] = {
-      scratchPath: driverWt,
-      generation: 0,
-      proposer: seat(),
-      evaluationKey: 'identity-test',
-    }
-
-    expect(runnerV1.toString()).toBe(runnerV2.toString())
-    expect((await runnerV1(smokeArgs)).resolved).toBe(false)
-    expect((await runnerV2(smokeArgs)).resolved).toBe(true)
-    expect(original).toMatch(
-      /^swe-arena-gepa-seat\|smoke=astropy__astropy-13033\|incumbent=[a-f0-9]{40,64}\|runner=sha256:[a-f0-9]{64}\|judge=sha256:[a-f0-9]{64}\|dispatchTimeoutMs=\d+$/,
-    )
-    expect(changedRunner).not.toBe(original)
-    expect(changedJudge).not.toBe(original)
-    expect(changedIncumbent).not.toBe(original)
-    expect(() =>
-      gepaSeatEvaluationId({
-        ...common,
-        ...IMPLEMENTATION_REFS,
-        runnerImplementationRef: 'runner-v2',
-      }),
-    ).toThrow(/runnerImplementationRef must be an immutable sha256 reference/)
-  })
-
-  it('records but rejects a winner when cost accounting is incomplete', async () => {
-    const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n`
-    const incomplete: GepaMethodFactory = () => ({
-      name: 'incomplete-cost',
-      async optimize(input) {
-        await input.dispatchWithSurface(WINNER, input.trainScenarios[0]!, fakeCtx)
-        return {
-          winnerSurface: WINNER,
-          cost: {
-            totalCostUsd: 0.25,
-            costProvenance: { kind: 'uncaptured', usd: null },
-            accountingComplete: false,
-            incompleteReasons: ['optimizer model receipt missing'],
-          },
-          durationMs: 1,
-          provenance: fullProvenance('incomplete-run', { evaluationCount: 1 }),
-        }
-      },
-    })
-    const gen = fanOutLoopsGenerator(baseConfig([seat()]), {
-      ...IMPLEMENTATION_REFS,
-      smokeRunner: async () => smokeVerdict({ resolved: true }),
-      smokeInstanceId: 'astropy__astropy-13033',
-      scoreSplit: null,
-      gepaMethodFactory: incomplete,
-    })
-
-    await expect(gen.generate(generatorArgs(0))).rejects.toThrow(
-      /cost accounting is incomplete: optimizer model receipt missing/,
-    )
-    expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(SEED)
-    const inner = JSON.parse(
-      await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'),
-    )
-    expect(inner).toMatchObject({
-      runId: 'incomplete-run',
-      totalCostUsd: 0.25,
-      accountingComplete: false,
-      incompleteReasons: ['optimizer model receipt missing'],
-    })
-  })
-
-  it('enforces the inner-call budget cap fail-closed', async () => {
-    const runaway: GepaMethodFactory = () => ({
-      name: 'runaway',
-      async optimize(input) {
-        const results = await Promise.allSettled(
-          Array.from({ length: 4 }, (_, index) =>
-            input.dispatchWithSurface(
-              `${SEED}candidate ${index}\n`,
-              input.trainScenarios[0]!,
-              fakeCtx,
-            ),
-          ),
-        )
-        const rejected = results.find(
-          (result): result is PromiseRejectedResult => result.status === 'rejected',
-        )
-        if (rejected) throw rejected.reason
-        return {
-          winnerSurface: SEED,
-          cost: {
-            totalCostUsd: 0,
-            costProvenance: { kind: 'uncaptured', usd: null },
-            accountingComplete: false,
-            incompleteReasons: [],
-          },
-          durationMs: 1,
-          provenance: fullProvenance('runaway'),
-        }
-      },
-    })
-    const gen = fanOutLoopsGenerator(baseConfig([seat({ maxMetricCalls: 3 })]), {
-      ...IMPLEMENTATION_REFS,
-      smokeRunner: async () => smokeVerdict(),
-      smokeInstanceId: 'astropy__astropy-13033',
-      scoreSplit: null,
-      gepaMethodFactory: runaway,
-    })
-    await expect(gen.generate(generatorArgs(0))).rejects.toThrow(/inner-call budget 3 exhausted/)
-  })
-
-  it('refuses to feed a PRIVATE smoke verdict to the bridge', async () => {
-    const gen = fanOutLoopsGenerator(baseConfig([seat()]), {
-      ...IMPLEMENTATION_REFS,
-      smokeRunner: async () => smokeVerdict({ iid: 'django__django-11532' }),
-      smokeInstanceId: 'astropy__astropy-13033',
-      scoreSplit: { privateInstances: ['django__django-11532'] },
-      gepaMethodFactory: fakeGepaFactory([`${SEED}x line long enough\n`]),
-    })
-    await expect(gen.generate(generatorArgs(0))).rejects.toThrow(/PRIVATE instance django__django-11532/)
-  })
-
-  it('declines the slot without a kill when GEPA returns the seed unchanged', async () => {
-    const gen = fanOutLoopsGenerator(baseConfig([seat()]), {
-      ...IMPLEMENTATION_REFS,
-      smokeRunner: async () => smokeVerdict(),
-      smokeInstanceId: 'astropy__astropy-13033',
-      scoreSplit: null,
-      gepaMethodFactory: fakeGepaFactory([]),
-    })
-    const result = await gen.generate(generatorArgs(0))
-    expect(result.applied).toBe(false)
-    expect(result.summary).toContain('equals the seed')
-    expect(gen.drainPrefilterKills()).toEqual([])
-    expect((await runOk('git', ['-C', driverWt, 'status', '--porcelain'])).stdout.trim()).toBe('')
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Integration: one real Node-to-Python-to-score roundtrip through the installed
-// bridge. The TypeScript adapter is a compile-time package dependency.
-// ---------------------------------------------------------------------------
-
-const pythonBridgeReady = (python: string): { ok: boolean; reason: string } => {
-  const probe = spawnSync(python, [
-    '-c',
-    'import agent_eval_rpc.gepa_bridge; from gepa.optimize_anything import optimize_anything, OptimizeAnythingConfig',
-  ])
-  if (probe.status !== 0) {
-    return { ok: false, reason: `python bridge unavailable: ${String(probe.stderr).trim().split('\n').pop()}` }
-  }
-  return { ok: true, reason: '' }
-}
-
-describe('integration: real adapter roundtrip', () => {
-  it('runs a metered optimizer through the real bridge and applies its winner', async (ctx) => {
-    const python = process.env.AGENT_EVAL_TEST_PYTHON ?? DEFAULT_GEPA_PYTHON
-    const pythonRuntime = pythonBridgeReady(python)
-    if (!pythonRuntime.ok) {
-      ctx.skip(`skip-with-reason: ${pythonRuntime.reason}`)
-      return
-    }
-
-    const winner = 'tiny synthetic surface\nAlways run the neighboring test before finalizing.'
-    const modelServer = createServer((_request, response) => {
-      response.writeHead(200, { 'content-type': 'application/json' })
-      response.end(
-        JSON.stringify({
-          choices: [
-            {
-              message: {
-                role: 'assistant',
-                content: `\`\`\`\n${winner}\`\`\``,
-              },
-            },
-          ],
-          usage: {
-            prompt_tokens: 20,
-            completion_tokens: 20,
-            total_tokens: 40,
-          },
-        }),
-      )
-    })
-    await new Promise<void>((resolve, reject) => {
-      modelServer.once('error', reject)
-      modelServer.listen(0, '127.0.0.1', resolve)
-    })
-    const address = modelServer.address()
-    if (!address || typeof address === 'string') {
-      throw new Error('test optimizer server did not bind')
-    }
-
-    const loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-int-repo-'))
-    const outDir = await mkdtemp(join(tmpdir(), 'gepa-int-out-'))
-    try {
-      await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
-      await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
-      await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
-      await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
-      await mkdir(join(loopsRepo, 'extensions', 'pi', 'prompts'), { recursive: true })
-      await writeFile(join(loopsRepo, SURFACE), 'tiny synthetic surface\n')
-      await runOk('git', ['-C', loopsRepo, 'add', '-A'])
-      await runOk('git', ['-C', loopsRepo, 'commit', '-q', '-m', 'init'])
-      const driverWt = join(outDir, 'driver-wt')
-      await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', driverWt, 'HEAD'])
-
-      const gen = fanOutLoopsGenerator(
-        {
-          ...defaultRound4Config(),
-          loopsRepo,
-          outDir,
-          populationSize: 1,
-          proposers: [seat({ maxMetricCalls: 10, python })],
-          prefilter: { enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false },
-        },
-        {
-          ...IMPLEMENTATION_REFS,
-          smokeRunner: async ({ scratchPath }) => {
-            const candidate = await readFile(join(scratchPath, SURFACE), 'utf8')
-            return {
-              iid: 'astropy__astropy-13033',
-              pass: true,
-              reason: 'stub smoke (integration)',
-              resolved: candidate === winner,
-              patchLines: 1,
-              wallS: 0,
-              verifyPass: true,
-            }
-          },
-          smokeInstanceId: 'astropy__astropy-13033',
-          scoreSplit: null,
-          gepaOptimizer: officialOptimizerModel({
-            env: testOptimizerEnv,
-            envPrefix: 'TEST_OPTIMIZER',
-            model: 'test-optimizer',
-            baseUrl: `http://127.0.0.1:${address.port}/v1`,
-            apiKey: 'local-test-key',
-            maxCostUsd: 1,
-            maxOutputTokensPerRequest: 2_000,
-            callRef: 'test:integration-optimizer',
-          }),
-        },
-      )
-      const result = await gen.generate({
-        worktreePath: driverWt,
-        findings: [],
-        maxShots: 1,
-        signal: new AbortController().signal,
-        generation: 0,
-        candidateIndex: 0,
-      })
-      const inner = JSON.parse(
-        await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'),
-      )
-      expect(result.applied, `${result.summary}\n${JSON.stringify(inner.innerScores, null, 2)}`).toBe(true)
-      expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(winner)
-      expect(inner.innerCallCount).toBeGreaterThanOrEqual(1)
-      expect(inner.accountingComplete).toBe(true)
-      expect(inner.incompleteReasons).toEqual([])
-      expect(inner.tokenUsage.calls).toBeGreaterThan(0)
-    } finally {
-      await new Promise<void>((resolve, reject) =>
-        modelServer.close((error) => (error ? reject(error) : resolve())),
-      )
-      await rm(outDir, { recursive: true, force: true })
-      await rm(loopsRepo, { recursive: true, force: true })
-    }
-  }, 300_000)
-})