@tangle-network/agent-bench 0.3.8 → 0.4.1

package/src/swe-arena/gepa-seat.test.mts

@@ -1,20 +1,25 @@
 import { spawnSync } from 'node:child_process'
 import { existsSync } from 'node:fs'
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
+import { createServer } from 'node:http'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type { DispatchContext } from '@tangle-network/agent-eval/campaign'
+import type {
+  DispatchContext,
+  OptimizationMethodProvenance,
+} from '@tangle-network/agent-eval/campaign'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import { ACTIVATION_PREDICATE_RELPATH, parseActivationPredicate } from './activation.mts'
 import {
   DEFAULT_GEPA_PYTHON,
   DEFAULT_MAX_METRIC_CALLS,
-  GEPA_ADAPTER_UPGRADE_HINT,
+  GEPA_INNER_RUNS_DIRNAME,
+  GEPA_PYTHON_INSTALL_HINT,
   gepaBridgeScenarios,
+  gepaSeatEvaluationId,
   innerSmokeComposite,
   innerSmokeJudge,
   isGepaSeat,
-  loadGepaMethodFactory,
   mechanicalActivationPredicate,
   probeGepaRuntime,
   recipeEvaluationBudget,
@@ -32,6 +37,10 @@ import { captureProposerProvenance } from './proposer-provenance.mts'
 import { runOk } from './proc.ts'
 
 const SURFACE = 'extensions/pi/prompts/worker-coding-system.md'
+const IMPLEMENTATION_REFS = {
+  runnerImplementationRef: `sha256:${'a'.repeat(64)}`,
+  judgeImplementationRef: `sha256:${'b'.repeat(64)}`,
+} as const
 
 const seat = (over: Partial<ProposerSpec> = {}): ProposerSpec => ({
   name: 'gepa-author',
@@ -45,7 +54,7 @@ const seat = (over: Partial<ProposerSpec> = {}): ProposerSpec => ({
 // ---------------------------------------------------------------------------
 
 describe('validateGepaSeat', () => {
-  it('accepts the gen-6 draft seat shape and isGepaSeat discriminates on engine', () => {
+  it('accepts an engine seat and isGepaSeat discriminates on engine', () => {
     expect(() => validateGepaSeat(seat())).not.toThrow()
     expect(() => validateGepaSeat(seat({ engine: 'omni', maxMetricCalls: 8 }))).not.toThrow()
     expect(isGepaSeat(seat())).toBe(true)
@@ -85,10 +94,10 @@ describe('recipeForSeat', () => {
     expect(recipeEvaluationBudget(recipe)).toBe(DEFAULT_MAX_METRIC_CALLS)
   })
 
-  it("'omni' is GEPA's best-of-then-continue shape and the four bounded runs sum EXACTLY to the budget", () => {
+  it("'omni' uses the official recipe and its four bounded runs preserve the budget", () => {
     const recipe = recipeForSeat(seat({ engine: 'omni', maxMetricCalls: 10 }) as GepaSeatSpec)
-    expect(recipe.kind).toBe('best-of-then-continue')
-    if (recipe.kind !== 'best-of-then-continue') throw new Error('unreachable')
+    expect(recipe.kind).toBe('omni')
+    if (recipe.kind !== 'omni') throw new Error('unreachable')
     expect(recipe.explore.map((r) => r.engine)).toEqual(['gepa', 'autoresearch', 'meta_harness'])
     expect(recipe.continueWith.engine).toBe('gepa')
     expect(recipeEvaluationBudget(recipe)).toBe(10)
@@ -153,7 +162,7 @@ describe('inner smoke score', () => {
     expect(innerSmokeComposite(verdict({ resolved: false, verifyPass: true }))).toBe(0.25)
     expect(innerSmokeComposite(verdict({ resolved: true, verifyPass: false }))).toBe(1)
     expect(innerSmokeComposite(verdict({ resolved: true, verifyPass: true }))).toBe(1.25)
-    // Pre-gen-6 verdicts without the field score as no verify signal.
+    // Older verdicts without the field score as no verify signal.
     expect(innerSmokeComposite(verdict({ resolved: true }))).toBe(1)
   })
 
@@ -169,21 +178,9 @@ describe('inner smoke score', () => {
 })
 
 // ---------------------------------------------------------------------------
-// Runtime seams: loud fails with exact instructions.
+// Optional Python runtime: loud failures with exact instructions.
 // ---------------------------------------------------------------------------
 
-describe('loadGepaMethodFactory', () => {
-  it('throws the upgrade instruction when the installed agent-eval predates the adapter', async () => {
-    await expect(loadGepaMethodFactory(async () => ({}))).rejects.toThrow(/#408/)
-    await expect(loadGepaMethodFactory(async () => ({}))).rejects.toThrow(/gepaOptimizationMethod/)
-  })
-
-  it('returns the export when present', async () => {
-    const factory = (() => ({})) as unknown as GepaMethodFactory
-    await expect(loadGepaMethodFactory(async () => ({ gepaOptimizationMethod: factory }))).resolves.toBe(factory)
-  })
-})
-
 describe('probeGepaRuntime', () => {
   const execFailingOn =
     (failFragment: string, stderr: string): ProbeExec =>
@@ -196,7 +193,9 @@ describe('probeGepaRuntime', () => {
 
   it('fails loud with pip install instructions when the bridge module is missing', async () => {
     const exec = execFailingOn('agent_eval_rpc.gepa_bridge', "ModuleNotFoundError: No module named 'agent_eval_rpc'")
-    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/pip install 'agent-eval-rpc\[gepa\]'/)
+    await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(
+      GEPA_PYTHON_INSTALL_HINT,
+    )
     await expect(probeGepaRuntime('python3', exec, 'gepa-author')).rejects.toThrow(/not installed/)
   })
 
@@ -234,15 +233,10 @@ describe('captureProposerProvenance with a gepa seat', () => {
     }
     return { code: 0, stdout: 'source', stderr: '' }
   }
-  const withAdapter = async (): Promise<Record<string, unknown>> => ({
-    gepaOptimizationMethod: () => ({}),
-  })
-
   it('records engine, surface, gepa version, bridge module, and the python runtime as harnessVersion', async () => {
     const record = await captureProposerProvenance([{ name: 'claude-author', harness: 'claude' }, seat()], {
       exec: okExec,
       readSettingsModel: () => 'settings-model',
-      importCampaign: withAdapter,
     })
     const gepa = record.proposers.find((p) => p.name === 'gepa-author')!
     expect(gepa).toMatchObject({
@@ -267,15 +261,9 @@ describe('captureProposerProvenance with a gepa seat', () => {
       args.join(' ').includes('gepa_bridge')
         ? { code: 1, stdout: '', stderr: 'ModuleNotFoundError' }
         : { code: 0, stdout: 'Python 3.12.3', stderr: '' }
-    await expect(
-      captureProposerProvenance([seat()], { exec, importCampaign: withAdapter }),
-    ).rejects.toThrow(/pip install 'agent-eval-rpc\[gepa\]'/)
-  })
-
-  it('fails LOUD at t=0 when the installed agent-eval predates the adapter export', async () => {
-    await expect(
-      captureProposerProvenance([seat()], { exec: okExec, importCampaign: async () => ({}) }),
-    ).rejects.toThrow(GEPA_ADAPTER_UPGRADE_HINT.slice(0, 40))
+    await expect(captureProposerProvenance([seat()], { exec })).rejects.toThrow(
+      GEPA_PYTHON_INSTALL_HINT,
+    )
   })
 })
 
@@ -284,7 +272,7 @@ describe('captureProposerProvenance with a gepa seat', () => {
 // ---------------------------------------------------------------------------
 
 describe('mechanicalActivationPredicate', () => {
-  it('targets the longest added line and produces a parseable v1 grep predicate', () => {
+  it('targets the longest added line and produces a parseable grep predicate', () => {
     const seed = 'alpha\nshared line stays here\n'
     const winner = 'alpha\nshared line stays here\nAlways run the neighboring test file before finalizing.\nshort\n'
     const predicate = mechanicalActivationPredicate(seed, winner, SURFACE)!
@@ -309,34 +297,128 @@ describe('mechanicalActivationPredicate', () => {
 })
 
 describe('recordGepaSeatInnerRun', () => {
-  it('appends to gepaInnerRuns while preserving the t=0 capture record', async () => {
+  const sourceHash = 'a'.repeat(64)
+  const bridgeHash = 'b'.repeat(64)
+  const moduleHash = 'c'.repeat(64)
+  const provenance = (runId: string): OptimizationMethodProvenance => ({
+    source: {
+      kind: 'package',
+      evidence: 'observed',
+      package: 'gepa',
+      version: '0.1.4',
+      sourceUrl: 'https://github.com/gepa-ai/gepa.git',
+      revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f',
+      sourceSha256: sourceHash,
+    },
+    bridge: {
+      kind: 'package',
+      evidence: 'observed',
+      package: 'agent-eval-rpc',
+      version: '0.126.1',
+      sourceSha256: bridgeHash,
+    },
+    modules: [{ module: 'example.engine', sourceSha256: moduleHash }],
+    python: { implementation: 'CPython', version: '3.12.3' },
+    runId,
+    compatibleRunId: 'compatible-run',
+    resumed: false,
+    evaluationCount: 3,
+    tokenUsage: {
+      inputTokens: 120,
+      cachedInputTokens: 20,
+      outputTokens: 30,
+      reasoningTokens: 10,
+      totalTokens: 150,
+      calls: 2,
+    },
+    artifactDir: `/tmp/${runId}`,
+  })
+  const run = (runId: string, over: Partial<GepaSeatInnerRun> = {}): GepaSeatInnerRun => {
+    const result = provenance(runId)
+    if (
+      result.source.revision === undefined ||
+      result.source.sourceSha256 === undefined ||
+      result.bridge?.sourceSha256 === undefined ||
+      result.modules === undefined ||
+      result.python === undefined ||
+      result.compatibleRunId === undefined ||
+      result.tokenUsage === undefined
+    ) {
+      throw new Error('invalid test provenance')
+    }
+    return {
+      seat: 'gepa-author',
+      engine: 'gepa',
+      surface: SURFACE,
+      generation: 0,
+      budget: 10,
+      innerCallCount: 2,
+      innerScores: [],
+      bestComposite: 1,
+      source: {
+        ...result.source,
+        revision: result.source.revision,
+        sourceSha256: result.source.sourceSha256,
+      },
+      bridge: { ...result.bridge, sourceSha256: result.bridge.sourceSha256 },
+      modules: result.modules,
+      python: result.python,
+      runId: result.runId,
+      compatibleRunId: result.compatibleRunId,
+      resumed: result.resumed,
+      evaluationCount: result.evaluationCount,
+      tokenUsage: result.tokenUsage,
+      artifactDir: result.artifactDir,
+      totalCostUsd: 0.25,
+      accountingComplete: true,
+      incompleteReasons: [],
+      durationMs: 5,
+      ...over,
+    }
+  }
+
+  it('writes collision-free immutable records in parallel and preserves the launch record', async () => {
     const dir = await mkdtemp(join(tmpdir(), 'gepa-prov-'))
     try {
-      await writeFile(join(dir, 'proposer-provenance.json'), JSON.stringify({ schema: 'swe-arena.proposer-provenance.v1', proposers: [] }))
-      const run: GepaSeatInnerRun = {
-        seat: 'gepa-author',
-        engine: 'gepa',
-        surface: SURFACE,
-        generation: 0,
-        budget: 10,
-        innerCallCount: 2,
-        innerScores: [],
-        bestComposite: 1,
-        adapterReportedCostUsd: 0,
-        adapterCostAccountingComplete: false,
-        durationMs: 5,
-      }
-      await recordGepaSeatInnerRun(dir, run)
-      await recordGepaSeatInnerRun(dir, { ...run, generation: 1 })
-      const record = JSON.parse(await readFile(join(dir, 'proposer-provenance.json'), 'utf8'))
-      expect(record.schema).toBe('swe-arena.proposer-provenance.v1')
-      expect(record.gepaInnerRuns).toHaveLength(2)
-      expect(record.gepaInnerRuns[0]).toMatchObject({ seat: 'gepa-author', innerCallCount: 2 })
-      expect(record.gepaInnerRuns[1]).toMatchObject({ generation: 1 })
+      const launchRecord = JSON.stringify({ capturedAt: '2026-07-24T00:00:00.000Z', proposers: [] }, null, 2)
+      await writeFile(join(dir, 'proposer-provenance.json'), launchRecord)
+      const paths = await Promise.all(
+        Array.from({ length: 24 }, (_, index) =>
+          recordGepaSeatInnerRun(dir, run(`run-${index}`, { generation: index })),
+        ),
+      )
+      expect(new Set(paths).size).toBe(24)
+      expect(await readFile(join(dir, 'proposer-provenance.json'), 'utf8')).toBe(launchRecord)
+
+      const names = (await readdir(join(dir, GEPA_INNER_RUNS_DIRNAME))).filter((name) => name.endsWith('.json'))
+      expect(names).toHaveLength(24)
+      const records = await Promise.all(
+        names.map(async (name) => JSON.parse(await readFile(join(dir, GEPA_INNER_RUNS_DIRNAME, name), 'utf8'))),
+      )
+      expect(new Set(records.map((record) => record.runId))).toEqual(
+        new Set(Array.from({ length: 24 }, (_, index) => `run-${index}`)),
+      )
+      expect(records.every((record) => record.source.sourceSha256 === sourceHash)).toBe(true)
     } finally {
       await rm(dir, { recursive: true, force: true })
     }
   })
+
+  it('fails on malformed existing launch or run data', async () => {
+    const launchDir = await mkdtemp(join(tmpdir(), 'gepa-prov-bad-launch-'))
+    const runDir = await mkdtemp(join(tmpdir(), 'gepa-prov-bad-run-'))
+    try {
+      await writeFile(join(launchDir, 'proposer-provenance.json'), '{not json')
+      await expect(recordGepaSeatInnerRun(launchDir, run('new-run'))).rejects.toThrow(/malformed JSON/)
+
+      await mkdir(join(runDir, GEPA_INNER_RUNS_DIRNAME), { recursive: true })
+      await writeFile(join(runDir, GEPA_INNER_RUNS_DIRNAME, 'broken.json'), '[]')
+      await expect(recordGepaSeatInnerRun(runDir, run('new-run'))).rejects.toThrow(/must contain a JSON object/)
+    } finally {
+      await rm(launchDir, { recursive: true, force: true })
+      await rm(runDir, { recursive: true, force: true })
+    }
+  })
 })
 
 // ---------------------------------------------------------------------------
@@ -344,6 +426,63 @@ describe('recordGepaSeatInnerRun', () => {
 // ---------------------------------------------------------------------------
 
 const fakeCtx = {} as unknown as DispatchContext
+const testOptimizer = {
+  model: 'optimizer-model',
+  baseUrl: 'http://127.0.0.1:1/v1',
+  apiKey: 'optimizer-key',
+  budget: {
+    maxCostUsd: 1,
+    maxRequests: 10,
+    maxRequestBytes: 100_000,
+    maxResponseBytes: 100_000,
+    maxOutputTokensPerRequest: 2_000,
+    pricing: {
+      inputUsdPerMillion: 1,
+      cachedInputUsdPerMillion: 0.1,
+      cacheWriteUsdPerMillion: 1.25,
+      outputUsdPerMillion: 5,
+    },
+  },
+}
+
+const fullProvenance = (
+  runId = 'gepa-run',
+  over: Partial<OptimizationMethodProvenance> = {},
+): OptimizationMethodProvenance => ({
+  source: {
+    kind: 'package',
+    evidence: 'observed',
+    package: 'gepa',
+    version: '0.1.4',
+    sourceUrl: 'https://github.com/gepa-ai/gepa.git',
+    revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f',
+    sourceSha256: '1'.repeat(64),
+  },
+  bridge: {
+    kind: 'package',
+    evidence: 'observed',
+    package: 'agent-eval-rpc',
+    version: '0.126.1',
+    sourceSha256: '2'.repeat(64),
+  },
+  modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }],
+  python: { implementation: 'CPython', version: '3.12.3' },
+  runId,
+  compatibleRunId: 'compatible-gepa-run',
+  resumed: false,
+  evaluationCount: 3,
+  tokenUsage: {
+    inputTokens: 100,
+    cachedInputTokens: 10,
+    cacheWriteInputTokens: 5,
+    outputTokens: 25,
+    reasoningTokens: 8,
+    totalTokens: 125,
+    calls: 2,
+  },
+  artifactDir: `/tmp/${runId}`,
+  ...over,
+})
 
 /** Mimics the adapter's loop: score the seed and each provided candidate via
  *  the seat's dispatch + judge, return the best-scoring candidate — exactly
@@ -365,8 +504,9 @@ const fakeGepaFactory =
         }
         return {
           winnerSurface: best.surface,
-          cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: ['fake factory'] },
+          cost: { totalCostUsd: 0.125, accountingComplete: true, incompleteReasons: [] },
           durationMs: 1,
+          provenance: fullProvenance(),
         }
       },
     }
@@ -434,16 +574,36 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
 
   it('construction fails loud without the smoke runner (the inner evaluator)', () => {
     expect(() => fanOutLoopsGenerator(baseConfig([seat()]))).toThrow(/inner evaluator/)
+    expect(() =>
+      fanOutLoopsGenerator(baseConfig([seat()]), {
+        smokeRunner: async () => smokeVerdict(),
+        smokeInstanceId: 'astropy__astropy-13033',
+      }),
+    ).toThrow(/immutable runner and judge references/)
     expect(() => fanOutLoopsGenerator(baseConfig([{ name: 'no-seat-kind' }]))).toThrow(/neither a harness nor an engine/)
   })
 
   it('materializes each candidate into the scratch surface, applies the winner through the normal prefilter path, and records provenance', async () => {
     const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n`
     const LOSER = `${SEED}delete all tests\n`
-    const seen: Array<{ content: string; scratch: string }> = []
-    const smokeRunner: SmokeRunner = async ({ scratchPath }) => {
+    const seen: Array<{
+      content: string
+      scratch: string
+      evaluationKey: string
+      hasCostLedger: boolean
+    }> = []
+    const smokeRunner: SmokeRunner = async ({
+      scratchPath,
+      evaluationKey,
+      costLedger,
+    }) => {
       const content = await readFile(join(scratchPath, SURFACE), 'utf8')
-      seen.push({ content, scratch: scratchPath })
+      seen.push({
+        content,
+        scratch: scratchPath,
+        evaluationKey,
+        hasCostLedger: costLedger !== undefined,
+      })
       // The winner candidate resolves; the seed gets verify-pass only; the
       // loser gets nothing — exercising resolve-dominates + tiebreak.
       if (content === WINNER) return smokeVerdict({ resolved: true, verifyPass: true })
@@ -453,21 +613,27 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
     const observed: { config?: unknown } = {}
     const config = baseConfig([seat({ maxMetricCalls: 5 })], { activationGate: true })
     const gen = fanOutLoopsGenerator(config, {
+      ...IMPLEMENTATION_REFS,
       smokeRunner,
       smokeInstanceId: 'astropy__astropy-13033',
       scoreSplit: { privateInstances: ['django__django-11532'] },
       gepaMethodFactory: fakeGepaFactory([LOSER, WINNER], observed),
+      gepaOptimizer: testOptimizer,
     })
 
     const result = await gen.generate(generatorArgs(0))
 
     expect(result).toMatchObject({ applied: true, label: 'gepa-author' })
     expect(result.rationale).toContain('engine gepa')
-    // 3 inner calls (seed, loser, winner) + 1 stage-B prefilter smoke on the
-    // final candidate — all in the seat's scratch worktree, never the driver.
+    // 3 isolated inner calls (seed, loser, winner) + 1 stage-B prefilter smoke
+    // on the final candidate, all outside the driver worktree.
     expect(seen).toHaveLength(4)
     for (const call of seen) expect(call.scratch).not.toBe(driverWt)
     expect(seen.map((c) => c.content)).toEqual([SEED, LOSER, WINNER, WINNER])
+    expect(new Set(seen.slice(0, 3).map((call) => call.scratch)).size).toBe(3)
+    expect(seen.slice(0, 3).every((call) => call.evaluationKey.startsWith('inner-'))).toBe(true)
+    expect(seen[3]!.evaluationKey).toBe('candidate-0')
+    expect(seen.slice(0, 3).every((call) => call.hasCostLedger)).toBe(true)
     // The winner landed on the driver worktree with the mechanical predicate.
     expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(WINNER)
     const predicate = parseActivationPredicate(await readFile(join(driverWt, ACTIVATION_PREDICATE_RELPATH), 'utf8'))
@@ -479,31 +645,243 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
       .filter(Boolean)
     expect(changed.sort()).toEqual([ACTIVATION_PREDICATE_RELPATH, SURFACE].sort())
     // Budget threaded into the adapter recipe.
-    expect(observed.config).toMatchObject({ recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } } })
-    // Inner-run provenance: per-seat file + the merged proposer-provenance.json.
+    const incumbentCommit = await git(['rev-parse', 'HEAD'], driverWt)
+    expect(observed.config).toMatchObject({
+      evaluationId: gepaSeatEvaluationId({
+        smokeInstanceId: 'astropy__astropy-13033',
+        dispatchTimeoutMs: config.dispatchTimeoutMs,
+        incumbentCommit,
+        ...IMPLEMENTATION_REFS,
+      }),
+      recipe: { kind: 'engine', run: { engine: 'gepa', maxEvaluations: 5 } },
+      optimizer: testOptimizer,
+      resume: 'if-compatible',
+      trustResumeState: true,
+    })
+    // Inner-run provenance: the seat-local file plus one immutable shared record.
     const inner = JSON.parse(await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'))
-    expect(inner).toMatchObject({ seat: 'gepa-author', engine: 'gepa', budget: 5, innerCallCount: 3, bestComposite: 1.25 })
+    expect(inner).toMatchObject({
+      seat: 'gepa-author',
+      engine: 'gepa',
+      budget: 5,
+      innerCallCount: 3,
+      bestComposite: 1.25,
+      source: {
+        package: 'gepa',
+        version: '0.1.4',
+        revision: 'f919db0a622e2e9f9204779b81fe00cc1b2d808f',
+        sourceSha256: '1'.repeat(64),
+      },
+      bridge: {
+        package: 'agent-eval-rpc',
+        version: '0.126.1',
+        sourceSha256: '2'.repeat(64),
+      },
+      modules: [{ module: 'custom_gepa_engines', sourceSha256: '3'.repeat(64) }],
+      python: { implementation: 'CPython', version: '3.12.3' },
+      runId: 'gepa-run',
+      compatibleRunId: 'compatible-gepa-run',
+      resumed: false,
+      evaluationCount: 3,
+      tokenUsage: {
+        inputTokens: 100,
+        cachedInputTokens: 10,
+        cacheWriteInputTokens: 5,
+        outputTokens: 25,
+        reasoningTokens: 8,
+        totalTokens: 125,
+        calls: 2,
+      },
+      artifactDir: '/tmp/gepa-run',
+      totalCostUsd: 0.125,
+      accountingComplete: true,
+      incompleteReasons: [],
+    })
     expect(inner.innerScores.map((s: { composite: number }) => s.composite)).toEqual([0.25, 0, 1.25])
-    const merged = JSON.parse(await readFile(join(outDir, 'proposer-provenance.json'), 'utf8'))
-    expect(merged.gepaInnerRuns).toHaveLength(1)
+    const records = await readdir(join(outDir, GEPA_INNER_RUNS_DIRNAME))
+    expect(records.filter((name) => name.endsWith('.json'))).toHaveLength(1)
+    expect(JSON.parse(await readFile(join(outDir, GEPA_INNER_RUNS_DIRNAME, records[0]!), 'utf8'))).toEqual(inner)
     expect(gen.drainPrefilterKills()).toEqual([])
   })
 
+  it('isolates concurrent candidate evaluations while preserving parallel execution', async () => {
+    const candidateA = `${SEED}candidate A keeps its own workspace\n`
+    const candidateB = `${SEED}candidate B keeps its own workspace\n`
+    let releaseBoth!: () => void
+    const bothStarted = new Promise<void>((resolve) => {
+      releaseBoth = resolve
+    })
+    let started = 0
+    const innerSeen: Array<{ content: string; scratchPath: string }> = []
+    const smokeRunner: SmokeRunner = async ({
+      scratchPath,
+      evaluationKey,
+    }) => {
+      if (evaluationKey.startsWith('inner-')) {
+        started += 1
+        if (started === 2) releaseBoth()
+        await bothStarted
+      }
+      const content = await readFile(join(scratchPath, SURFACE), 'utf8')
+      if (evaluationKey.startsWith('inner-')) {
+        innerSeen.push({ content, scratchPath })
+      }
+      return smokeVerdict({ resolved: content === candidateA })
+    }
+    const parallelFactory: GepaMethodFactory = () => ({
+      name: 'parallel-gepa',
+      async optimize(input) {
+        const scenario = input.trainScenarios[0]!
+        await Promise.all([
+          input.dispatchWithSurface(candidateA, scenario, fakeCtx),
+          input.dispatchWithSurface(candidateB, scenario, fakeCtx),
+        ])
+        return {
+          winnerSurface: candidateA,
+          cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+          durationMs: 1,
+          provenance: fullProvenance('parallel-gepa', { evaluationCount: 2 }),
+        }
+      },
+    })
+    const gen = fanOutLoopsGenerator(baseConfig([seat({ maxMetricCalls: 2 })]), {
+      ...IMPLEMENTATION_REFS,
+      smokeRunner,
+      smokeInstanceId: 'astropy__astropy-13033',
+      scoreSplit: null,
+      gepaMethodFactory: parallelFactory,
+    })
+
+    const result = await gen.generate(generatorArgs(0))
+
+    expect(result.applied).toBe(true)
+    expect(innerSeen.map((entry) => entry.content).sort()).toEqual(
+      [candidateA, candidateB].sort(),
+    )
+    expect(new Set(innerSeen.map((entry) => entry.scratchPath)).size).toBe(2)
+  })
+
+  it('uses explicit immutable refs so captured runner or judge behavior cannot share resume state', async () => {
+    const config = baseConfig([seat()])
+    const incumbentCommit = await git(['rev-parse', 'HEAD'], driverWt)
+    const makeRunner = (resolved: boolean): SmokeRunner =>
+      async () => smokeVerdict({ resolved })
+    const runnerV1 = makeRunner(false)
+    const runnerV2 = makeRunner(true)
+    const common = {
+      smokeInstanceId: 'astropy__astropy-13033',
+      dispatchTimeoutMs: config.dispatchTimeoutMs,
+      incumbentCommit,
+    }
+    const original = gepaSeatEvaluationId({ ...common, ...IMPLEMENTATION_REFS })
+    const changedRunner = gepaSeatEvaluationId({
+      ...common,
+      ...IMPLEMENTATION_REFS,
+      runnerImplementationRef: `sha256:${'c'.repeat(64)}`,
+    })
+    const changedJudge = gepaSeatEvaluationId({
+      ...common,
+      ...IMPLEMENTATION_REFS,
+      judgeImplementationRef: `sha256:${'d'.repeat(64)}`,
+    })
+    const changedIncumbent = gepaSeatEvaluationId({
+      ...common,
+      ...IMPLEMENTATION_REFS,
+      incumbentCommit: 'e'.repeat(40),
+    })
+    const smokeArgs: Parameters<SmokeRunner>[0] = {
+      scratchPath: driverWt,
+      generation: 0,
+      proposer: seat(),
+      evaluationKey: 'identity-test',
+    }
+
+    expect(runnerV1.toString()).toBe(runnerV2.toString())
+    expect((await runnerV1(smokeArgs)).resolved).toBe(false)
+    expect((await runnerV2(smokeArgs)).resolved).toBe(true)
+    expect(original).toMatch(
+      /^swe-arena-gepa-seat\|smoke=astropy__astropy-13033\|incumbent=[a-f0-9]{40,64}\|runner=sha256:[a-f0-9]{64}\|judge=sha256:[a-f0-9]{64}\|dispatchTimeoutMs=\d+$/,
+    )
+    expect(changedRunner).not.toBe(original)
+    expect(changedJudge).not.toBe(original)
+    expect(changedIncumbent).not.toBe(original)
+    expect(() =>
+      gepaSeatEvaluationId({
+        ...common,
+        ...IMPLEMENTATION_REFS,
+        runnerImplementationRef: 'runner-v2',
+      }),
+    ).toThrow(/runnerImplementationRef must be an immutable sha256 reference/)
+  })
+
+  it('records but rejects a winner when cost accounting is incomplete', async () => {
+    const WINNER = `${SEED}Always run the neighboring test file before finalizing.\n`
+    const incomplete: GepaMethodFactory = () => ({
+      name: 'incomplete-cost',
+      async optimize(input) {
+        await input.dispatchWithSurface(WINNER, input.trainScenarios[0]!, fakeCtx)
+        return {
+          winnerSurface: WINNER,
+          cost: {
+            totalCostUsd: 0.25,
+            accountingComplete: false,
+            incompleteReasons: ['optimizer model receipt missing'],
+          },
+          durationMs: 1,
+          provenance: fullProvenance('incomplete-run', { evaluationCount: 1 }),
+        }
+      },
+    })
+    const gen = fanOutLoopsGenerator(baseConfig([seat()]), {
+      ...IMPLEMENTATION_REFS,
+      smokeRunner: async () => smokeVerdict({ resolved: true }),
+      smokeInstanceId: 'astropy__astropy-13033',
+      scoreSplit: null,
+      gepaMethodFactory: incomplete,
+    })
+
+    await expect(gen.generate(generatorArgs(0))).rejects.toThrow(
+      /cost accounting is incomplete: optimizer model receipt missing/,
+    )
+    expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(SEED)
+    const inner = JSON.parse(
+      await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'),
+    )
+    expect(inner).toMatchObject({
+      runId: 'incomplete-run',
+      totalCostUsd: 0.25,
+      accountingComplete: false,
+      incompleteReasons: ['optimizer model receipt missing'],
+    })
+  })
+
   it('enforces the inner-call budget cap fail-closed', async () => {
     const runaway: GepaMethodFactory = () => ({
       name: 'runaway',
       async optimize(input) {
-        for (let i = 0; i < 4; i++) {
-          await input.dispatchWithSurface(`${SEED}candidate ${i}\n`, input.trainScenarios[0]!, fakeCtx)
-        }
+        const results = await Promise.allSettled(
+          Array.from({ length: 4 }, (_, index) =>
+            input.dispatchWithSurface(
+              `${SEED}candidate ${index}\n`,
+              input.trainScenarios[0]!,
+              fakeCtx,
+            ),
+          ),
+        )
+        const rejected = results.find(
+          (result): result is PromiseRejectedResult => result.status === 'rejected',
+        )
+        if (rejected) throw rejected.reason
         return {
           winnerSurface: SEED,
           cost: { totalCostUsd: 0, accountingComplete: false, incompleteReasons: [] },
           durationMs: 1,
+          provenance: fullProvenance('runaway'),
         }
       },
     })
     const gen = fanOutLoopsGenerator(baseConfig([seat({ maxMetricCalls: 3 })]), {
+      ...IMPLEMENTATION_REFS,
       smokeRunner: async () => smokeVerdict(),
       smokeInstanceId: 'astropy__astropy-13033',
       scoreSplit: null,
@@ -514,6 +892,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
 
   it('refuses to feed a PRIVATE smoke verdict to the bridge', async () => {
     const gen = fanOutLoopsGenerator(baseConfig([seat()]), {
+      ...IMPLEMENTATION_REFS,
       smokeRunner: async () => smokeVerdict({ iid: 'django__django-11532' }),
       smokeInstanceId: 'astropy__astropy-13033',
       scoreSplit: { privateInstances: ['django__django-11532'] },
@@ -524,6 +903,7 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
 
   it('declines the slot without a kill when GEPA returns the seed unchanged', async () => {
     const gen = fanOutLoopsGenerator(baseConfig([seat()]), {
+      ...IMPLEMENTATION_REFS,
       smokeRunner: async () => smokeVerdict(),
       smokeInstanceId: 'astropy__astropy-13033',
       scoreSplit: null,
@@ -538,13 +918,12 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
 })
 
 // ---------------------------------------------------------------------------
-// Integration: ONE real Node→Python→score roundtrip through the installed
-// adapter + bridge. Skips with an exact reason when either runtime is absent
-// (this is the same condition the t=0 provenance capture enforces loud).
+// Integration: one real Node-to-Python-to-score roundtrip through the installed
+// bridge. The TypeScript adapter is a compile-time package dependency.
 // ---------------------------------------------------------------------------
 
-const pythonBridgeReady = (): { ok: boolean; reason: string } => {
-  const probe = spawnSync(DEFAULT_GEPA_PYTHON, [
+const pythonBridgeReady = (python: string): { ok: boolean; reason: string } => {
+  const probe = spawnSync(python, [
     '-c',
     'import agent_eval_rpc.gepa_bridge; from gepa.optimize_anything import optimize_anything, OptimizeAnythingConfig',
   ])
@@ -555,23 +934,44 @@ const pythonBridgeReady = (): { ok: boolean; reason: string } => {
 }
 
 describe('integration: real adapter roundtrip', () => {
-  it('runs one inner smoke roundtrip through gepaOptimizationMethod when the full runtime is installed', async (ctx) => {
-    const campaign = (await import('@tangle-network/agent-eval/campaign')) as Record<string, unknown>
-    const missing: string[] = []
-    if (typeof campaign.gepaOptimizationMethod !== 'function') {
-      missing.push(
-        'installed @tangle-network/agent-eval lacks gepaOptimizationMethod (needs a release after 0.123.5 containing PRs #408/#409)',
-      )
-    }
-    const python = pythonBridgeReady()
-    if (!python.ok) missing.push(python.reason)
-    if (missing.length > 0) {
-      ctx.skip(`skip-with-reason: ${missing.join('; ')}`)
+  it('runs a metered optimizer through the real bridge and applies its winner', async (ctx) => {
+    const python = process.env.AGENT_EVAL_TEST_PYTHON ?? DEFAULT_GEPA_PYTHON
+    const pythonRuntime = pythonBridgeReady(python)
+    if (!pythonRuntime.ok) {
+      ctx.skip(`skip-with-reason: ${pythonRuntime.reason}`)
       return
     }
 
-    // Full runtime present: drive the REAL adapter with a stub smoke runner
-    // (no arm spend) over a synthetic tiny surface in a temp repo.
+    const winner = 'tiny synthetic surface\nAlways run the neighboring test before finalizing.'
+    const modelServer = createServer((_request, response) => {
+      response.writeHead(200, { 'content-type': 'application/json' })
+      response.end(
+        JSON.stringify({
+          choices: [
+            {
+              message: {
+                role: 'assistant',
+                content: `\`\`\`\n${winner}\`\`\``,
+              },
+            },
+          ],
+          usage: {
+            prompt_tokens: 20,
+            completion_tokens: 20,
+            total_tokens: 40,
+          },
+        }),
+      )
+    })
+    await new Promise<void>((resolve, reject) => {
+      modelServer.once('error', reject)
+      modelServer.listen(0, '127.0.0.1', resolve)
+    })
+    const address = modelServer.address()
+    if (!address || typeof address === 'string') {
+      throw new Error('test optimizer server did not bind')
+    }
+
     const loopsRepo = await mkdtemp(join(tmpdir(), 'gepa-int-repo-'))
     const outDir = await mkdtemp(join(tmpdir(), 'gepa-int-out-'))
     try {
@@ -585,24 +985,24 @@ describe('integration: real adapter roundtrip', () => {
       const driverWt = join(outDir, 'driver-wt')
       await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', driverWt, 'HEAD'])
 
-      let innerCalls = 0
       const gen = fanOutLoopsGenerator(
         {
           ...defaultRound4Config(),
           loopsRepo,
           outDir,
           populationSize: 1,
-          proposers: [seat({ maxMetricCalls: 4 })],
+          proposers: [seat({ maxMetricCalls: 10, python })],
           prefilter: { enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false },
         },
         {
-          smokeRunner: async () => {
-            innerCalls += 1
+          ...IMPLEMENTATION_REFS,
+          smokeRunner: async ({ scratchPath }) => {
+            const candidate = await readFile(join(scratchPath, SURFACE), 'utf8')
             return {
               iid: 'astropy__astropy-13033',
               pass: true,
               reason: 'stub smoke (integration)',
-              resolved: innerCalls > 1,
+              resolved: candidate === winner,
               patchLines: 1,
               wallS: 0,
               verifyPass: true,
@@ -610,6 +1010,24 @@ describe('integration: real adapter roundtrip', () => {
           },
           smokeInstanceId: 'astropy__astropy-13033',
           scoreSplit: null,
+          gepaOptimizer: {
+            model: 'test-optimizer',
+            baseUrl: `http://127.0.0.1:${address.port}/v1`,
+            apiKey: 'local-test-key',
+            budget: {
+              maxCostUsd: 1,
+              maxRequests: 10,
+              maxRequestBytes: 100_000,
+              maxResponseBytes: 100_000,
+              maxOutputTokensPerRequest: 2_000,
+              pricing: {
+                inputUsdPerMillion: 1,
+                cachedInputUsdPerMillion: 0.1,
+                cacheWriteUsdPerMillion: 1.25,
+                outputUsdPerMillion: 5,
+              },
+            },
+          },
         },
       )
       const result = await gen.generate({
@@ -621,13 +1039,19 @@ describe('integration: real adapter roundtrip', () => {
         generation: 0,
         candidateIndex: 0,
       })
-      // The bridge ran: inner provenance must show >= 1 scored callback.
       const inner = JSON.parse(
         await readFile(join(outDir, 'gepa-seat', 'gen0-gepa-author', 'inner-provenance.json'), 'utf8'),
       )
+      expect(result.applied, `${result.summary}\n${JSON.stringify(inner.innerScores, null, 2)}`).toBe(true)
+      expect(await readFile(join(driverWt, SURFACE), 'utf8')).toBe(winner)
       expect(inner.innerCallCount).toBeGreaterThanOrEqual(1)
-      expect(typeof result.applied).toBe('boolean')
+      expect(inner.accountingComplete).toBe(true)
+      expect(inner.incompleteReasons).toEqual([])
+      expect(inner.tokenUsage.calls).toBeGreaterThan(0)
     } finally {
+      await new Promise<void>((resolve, reject) =>
+        modelServer.close((error) => (error ? reject(error) : resolve())),
+      )
       await rm(outDir, { recursive: true, force: true })
       await rm(loopsRepo, { recursive: true, force: true })
     }