@tangle-network/agent-bench 0.3.8 → 0.4.0

package/src/swe-arena/proposer-provenance.mts

@@ -27,21 +27,19 @@ import { join } from 'node:path'
 import {
   DEFAULT_GEPA_PYTHON,
   isGepaSeat,
-  loadGepaMethodFactory,
   probeGepaRuntime,
-  type CampaignModuleImport,
 } from './gepa-seat.mts'
 import { run } from './proc.ts'
 import type { ProposerSpec } from './proposer-fanout.mts'
 
 export interface ProposerModelProvenance {
   name: string
-  /** Absent on a GEN-6 engine seat (see `engine`). */
+  /** Absent on an engine seat (see `engine`). */
   harness: ProposerSpec['harness']
   /** Explicit model pin from the spec (threaded as `-m`), or null when the
    *  seat runs the CLI's own resolved default. */
   pinnedModel: string | null
-  /** `<harness> --version` stdout (trimmed). For a GEN-6 gepa seat this is
+  /** `<harness> --version` stdout (trimmed). For a GEPA seat this is
    *  the bridge python's `--version` output — the runtime that authors. */
   harnessVersion: string
   /** claude seats only: the settings default model the logged-in CLI resolves
@@ -51,13 +49,13 @@ export interface ProposerModelProvenance {
   /** codex seats only: `codex login status` stdout (trimmed). */
   authStatus: string | null
   merge: boolean
-  /** GEN-6 gepa seat: the engine name from the spec. */
+  /** GEPA seat: the engine name from the spec. */
   engine?: 'gepa' | 'omni'
-  /** GEN-6 gepa seat: the ONE change-space file GEPA optimizes. */
+  /** GEPA seat: the one change-space file GEPA optimizes. */
   surface?: string
-  /** GEN-6 gepa seat: installed gepa version ('source' for a source pin). */
+  /** GEPA seat: installed GEPA version ('source' for a source pin). */
   gepaVersion?: string
-  /** GEN-6 gepa seat: the Python bridge module the seat runs. */
+  /** GEPA seat: the Python bridge module the seat runs. */
   bridge?: string
 }
 
@@ -92,17 +90,16 @@ export function claudeSettingsModel(
 }
 
 /** Capture per-proposer model provenance. Throws when any configured harness
- *  binary is missing/broken, when a codex seat is not logged in, or — GEN-6 —
- *  when a gepa seat's runtime is incomplete: the installed agent-eval must
- *  export `gepaOptimizationMethod` and the Python bridge + GEPA engine must
- *  import (probeGepaRuntime carries the exact install instructions). A dead
- *  seat fails the launch at t=0, never a mid-run candidate slot. */
+ *  binary is missing/broken, when a codex seat is not logged in, or when
+ *  a GEPA seat's Python bridge or engine cannot import
+ *  (`probeGepaRuntime` carries the exact install instructions). A dead seat
+ *  fails the launch at t=0, never a mid-run candidate slot. The TypeScript
+ *  adapter is a pinned package dependency and therefore checked at install. */
 export async function captureProposerProvenance(
   proposers: ProposerSpec[],
   deps: {
     exec?: VersionExec
     readSettingsModel?: () => string | null
-    importCampaign?: CampaignModuleImport
   } = {},
 ): Promise<ProvenanceCaptureRecord> {
   const exec = deps.exec ?? defaultExec
@@ -112,8 +109,6 @@ export async function captureProposerProvenance(
   const gepaBySeat = new Map<string, { pythonVersion: string; gepaVersion: string }>()
   const gepaSeats = proposers.filter(isGepaSeat)
   if (gepaSeats.length > 0) {
-    // Node side first: the adapter export (fails loud with upgrade hint).
-    await loadGepaMethodFactory(...(deps.importCampaign ? [deps.importCampaign] : []))
     for (const seat of gepaSeats) {
       gepaBySeat.set(seat.name, await probeGepaRuntime(seat.python ?? DEFAULT_GEPA_PYTHON, exec, seat.name))
     }

package/src/swe-arena/scratch-worktree.test.mts

@@ -0,0 +1,55 @@
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { runOk } from './proc.ts'
+import {
+  createDetachedWorktree,
+  pruneDetachedWorktrees,
+  removeDetachedWorktree,
+} from './scratch-worktree.ts'
+
+describe('scratch worktrees', () => {
+  const roots: string[] = []
+
+  afterEach(async () => {
+    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
+  })
+
+  it('creates and removes a parallel batch without pruning live metadata', async () => {
+    const repository = await mkdtemp(join(tmpdir(), 'scratch-worktree-repo-'))
+    const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-'))
+    roots.push(repository, output)
+    await runOk('git', ['init', '-q', '-b', 'main', repository])
+    await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com'])
+    await runOk('git', ['-C', repository, 'config', 'user.name', 'Test'])
+    await writeFile(join(repository, 'seed.txt'), 'seed\n')
+    await runOk('git', ['-C', repository, 'add', 'seed.txt'])
+    await runOk('git', ['-C', repository, 'commit', '-q', '-m', 'seed'])
+    const commit = (
+      await runOk('git', ['-C', repository, 'rev-parse', 'HEAD'])
+    ).stdout.trim()
+    for (let batch = 0; batch < 5; batch += 1) {
+      const worktrees = Array.from(
+        { length: 8 },
+        (_, index) => join(output, `batch-${batch}-candidate-${index}`),
+      )
+      await pruneDetachedWorktrees(repository)
+      await Promise.all(
+        worktrees.map((worktree) => createDetachedWorktree(repository, commit, worktree)),
+      )
+      expect(
+        await Promise.all(
+          worktrees.map((worktree) => readFile(join(worktree, 'seed.txt'), 'utf8')),
+        ),
+      ).toEqual(Array.from({ length: 8 }, () => 'seed\n'))
+      await Promise.all(
+        worktrees.map((worktree) => removeDetachedWorktree(repository, worktree)),
+      )
+    }
+    const listed = (
+      await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain'])
+    ).stdout
+    expect(listed.match(/^worktree /gmu)).toHaveLength(1)
+  })
+})

package/src/swe-arena/scratch-worktree.ts

@@ -0,0 +1,34 @@
+import { rm } from 'node:fs/promises'
+import { run, runOk } from './proc'
+
+export async function createDetachedWorktree(
+  repository: string,
+  commit: string,
+  destination: string,
+): Promise<void> {
+  await run('git', ['-C', repository, 'worktree', 'remove', '--force', '--', destination])
+  await rm(destination, { recursive: true, force: true })
+  await runOk('git', ['-C', repository, 'worktree', 'add', '--detach', destination, commit])
+}
+
+export async function pruneDetachedWorktrees(repository: string): Promise<void> {
+  await runOk('git', ['-C', repository, 'worktree', 'prune'])
+}
+
+export async function removeDetachedWorktree(
+  repository: string,
+  destination: string,
+): Promise<void> {
+  const result = await run('git', [
+    '-C',
+    repository,
+    'worktree',
+    'remove',
+    '--force',
+    '--',
+    destination,
+  ])
+  if (result.code !== 0) {
+    await rm(destination, { recursive: true, force: true })
+  }
+}

package/src/swe-code-improve.mts

@@ -33,7 +33,6 @@ import { spawn, spawnSync } from 'node:child_process'
 import { existsSync, mkdirSync, symlinkSync } from 'node:fs'
 import { join } from 'node:path'
 import { improve, agenticGenerator } from '@tangle-network/agent-runtime'
-import type { AgentProfile } from '@tangle-network/agent-interface'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
 import { createSweBenchAdapter } from './benchmarks/swe-bench'
 import type { BenchTask } from './benchmarks/types'
@@ -108,7 +107,6 @@ async function main(): Promise<void> {
   if (!routerKey) throw new Error('TANGLE_API_KEY required (worker calls the router)')
   const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
   const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6'
-  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
   const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931,pallets__flask-5014').split(',').map((s) => s.trim()).filter(Boolean)
   const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142,psf__requests-1921').split(',').map((s) => s.trim()).filter(Boolean)
   const generations = Number(process.env.GENERATIONS ?? 1)
@@ -127,7 +125,7 @@ async function main(): Promise<void> {
   const allIds = [...new Set([...trainIds, ...holdoutIds])]
 
   console.log('=== META-HARNESS on the SWE scaffold — improve(surface:code) ===')
-  console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl} runTool=${enableRun}`)
+  console.log(`worker=${workerModel} router=${routerBaseUrl} runTool=${enableRun}`)
   console.log(`train=[${trainIds.join(', ')}] holdout=[${holdoutIds.join(', ')}]`)
   console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} maxTokens=${maxTokens}`)
   console.log(`repoRoot=${REPO_ROOT} baseRef=${baseRef}`)
@@ -156,15 +154,25 @@ async function main(): Promise<void> {
     const worktreeRef = isCode ? String((surface as { worktreeRef?: string }).worktreeRef ?? '') : ''
     const rootDir = isCode && worktreeRef && existsSync(worktreeRef) ? worktreeRef : SWE_MAIN_ROOT
     const t0 = Date.now()
-    const r = await runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs)
+    const paid = await ctx.cost.runPaidCall({
+      channel: 'agent',
+      actor: 'swe-scaffold-worker',
+      model: workerModel,
+      execute: () => runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs),
+      receipt: (result) => {
+        const usageUnknown = result.tokIn === 0 && result.tokOut === 0
+        return {
+          model: workerModel,
+          inputTokens: result.tokIn,
+          outputTokens: result.tokOut,
+          ...(result.usd > 0 ? { actualCostUsd: result.usd } : {}),
+          ...(usageUnknown ? { usageUnknown: true, costUnknown: result.usd <= 0 } : {}),
+        }
+      },
+    })
+    if (!paid.succeeded) throw paid.error
+    const r = paid.value
     const hasPatch = r.patch.trim().length > 0
-    // Report real usage; floor a patch-bearing zero-usage cell so the stub-guard cannot abort on a
-    // router telemetry gap (lift is judge-derived, so this only affects cost accounting).
-    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
-    ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
-    ctx.cost.observeTokens(
-      zeroUsage && hasPatch ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
-    )
     const files = hasPatch ? [...r.patch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
     console.log(
       `  [measure] ${isCode ? 'cand' : 'base'} ${scenario.id} patch=${r.patch.length}b files=[${files.join(', ') || 'none'}] ` +
@@ -173,7 +181,7 @@ async function main(): Promise<void> {
     return hasPatch ? r.patch : null
   }
 
-  const judge: JudgeConfig<string, Scenario> = {
+  const judge: JudgeConfig<string | null, Scenario> = {
     name: 'swebench-docker',
     dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }],
     async score({ artifact, scenario }) {
@@ -286,11 +294,10 @@ async function main(): Promise<void> {
     runHarness: runHarness as any,
   })
 
-  const profile: AgentProfile = { name: 'swe-scaffold', prompt: { systemPrompt: '' } }
   const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
   const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
 
-  const out = await improve(profile, [], {
+  const out = await improve({
     surface: 'code',
     gate: 'holdout',
     code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, generator },
@@ -301,25 +308,17 @@ async function main(): Promise<void> {
     agent,
     expectUsage: 'warn',
     budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency: 1, reps: 1 },
-    llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel },
   })
 
   console.log('\n=== RESULT ===')
-  console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`)
+  console.log(`decision=${out.decision} lift=${out.lift}`)
   console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`)
   console.log(`winner   holdout composite = ${out.raw.winner.compositeMean}`)
   console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
   console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
   if (out.raw.winner.label) console.log(`winner label: ${out.raw.winner.label}`)
-  if (out.raw.winner.summary) console.log(`winner summary: ${out.raw.winner.summary}`)
-  for (const gen of out.raw.generations ?? []) {
-    console.log(`\n-- generation ${gen.record.generationIndex} candidates --`)
-    for (const c of gen.record.candidates) {
-      const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? []
-      const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ')
-      console.log(`  candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''} [${detail}]`)
-    }
-  }
+  if (out.raw.winner.rationale) console.log(`winner rationale: ${out.raw.winner.rationale}`)
+  console.log(`generations explored: ${out.generationsExplored ?? 0}`)
 }
 
 main().catch((e) => {

package/src/swe-improve.mts

@@ -1,10 +1,9 @@
 /**
- * SELF-IMPROVEMENT on the SEE-able LOCAL SWE-bench path — NO tangle sandbox.
+ * Official GEPA prompt optimization on the local SWE-bench path.
  *
- * Composes the three proven pieces into ONE held-out-gated improvement generation:
- *   1. `improve({ surface: 'prompt' })` (agent-runtime) drives the loop: it asks
- *      `gepaProposer` to EVOLVE the SWE agent's system prompt, then measures each
- *      candidate prompt on real instances and gates the winner on a held-out split.
+ * Composes three pieces:
+ *   1. `improve({ method: officialGepa(...) })` runs GEPA's upstream
+ *      Optimize Anything engine on explicit train and selection partitions.
  *   2. Per candidate + scenario, the `agent` fn runs the LOCAL SWE env
  *      (`createSweBenchEnvironment` + `runAgentic`): clone the instance repo to a
  *      host tmpdir, run the jailed list/read/edit tool loop with the CANDIDATE
@@ -16,23 +15,33 @@
  * IN-LOOP score is a cheap patch-exists proxy (NOT the Docker judge) so the ONLY
  * Docker run per cell is the improve judge — one deterministic verdict per cell.
  *
- * Cost per run = T·(1 + G·P) + 2·H cells, each = 1 clone + 1 runAgentic + 1 judge.
- *
- *   TANGLE_API_KEY=… dotenvx run -f …/agent-state.env -- \
- *     TRAIN_IDS=psf__requests-2931 HOLDOUT_IDS=psf__requests-1142 \
- *     GENERATIONS=1 POPULATION=1 WORKER_MODEL=glm-4.6 REFLECT_MODEL=glm-4.6 \
+ *   TANGLE_API_KEY=... dotenvx run -f .../agent-state.env -- \
+ *     TRAIN_IDS=psf__requests-2931 SELECTION_IDS=pallets__flask-5014 \
+ *     TEST_IDS=psf__requests-1142,psf__requests-1921 \
+ *     MAX_EVALUATIONS=4 MAX_PROPOSER_COST_USD=2 \
  *     node_modules/.bin/tsx bench/src/swe-improve.mts
  */
 import { execFile } from 'node:child_process'
 import { promisify } from 'node:util'
-import { improve } from '@tangle-network/agent-runtime'
-import type { AgentProfile } from '@tangle-network/agent-interface'
+import {
+  improve,
+  officialGepa,
+  type ReadonlyAgentProfile,
+} from '@tangle-network/agent-runtime'
+import {
+  canonicalCandidateDigest,
+  type AgentProfile,
+} from '@tangle-network/agent-interface'
 import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops'
 import { refine, runAgentic } from '@tangle-network/agent-runtime/loops'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
-import { gepaProposer } from '@tangle-network/agent-eval/campaign'
 import { createSweBenchAdapter } from './benchmarks/swe-bench'
 import type { BenchTask } from './benchmarks/types'
+import {
+  assertCompleteCost,
+  officialOptimizerModel,
+  requiredTokenPricing,
+} from './official-optimizer-config.mjs'
 import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env'
 
 const exec = promisify(execFile)
@@ -42,36 +51,47 @@ async function main(): Promise<void> {
   if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker + reflection call the router)')
   const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
   const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6'
+  const reflectBase = process.env.REFLECT_BASE ?? routerBaseUrl
+  const reflectKey = process.env.REFLECT_KEY ?? routerKey
   const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
   const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931').split(',').map((s) => s.trim()).filter(Boolean)
-  const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142').split(',').map((s) => s.trim()).filter(Boolean)
-  const generations = Number(process.env.GENERATIONS ?? 1)
-  const population = Number(process.env.POPULATION ?? 1)
+  const selectionIds = (process.env.SELECTION_IDS ?? 'pallets__flask-5014').split(',').map((s) => s.trim()).filter(Boolean)
+  const testIds = (process.env.TEST_IDS ?? 'psf__requests-1142,psf__requests-1921').split(',').map((s) => s.trim()).filter(Boolean)
+  const maxEvaluations = Number(process.env.MAX_EVALUATIONS ?? 4)
+  const maxProposerCostUsd = Number(process.env.MAX_PROPOSER_COST_USD ?? 2)
   const innerTurns = Number(process.env.INNER_TURNS ?? 40)
   const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 8000)
   const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 12000)
   const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 1)
   const budgetShots = Number(process.env.BUDGET ?? 1)
+  const runDir = process.env.RUN_DIR ?? '.runs/swe-official-gepa'
   // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool AND swaps the seed to the run-aware prompt.
   // Default OFF ⇒ reproduces the read/edit-only baseline denominator unchanged.
   const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
   const SEED_PROMPT = enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT
+  const allIds = [...new Set([...trainIds, ...selectionIds, ...testIds])]
 
-  const allIds = [...new Set([...trainIds, ...holdoutIds])]
-  const cellsMax = trainIds.length * (1 + generations * population) + 2 * holdoutIds.length
-
-  console.log('═══ SWE-bench self-improvement — SEE-able LOCAL (no tangle sandbox) ═══')
-  console.log(`worker=${workerModel}  reflect=${reflectModel}  router=${routerBaseUrl}`)
-  console.log(`train=[${trainIds.join(', ')}]  holdout=[${holdoutIds.join(', ')}]`)
-  console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} workerMaxTokens=${workerMaxTokens} reflectMaxTokens=${reflectMaxTokens} runTool=${enableRun}`)
-  console.log(`≈ ${cellsMax} cells max (each = 1 clone + 1 runAgentic + 1 Docker judge)\n`)
+  console.log('=== SWE-bench prompt optimization with official GEPA ===')
+  console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl}`)
+  console.log(`train=[${trainIds.join(', ')}] selection=[${selectionIds.join(', ')}] test=[${testIds.join(', ')}]`)
+  console.log(`maxEvaluations=${maxEvaluations} maxProposerCostUsd=${maxProposerCostUsd} innerTurns=${innerTurns} workerMaxTokens=${workerMaxTokens} runTool=${enableRun}`)
+  console.log(`runDir=${runDir}\n`)
 
   if (process.env.DRYRUN) {
     // Import + wiring smoke: prove every module resolves and the plan is well-formed
     // WITHOUT paying for a clone / model call / Docker judge.
-    console.log(`DRYRUN: imports OK (improve=${typeof improve}, gepaProposer=${typeof gepaProposer}, runAgentic=${typeof runAgentic}, refine=${typeof refine})`)
+    console.log(`DRYRUN: imports OK (improve=${typeof improve}, officialGepa=${typeof officialGepa}, runAgentic=${typeof runAgentic}, refine=${typeof refine})`)
     return
   }
+  const workerPricing = requiredTokenPricing(process.env, 'WORKER')
+  const optimizer = officialOptimizerModel({
+    env: process.env,
+    model: reflectModel,
+    baseUrl: reflectBase,
+    apiKey: reflectKey,
+    maxCostUsd: maxProposerCostUsd,
+    maxOutputTokensPerRequest: reflectMaxTokens,
+  })
 
   const { environment, adapter } = await createSweBenchEnvironment(allIds.length, { ids: allIds, enableRun })
   const pool = await adapter.loadTasks({ ids: allIds, split: 'test' })
@@ -82,8 +102,9 @@ async function main(): Promise<void> {
   // one instance, return the git-diff patch. A per-call proxy captures the patch in
   // score() BEFORE runAgentic closes (rm) the workspace; its score is a cheap
   // patch-exists proxy so the ONLY Docker run per cell is the improve judge.
-  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
-    const promptText = String(surface)
+  const agent = async (candidate: ReadonlyAgentProfile, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const promptText = candidate.prompt?.systemPrompt
+    if (promptText === undefined) throw new Error('agent: candidate profile has no system prompt')
     const bt = byId.get(scenario.id)
     if (!bt) throw new Error(`agent: unknown scenario ${scenario.id}`)
     const task = { id: bt.id, systemPrompt: promptText, userPrompt: bt.prompt, meta: { instanceId: bt.id } }
@@ -111,40 +132,44 @@ async function main(): Promise<void> {
       },
     }
     const t0 = Date.now()
-    const r = await runAgentic({
-      surface: proxy,
-      task,
-      strategy: refine,
-      routerBaseUrl,
-      routerKey,
+    const paid = await ctx.cost.runPaidCall({
+      channel: 'agent',
+      actor: 'swe-worker',
       model: workerModel,
-      maxTokens: workerMaxTokens,
-      innerTurns,
-      budget: budgetShots,
+      execute: () =>
+        runAgentic({
+          surface: proxy,
+          task,
+          strategy: refine,
+          routerBaseUrl,
+          routerKey,
+          model: workerModel,
+          maxTokens: workerMaxTokens,
+          innerTurns,
+          budget: budgetShots,
+        }),
+      receipt: (result) => {
+        const inputTokens = result.tokens.input ?? 0
+        const outputTokens = result.tokens.output ?? 0
+        const usageUnknown = inputTokens === 0 && outputTokens === 0
+        return {
+          model: workerModel,
+          inputTokens,
+          outputTokens,
+          customTokenPricing: workerPricing,
+          ...(usageUnknown ? { usageUnknown: true } : {}),
+        }
+      },
     })
-    // Report REAL cost/tokens so the backend-integrity guard sees a real backend
-    // rather than a silent-zero stub. A glm-5.2 turn occasionally returns a real
-    // patch with an UNPOPULATED usage block (a router telemetry gap on some
-    // reasoning-model responses — NOT a stub: the cell made real tool calls and
-    // produced a patch). In that gap case report a nominal floor so the stub-guard
-    // (artifact + zero usage) cannot abort the whole campaign on a telemetry gap.
-    // The lift metric is judge-derived, so a floored count does not distort it; only
-    // cost accounting undercounts those few cells (disclosed). No-patch cells return
-    // null below and are skipped by the guard's own contract, so this floor only
-    // ever applies to a cell that genuinely produced a patch.
+    if (!paid.succeeded) throw paid.error
+    const r = paid.value
     const zeroUsage = (r.tokens.input ?? 0) === 0 && (r.tokens.output ?? 0) === 0
     const hasPatch = capturedPatch.trim().length > 0
-    ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd ?? 0, 0.0001) : r.usd ?? 0, workerModel)
-    ctx.cost.observeTokens(
-      zeroUsage && hasPatch
-        ? { input: Math.max(r.tokens.input ?? 0, 1), output: Math.max(r.tokens.output ?? 0, 1) }
-        : { input: r.tokens.input, output: r.tokens.output },
-    )
     const files = capturedPatch ? [...capturedPatch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
     console.log(
       `  [agent] ${scenario.id} prompt=${promptText.length}c tools(l/r/e+/e-/run/run!)=${stats.list}/${stats.read}/${stats.edit_ok}/${stats.edit_fail}/${stats.run}/${stats.run_err} ` +
-        `patch=${capturedPatch.length}b files=[${files.join(', ') || 'none'}] tok=in:${r.tokens.input}/out:${r.tokens.output} usd=${r.usd} ${Math.round((Date.now() - t0) / 1000)}s` +
-        `${zeroUsage ? (hasPatch ? ' [zero-usage telemetry gap: patch kept, usage floored]' : ' [zero-usage cell: empty completion — scored as no-patch]') : ''}`,
+        `patch=${capturedPatch.length}b files=[${files.join(', ') || 'none'}] tok=in:${r.tokens.input}/out:${r.tokens.output} ${Math.round((Date.now() - t0) / 1000)}s` +
+        `${zeroUsage ? ' [provider usage unavailable]' : ''}`,
     )
     // A cell with no patch produced NO artifact. Return null (not '') so the
     // backend-integrity guard's own contract (`artifact == null → skip`) applies:
@@ -156,7 +181,7 @@ async function main(): Promise<void> {
 
   // The judge: the OFFICIAL swebench Docker harness. Deterministic FAIL_TO_PASS +
   // PASS_TO_PASS → resolved 0/1. This is the held-out gate's scoring axis.
-  const judge: JudgeConfig<string, Scenario> = {
+  const judge: JudgeConfig<string | null, Scenario> = {
     name: 'swebench-docker',
     dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }],
     async score({ artifact, scenario }) {
@@ -177,53 +202,61 @@ async function main(): Promise<void> {
   }
 
   const profile: AgentProfile = { name: 'swe-agent-glm46', prompt: { systemPrompt: SEED_PROMPT } }
-  const proposer = gepaProposer({
-    llm: { baseUrl: routerBaseUrl, apiKey: routerKey },
-    model: reflectModel,
-    target: 'the system prompt of a coding agent that fixes real GitHub bugs via list_files/read_file/edit_file tools',
-    maxTokens: reflectMaxTokens,
-    temperature: 0.7,
-  })
-
-  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
-  const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
+  const scenario = (id: string): Scenario => ({ id, kind: 'swe-bench-verified' })
 
-  const out = await improve(profile, [], {
+  const out = await improve(profile, {
     surface: 'prompt',
-    gate: 'holdout',
-    generator: proposer,
-    scenarios,
-    judge,
+    executionRef: canonicalCandidateDigest({
+      callback: 'bench/swe-improve',
+      model: workerModel,
+      endpoint: new URL(routerBaseUrl).origin,
+      innerTurns,
+      maxTokens: workerMaxTokens,
+      budgetShots,
+      enableRun,
+    }),
+    method: officialGepa<Scenario, string | null>({
+      objective:
+        'Improve the system prompt of a coding agent that fixes real GitHub bugs with list_files, read_file, edit_file, and optional run tools.',
+      background:
+        'Return the complete system prompt. Preserve tool names and require evidence from repository files and tests.',
+      recipe: {
+        kind: 'engine',
+        run: {
+          engine: 'gepa',
+          maxEvaluations,
+          maxProposerCostUsd,
+        },
+      },
+      optimizer,
+      resume: 'if-compatible',
+      trustResumeState: true,
+      describeScenario: (item) => ({ prompt: byId.get(item.id)?.prompt ?? item.id }),
+    }),
+    trainScenarios: trainIds.map(scenario),
+    selectionScenarios: selectionIds.map(scenario),
+    testScenarios: testIds.map(scenario),
+    judges: [judge],
     agent,
-    // glm-5.2 occasionally returns a real patch with an unpopulated usage block
-    // (a router telemetry gap on some reasoning-model responses — NOT a stub: the
-    // cell made real tool calls and produced a patch). 'assert' would abort the
-    // whole campaign on such a cell; 'warn' logs it and continues. The lift metric
-    // (resolved) is judge-derived, so a missing token count does not distort it —
-    // only the cost accounting undercounts those cells, which is disclosed.
     expectUsage: 'warn',
-    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
-    llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel },
+    maxConcurrency,
+    reps: 1,
+    runDir,
+    optimizationRunOptions: {
+      expectUsage: 'warn',
+      maxConcurrency,
+      reps: 1,
+    },
   })
 
-  console.log('\n═══ RESULT ═══')
-  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
-  console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`)
-  console.log(`winner   holdout composite = ${out.raw.winner.compositeMean}`)
-  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
-  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
-  if (out.raw.winner.label) console.log(`winner label   : ${out.raw.winner.label}`)
-  if (out.raw.winner.rationale) console.log(`winner rationale: ${out.raw.winner.rationale}`)
-
-  // Per-candidate verdicts on the train set (the "real swebench verdict per candidate").
-  for (const gen of out.raw.generations ?? []) {
-    console.log(`\n── generation ${gen.record.generationIndex} candidates ──`)
-    for (const c of gen.record.candidates) {
-      const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? []
-      const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ')
-      console.log(`  candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''}  [${detail}]`)
-    }
-  }
+  assertCompleteCost('SWE-bench official GEPA run', out.cost)
+  console.log('\n=== RESULT ===')
+  console.log(`decision=${out.decision} lift=${out.lift} interval=[${out.liftInterval.low}, ${out.liftInterval.high}]`)
+  console.log(`baseline test composite=${out.raw.best.baselineComposite}`)
+  console.log(`winner test composite=${out.raw.best.winnerComposite}`)
+  console.log(`test scenarios=${JSON.stringify(out.raw.best.scenarioScores)}`)
+  console.log(`cost=${JSON.stringify(out.cost)}`)
+  console.log(`candidate prompt:\n${String(out.candidate.value).slice(0, 2000)}`)
 }
 
 main().catch((e) => {

package/src/swe-local-proof.mts

@@ -46,6 +46,9 @@ async function main(): Promise<void> {
 
   const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun })
   const taskList = await tasks(0, ids.length)
+  const benchTaskById = new Map(
+    (await adapter.loadTasks({ ids, split: 'test' })).map((task) => [task.id, task]),
+  )
 
   // One shot per pinned id: proxy score() to capture the exact judged bytes + a NON-DESTRUCTIVE
   // apply-coherence check, then delegate the verdict to the real Docker judge.
@@ -123,7 +126,9 @@ async function main(): Promise<void> {
       // Cached judge: identical patch ⇒ identical verdict; don't pay for a second Docker run.
       let s = judged.get(effPatch)
       if (!s) {
-        s = await adapter.judge(task, effPatch)
+        const benchTask = benchTaskById.get(task.id)
+        if (!benchTask) throw new Error(`swe-local-proof: unknown benchmark task ${task.id}`)
+        s = await adapter.judge(benchTask, effPatch)
         judged.set(effPatch, s)
       }
       rec.score = s

package/src/swe-stream.mts

@@ -700,6 +700,8 @@ async function acquireRepro(
 // hands the worker a no-code plan. Primitives replicated from supervisor-arena.mts (that file runs
 // main() on import, so it cannot be imported) — evidence from execution-verified inputs only. ----------
 
+type RepairFailureKind = 'wrong-fix' | 'apply-failed' | 'empty-diff'
+
 /** Evidence for the supervisor: the issue, the worker's own candidate diff, and the tail of the
  *  gold-verified reproduction's output on that diff. Execution-verified / model-visible ONLY — never
  *  FAIL_TO_PASS, never the gold patch, never any worker self-report. Bounded to maxChars. */
@@ -709,7 +711,7 @@ function renderRepairEvidence(
   reproTail: string,
   reproExit: number | null,
   maxChars: number,
-  failureKind: 'wrong-fix' | 'apply-failed' = 'wrong-fix',
+  failureKind: RepairFailureKind = 'wrong-fix',
 ): string {
   const header =
     failureKind === 'apply-failed'
@@ -789,7 +791,7 @@ async function superviseRepair(
   reproExit: number | null,
   marks: readonly string[],
   deadlineAt: number,
-  failureKind: 'wrong-fix' | 'apply-failed' = 'wrong-fix',
+  failureKind: RepairFailureKind = 'wrong-fix',
 ): Promise<SupervisorPlanReceipt> {
   const md = bt.metadata as Record<string, string>
   const evidence = renderRepairEvidence(String(md.problem_statement ?? ''), candidateDiff, reproTail, reproExit, 14_000, failureKind)