@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-improve.mts

@@ -0,0 +1,232 @@
+/**
+ * SELF-IMPROVEMENT on the SEE-able LOCAL SWE-bench path — NO tangle sandbox.
+ *
+ * Composes the three proven pieces into ONE held-out-gated improvement generation:
+ *   1. `improve({ surface: 'prompt' })` (agent-runtime) drives the loop: it asks
+ *      `gepaProposer` to EVOLVE the SWE agent's system prompt, then measures each
+ *      candidate prompt on real instances and gates the winner on a held-out split.
+ *   2. Per candidate + scenario, the `agent` fn runs the LOCAL SWE env
+ *      (`createSweBenchEnvironment` + `runAgentic`): clone the instance repo to a
+ *      host tmpdir, run the jailed list/read/edit tool loop with the CANDIDATE
+ *      prompt as the system prompt, and return the `git diff` as the artifact.
+ *   3. The `judge` scores that patch with the OFFICIAL swebench Docker harness
+ *      (`adapter.judge` → resolved 0/1). The only remote call is the model
+ *      completion via the router; nothing touches sandbox.tangle.tools.
+ *
+ * IN-LOOP score is a cheap patch-exists proxy (NOT the Docker judge) so the ONLY
+ * Docker run per cell is the improve judge — one deterministic verdict per cell.
+ *
+ * Cost per run = T·(1 + G·P) + 2·H cells, each = 1 clone + 1 runAgentic + 1 judge.
+ *
+ *   TANGLE_API_KEY=… dotenvx run -f …/agent-state.env -- \
+ *     TRAIN_IDS=psf__requests-2931 HOLDOUT_IDS=psf__requests-1142 \
+ *     GENERATIONS=1 POPULATION=1 WORKER_MODEL=glm-4.6 REFLECT_MODEL=glm-4.6 \
+ *     node_modules/.bin/tsx bench/src/swe-improve.mts
+ */
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import { improve } from '@tangle-network/agent-runtime'
+import type { AgentProfile } from '@tangle-network/agent-interface'
+import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops'
+import { refine, runAgentic } from '@tangle-network/agent-runtime/loops'
+import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
+import { gepaProposer } from '@tangle-network/agent-eval/campaign'
+import { createSweBenchAdapter } from './benchmarks/swe-bench'
+import type { BenchTask } from './benchmarks/types'
+import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env'
+
+const exec = promisify(execFile)
+
+async function main(): Promise<void> {
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker + reflection call the router)')
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6'
+  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
+  const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931').split(',').map((s) => s.trim()).filter(Boolean)
+  const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142').split(',').map((s) => s.trim()).filter(Boolean)
+  const generations = Number(process.env.GENERATIONS ?? 1)
+  const population = Number(process.env.POPULATION ?? 1)
+  const innerTurns = Number(process.env.INNER_TURNS ?? 40)
+  const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 8000)
+  const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 12000)
+  const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 1)
+  const budgetShots = Number(process.env.BUDGET ?? 1)
+  // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool AND swaps the seed to the run-aware prompt.
+  // Default OFF ⇒ reproduces the read/edit-only baseline denominator unchanged.
+  const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+  const SEED_PROMPT = enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT
+
+  const allIds = [...new Set([...trainIds, ...holdoutIds])]
+  const cellsMax = trainIds.length * (1 + generations * population) + 2 * holdoutIds.length
+
+  console.log('═══ SWE-bench self-improvement — SEE-able LOCAL (no tangle sandbox) ═══')
+  console.log(`worker=${workerModel}  reflect=${reflectModel}  router=${routerBaseUrl}`)
+  console.log(`train=[${trainIds.join(', ')}]  holdout=[${holdoutIds.join(', ')}]`)
+  console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} workerMaxTokens=${workerMaxTokens} reflectMaxTokens=${reflectMaxTokens} runTool=${enableRun}`)
+  console.log(`≈ ${cellsMax} cells max (each = 1 clone + 1 runAgentic + 1 Docker judge)\n`)
+
+  if (process.env.DRYRUN) {
+    // Import + wiring smoke: prove every module resolves and the plan is well-formed
+    // WITHOUT paying for a clone / model call / Docker judge.
+    console.log(`DRYRUN: imports OK (improve=${typeof improve}, gepaProposer=${typeof gepaProposer}, runAgentic=${typeof runAgentic}, refine=${typeof refine})`)
+    return
+  }
+
+  const { environment, adapter } = await createSweBenchEnvironment(allIds.length, { ids: allIds, enableRun })
+  const pool = await adapter.loadTasks({ ids: allIds, split: 'test' })
+  const byId = new Map<string, BenchTask>(pool.map((t) => [t.id, t]))
+  for (const id of allIds) if (!byId.has(id)) throw new Error(`instance not found in Verified: ${id}`)
+
+  // The agent under improvement: run the LOCAL SWE env with the CANDIDATE prompt on
+  // one instance, return the git-diff patch. A per-call proxy captures the patch in
+  // score() BEFORE runAgentic closes (rm) the workspace; its score is a cheap
+  // patch-exists proxy so the ONLY Docker run per cell is the improve judge.
+  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const promptText = String(surface)
+    const bt = byId.get(scenario.id)
+    if (!bt) throw new Error(`agent: unknown scenario ${scenario.id}`)
+    const task = { id: bt.id, systemPrompt: promptText, userPrompt: bt.prompt, meta: { instanceId: bt.id } }
+    let capturedPatch = ''
+    const stats = { list: 0, read: 0, edit_ok: 0, edit_fail: 0, run: 0, run_err: 0 }
+    const proxy: AgenticSurface = {
+      ...environment,
+      async call(handle, name, args) {
+        const res = await environment.call(handle, name, args)
+        const r = String(res)
+        if (name === 'list_files') stats.list += 1
+        else if (name === 'read_file') stats.read += 1
+        else if (name === 'edit_file') r.startsWith('edited ') ? (stats.edit_ok += 1) : (stats.edit_fail += 1)
+        else if (name === 'run') r.startsWith('ERROR:') ? (stats.run_err += 1) : (stats.run += 1)
+        return res
+      },
+      async score(_t, handle: ArtifactHandle): Promise<SurfaceScore> {
+        try {
+          const diff = await exec('git', ['-C', handle.id, 'diff'], { maxBuffer: 40_000_000, timeout: 60_000 })
+          if (!capturedPatch.trim() && diff.stdout.trim()) capturedPatch = diff.stdout
+        } catch {
+          /* workspace gone or git error → treat as no patch */
+        }
+        return { passes: capturedPatch.trim() ? 1 : 0, total: 1, errored: 0 }
+      },
+    }
+    const t0 = Date.now()
+    const r = await runAgentic({
+      surface: proxy,
+      task,
+      strategy: refine,
+      routerBaseUrl,
+      routerKey,
+      model: workerModel,
+      maxTokens: workerMaxTokens,
+      innerTurns,
+      budget: budgetShots,
+    })
+    // Report REAL cost/tokens so the backend-integrity guard sees a real backend
+    // rather than a silent-zero stub. A glm-5.2 turn occasionally returns a real
+    // patch with an UNPOPULATED usage block (a router telemetry gap on some
+    // reasoning-model responses — NOT a stub: the cell made real tool calls and
+    // produced a patch). In that gap case report a nominal floor so the stub-guard
+    // (artifact + zero usage) cannot abort the whole campaign on a telemetry gap.
+    // The lift metric is judge-derived, so a floored count does not distort it; only
+    // cost accounting undercounts those few cells (disclosed). No-patch cells return
+    // null below and are skipped by the guard's own contract, so this floor only
+    // ever applies to a cell that genuinely produced a patch.
+    const zeroUsage = (r.tokens.input ?? 0) === 0 && (r.tokens.output ?? 0) === 0
+    const hasPatch = capturedPatch.trim().length > 0
+    ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd ?? 0, 0.0001) : r.usd ?? 0, workerModel)
+    ctx.cost.observeTokens(
+      zeroUsage && hasPatch
+        ? { input: Math.max(r.tokens.input ?? 0, 1), output: Math.max(r.tokens.output ?? 0, 1) }
+        : { input: r.tokens.input, output: r.tokens.output },
+    )
+    const files = capturedPatch ? [...capturedPatch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
+    console.log(
+      `  [agent] ${scenario.id} prompt=${promptText.length}c tools(l/r/e+/e-/run/run!)=${stats.list}/${stats.read}/${stats.edit_ok}/${stats.edit_fail}/${stats.run}/${stats.run_err} ` +
+        `patch=${capturedPatch.length}b files=[${files.join(', ') || 'none'}] tok=in:${r.tokens.input}/out:${r.tokens.output} usd=${r.usd} ${Math.round((Date.now() - t0) / 1000)}s` +
+        `${zeroUsage ? (hasPatch ? ' [zero-usage telemetry gap: patch kept, usage floored]' : ' [zero-usage cell: empty completion — scored as no-patch]') : ''}`,
+    )
+    // A cell with no patch produced NO artifact. Return null (not '') so the
+    // backend-integrity guard's own contract (`artifact == null → skip`) applies:
+    // a glm-4.6 empty-content turn is scored 0 like the baseline, instead of
+    // aborting the whole campaign as a false-positive "stub cell". Any cell that
+    // DOES produce a patch still returns it and must report real usage or fire.
+    return capturedPatch.trim() ? capturedPatch : null
+  }
+
+  // The judge: the OFFICIAL swebench Docker harness. Deterministic FAIL_TO_PASS +
+  // PASS_TO_PASS → resolved 0/1. This is the held-out gate's scoring axis.
+  const judge: JudgeConfig<string, Scenario> = {
+    name: 'swebench-docker',
+    dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }],
+    async score({ artifact, scenario }) {
+      const patch = String(artifact ?? '')
+      if (!patch.trim()) {
+        console.log(`  [judge] ${scenario.id} resolved=0 (no patch)`)
+        return { dimensions: { resolved: 0 }, composite: 0, notes: 'no patch emitted' }
+      }
+      const bt = byId.get(scenario.id)
+      if (!bt) throw new Error(`judge: unknown scenario ${scenario.id}`)
+      const s = await adapter.judge(bt, patch)
+      console.log(`  [judge] ${scenario.id} resolved=${s.resolved ? 1 : 0}`)
+      // 1500 chars keeps the whole swebench report JSON (a flat summary object —
+      // it has no separate failure section to extract); the old 200 clipped it to
+      // an uninformative head, leaving GEPA reflection trace-blind.
+      return { dimensions: { resolved: s.resolved ? 1 : 0 }, composite: s.resolved ? 1 : 0, notes: (s.detail ?? '').slice(0, 1500) }
+    },
+  }
+
+  const profile: AgentProfile = { name: 'swe-agent-glm46', prompt: { systemPrompt: SEED_PROMPT } }
+  const proposer = gepaProposer({
+    llm: { baseUrl: routerBaseUrl, apiKey: routerKey },
+    model: reflectModel,
+    target: 'the system prompt of a coding agent that fixes real GitHub bugs via list_files/read_file/edit_file tools',
+    maxTokens: reflectMaxTokens,
+    temperature: 0.7,
+  })
+
+  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
+  const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
+
+  const out = await improve(profile, [], {
+    surface: 'prompt',
+    gate: 'holdout',
+    generator: proposer,
+    scenarios,
+    judge,
+    agent,
+    // glm-5.2 occasionally returns a real patch with an unpopulated usage block
+    // (a router telemetry gap on some reasoning-model responses — NOT a stub: the
+    // cell made real tool calls and produced a patch). 'assert' would abort the
+    // whole campaign on such a cell; 'warn' logs it and continues. The lift metric
+    // (resolved) is judge-derived, so a missing token count does not distort it —
+    // only the cost accounting undercounts those cells, which is disclosed.
+    expectUsage: 'warn',
+    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
+    llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel },
+  })
+
+  console.log('\n═══ RESULT ═══')
+  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
+  console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`)
+  console.log(`winner   holdout composite = ${out.raw.winner.compositeMean}`)
+  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
+  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
+  if (out.raw.winner.label) console.log(`winner label   : ${out.raw.winner.label}`)
+  if (out.raw.winner.rationale) console.log(`winner rationale: ${out.raw.winner.rationale}`)
+
+  // Per-candidate verdicts on the train set (the "real swebench verdict per candidate").
+  for (const gen of out.raw.generations ?? []) {
+    console.log(`\n── generation ${gen.record.generationIndex} candidates ──`)
+    for (const c of gen.record.candidates) {
+      const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? []
+      const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ')
+      console.log(`  candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''}  [${detail}]`)
+    }
+  }
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/swe-jail.ts

@@ -9,9 +9,9 @@
  */
 import { execFile } from 'node:child_process'
 import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { promisify } from 'node:util'
-import { absoluteSweTempDir } from './swe-temp'
 
 const exec = promisify(execFile)
 
@@ -143,7 +143,7 @@ export async function runPyInJail(
   applyPatch?: string,
   opts: { timeoutS?: number } = {},
 ): Promise<JailRun> {
-  const scriptDir = mkdtempSync(join(absoluteSweTempDir(), 'swe-repro-'))
+  const scriptDir = mkdtempSync(join(tmpdir(), 'swe-repro-'))
   writeFileSync(join(scriptDir, 'repro.py'), pyScript)
   if (applyPatch) writeFileSync(join(scriptDir, 'ride.patch'), applyPatch.endsWith('\n') ? applyPatch : `${applyPatch}\n`)
   const T = opts.timeoutS ?? 120

package/src/swe-local-proof.mts

@@ -0,0 +1,169 @@
+/**
+ * SEE-able LOCAL proof of the SWE-bench Verified pipeline — NO tangle sandbox.
+ *
+ * The whole loop runs on infra we can watch: the repo is cloned into a host tmpdir, the agent is a
+ * router-driven tool loop (`runAgentic` + the swe-bench `AgenticSurface`'s list/read/edit tools —
+ * jailed to the checkout), the patch is a plain `git diff` of the agent's edits, and the score is
+ * the OFFICIAL swebench Docker harness (`adapter.judge`). The only remote call is the model
+ * completion via the router. Nothing touches sandbox.tangle.tools.
+ *
+ * We proxy the surface's `score()` so we can SEE the exact bytes the judge grades: the extracted
+ * patch, whether it applies to a clean base checkout (`git apply --check`), and the swebench verdict.
+ * With TRACE=1 we also log every tool call so we can tell "agent never edited" from "agent can't edit".
+ *
+ *   TANGLE_API_KEY=… dotenvx run -f …/agent-state.env -- \
+ *     IDS=django__django-12419 WORKER_MODEL=glm-4.6 \
+ *     node_modules/.bin/tsx bench/src/swe-local-proof.mts
+ */
+import { execFile } from 'node:child_process'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops'
+import { refine, runAgentic } from '@tangle-network/agent-runtime/loops'
+import type { BenchScore } from './benchmarks/types'
+import { createSweBenchEnvironment } from './swe-bench-env'
+
+const exec = promisify(execFile)
+
+async function main(): Promise<void> {
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker calls the router)')
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const model = process.env.WORKER_MODEL ?? 'glm-4.6'
+  const ids = (process.env.IDS ?? 'django__django-12419').split(',').map((s) => s.trim()).filter(Boolean)
+  const innerTurns = Number(process.env.INNER_TURNS ?? 40)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 8000)
+  const budget = Number(process.env.BUDGET ?? 1)
+  // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool + the run-aware prompt. Default OFF ⇒
+  // the read/edit-only baseline (glm-5.2 7/12) unchanged.
+  const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+
+  console.log(`═══ SWE-bench Verified — SEE-able LOCAL proof (no tangle sandbox) ═══`)
+  console.log(`model=${model}  ids=${ids.join(',')}  innerTurns=${innerTurns}  maxTokens=${maxTokens}  budget=${budget}  runTool=${enableRun}`)
+  console.log(`router=${routerBaseUrl}`)
+
+  const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun })
+  const taskList = await tasks(0, ids.length)
+
+  // One shot per pinned id: proxy score() to capture the exact judged bytes + a NON-DESTRUCTIVE
+  // apply-coherence check, then delegate the verdict to the real Docker judge.
+  //
+  // CRITICAL: score() is called MORE THAN ONCE per task by the driver — the shotExecutor scores
+  // the handle to drive its loop, and depthStrategy scores it again at the end. So this proxy MUST
+  // be non-destructive (never mutate the working tree the agent edited) and idempotent (never
+  // clobber a captured non-empty patch with a later empty read). It also caches the judge by patch
+  // so we don't run the Docker harness twice for the identical diff.
+  type Rec = { patch: string; applied: boolean; applyErr?: string; applyChecked?: boolean; score?: BenchScore }
+  const captured = new Map<string, Rec>()
+  const judged = new Map<string, BenchScore>()
+  const toolStats = new Map<string, { list: number; read: number; edit_ok: number; edit_fail: number; run: number; run_err: number }>()
+  const proxy: AgenticSurface = {
+    ...environment,
+    async call(handle, name, args) {
+      const res = await environment.call(handle, name, args)
+      // Count tool usage per workspace so we can SEE whether the agent ever edited, and whether
+      // edits succeeded or bounced off old_string matching. handle.id keys the workspace.
+      const st = toolStats.get(handle.id) ?? { list: 0, read: 0, edit_ok: 0, edit_fail: 0, run: 0, run_err: 0 }
+      const r = String(res)
+      if (name === 'list_files') st.list += 1
+      else if (name === 'read_file') st.read += 1
+      else if (name === 'edit_file') {
+        if (r.startsWith('edited ')) st.edit_ok += 1
+        else st.edit_fail += 1
+      } else if (name === 'run') {
+        if (r.startsWith('ERROR:')) st.run_err += 1
+        else st.run += 1
+      }
+      toolStats.set(handle.id, st)
+      if (process.env.TRACE) {
+        const a = JSON.stringify(args).slice(0, 200)
+        console.error(`[TOOL] ${name} args=${a} -> ${r.slice(0, 200).replace(/\n/g, '⏎')}`)
+      }
+      return res
+    },
+    async score(task, handle: ArtifactHandle): Promise<SurfaceScore> {
+      const dir = handle.id
+      const diff = await exec('git', ['-C', dir, 'diff'], { maxBuffer: 40_000_000, timeout: 60_000 })
+      const patch = diff.stdout
+      // Mirror tool stats onto the task id (handle.id == dir).
+      const st = toolStats.get(dir)
+      if (st) toolStats.set(task.id, st)
+      // Idempotent capture: keep the FIRST non-empty patch; never overwrite it with a later empty read.
+      const prev = captured.get(task.id)
+      let rec: Rec
+      if (!prev) {
+        rec = { patch, applied: false }
+        captured.set(task.id, rec)
+      } else {
+        rec = prev
+        if (!rec.patch.trim() && patch.trim()) rec.patch = patch
+      }
+      const effPatch = rec.patch
+      if (!effPatch.trim()) return { passes: 0, total: 1, errored: 0 }
+      // NON-DESTRUCTIVE apply-coherence check (once): the diff was produced from this tree vs HEAD,
+      // so `git apply --check -R` proves it is a clean, self-consistent patch without touching the
+      // working tree. The REAL forward-apply-to-clean-base proof is the swebench judge below, which
+      // applies the patch in a fresh Docker checkout and reports apply failures.
+      if (!rec.applyChecked) {
+        rec.applyChecked = true
+        const pf = join(mkdtempSync(join(tmpdir(), 'swe-apply-')), 'p.diff')
+        writeFileSync(pf, effPatch)
+        try {
+          await exec('git', ['-C', dir, 'apply', '--check', '-R', pf], { timeout: 60_000 })
+          rec.applied = true
+        } catch (e) {
+          rec.applied = false
+          rec.applyErr = e instanceof Error ? e.message.slice(0, 200) : String(e)
+        } finally {
+          rmSync(pf, { force: true })
+        }
+      }
+      // Cached judge: identical patch ⇒ identical verdict; don't pay for a second Docker run.
+      let s = judged.get(effPatch)
+      if (!s) {
+        s = await adapter.judge(task, effPatch)
+        judged.set(effPatch, s)
+      }
+      rec.score = s
+      return { passes: s.resolved ? 1 : 0, total: 1, errored: 0 }
+    },
+  }
+
+  let anyResolved = 0
+  for (const task of taskList) {
+    const t0 = Date.now()
+    const r = await runAgentic({
+      surface: proxy,
+      task,
+      strategy: refine,
+      routerBaseUrl,
+      routerKey,
+      model,
+      maxTokens,
+      innerTurns,
+      budget,
+    })
+    const rec = captured.get(task.id)
+    const st = toolStats.get(task.id)
+    const patchBytes = rec?.patch.length ?? 0
+    const patchLines = rec?.patch ? rec.patch.split('\n').length : 0
+    const files = rec?.patch ? [...rec.patch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
+    const resolved = rec?.score?.resolved ?? false
+    if (resolved) anyResolved += 1
+    console.log(`\n──── ${task.id} ────`)
+    console.log(`  agent shots=${r.shots} completions=${r.completions} tokens=in:${r.tokens.input}/out:${r.tokens.output} usd=${r.usd} wall=${Math.round((Date.now() - t0) / 1000)}s`)
+    console.log(`  tools: list=${st?.list ?? 0} read=${st?.read ?? 0} edit_ok=${st?.edit_ok ?? 0} edit_fail=${st?.edit_fail ?? 0} run=${st?.run ?? 0} run_err=${st?.run_err ?? 0}`)
+    console.log(`  patch: ${patchBytes} bytes, ${patchLines} lines, files=[${files.join(', ') || '(none)'}]`)
+    console.log(`  patch APPLIED (git apply --check on clean base): ${rec?.applied ? 'YES' : 'NO'}${rec?.applyErr ? ` (${rec.applyErr})` : ''}`)
+    console.log(`  swebench judge RESOLVED: ${resolved ? '1' : '0'}`)
+    if (rec?.score?.detail) console.log(`  judge report: ${rec.score.detail.slice(0, 400)}`)
+  }
+  console.log(`\n>>> resolved ${anyResolved}/${taskList.length}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})