@tangle-network/agent-bench 0.3.6 → 0.3.7

package/src/swe-jail.ts

@@ -9,9 +9,9 @@
  */
 import { execFile } from 'node:child_process'
 import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { promisify } from 'node:util'
-import { absoluteSweTempDir } from './swe-temp'
 
 const exec = promisify(execFile)
 
@@ -143,7 +143,7 @@ export async function runPyInJail(
   applyPatch?: string,
   opts: { timeoutS?: number } = {},
 ): Promise<JailRun> {
-  const scriptDir = mkdtempSync(join(absoluteSweTempDir(), 'swe-repro-'))
+  const scriptDir = mkdtempSync(join(tmpdir(), 'swe-repro-'))
   writeFileSync(join(scriptDir, 'repro.py'), pyScript)
   if (applyPatch) writeFileSync(join(scriptDir, 'ride.patch'), applyPatch.endsWith('\n') ? applyPatch : `${applyPatch}\n`)
   const T = opts.timeoutS ?? 120

package/src/swe-local-proof.mts

@@ -0,0 +1,169 @@
+/**
+ * SEE-able LOCAL proof of the SWE-bench Verified pipeline — NO tangle sandbox.
+ *
+ * The whole loop runs on infra we can watch: the repo is cloned into a host tmpdir, the agent is a
+ * router-driven tool loop (`runAgentic` + the swe-bench `AgenticSurface`'s list/read/edit tools —
+ * jailed to the checkout), the patch is a plain `git diff` of the agent's edits, and the score is
+ * the OFFICIAL swebench Docker harness (`adapter.judge`). The only remote call is the model
+ * completion via the router. Nothing touches sandbox.tangle.tools.
+ *
+ * We proxy the surface's `score()` so we can SEE the exact bytes the judge grades: the extracted
+ * patch, whether it applies to a clean base checkout (`git apply --check`), and the swebench verdict.
+ * With TRACE=1 we also log every tool call so we can tell "agent never edited" from "agent can't edit".
+ *
+ *   TANGLE_API_KEY=… dotenvx run -f …/agent-state.env -- \
+ *     IDS=django__django-12419 WORKER_MODEL=glm-4.6 \
+ *     node_modules/.bin/tsx bench/src/swe-local-proof.mts
+ */
+import { execFile } from 'node:child_process'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops'
+import { refine, runAgentic } from '@tangle-network/agent-runtime/loops'
+import type { BenchScore } from './benchmarks/types'
+import { createSweBenchEnvironment } from './swe-bench-env'
+
+const exec = promisify(execFile)
+
+async function main(): Promise<void> {
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker calls the router)')
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const model = process.env.WORKER_MODEL ?? 'glm-4.6'
+  const ids = (process.env.IDS ?? 'django__django-12419').split(',').map((s) => s.trim()).filter(Boolean)
+  const innerTurns = Number(process.env.INNER_TURNS ?? 40)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 8000)
+  const budget = Number(process.env.BUDGET ?? 1)
+  // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool + the run-aware prompt. Default OFF ⇒
+  // the read/edit-only baseline (glm-5.2 7/12) unchanged.
+  const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+
+  console.log(`═══ SWE-bench Verified — SEE-able LOCAL proof (no tangle sandbox) ═══`)
+  console.log(`model=${model}  ids=${ids.join(',')}  innerTurns=${innerTurns}  maxTokens=${maxTokens}  budget=${budget}  runTool=${enableRun}`)
+  console.log(`router=${routerBaseUrl}`)
+
+  const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun })
+  const taskList = await tasks(0, ids.length)
+
+  // One shot per pinned id: proxy score() to capture the exact judged bytes + a NON-DESTRUCTIVE
+  // apply-coherence check, then delegate the verdict to the real Docker judge.
+  //
+  // CRITICAL: score() is called MORE THAN ONCE per task by the driver — the shotExecutor scores
+  // the handle to drive its loop, and depthStrategy scores it again at the end. So this proxy MUST
+  // be non-destructive (never mutate the working tree the agent edited) and idempotent (never
+  // clobber a captured non-empty patch with a later empty read). It also caches the judge by patch
+  // so we don't run the Docker harness twice for the identical diff.
+  type Rec = { patch: string; applied: boolean; applyErr?: string; applyChecked?: boolean; score?: BenchScore }
+  const captured = new Map<string, Rec>()
+  const judged = new Map<string, BenchScore>()
+  const toolStats = new Map<string, { list: number; read: number; edit_ok: number; edit_fail: number; run: number; run_err: number }>()
+  const proxy: AgenticSurface = {
+    ...environment,
+    async call(handle, name, args) {
+      const res = await environment.call(handle, name, args)
+      // Count tool usage per workspace so we can SEE whether the agent ever edited, and whether
+      // edits succeeded or bounced off old_string matching. handle.id keys the workspace.
+      const st = toolStats.get(handle.id) ?? { list: 0, read: 0, edit_ok: 0, edit_fail: 0, run: 0, run_err: 0 }
+      const r = String(res)
+      if (name === 'list_files') st.list += 1
+      else if (name === 'read_file') st.read += 1
+      else if (name === 'edit_file') {
+        if (r.startsWith('edited ')) st.edit_ok += 1
+        else st.edit_fail += 1
+      } else if (name === 'run') {
+        if (r.startsWith('ERROR:')) st.run_err += 1
+        else st.run += 1
+      }
+      toolStats.set(handle.id, st)
+      if (process.env.TRACE) {
+        const a = JSON.stringify(args).slice(0, 200)
+        console.error(`[TOOL] ${name} args=${a} -> ${r.slice(0, 200).replace(/\n/g, '⏎')}`)
+      }
+      return res
+    },
+    async score(task, handle: ArtifactHandle): Promise<SurfaceScore> {
+      const dir = handle.id
+      const diff = await exec('git', ['-C', dir, 'diff'], { maxBuffer: 40_000_000, timeout: 60_000 })
+      const patch = diff.stdout
+      // Mirror tool stats onto the task id (handle.id == dir).
+      const st = toolStats.get(dir)
+      if (st) toolStats.set(task.id, st)
+      // Idempotent capture: keep the FIRST non-empty patch; never overwrite it with a later empty read.
+      const prev = captured.get(task.id)
+      let rec: Rec
+      if (!prev) {
+        rec = { patch, applied: false }
+        captured.set(task.id, rec)
+      } else {
+        rec = prev
+        if (!rec.patch.trim() && patch.trim()) rec.patch = patch
+      }
+      const effPatch = rec.patch
+      if (!effPatch.trim()) return { passes: 0, total: 1, errored: 0 }
+      // NON-DESTRUCTIVE apply-coherence check (once): the diff was produced from this tree vs HEAD,
+      // so `git apply --check -R` proves it is a clean, self-consistent patch without touching the
+      // working tree. The REAL forward-apply-to-clean-base proof is the swebench judge below, which
+      // applies the patch in a fresh Docker checkout and reports apply failures.
+      if (!rec.applyChecked) {
+        rec.applyChecked = true
+        const pf = join(mkdtempSync(join(tmpdir(), 'swe-apply-')), 'p.diff')
+        writeFileSync(pf, effPatch)
+        try {
+          await exec('git', ['-C', dir, 'apply', '--check', '-R', pf], { timeout: 60_000 })
+          rec.applied = true
+        } catch (e) {
+          rec.applied = false
+          rec.applyErr = e instanceof Error ? e.message.slice(0, 200) : String(e)
+        } finally {
+          rmSync(pf, { force: true })
+        }
+      }
+      // Cached judge: identical patch ⇒ identical verdict; don't pay for a second Docker run.
+      let s = judged.get(effPatch)
+      if (!s) {
+        s = await adapter.judge(task, effPatch)
+        judged.set(effPatch, s)
+      }
+      rec.score = s
+      return { passes: s.resolved ? 1 : 0, total: 1, errored: 0 }
+    },
+  }
+
+  let anyResolved = 0
+  for (const task of taskList) {
+    const t0 = Date.now()
+    const r = await runAgentic({
+      surface: proxy,
+      task,
+      strategy: refine,
+      routerBaseUrl,
+      routerKey,
+      model,
+      maxTokens,
+      innerTurns,
+      budget,
+    })
+    const rec = captured.get(task.id)
+    const st = toolStats.get(task.id)
+    const patchBytes = rec?.patch.length ?? 0
+    const patchLines = rec?.patch ? rec.patch.split('\n').length : 0
+    const files = rec?.patch ? [...rec.patch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
+    const resolved = rec?.score?.resolved ?? false
+    if (resolved) anyResolved += 1
+    console.log(`\n──── ${task.id} ────`)
+    console.log(`  agent shots=${r.shots} completions=${r.completions} tokens=in:${r.tokens.input}/out:${r.tokens.output} usd=${r.usd} wall=${Math.round((Date.now() - t0) / 1000)}s`)
+    console.log(`  tools: list=${st?.list ?? 0} read=${st?.read ?? 0} edit_ok=${st?.edit_ok ?? 0} edit_fail=${st?.edit_fail ?? 0} run=${st?.run ?? 0} run_err=${st?.run_err ?? 0}`)
+    console.log(`  patch: ${patchBytes} bytes, ${patchLines} lines, files=[${files.join(', ') || '(none)'}]`)
+    console.log(`  patch APPLIED (git apply --check on clean base): ${rec?.applied ? 'YES' : 'NO'}${rec?.applyErr ? ` (${rec.applyErr})` : ''}`)
+    console.log(`  swebench judge RESOLVED: ${resolved ? '1' : '0'}`)
+    if (rec?.score?.detail) console.log(`  judge report: ${rec.score.detail.slice(0, 400)}`)
+  }
+  console.log(`\n>>> resolved ${anyResolved}/${taskList.length}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/swe-repro-calibrate.mts

@@ -0,0 +1,446 @@
+/**
+ * Stage 0 of the SWE-bench frontier push — the REPRODUCTION-ORACLE CALIBRATOR
+ * (contract: supervisor-lab/docs/results/PREREG-swe-frontier.md, Stage 0 only).
+ *
+ * Question it answers: can glm-5.2, given ONLY the issue text (+ up to 3 requested file reads),
+ * author a self-contained repro script that (a) FAILS on the unpatched tree — "validity" — and
+ * (b) PASSES after the GOLD patch — "soundness"? GATE: ≥60% of instances valid+sound (1 retry).
+ *
+ * No cheating by construction: the GOLD patch is used strictly script-side (applied host-side to a
+ * COPY of the tree, then run in the jail); a hard assert refuses to send any model message that
+ * contains gold-patch content. The repro runs in the instance's cached swebench Docker image with
+ * the tree mounted READ-ONLY — the byte-identical invocation pattern of swe-bench-env's `run` tool
+ * (conda testbed, cwd=/testbed, --network none, dual timeout) — so calibration measures the same
+ * substrate Stage 1/2 will select patches on.
+ *
+ *   cd ~/company/devops/secrets && dotenvx run -f agent-state.env -f tangle-router.env -- bash -c \
+ *     'cd ~/code/agent-runtime-swe && OUT=/path/swe-stage0.jsonl node_modules/.bin/tsx bench/src/swe-repro-calibrate.mts'
+ *
+ * Env: ZAI_API_KEY (required unless CANARY_ONLY), ZAI_BASE, MODEL=glm-5.2, MAX_TOKENS=12000, TEMP=0.2,
+ *      CONC=3, REPRO_TIMEOUT=120 (s), LLM_TIMEOUT_MS=480000, IDS=comma-list override, OUT=jsonl path,
+ *      REPRO_EXEC=mount|image (execution substrate; see the constant below),
+ *      CANARY_ONLY=1 (run ONLY the per-instance execution canary — no model calls, no grading —
+ *      to decide which substrate is trustworthy per instance before spending on authoring).
+ */
+import { execFile } from 'node:child_process'
+import { appendFileSync, cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import type { AgenticTask, ArtifactHandle } from '@tangle-network/agent-runtime/loops'
+import type { BenchTask } from './benchmarks/types'
+import { createSweBenchEnvironment, resolveImageForMetadata } from './swe-bench-env'
+import {
+  APPLY_SENTINEL,
+  cachedInstanceIds,
+  extractReadRequests as extractReads,
+  extractReproScript as extractScript,
+  IMPORT_NAME,
+  importCanaryScript,
+  type JailRun,
+  reproAuthorSystem,
+  runPyInJail,
+  tail,
+  zaiChatRaw,
+} from './swe-jail'
+
+const exec = promisify(execFile)
+
+// ---------- config ----------
+
+const ZAI_BASE = process.env.ZAI_BASE ?? 'https://api.z.ai/api/coding/paas/v4'
+const ZAI_KEY = process.env.ZAI_API_KEY ?? ''
+const MODEL = process.env.MODEL ?? 'glm-5.2'
+// glm-5.2 is a reasoning model: hidden reasoning consumes max_tokens, so <8000 starves content.
+const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 12_000)
+const TEMP = Number(process.env.TEMP ?? 0.2)
+const CONC = Math.max(1, Math.min(3, Number(process.env.CONC ?? 3)))
+const REPRO_TIMEOUT_S = Number(process.env.REPRO_TIMEOUT ?? 120)
+const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 480_000)
+const OUT = process.env.OUT ?? 'swe-stage0.jsonl'
+/** Repro EXECUTION substrate. `mount` (the prereg default) mounts the fresh host clone :ro over
+ *  /testbed — the run tool's exact pattern. `image` executes against the image's OWN /testbed
+ *  (base_commit, BUILT — compiled extensions and generated version files present), applying the
+ *  gold patch in-container instead of host-side. Measured on the 23 cached instances: `mount`
+ *  kills 6 (astropy×2/matplotlib/sklearn×2/pytest) with import errors a fresh un-built clone
+ *  cannot avoid; the SAME scripts were all valid+sound under `image`. */
+const EXEC = process.env.REPRO_EXEC ?? 'mount'
+if (EXEC !== 'mount' && EXEC !== 'image') throw new Error(`REPRO_EXEC must be mount|image, got ${EXEC}`)
+/** Canary-only sweep: measure per-instance substrate trustworthiness (gold applied, import resolves
+ *  into the patched tree) across all instances with ZERO model calls. */
+const CANARY_ONLY = process.env.CANARY_ONLY === '1'
+
+// ---------- model client (swe-jail's zaiChatRaw: patient 429 ladder + empty-content retry —
+// the glm reasoning path starves `content` when reasoning eats max_tokens) ----------
+
+interface ChatMsg {
+  role: 'system' | 'user' | 'assistant'
+  content: string
+}
+
+interface Completion {
+  content: string
+  attempts: number
+  tokensIn: number
+  tokensOut: number
+}
+
+async function complete(messages: ChatMsg[]): Promise<Completion> {
+  const { json, attempts } = await zaiChatRaw(
+    { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS },
+    { model: MODEL, max_tokens: MAX_TOKENS, temperature: TEMP, messages },
+  )
+  const d = json as {
+    choices?: Array<{ message?: { content?: string } }>
+    usage?: { prompt_tokens?: number; completion_tokens?: number }
+  }
+  return {
+    content: d.choices?.[0]?.message?.content ?? '',
+    attempts,
+    tokensIn: d.usage?.prompt_tokens ?? 0,
+    tokensOut: d.usage?.completion_tokens ?? 0,
+  }
+}
+
+// ---------- authoring protocol (one optional read round, plain-text READ: lines) ----------
+// The protocol constants/parsers now live in swe-jail.ts (shared with the stream driver);
+// behavior here is byte-identical to the in-file originals.
+
+const AUTHOR_SYSTEM = reproAuthorSystem(REPRO_TIMEOUT_S)
+
+// ---------- per-instance row ----------
+
+interface Row {
+  instanceId: string
+  repo: string
+  execMode: string
+  image: string | null
+  imagePresent: boolean
+  canaryExit: number | null
+  canaryOut: string
+  /** true = gold applied AND import resolved into the patched tree; false = this substrate cannot
+   *  grade this instance; null = repo not in IMPORT_NAME (canary not applicable). */
+  canaryPass: boolean | null
+  readsRequested: string[]
+  authorCalls: number
+  retryUsed: boolean
+  preExitFirst: number | null
+  preExitFinal: number | null
+  preOut: string
+  valid: boolean
+  goldApplyOk: boolean | null
+  postExit: number | null
+  postOut: string
+  sound: boolean
+  validAndSound: boolean
+  autoClass: string
+  script: string | null
+  scriptFirst: string | null
+  tokensIn: number
+  tokensOut: number
+  wallMs: number
+  error?: string
+}
+
+async function calibrateInstance(
+  env: Awaited<ReturnType<typeof createSweBenchEnvironment>>,
+  bt: BenchTask,
+): Promise<Row> {
+  const t0 = Date.now()
+  const id = bt.id
+  const row: Row = {
+    instanceId: id, repo: '', execMode: EXEC, image: null, imagePresent: false, canaryExit: null, canaryOut: '',
+    canaryPass: null, readsRequested: [], authorCalls: 0, retryUsed: false, preExitFirst: null, preExitFinal: null,
+    preOut: '', valid: false, goldApplyOk: null, postExit: null, postOut: '', sound: false,
+    validAndSound: false, autoClass: 'infra-error', script: null, scriptFirst: null,
+    tokensIn: 0, tokensOut: 0, wallMs: 0,
+  }
+  let handle: ArtifactHandle | null = null
+  let patchedDir: string | null = null
+  try {
+    // Metadata (issue + gold) comes from the adapter's task row, never the model path.
+    const md = bt.metadata as Record<string, string>
+    row.repo = md.repo
+    const gold = String(md.patch ?? '')
+    if (!gold.trim()) throw new Error('gold patch missing from metadata')
+
+    // 1. Hard-assert the cached docker image BEFORE paying for a clone or a model call.
+    const img = await resolveImageForMetadata(bt.metadata ?? {})
+    if (!img.ok) {
+      row.autoClass = 'image-missing'
+      row.error = img.reason
+      return row
+    }
+    row.image = img.tag
+    row.imagePresent = true
+
+    // 2. Open the environment: host clone at base_commit. The `mount` substrate jails against this
+    // tree :ro; the `image` substrate uses it only for list_files/read_file during authoring — so a
+    // canary-only image sweep skips the clone (and its network cost) entirely.
+    let treeDir: string | null = null
+    if (EXEC === 'mount' || !CANARY_ONLY) {
+      const h = await env.environment.open({ id, systemPrompt: '', userPrompt: '', meta: {} } as AgenticTask)
+      handle = h
+      treeDir = h.id
+    }
+    const execTree = EXEC === 'image' ? null : treeDir
+
+    // Mount substrate: apply the gold patch host-side to a COPY of the tree UP FRONT — the canary
+    // must observe the tree exactly as the soundness run will mount it. A failed apply is terminal
+    // before any model call is spent.
+    if (EXEC === 'mount') {
+      patchedDir = mkdtempSync(join(tmpdir(), 'swe-gold-'))
+      cpSync(treeDir as string, patchedDir, { recursive: true })
+      const goldFile = join(patchedDir, '.swe-gold.patch')
+      writeFileSync(goldFile, gold.endsWith('\n') ? gold : `${gold}\n`)
+      try {
+        await exec('git', ['-C', patchedDir, 'apply', '--whitespace=nowarn', goldFile], { timeout: 60_000 })
+        row.goldApplyOk = true
+      } catch {
+        // Official-harness fallback: GNU patch with fuzz.
+        try {
+          await exec('patch', ['-p1', '--fuzz=5', '-i', goldFile], { cwd: patchedDir, timeout: 60_000 })
+          row.goldApplyOk = true
+        } catch (e2) {
+          row.goldApplyOk = false
+          row.autoClass = 'gold-apply-failed'
+          row.error = `gold patch failed to apply: ${(e2 as Error).message.slice(0, 200)}`
+          return row
+        }
+      }
+      rmSync(goldFile, { force: true })
+    }
+
+    // EXECUTION CANARY (mode-deciding, zero model calls): with the gold patch applied to the tree
+    // under test, `import <pkg>` must succeed AND resolve INTO that tree. PYTHONPATH=/testbed is
+    // already pinned by runPyInJail, so a site-packages resolution here (exit 3) means this substrate
+    // would grade code the patch never reaches — the instance is env-unresolvable in this mode.
+    const pkg = IMPORT_NAME[md.repo]
+    if (pkg) {
+      const canaryScript = importCanaryScript(pkg)
+      const c = EXEC === 'image'
+        ? await runPyInJail(img.tag, null, canaryScript, gold, { timeoutS: REPRO_TIMEOUT_S })
+        : await runPyInJail(img.tag, patchedDir, canaryScript, undefined, { timeoutS: REPRO_TIMEOUT_S })
+      if (c.infraError) throw new Error(c.infraError)
+      row.canaryExit = c.code
+      row.canaryOut = tail(c.out, 200)
+      if (EXEC === 'image') {
+        row.goldApplyOk = c.out.includes(APPLY_SENTINEL)
+        if (!row.goldApplyOk) {
+          row.autoClass = 'gold-apply-failed'
+          row.error = `gold patch failed to apply in-container: ${tail(c.out, 200)}`
+          return row
+        }
+      }
+      row.canaryPass = c.code === 0
+      if (!row.canaryPass) {
+        row.autoClass = 'env-unresolvable'
+        row.error = `canary: import ${pkg} did not resolve into the patched tree (exit ${c.code})`
+        return row
+      }
+    }
+    if (CANARY_ONLY) {
+      row.autoClass = row.canaryPass === true ? 'canary-pass' : 'canary-unknown'
+      return row
+    }
+    const h = handle as ArtifactHandle
+
+    // 3. Author the repro: issue text + top-level listing; ONE optional read round (≤3 files).
+    const listing = String(await env.environment.call(h, 'list_files', { dir: '' })).slice(0, 5_000)
+    const issue = String(md.problem_statement ?? '').slice(0, 20_000)
+    const messages: ChatMsg[] = [
+      { role: 'system', content: AUTHOR_SYSTEM },
+      {
+        role: 'user',
+        content:
+          `Repository: ${md.repo} (checked out at the commit where the bug is PRESENT).\n\n` +
+          `Repository file listing (top levels):\n${listing}\n\n--- Issue ---\n${issue}\n\n--- Instructions ---\n` +
+          'If you need to see specific source files before writing the script, reply with ONLY read requests, ' +
+          'one per line, at most 3, in the form:\nREAD: path/relative/to/repo/root\n' +
+          'Otherwise reply now with the final script in a single ```python fenced block.',
+      },
+    ]
+    // Leak guard: no model message may carry gold-patch content. A distinctive added line is the sentinel.
+    const goldMark = gold.split('\n').find((l) => l.startsWith('+') && !l.startsWith('+++') && l.trim().length > 12)?.slice(0, 80)
+    const guardedComplete = async (msgs: ChatMsg[]): Promise<Completion> => {
+      if (goldMark && msgs.some((m) => m.content.includes(goldMark))) {
+        throw new Error('REFUSED: gold patch content leaked into model messages')
+      }
+      const c = await complete(msgs)
+      row.authorCalls += 1
+      row.tokensIn += c.tokensIn
+      row.tokensOut += c.tokensOut
+      return c
+    }
+
+    let resp = await guardedComplete(messages)
+    let script = extractScript(resp.content)
+    if (!script) {
+      const reads = extractReads(resp.content)
+      row.readsRequested = reads
+      messages.push({ role: 'assistant', content: resp.content })
+      if (reads.length) {
+        const bodies: string[] = []
+        for (const p of reads) {
+          const c = String(await env.environment.call(h, 'read_file', { path: p }))
+          bodies.push(`----- ${p} -----\n${c.slice(0, 12_000)}${c.length > 12_000 ? '\n…[truncated]' : ''}`)
+        }
+        messages.push({
+          role: 'user',
+          content: `${bodies.join('\n\n')}\n\nNow reply with the final script in a single \`\`\`python fenced block.`,
+        })
+      } else {
+        messages.push({ role: 'user', content: 'Reply with ONLY the final Python script in a single ```python fenced block.' })
+      }
+      resp = await guardedComplete(messages)
+      script = extractScript(resp.content)
+    }
+    if (!script) {
+      row.autoClass = 'authoring-failed'
+      row.error = `no script in response: ${resp.content.slice(0, 200)}`
+      return row
+    }
+    row.script = script
+
+    // 4. VALIDITY: the script must indicate bug-present (nonzero, non-timeout) on the UNPATCHED tree.
+    let pre = await runPyInJail(img.tag, execTree, script, undefined, { timeoutS: REPRO_TIMEOUT_S })
+    if (pre.infraError) throw new Error(pre.infraError)
+    row.preExitFirst = pre.code
+    let detected = pre.code !== 0 && !pre.timedOut
+    if (!detected) {
+      // One retry with feedback, per the prereg.
+      row.retryUsed = true
+      row.scriptFirst = script
+      const feedback = pre.timedOut
+        ? `your script timed out after ${REPRO_TIMEOUT_S}s on the known-buggy code. Write a faster, simpler script that still detects the bug.`
+        : 'your script did not detect the bug on the known-buggy code: it exited 0 on the UNPATCHED repository. ' +
+          `Its output was:\n${tail(pre.out, 1_500)}\nWrite a corrected script that FAILS (nonzero exit) on the buggy code.`
+      messages.push({ role: 'assistant', content: `\`\`\`python\n${script}\n\`\`\`` }, { role: 'user', content: feedback })
+      const retry = await guardedComplete(messages)
+      const script2 = extractScript(retry.content)
+      if (script2) {
+        script = script2
+        row.script = script2
+        pre = await runPyInJail(img.tag, execTree, script2, undefined, { timeoutS: REPRO_TIMEOUT_S })
+        if (pre.infraError) throw new Error(pre.infraError)
+        detected = pre.code !== 0 && !pre.timedOut
+      }
+    }
+    row.preExitFinal = pre.code
+    row.preOut = tail(pre.out, 1_500)
+    row.valid = detected
+    if (!row.valid) {
+      row.autoClass = pre.timedOut ? 'invalid-timeout' : 'invalid-exit0'
+      return row
+    }
+
+    // 5. SOUNDNESS (script-side only): gold patch applied, same jail, must exit 0.
+    let post: JailRun
+    if (EXEC === 'image') {
+      // Gold applied in-container to the image's built /testbed; a failed apply surfaces as exit≠0
+      // with git's message in the output.
+      post = await runPyInJail(img.tag, null, script, gold, { timeoutS: REPRO_TIMEOUT_S })
+      if (post.infraError) throw new Error(post.infraError)
+      row.goldApplyOk = post.out.includes(APPLY_SENTINEL)
+      if (!row.goldApplyOk) {
+        row.autoClass = 'gold-apply-failed'
+        row.error = `gold patch failed to apply in-container: ${tail(post.out, 200)}`
+        return row
+      }
+    } else {
+      // Mount substrate: gold was already applied host-side to the canary-verified copy up front.
+      post = await runPyInJail(img.tag, patchedDir as string, script, undefined, { timeoutS: REPRO_TIMEOUT_S })
+      if (post.infraError) throw new Error(post.infraError)
+    }
+    row.postExit = post.code
+    row.postOut = tail(post.out, 1_500)
+    row.sound = post.code === 0
+    row.validAndSound = row.valid && row.sound
+    if (row.validAndSound) row.autoClass = 'ok'
+    else if (post.timedOut) row.autoClass = 'unsound-timeout'
+    else if (/ModuleNotFoundError|ImportError/.test(post.out)) row.autoClass = 'unsound-import-error'
+    else row.autoClass = 'unsound-still-failing'
+    return row
+  } catch (e) {
+    row.autoClass = row.autoClass === 'ok' ? 'infra-error' : row.autoClass
+    row.error = e instanceof Error ? e.message.slice(0, 400) : String(e).slice(0, 400)
+    return row
+  } finally {
+    row.wallMs = Date.now() - t0
+    if (patchedDir) rmSync(patchedDir, { recursive: true, force: true })
+    if (handle) await env.environment.close(handle).catch(() => {})
+  }
+}
+
+// ---------- driver ----------
+
+async function main(): Promise<void> {
+  if (!ZAI_KEY && !CANARY_ONLY) throw new Error('ZAI_API_KEY required (run under dotenvx: agent-state.env)')
+  const ids = process.env.IDS
+    ? process.env.IDS.split(',').map((s) => s.trim()).filter(Boolean)
+    : await cachedInstanceIds()
+  if (!ids.length) throw new Error('no cached sweb.eval images found and no IDS given')
+
+  console.log(`═══ SWE-bench Stage 0 — ${CANARY_ONLY ? 'execution-canary sweep (no model calls)' : 'reproduction-oracle calibration'} ═══`)
+  console.log(`model=${MODEL} base=${ZAI_BASE} maxTokens=${MAX_TOKENS} temp=${TEMP} conc=${CONC} reproTimeout=${REPRO_TIMEOUT_S}s exec=${EXEC}`)
+  console.log(`instances (${ids.length}): ${ids.join(', ')}`)
+  console.log(`out=${OUT}`)
+
+  const env = await createSweBenchEnvironment(ids.length, { ids })
+  // One dataset scan for all instances; per-instance metadata (issue + gold) rides the BenchTask.
+  const taskById = new Map((await env.adapter.loadTasks({ ids, split: 'test' })).map((t) => [t.id, t]))
+  const missing = ids.filter((id) => !taskById.has(id))
+  if (missing.length) throw new Error(`instances not found in SWE-bench_Verified: ${missing.join(', ')}`)
+  const rows: Row[] = []
+  let next = 0
+  const worker = async (): Promise<void> => {
+    while (next < ids.length) {
+      const i = next++
+      const id = ids[i]
+      console.log(`[${i + 1}/${ids.length}] ${id} …`)
+      const row = await calibrateInstance(env, taskById.get(id) as BenchTask)
+      rows.push(row)
+      appendFileSync(OUT, `${JSON.stringify(row)}\n`)
+      console.log(
+        `[${i + 1}/${ids.length}] ${id} → ${row.autoClass}` +
+          ` (pre=${row.preExitFinal ?? '-'} post=${row.postExit ?? '-'} retry=${row.retryUsed ? 'y' : 'n'}` +
+          ` calls=${row.authorCalls} tok=${row.tokensIn}/${row.tokensOut} wall=${Math.round(row.wallMs / 1000)}s)` +
+          (row.error ? ` err=${row.error.slice(0, 120)}` : ''),
+      )
+    }
+  }
+  await Promise.all(Array.from({ length: CONC }, () => worker()))
+
+  // ---------- summary ----------
+  const n = rows.length
+  const present = rows.filter((r) => r.imagePresent).length
+  const valid = rows.filter((r) => r.valid).length
+  const sound = rows.filter((r) => r.validAndSound).length
+  const retried = rows.filter((r) => r.retryUsed).length
+  const classes = new Map<string, number>()
+  for (const r of rows) classes.set(r.autoClass, (classes.get(r.autoClass) ?? 0) + 1)
+
+  console.log('\n══ per-instance ══')
+  console.log('instance | class | canary | pre1 | preF | valid | goldApply | post | sound | retry | calls | tokIn/out | wall_s')
+  for (const r of [...rows].sort((a, b) => a.instanceId.localeCompare(b.instanceId))) {
+    const canary = r.canaryPass === null ? `?(${r.canaryExit ?? '-'})` : r.canaryPass ? 'pass' : `FAIL(${r.canaryExit})`
+    console.log(
+      `${r.instanceId} | ${r.autoClass} | ${canary} | ${r.preExitFirst ?? '-'} | ${r.preExitFinal ?? '-'} | ` +
+        `${r.valid ? 1 : 0} | ${r.goldApplyOk === null ? '-' : r.goldApplyOk ? 1 : 0} | ${r.postExit ?? '-'} | ${r.sound ? 1 : 0} | ` +
+        `${r.retryUsed ? 1 : 0} | ${r.authorCalls} | ${r.tokensIn}/${r.tokensOut} | ${Math.round(r.wallMs / 1000)}`,
+    )
+  }
+  console.log('\n══ summary ══')
+  console.log(`n=${n} imagePresent=${present} canaryPass=${rows.filter((r) => r.canaryPass === true).length} valid(bug detected pre-patch)=${valid} valid+sound=${sound} retryUsed=${retried}`)
+  console.log(`failure modes: ${[...classes.entries()].map(([k, v]) => `${k}=${v}`).join('  ')}`)
+  if (!CANARY_ONLY) {
+    const rate = n ? sound / n : 0
+    const pct = (100 * rate).toFixed(1)
+    console.log(`\nSTAGE-0 GATE (>=60% valid+sound): ${rate >= 0.6 ? 'PASS' : 'FAIL'} — ${sound}/${n} = ${pct}%`)
+  }
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})