@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/types.ts

@@ -0,0 +1,218 @@
+/**
+ * swe-arena — typed replay of the committed SOLO-vs-SUPERVISOR head-to-head
+ * artifacts (SWE-bench Verified, glm-5.2 both arms).
+ *
+ * MILESTONE 1: these types mirror the fixture files byte-for-byte semantics.
+ * They are the proof-of-faithfulness layer: `reconcile.ts` + `analyze.ts`
+ * must reproduce the reference `fixtures/analyze.py` output exactly (pinned in
+ * `replay.test.mts`) before any typed execution path is built on top.
+ */
+
+/**
+ * The fields of one SWE-bench Verified instance we actually consume from
+ * `task-meta.json` (generated by the experiment's `load_meta.py` from
+ * princeton-nlp/SWE-bench_Verified). `FAIL_TO_PASS` / `PASS_TO_PASS` are
+ * JSON-encoded string arrays as shipped by the HF dataset — kept as raw
+ * strings here; decode at the point of use.
+ */
+export interface SweInstance {
+  instance_id: string
+  repo: string
+  base_commit: string
+  problem_statement: string
+  patch: string
+  test_patch: string
+  /** JSON-encoded string[] (raw HF dataset encoding). */
+  FAIL_TO_PASS: string
+  /** JSON-encoded string[] (raw HF dataset encoding). */
+  PASS_TO_PASS: string
+  version: string | null
+  environment_setup_commit: string | null
+}
+
+/**
+ * One factory-bench instance — a merged feature PR from our own repo history
+ * turned into a gradable end-to-end feature-building task (see
+ * supervisor-lab/factory-bench/docs/design.md). The worker sees only the tree
+ * at `base_commit` (archive export, synthetic git history) plus the rewritten
+ * spec; the PR's own added test files are the hidden judge, overlaid from
+ * `judge_ref` at judge time. Field names mirror the instance dirs'
+ * `manifest.json` byte-for-byte, same policy as SweInstance vs task-meta.json.
+ */
+export interface FactoryInstance {
+  /** `factory.<repo>.<pr>` */
+  id: string
+  /** `owner/name` */
+  repo: string
+  /** Judge-side local mirror; NEVER exposed to the worker workspace. */
+  repo_local_mirror: string
+  /** The worker's world — the PR's base commit. */
+  base_commit: string
+  /** Merge commit the judge tests are read from (`git show <judge_ref>:<path>`). */
+  judge_ref: string
+  /** Instance-dir-relative spec file (PM-ticket grade rewrite of the PR body). */
+  spec_md: string
+  /** Hidden judge test files, overlaid at judge time only. */
+  judge_tests: string[]
+  /** Flaky/env-dependent tests excluded at calibration, reasons in calibration.md. */
+  excluded_tests: string[]
+  /** Immutable Node container image used for setup and judge commands. */
+  command_image: string
+  setup_cmds: string[]
+  judge_cmds: string[]
+  /** e.g. "all 30 judge tests pass; partial score = passed/30" — the /NN is parsed. */
+  resolved_criterion: string
+  timeout_s: number
+  worker_visible_paths_note?: string
+  runtime?: string
+  calibration?: { gold: string; base: string; receipts: string }
+}
+
+/**
+ * Discriminated instance union for the seams that used to assume SweInstance.
+ * A tagged wrapper (not a structural union) because both shapes mirror their
+ * on-disk artifacts byte-for-byte and neither may grow a discriminant field.
+ */
+export type ArenaInstance =
+  | { kind: 'swe'; instance: SweInstance }
+  | { kind: 'factory'; instance: FactoryInstance }
+
+/** The ledger/judge identity: SWE `instance_id` or factory `id`. */
+export function arenaInstanceId(a: ArenaInstance): string {
+  return a.kind === 'swe' ? a.instance.instance_id : a.instance.id
+}
+
+/** Per-run opencode usage breakdown captured on the SOLO arm. */
+export interface SoloUsage {
+  steps: number
+  in: number
+  out: number
+  reasoning: number
+  cache_w: number
+  cache_r: number
+  max_ctx: number
+  oc_cost: number
+  total_io: number
+}
+
+/**
+ * One paired row of `ledger.jsonl` — exactly the schema the experiment wrote.
+ * `null` values are real telemetry gaps (e.g. `sup_spentTokens: null` when the
+ * supervisor driver exited rc=3 mid-run), not absent data to be defaulted.
+ */
+export interface LedgerRow {
+  iid: string
+  solo_resolved: boolean
+  sup_resolved: boolean
+  solo_verify_pass: boolean
+  sup_verify_pass: boolean
+  solo_patch_lines: number
+  sup_patch_lines: number
+  solo_wall_s: number
+  sup_wall_s: number
+  solo_tokens: number
+  solo_usage: SoloUsage
+  sup_spentTokens: number | null
+  sup_spentUsd: number | null
+  sup_spawned: number
+  /** Missing on rows written before the field was added (pallets__flask-5014). */
+  sup_workers?: number
+  sup_settled: number
+  sup_subtasks: string[]
+  sup_delivered: boolean | null
+  /**
+   * The fixture ledger observed only completed/running; the driver can also
+   * settle failed/cancelled (M2 widened the union — the typed execution path
+   * records those honestly instead of coercing them).
+   */
+  sup_status: 'completed' | 'running' | 'failed' | 'cancelled' | null
+  sup_verdict: 'delivered' | 'no-winner' | 'best-effort' | null
+  solo_oc_rc: number
+  sup_driver_rc: number
+  solo_patch: string
+  sup_patch: string
+  /** Free-form correction note (e.g. psf__requests-1766 sup re-judge). */
+  _note?: string
+}
+
+/**
+ * Tags carried by `rejudge.jsonl` rows. Gold-family tags grade the OFFICIAL
+ * gold patch (calibrating the judge itself); final/my-reverify tags are the
+ * personal authoritative re-judgements of arm patches that override the
+ * automated ledger verdicts.
+ */
+export type RejudgeTag =
+  | 'gold-control'
+  | 'gold'
+  | 'gold2'
+  | 'solo-final'
+  | 'sup-final'
+  | 'solo-final2'
+  | 'sup-final2'
+  | 'my-reverify'
+
+/** One row of `rejudge.jsonl`. `resolved: null` = the judge run failed to parse (inconclusive). */
+export interface RejudgeRow {
+  iid: string
+  tag: RejudgeTag
+  patch: string
+  resolved: boolean | null
+  score?: number
+  secs?: number
+  patch_bytes?: number
+  error?: string
+  raw?: string
+}
+
+/** Evolution-round labels: SUP2/3/4 = rematch.jsonl / rematch2.jsonl / rematch3.jsonl. */
+export type RematchArm = 'SUP2' | 'SUP3' | 'SUP4'
+
+/** One row of a rematch*.jsonl evolution round. */
+export interface RematchRow {
+  iid: string
+  arm: RematchArm
+  resolved: boolean
+  verify_pass: boolean
+  patch_lines: number
+  wall_s: number
+  spawned: number
+  workers: number
+  sup_status: 'completed' | 'running'
+  sup_verdict: 'delivered' | 'no-winner' | 'best-effort' | null
+  delivered: boolean | null
+  spentTokens: number | null
+}
+
+/** Declarative arm identity for typed execution paths built on this module. */
+export interface ArmSpec {
+  name: string
+  kind: 'solo' | 'supervisor'
+  env: Record<string, string>
+  provenance: { repo: string; commit: string }
+}
+
+/** One pre-registered holdout instance from `holdout.json`. */
+export interface HoldoutEntry {
+  iid: string
+  repo: string
+  gold_official_resolved: boolean
+  verify_calibrated: boolean
+  selected_at_commit: string
+}
+
+/**
+ * The holdout registry: instances selected BEFORE any arm ran (see
+ * `holdout-preregister.log`), each with its gold patch verified against the
+ * calibrated judge. Only entries passing both checks are usable.
+ */
+export interface HoldoutRegistry {
+  entries: HoldoutEntry[]
+  /** Commit of the supervisor runtime (`loops`) at selection time. */
+  selectedAtCommit: string
+}
+
+/** Per-instance supervisor worker-session spend from `worker-tokens.json`. */
+export interface WorkerTokens {
+  worker_sessions: number
+  worker_tok: number
+}

package/src/swe-code-improve.mts

@@ -0,0 +1,328 @@
+/**
+ * META-HARNESS on the SWE scaffold — improve({ surface: 'code' }).
+ *
+ * A coding agent (Claude Code) REWRITES the scaffold LOGIC under bench/src (the seed prompt/playbook,
+ * runAgentic strategy/params, context handling, retry/patch synthesis) with the MODEL (glm worker) +
+ * the TOOL surface (list/read/edit[/run]) + the JUDGE held FIXED, judged on the official swebench
+ * Docker verdict, gated on a held-out instance split. This is the DGM/meta-harness recipe: let the
+ * SYSTEM find the scaffold lever from the RAW failure traces, not hand-build it.
+ *
+ * Wiring (all verified in this worktree):
+ *   - improve()/codeProposerFor + rawTraceContext come from the LOCAL agent-runtime build, linked into
+ *     this bench's node_modules (bench/node_modules/@tangle-network/agent-runtime -> /home/drew/code/agent-runtime).
+ *   - The candidate proposer is agenticGenerator(harness:'claude'), BUT the shipped runLocalHarness
+ *     spawns `claude --headless -p` and --headless is an unknown option on the current CLI (exit 1, no
+ *     edits ever). We pass code.generator with a corrected runHarness that spawns
+ *     `claude -p <prompt> --dangerously-skip-permissions` so the coding agent can actually edit the
+ *     worktree. This is a harness-spawn fix, NOT a hand-authored scaffold edit — Claude still finds the
+ *     lever itself from the traces.
+ *   - Each candidate is a git worktree the driver forks off baseRef; Claude edits bench/src in place;
+ *     `verify` (an import smoke of the edited scaffold) gates it before the expensive measurement.
+ *   - MEASUREMENT: the code-aware agent fn shells into the candidate scaffold's OWN judge-free emit
+ *     entrypoint (swe-emit-patch.mts) with cwd = the worktree, captures the unified diff, and returns
+ *     it. improve()'s FIXED swebench judge scores that diff OUTSIDE the candidate, so the scaffold can
+ *     never game its own axis. Baseline (empty surface) runs the UNEDITED scaffold from the main tree.
+ *
+ * Run (router WAF: keep example commands OUT of backticks):
+ *   TANGLE_API_KEY=... dotenvx run --quiet -f .../agent-state.env --
+ *     TRAIN_IDS=... HOLDOUT_IDS=... WORKER_MODEL=glm-4.6 GENERATIONS=1 POPULATION=2
+ *     RUN_DIR=/abs/run BASE_REF=meta/swe-scaffold-baseline
+ *     node_modules/.bin/tsx bench/src/swe-code-improve.mts
+ */
+import { spawn, spawnSync } from 'node:child_process'
+import { existsSync, mkdirSync, symlinkSync } from 'node:fs'
+import { join } from 'node:path'
+import { improve, agenticGenerator } from '@tangle-network/agent-runtime'
+import type { AgentProfile } from '@tangle-network/agent-interface'
+import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
+import { createSweBenchAdapter } from './benchmarks/swe-bench'
+import type { BenchTask } from './benchmarks/types'
+
+/** The agent-runtime repo the worktree adapter forks candidate checkouts from (agent-runtime-swe is a
+ *  worktree of it, so bench/src is tracked here). */
+const REPO_ROOT = process.env.REPO_ROOT ?? '/home/drew/code/agent-runtime'
+/** The main working tree = the baseline scaffold (its bench has the linked node_modules). */
+const SWE_MAIN_ROOT = process.env.SWE_MAIN_ROOT ?? '/home/drew/code/agent-runtime-swe'
+const MAIN_BENCH = join(SWE_MAIN_ROOT, 'bench')
+const TSX_BIN = join(MAIN_BENCH, 'node_modules/.bin/tsx')
+
+/** A candidate worktree is a fresh checkout with NO node_modules. Symlink the main bench's (already
+ *  carrying the linked local runtime) so both the verifier and the measurement can run the scaffold. */
+function ensureNodeModules(rootDir: string): void {
+  const target = join(rootDir, 'bench', 'node_modules')
+  if (existsSync(target)) return
+  try {
+    symlinkSync(join(MAIN_BENCH, 'node_modules'), target, 'dir')
+  } catch {
+    /* concurrent create / already exists — fine */
+  }
+}
+
+/** Run the scaffold's judge-free emit entrypoint on ONE instance, from `rootDir`'s bench/src. Returns
+ *  the unified diff (stdout, kept clean of any banner) + the tokens the worker reported (parsed from
+ *  the [emit] stderr line). NO judge — the judge is held outside, in this file. */
+function runEmit(
+  rootDir: string,
+  id: string,
+  env: Record<string, string>,
+  timeoutMs: number,
+): Promise<{ patch: string; tokIn: number; tokOut: number; usd: number; code: number | null }> {
+  ensureNodeModules(rootDir)
+  const benchDir = join(rootDir, 'bench')
+  return new Promise((resolve) => {
+    const child = spawn(TSX_BIN, [join(benchDir, 'src/swe-emit-patch.mts')], {
+      cwd: benchDir,
+      env: { ...process.env, ...env, IDS: id },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    })
+    let out = ''
+    let err = ''
+    child.stdout.on('data', (d) => (out += String(d)))
+    child.stderr.on('data', (d) => (err += String(d)))
+    const timer = setTimeout(() => {
+      if (!child.killed) child.kill('SIGKILL')
+    }, timeoutMs)
+    ;(timer as { unref?: () => void }).unref?.()
+    child.on('error', () => {
+      clearTimeout(timer)
+      resolve({ patch: '', tokIn: 0, tokOut: 0, usd: 0, code: 1 })
+    })
+    child.on('close', (code) => {
+      clearTimeout(timer)
+      const m = /tok=in:(\d+)\/out:(\d+)/.exec(err)
+      const u = /usd=([0-9.]+)/.exec(err)
+      if (code !== 0) console.error(`  [emit ${id}] exit=${code} stderr: ${err.slice(-400).replace(/\n/g, ' ')}`)
+      resolve({
+        patch: out,
+        tokIn: m ? Number(m[1]) : 0,
+        tokOut: m ? Number(m[2]) : 0,
+        usd: u ? Number(u[1]) : 0,
+        code,
+      })
+    })
+  })
+}
+
+async function main(): Promise<void> {
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('TANGLE_API_KEY required (worker calls the router)')
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6'
+  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
+  const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931,pallets__flask-5014').split(',').map((s) => s.trim()).filter(Boolean)
+  const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142,psf__requests-1921').split(',').map((s) => s.trim()).filter(Boolean)
+  const generations = Number(process.env.GENERATIONS ?? 1)
+  const population = Number(process.env.POPULATION ?? 2)
+  const innerTurns = Number(process.env.INNER_TURNS ?? 40)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 12000)
+  const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+  const baseRef = process.env.BASE_REF ?? 'meta/swe-scaffold-baseline'
+  const worktreeDir = process.env.WORKTREE_DIR ?? '/tmp/claude-1000/-home-drew-code-supervisor-lab/9ee6a456-a94f-474c-9888-b4afc9bc26bd/scratchpad/mh-worktrees'
+  const runDir = process.env.RUN_DIR ?? '/tmp/claude-1000/-home-drew-code-supervisor-lab/9ee6a456-a94f-474c-9888-b4afc9bc26bd/scratchpad/mh-run'
+  const emitTimeoutMs = Number(process.env.EMIT_TIMEOUT_MS ?? 600_000)
+  const harnessTimeoutMs = Number(process.env.HARNESS_TIMEOUT_MS ?? 900_000)
+  mkdirSync(worktreeDir, { recursive: true })
+  mkdirSync(runDir, { recursive: true })
+
+  const allIds = [...new Set([...trainIds, ...holdoutIds])]
+
+  console.log('=== META-HARNESS on the SWE scaffold — improve(surface:code) ===')
+  console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl} runTool=${enableRun}`)
+  console.log(`train=[${trainIds.join(', ')}] holdout=[${holdoutIds.join(', ')}]`)
+  console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} maxTokens=${maxTokens}`)
+  console.log(`repoRoot=${REPO_ROOT} baseRef=${baseRef}`)
+  console.log(`runDir=${runDir} worktreeDir=${worktreeDir}`)
+
+  // Fixed swebench Docker judge (held OUTSIDE every candidate) + task pool.
+  const adapter = createSweBenchAdapter()
+  const pool = await adapter.loadTasks({ ids: allIds, split: 'test' })
+  const byId = new Map<string, BenchTask>(pool.map((t) => [t.id, t]))
+  for (const id of allIds) if (!byId.has(id)) throw new Error(`instance not found in Verified: ${id}`)
+
+  const workerEnv: Record<string, string> = {
+    WORKER_MODEL: workerModel,
+    MAX_TOKENS: String(maxTokens),
+    INNER_TURNS: String(innerTurns),
+    ROUTER_BASE: routerBaseUrl,
+    TANGLE_API_KEY: routerKey,
+    RUN_TOOL: enableRun ? '1' : '0',
+  }
+
+  // The code-aware measurement agent. A CodeSurface -> run the candidate scaffold in its worktree; an
+  // empty/string surface (the baseline arm) -> run the UNEDITED scaffold from the main tree. Same path,
+  // so baseline and candidates are measured identically.
+  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const isCode = !!surface && typeof surface === 'object' && (surface as { kind?: string }).kind === 'code'
+    const worktreeRef = isCode ? String((surface as { worktreeRef?: string }).worktreeRef ?? '') : ''
+    const rootDir = isCode && worktreeRef && existsSync(worktreeRef) ? worktreeRef : SWE_MAIN_ROOT
+    const t0 = Date.now()
+    const r = await runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs)
+    const hasPatch = r.patch.trim().length > 0
+    // Report real usage; floor a patch-bearing zero-usage cell so the stub-guard cannot abort on a
+    // router telemetry gap (lift is judge-derived, so this only affects cost accounting).
+    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
+    ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
+    ctx.cost.observeTokens(
+      zeroUsage && hasPatch ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
+    )
+    const files = hasPatch ? [...r.patch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
+    console.log(
+      `  [measure] ${isCode ? 'cand' : 'base'} ${scenario.id} patch=${r.patch.length}b files=[${files.join(', ') || 'none'}] ` +
+        `tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s${isCode ? ` @ ${worktreeRef}` : ''}`,
+    )
+    return hasPatch ? r.patch : null
+  }
+
+  const judge: JudgeConfig<string, Scenario> = {
+    name: 'swebench-docker',
+    dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }],
+    async score({ artifact, scenario }) {
+      const patch = String(artifact ?? '')
+      if (!patch.trim()) {
+        console.log(`  [judge] ${scenario.id} resolved=0 (no patch)`)
+        return { dimensions: { resolved: 0 }, composite: 0, notes: 'no patch emitted' }
+      }
+      const bt = byId.get(scenario.id)
+      if (!bt) throw new Error(`judge: unknown scenario ${scenario.id}`)
+      const s = await adapter.judge(bt, patch)
+      console.log(`  [judge] ${scenario.id} resolved=${s.resolved ? 1 : 0}`)
+      return { dimensions: { resolved: s.resolved ? 1 : 0 }, composite: s.resolved ? 1 : 0, notes: (s.detail ?? '').slice(0, 200) }
+    },
+  }
+
+  // The corrected coding-harness spawn: `claude -p <prompt> --dangerously-skip-permissions`. The shipped
+  // runLocalHarness uses `claude --headless -p` (unknown option on this CLI). agenticGenerator ignores
+  // the return value (it reads worktree dirtiness), so a minimal result shape is enough.
+  const runHarness = (o: { cwd: string; taskPrompt: string; timeoutMs?: number; signal?: AbortSignal }): Promise<{ exitCode: number | null; stdout: string; stderr: string; killedBySignal: NodeJS.Signals | null; durationMs: number; timedOut: boolean }> => {
+    const started = Date.now()
+    return new Promise((resolve) => {
+      const child = spawn('claude', ['-p', o.taskPrompt, '--dangerously-skip-permissions'], {
+        cwd: o.cwd,
+        env: process.env,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      })
+      let stdout = ''
+      let stderr = ''
+      let timedOut = false
+      child.stdout?.on('data', (d) => (stdout += String(d)))
+      child.stderr?.on('data', (d) => (stderr += String(d)))
+      const timer = setTimeout(() => {
+        timedOut = true
+        if (!child.killed) child.kill('SIGTERM')
+      }, o.timeoutMs ?? harnessTimeoutMs)
+      ;(timer as { unref?: () => void }).unref?.()
+      const onAbort = () => {
+        if (!child.killed) child.kill('SIGTERM')
+      }
+      o.signal?.addEventListener('abort', onAbort, { once: true })
+      child.on('error', () => {
+        clearTimeout(timer)
+        resolve({ exitCode: 1, stdout, stderr: `${stderr}\n[spawn error]`, killedBySignal: null, durationMs: Date.now() - started, timedOut })
+      })
+      child.on('close', (code, signal) => {
+        clearTimeout(timer)
+        o.signal?.removeEventListener('abort', onAbort)
+        console.error(`  [proposer:claude] exit=${code} wall=${Math.round((Date.now() - started) / 1000)}s out=${stdout.length}b`)
+        resolve({ exitCode: code, stdout, stderr, killedBySignal: signal, durationMs: Date.now() - started, timedOut })
+      })
+    })
+  }
+
+  // Domain prompt: name the EDIT BOUNDARY (scaffold logic only) + keep the raw-trace evidence discipline
+  // (agenticGenerator discards a raw-trace candidate that doesn't inspect a trace + write the diagnosis).
+  const buildPrompt = (args: { report: unknown; findings: Array<{ severity?: string; claim?: string; recommended_action?: string }> }): string => {
+    const lines: string[] = [
+      'You are improving a SWE-bench coding SCAFFOLD: a harness that drives a FIXED worker model to fix real GitHub bugs via list_files/read_file/edit_file tools. Your job is to rewrite the SCAFFOLD LOGIC so the SAME worker model resolves MORE instances on a held-out split.',
+      '',
+      'EDIT ONLY the scaffold logic under bench/src:',
+      '  - the seed prompt / playbook: SWE_SEED_PROMPT and SWE_SEED_PROMPT_WITH_RUN in bench/src/swe-bench-env.ts',
+      '  - the exploration/context handling in bench/src/swe-bench-env.ts: list_files walk depth and 240-entry cap, the read_file 24000-char truncation, edit_file retry messaging, patch synthesis (git diff)',
+      '  - the runAgentic strategy/params in bench/src/swe-emit-patch.mts: innerTurns default, budget, how the patch is captured',
+      '',
+      'DO NOT change (FIXED for this search):',
+      '  - the TOOL surface: the list_files/read_file/edit_file/run tool NAMES, JSON signatures, or the path jail. Do not add or remove a tool.',
+      '  - the worker MODEL or MAX_TOKENS (passed via env — never hardcode a different model).',
+      '  - the swebench Docker JUDGE.',
+      '  - the swe-emit-patch.mts I/O contract: it still reads IDS + WORKER_MODEL + router env and prints ONLY the unified diff to stdout (diagnostics to stderr). You may change HOW the patch is produced, never this stdin/stdout contract.',
+      '',
+      'Make the smallest set of edits that addresses the failure evidence below, then stop. Leave changes in the working tree; do NOT commit.',
+      '',
+      'Failure evidence from the previous generation (its RAW run traces are on disk):',
+    ]
+    for (const f of args.findings) {
+      lines.push(`- (${f.severity ?? 'info'}) ${f.claim ?? ''}`)
+      if (f.recommended_action) lines.push(`    -> ${f.recommended_action}`)
+    }
+    lines.push(
+      '',
+      'Raw-trace evidence requirement (enforced — a candidate that skips this is discarded):',
+      '  - Inspect at least one raw trace path named above (grep/cat/ls it) BEFORE editing.',
+      '  - Write .improve/raw-trace-diagnosis.md in this worktree containing: the exact trace path(s) you inspected, the failure mechanism you found, and the scaffold-logic change you made.',
+      '  - A candidate with ONLY that file changed (no real scaffold edit) is discarded.',
+    )
+    return lines.join('\n')
+  }
+
+  const verify = (worktreePath: string): { ok: boolean; feedback?: string } => {
+    ensureNodeModules(worktreePath)
+    const res = spawnSync(TSX_BIN, [join(worktreePath, 'bench/src/swe-emit-patch.mts')], {
+      cwd: join(worktreePath, 'bench'),
+      env: { ...process.env, SWE_EMIT_SMOKE: '1' },
+      encoding: 'utf-8',
+      timeout: 180_000,
+    })
+    if (res.status === 0) return { ok: true }
+    const out = `${res.stdout ?? ''}${res.stderr ?? ''}`.trim()
+    return { ok: false, feedback: `edited scaffold failed import smoke (tsx swe-emit-patch SWE_EMIT_SMOKE=1):\n${out.slice(0, 3000)}` }
+  }
+
+  const generator = agenticGenerator({
+    harness: 'claude',
+    verify,
+    timeoutMs: harnessTimeoutMs,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    buildPrompt: buildPrompt as any,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    runHarness: runHarness as any,
+  })
+
+  const profile: AgentProfile = { name: 'swe-scaffold', prompt: { systemPrompt: '' } }
+  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
+  const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
+
+  const out = await improve(profile, [], {
+    surface: 'code',
+    gate: 'holdout',
+    code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, generator },
+    rawTraceContext: true,
+    runDir,
+    scenarios,
+    judge,
+    agent,
+    expectUsage: 'warn',
+    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency: 1, reps: 1 },
+    llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel },
+  })
+
+  console.log('\n=== RESULT ===')
+  console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`)
+  console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`)
+  console.log(`winner   holdout composite = ${out.raw.winner.compositeMean}`)
+  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
+  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
+  if (out.raw.winner.label) console.log(`winner label: ${out.raw.winner.label}`)
+  if (out.raw.winner.summary) console.log(`winner summary: ${out.raw.winner.summary}`)
+  for (const gen of out.raw.generations ?? []) {
+    console.log(`\n-- generation ${gen.record.generationIndex} candidates --`)
+    for (const c of gen.record.candidates) {
+      const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? []
+      const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ')
+      console.log(`  candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''} [${detail}]`)
+    }
+  }
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/swe-emit-patch.mts

@@ -0,0 +1,104 @@
+/**
+ * Judge-FREE patch emitter for the SWE scaffold — the FIXED measurement entrypoint.
+ *
+ * Given ONE instance id (IDS) + worker model + router env, this runs the candidate scaffold's OWN
+ * `createSweBenchEnvironment` + `runAgentic` IN THIS worktree and prints the unified `git diff` of
+ * the agent's edits to STDOUT. It does NOT judge — the swebench Docker judge is held OUTSIDE the
+ * candidate (in swe-code-improve.mts) so the scaffold can never grade its own axis.
+ *
+ * This is the seam the code-aware improve() agent shells into. Its I/O contract is FIXED and the
+ * scaffold proposer must NOT change it:
+ *   IN  (env):  IDS=<one instance id>, WORKER_MODEL, MAX_TOKENS, ROUTER_BASE, TANGLE_API_KEY,
+ *               INNER_TURNS, BUDGET, RUN_TOOL
+ *   OUT (fd1):  the unified diff (empty string when the agent made no edit)
+ *   diagnostics go to STDERR only, so stdout stays a clean patch.
+ *
+ * SWE_EMIT_SMOKE=1 → import/wiring check only: the module (and swe-bench-env) loaded, print READY on
+ * stderr and exit 0 WITHOUT a clone / model call / dataset read. This is what the candidate verifier
+ * runs to discard a scaffold edit that no longer imports.
+ *
+ * Command examples in prompts are kept OUT of backticks on purpose (router WAF 403s backtick-wrapped
+ * command text) — see swe-bench-env.ts.
+ */
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import type { AgenticSurface, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops'
+import { refine, runAgentic } from '@tangle-network/agent-runtime/loops'
+import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env'
+
+const exec = promisify(execFile)
+
+async function main(): Promise<void> {
+  const smoke = ['1', 'true', 'yes'].includes((process.env.SWE_EMIT_SMOKE ?? '').toLowerCase())
+  if (smoke) {
+    // The import graph (this file + swe-bench-env + the linked runtime) resolved by the time we get
+    // here. That is the whole check — no clone, no model call, no dataset read.
+    console.error('SWE_EMIT_SMOKE ok: swe-emit-patch + swe-bench-env import graph loaded')
+    return
+  }
+
+  const routerKey = process.env.TANGLE_API_KEY
+  if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker calls the router)')
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const model = process.env.WORKER_MODEL ?? 'glm-5.2'
+  const ids = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+  if (ids.length !== 1) throw new Error(`swe-emit-patch: exactly one IDS required, got [${ids.join(', ')}]`)
+  const id = ids[0]!
+  const innerTurns = Number(process.env.INNER_TURNS ?? 40)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 12000)
+  const budget = Number(process.env.BUDGET ?? 1)
+  const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+
+  const { environment, adapter } = await createSweBenchEnvironment(1, { ids, enableRun })
+  const pool = await adapter.loadTasks({ ids, split: 'test' })
+  const bt = pool.find((t) => t.id === id)
+  if (!bt) throw new Error(`swe-emit-patch: instance not found in Verified: ${id}`)
+
+  const task = {
+    id: bt.id,
+    systemPrompt: enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT,
+    userPrompt: bt.prompt,
+    meta: { instanceId: bt.id },
+  }
+
+  // Capture the patch from inside score() (called during the refine loop, BEFORE the surface closes
+  // and rms the checkout). Keep the LATEST non-empty diff so accumulated refinements win and a later
+  // empty read never clobbers a real patch.
+  let capturedPatch = ''
+  const proxy: AgenticSurface = {
+    ...environment,
+    async score(_t, handle: ArtifactHandle): Promise<SurfaceScore> {
+      try {
+        const d = await exec('git', ['-C', handle.id, 'diff'], { maxBuffer: 40_000_000, timeout: 60_000 })
+        if (d.stdout.trim()) capturedPatch = d.stdout
+      } catch {
+        /* workspace gone or git error → keep whatever we already captured */
+      }
+      return { passes: capturedPatch.trim() ? 1 : 0, total: 1, errored: 0 }
+    },
+  }
+
+  const t0 = Date.now()
+  const r = await runAgentic({
+    surface: proxy,
+    task,
+    strategy: refine,
+    routerBaseUrl,
+    routerKey,
+    model,
+    maxTokens,
+    innerTurns,
+    budget,
+  })
+  const files = capturedPatch ? [...capturedPatch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
+  console.error(
+    `[emit] ${id} shots=${r.shots} completions=${r.completions} tok=in:${r.tokens.input}/out:${r.tokens.output} ` +
+      `patch=${capturedPatch.length}b files=[${files.join(', ') || 'none'}] ${Math.round((Date.now() - t0) / 1000)}s`,
+  )
+  process.stdout.write(capturedPatch)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})