@tangle-network/agent-bench 0.4.4 → 0.4.6

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.4.6
+
+- Require typed proposal findings with explicit search or production origin throughout the SWE improvement loop.
+- Consume Runtime 0.108.0 with Eval 0.134.1, Interface 0.36.0, Knowledge 6.1.7, Materialize 0.9.2, and Sandbox 0.15.2.
+
+## 0.4.5
+
+- Allow the zero-model Pier proof to complete a cold separate-verifier image build before its task and overall execution deadlines.
+- Consume Runtime 0.107.5 with Eval 0.133.3, Interface 0.36.0, Knowledge 6.1.5, and Sandbox 0.15.1.
+
 ## 0.4.4
 
 - Consume Runtime 0.107.2 and Sandbox 0.15.0 with the current Eval, Interface, and Knowledge packages.

package/fixtures/pier-agent/no-model-task/task.toml

@@ -16,7 +16,7 @@ tags = ["no-model", "separate-verifier"]
 timeout_sec = 60.0
 
 [verifier]
-timeout_sec = 60.0
+timeout_sec = 300.0
 environment_mode = "separate"
 
 [verifier.environment]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.4.4",
+  "version": "0.4.6",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.133.0",
+    "@tangle-network/agent-eval": "0.134.1",
     "@tangle-network/agent-interface": "0.36.0",
-    "@tangle-network/agent-knowledge": "6.1.2",
-    "@tangle-network/sandbox": "0.15.0",
-    "@tangle-network/agent-runtime": "0.107.2"
+    "@tangle-network/agent-knowledge": "6.1.7",
+    "@tangle-network/sandbox": "0.15.2",
+    "@tangle-network/agent-runtime": "0.108.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/scripts/verify-pier-agent.mts

@@ -435,7 +435,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
     workspace: taskWorkspace,
     evaluatorTaskContainer: container,
     limits: {
-      timeoutMs: 180_000,
+      timeoutMs: 600_000,
       maxSteps: 8,
       maxModelCalls: 0,
       maxInputTokens: 0,

package/src/swe-arena/activation.test.mts

@@ -9,7 +9,7 @@ import { existsSync } from 'node:fs'
 import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type { AnalystFinding } from '@tangle-network/agent-eval'
+import type { ProposalFinding } from '@tangle-network/agent-eval'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import {
   ACTIVATION_PREDICATE_RELPATH,
@@ -236,8 +236,7 @@ describe('activation-predicate prefilter', () => {
 
   const generatorArgs = (candidateIndex: number) => ({
     worktreePath: driverWt,
-    report: undefined,
-    findings: [] as AnalystFinding[],
+    findings: [] as ProposalFinding[],
     maxShots: 1,
     signal: new AbortController().signal,
     generation: 0,

package/src/swe-arena/gepa-seat.test.mts

@@ -553,7 +553,6 @@ describe('fanOutLoopsGenerator with the gepa seat', () => {
 
   const generatorArgs = (candidateIndex: number) => ({
     worktreePath: driverWt,
-    report: undefined,
     findings: [],
     maxShots: 1,
     signal: new AbortController().signal,
@@ -1032,7 +1031,6 @@ describe('integration: real adapter roundtrip', () => {
       )
       const result = await gen.generate({
         worktreePath: driverWt,
-        report: undefined,
         findings: [],
         maxShots: 1,
         signal: new AbortController().signal,

package/src/swe-arena/outer-loop.mts

@@ -73,7 +73,11 @@ import {
 } from '@tangle-network/agent-runtime'
 import { runLocalHarness } from '@tangle-network/agent-runtime/mcp'
 import { canonicalCandidateDigest } from '@tangle-network/agent-interface'
-import { makeFinding } from '@tangle-network/agent-eval'
+import {
+  makeProposalFinding,
+  type AnalystFinding,
+  type ProposalFinding,
+} from '@tangle-network/agent-eval'
 import {
   FsLabeledScenarioStore,
   surfaceHash,
@@ -1111,6 +1115,10 @@ export async function removeEvalWorktree(loopsRepo: string, dest: string): Promi
  *  artifact path its gate checks for. */
 export const RAW_TRACE_DIAGNOSIS_PATH = '.improve/raw-trace-diagnosis.md'
 
+function asSearchProposalFinding(finding: AnalystFinding): ProposalFinding {
+  return { ...finding, proposal_origin: 'search' }
+}
+
 export function changeSpaceInstruction(space: ChangeSpace = LOOPS_CHANGE_SPACE): string {
   return [
     'DECLARED CHANGE-SPACE (hard constraint, enforced by an automated gate):',
@@ -1123,7 +1131,7 @@ export function changeSpaceInstruction(space: ChangeSpace = LOOPS_CHANGE_SPACE):
   ].join('\n')
 }
 
-export function round4BuildPrompt(args: { report: unknown; findings: Array<Record<string, unknown>> }): string {
+export function round4BuildPrompt(args: { findings: ReadonlyArray<ProposalFinding> }): string {
   const lines: string[] = [
     'You are the optimizer of the "loops" pi SUPERVISOR — an agent that plans, spawns sandboxed coding',
     'workers, and settles a delivered patch for SWE-bench Verified instances (glm-5.2 in both seats, frozen).',
@@ -1274,8 +1282,7 @@ export function constrainedLoopsGenerator(config: OuterLoopConfig): CandidateGen
   const inner = agenticGenerator({
     harness: config.proposerHarness,
     timeoutMs: config.proposerTimeoutMs,
-    buildPrompt: (args) =>
-      round4BuildPrompt(args as unknown as { report: unknown; findings: Array<Record<string, unknown>> }),
+    buildPrompt: round4BuildPrompt,
     verify: loopsCandidateVerifier(config.loopsRepo),
     runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(config.proposerHarness) }),
     // Three runs died as "author shot exited with code 1" with the shot's
@@ -1982,7 +1989,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P
 
   // ── diagnosis at the analyzeGeneration seam ──────────────────────────
   const rawTrace = rawTraceDistiller<Scenario, R4Artifact>({ fallbackFindings: [] })
-  const steeringFinding = makeFinding({
+  const steeringFinding = makeProposalFinding({
     analyst_id: 'round4-protocol',
     severity: 'high',
     area: 'constraint',
@@ -1992,13 +1999,14 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P
       '(plus the .improve/ diagnosis artifact). Judge, verify scripts, task prompts, model ids and budgets are immutable.',
     recommended_action: 'Keep every edit inside the change-space; out-of-space candidate diffs are rejected before evaluation.',
     evidence_refs: [],
+    proposal_origin: 'search',
   })
   const analyzeGeneration = async (input: {
     generation: number
     runDir: string
     candidates: Array<{ surfaceHash: string; composite: number; campaign: unknown }>
     history: unknown[]
-  }): Promise<unknown[]> => {
+  }): Promise<ProposalFinding[]> => {
     signal?.throwIfAborted()
     const runs: SupRunArtifacts[] = []
     if (input.generation === -1) {
@@ -2038,7 +2046,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P
         })
       }
     }
-    let ensembleFindings: unknown[] = []
+    let ensembleFindings: ProposalFinding[] = []
     if (runs.length > 0) {
       try {
         const scratch = join(config.outDir, 'diagnosis', `gen-${input.generation}`)
@@ -2058,7 +2066,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P
         ensembleFindings = fusedToAnalystFindings(ensemble.fused, {
           dirs: [...new Set(runs.map((r) => r.dir))],
           totalAnalysts: analysts.length,
-        })
+        }).map(asSearchProposalFinding)
       } catch (cause) {
         if (signal?.aborted) throw signal.reason
         // A dead router must not kill the round: the raw-trace context below
@@ -2084,7 +2092,7 @@ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): P
             }),
           }
     signal?.throwIfAborted()
-    const rawFindings = (await rawTrace(censoredInput as Parameters<typeof rawTrace>[0])) as unknown[]
+    const rawFindings = await rawTrace(censoredInput as Parameters<typeof rawTrace>[0])
     signal?.throwIfAborted()
     return [steeringFinding, ...ensembleFindings, ...rawFindings]
   }

package/src/swe-arena/outer-loop.test.mts

@@ -9,6 +9,7 @@ import { existsSync } from 'node:fs'
 import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
+import { makeProposalFinding } from '@tangle-network/agent-eval'
 import { describe, expect, it } from 'vitest'
 import { runOk } from './proc.ts'
 import {
@@ -574,9 +575,17 @@ describe('launch guards', () => {
 describe('round4BuildPrompt', () => {
   it('declares the change-space and renders findings', () => {
     const prompt = round4BuildPrompt({
-      report: undefined,
       findings: [
-        { severity: 'high', claim: 'fix placement mismatch', recommended_action: 'settle where maintainers expect' },
+        makeProposalFinding({
+          analyst_id: 'test',
+          severity: 'high',
+          area: 'mechanism',
+          claim: 'fix placement mismatch',
+          recommended_action: 'settle where maintainers expect',
+          evidence_refs: [],
+          confidence: 1,
+          proposal_origin: 'search',
+        }),
       ],
     })
     expect(prompt).toContain('DECLARED CHANGE-SPACE')
@@ -591,8 +600,17 @@ describe('round4BuildPrompt', () => {
 
   it('adds the raw-trace evidence contract when raw-trace findings are present', () => {
     const prompt = round4BuildPrompt({
-      report: undefined,
-      findings: [{ severity: 'high', area: 'raw-trace-context', claim: 'traces at /run/gen-0' }],
+      findings: [
+        makeProposalFinding({
+          analyst_id: 'raw-trace-distiller',
+          severity: 'high',
+          area: 'raw-trace-context',
+          claim: 'traces at /run/gen-0',
+          evidence_refs: [],
+          confidence: 1,
+          proposal_origin: 'search',
+        }),
+      ],
     })
     expect(prompt).toContain('Raw trace evidence requirement')
     expect(prompt).toContain(RAW_TRACE_DIAGNOSIS_PATH)

package/src/swe-arena/proposer-fanout.mts

@@ -49,7 +49,7 @@ import {
 } from '@tangle-network/agent-runtime'
 import { runLocalHarness } from '@tangle-network/agent-runtime/mcp'
 import type { AgentProfile } from '@tangle-network/agent-interface'
-import type { AnalystFinding, CostLedgerHandle } from '@tangle-network/agent-eval'
+import type { CostLedgerHandle, ProposalFinding } from '@tangle-network/agent-eval'
 import {
   changeSpaceViolations,
   loopsCandidateVerifier,
@@ -297,7 +297,7 @@ export function parentsPromptSection(parents: ParetoParentContext[]): string {
  *  stay (round4BuildPrompt), and the TASK is replaced with an explicit
  *  coherent-union merge of the parents' diffs. */
 export function mergeAuthorPrompt(
-  args: { report: unknown; findings: Array<Record<string, unknown>> },
+  args: { findings: ReadonlyArray<ProposalFinding> },
   spec: ProposerSpec,
   parents: ParetoParentContext[],
 ): string {
@@ -349,7 +349,10 @@ function isSteeringOrRawTrace(f: Record<string, unknown>): boolean {
 }
 
 /** Slice the diagnosis findings for one proposer. Pure. */
-export function sliceFindings(findings: AnalystFinding[], slice: ProposerSpec['diagnosisSlice']): AnalystFinding[] {
+export function sliceFindings(
+  findings: ReadonlyArray<ProposalFinding>,
+  slice: ProposerSpec['diagnosisSlice'],
+): ProposalFinding[] {
   if (slice === undefined || slice === 'all') return findings
   return findings.filter((finding) => {
     const f = finding as unknown as Record<string, unknown>
@@ -380,7 +383,7 @@ function appendGen5Sections(prompt: string, extras: Gen5PromptExtras): string {
  *  gen-5 briefing/activation sections when configured. A merge-seat spec gets
  *  the dedicated merge prompt instead (gen-5 sections still apply). */
 export function proposerBuildPrompt(
-  args: { report: unknown; findings: Array<Record<string, unknown>> },
+  args: { findings: ReadonlyArray<ProposalFinding> },
   spec: ProposerSpec,
   parents: ParetoParentContext[] = [],
   extras: Gen5PromptExtras = {},
@@ -543,16 +546,11 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn {
         harness,
         ...(profile ? { profile } : {}),
         timeoutMs: config.proposerTimeoutMs,
-        buildPrompt: (a) =>
-          proposerBuildPrompt(
-            a as unknown as { report: unknown; findings: Array<Record<string, unknown>> },
-            proposer,
-            deps.parents ?? [],
-            {
-              ...(deps.briefing ? { briefing: deps.briefing } : {}),
-              ...(config.activationGate === true ? { activationGate: true } : {}),
-            },
-          ),
+        buildPrompt: (args) =>
+          proposerBuildPrompt(args, proposer, deps.parents ?? [], {
+            ...(deps.briefing ? { briefing: deps.briefing } : {}),
+            ...(config.activationGate === true ? { activationGate: true } : {}),
+          }),
         verify: loopsCandidateVerifier(config.loopsRepo),
         runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(harness) }),
         onShotCompleted: proposerShotHooks({

package/src/swe-arena/proposer-fanout.test.mts

@@ -2,7 +2,7 @@ import { existsSync } from 'node:fs'
 import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type { AnalystFinding } from '@tangle-network/agent-eval'
+import { makeProposalFinding, type ProposalFinding } from '@tangle-network/agent-eval'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import {
   defaultGen3Config,
@@ -37,19 +37,20 @@ import { runOk } from './proc.ts'
 // Pure pieces.
 // ---------------------------------------------------------------------------
 
-const finding = (over: Record<string, unknown>): AnalystFinding =>
-  ({
-    schema_version: '1.0.0',
-    finding_id: `f-${Math.random().toString(36).slice(2)}`,
+const finding = (
+  over: Partial<Pick<ProposalFinding, 'analyst_id' | 'area' | 'claim'>>,
+): ProposalFinding =>
+  makeProposalFinding({
     analyst_id: 'analyst',
-    produced_at: new Date(0).toISOString(),
     severity: 'high',
     area: 'mechanism',
     claim: 'workers drop build artifacts on clone',
     evidence_refs: [],
     confidence: 0.9,
     ...over,
-  }) as unknown as AnalystFinding
+    proposal_origin: 'search',
+    produced_at: new Date(0).toISOString(),
+  })
 
 describe('sliceFindings', () => {
   const mech = finding({ claim: 'sandbox clone drops untracked build artifacts' })
@@ -73,7 +74,7 @@ describe('sliceFindings', () => {
 describe('proposerBuildPrompt', () => {
   it('appends the lens AFTER the shared protocol prompt, leaving the change-space text intact', () => {
     const spec: ProposerSpec = { name: 'x', harness: 'claude', lens: 'Prefer code-path fixes.' }
-    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec)
+    const prompt = proposerBuildPrompt({ findings: [] }, spec)
     expect(prompt).toContain('DECLARED CHANGE-SPACE')
     expect(prompt.indexOf('DECLARED CHANGE-SPACE')).toBeLessThan(prompt.indexOf('YOUR AUTHORING LENS (x)'))
     expect(prompt).toContain('Prefer code-path fixes.')
@@ -81,7 +82,7 @@ describe('proposerBuildPrompt', () => {
 
   it('is the bare round prompt without a lens', () => {
     const spec: ProposerSpec = { name: 'x', harness: 'claude' }
-    expect(proposerBuildPrompt({ report: undefined, findings: [] }, spec)).not.toContain('AUTHORING LENS')
+    expect(proposerBuildPrompt({ findings: [] }, spec)).not.toContain('AUTHORING LENS')
   })
 })
 
@@ -134,7 +135,7 @@ describe('resolveAuthorProfile (pinned models)', () => {
 describe('proposerBuildPrompt with pareto parents', () => {
   it('appends the parents section (evidence + diffs) after the protocol prompt and lens', () => {
     const spec: ProposerSpec = { name: 'x', harness: 'claude', lens: 'Prefer code-path fixes.' }
-    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS)
+    const prompt = proposerBuildPrompt({ findings: [] }, spec, PARENTS)
     expect(prompt).toContain('DECLARED CHANGE-SPACE')
     expect(prompt).toContain('PARETO PARENTS')
     expect(prompt.indexOf('YOUR AUTHORING LENS')).toBeLessThan(prompt.indexOf('PARETO PARENTS'))
@@ -146,7 +147,7 @@ describe('proposerBuildPrompt with pareto parents', () => {
 
   it('leaves the prompt untouched when no parents are seeded (gen-3 behavior)', () => {
     const spec: ProposerSpec = { name: 'x', harness: 'claude' }
-    expect(proposerBuildPrompt({ report: undefined, findings: [] }, spec)).not.toContain('PARETO PARENTS')
+    expect(proposerBuildPrompt({ findings: [] }, spec)).not.toContain('PARETO PARENTS')
     expect(parentsPromptSection(PARENTS)).toContain('measured evidence')
   })
 })
@@ -155,7 +156,7 @@ describe('mergeAuthorPrompt', () => {
   const spec: ProposerSpec = { name: 'merge-author', harness: 'claude', merge: true }
 
   it('keeps the change-space contract and presents BOTH parent diffs with the coherent-union task', () => {
-    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS)
+    const prompt = proposerBuildPrompt({ findings: [] }, spec, PARENTS)
     expect(prompt).toContain('DECLARED CHANGE-SPACE')
     expect(prompt).toContain('MERGE SEAT')
     expect(prompt).toContain('UNION of the')
@@ -169,8 +170,8 @@ describe('mergeAuthorPrompt', () => {
   })
 
   it('fails loud with fewer than two parents', () => {
-    expect(() => mergeAuthorPrompt({ report: undefined, findings: [] }, spec, [PARENTS[0]!])).toThrow(/>=2/)
-    expect(() => proposerBuildPrompt({ report: undefined, findings: [] }, spec, [])).toThrow(/>=2/)
+    expect(() => mergeAuthorPrompt({ findings: [] }, spec, [PARENTS[0]!])).toThrow(/>=2/)
+    expect(() => proposerBuildPrompt({ findings: [] }, spec, [])).toThrow(/>=2/)
   })
 })
 
@@ -347,8 +348,7 @@ describe('fanOutLoopsGenerator', () => {
 
   const generatorArgs = (candidateIndex: number) => ({
     worktreePath: driverWt,
-    report: undefined,
-    findings: [] as AnalystFinding[],
+    findings: [] as ProposalFinding[],
     maxShots: 1,
     signal: new AbortController().signal,
     generation: 0,
@@ -586,7 +586,7 @@ describe('gen-5 prompt sections', () => {
 
   it('appends EVIDENCE MAP + briefing + activation contract after the protocol prompt', () => {
     const spec: ProposerSpec = { name: 'x', harness: 'claude' }
-    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, [], {
+    const prompt = proposerBuildPrompt({ findings: [] }, spec, [], {
       briefing,
       activationGate: true,
     })
@@ -599,7 +599,7 @@ describe('gen-5 prompt sections', () => {
 
   it('the merge seat gets the gen-5 sections too', () => {
     const spec: ProposerSpec = { name: 'merge-author', harness: 'claude', merge: true }
-    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS, {
+    const prompt = proposerBuildPrompt({ findings: [] }, spec, PARENTS, {
       briefing,
       activationGate: true,
     })
@@ -610,9 +610,9 @@ describe('gen-5 prompt sections', () => {
 
   it('leaves gen-3/gen-4 prompts byte-identical when no extras are passed', () => {
     const spec: ProposerSpec = { name: 'x', harness: 'claude' }
-    const legacy = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS)
+    const legacy = proposerBuildPrompt({ findings: [] }, spec, PARENTS)
     expect(legacy).not.toContain('EVIDENCE MAP')
     expect(legacy).not.toContain('ACTIVATION PREDICATE')
-    expect(proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS, {})).toBe(legacy)
+    expect(proposerBuildPrompt({ findings: [] }, spec, PARENTS, {})).toBe(legacy)
   })
 })

package/src/swe-arena/run-supervisor.test.mts

@@ -3,11 +3,25 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterAll, describe, expect, it } from 'vitest'
 import { runSupervisorShim, supervisorDriverKillGraceMs } from './arms.ts'
-import { run, TIMEOUT_RC } from './proc.ts'
+import { ABORT_RC, run } from './proc.ts'
 
 const dirs: string[] = []
 const workerPids: number[] = []
 
+async function waitForFile(path: string, timeoutMs = 10_000): Promise<void> {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    try {
+      await access(path)
+      return
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
+    }
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+  throw new Error(`timed out waiting for ${path}`)
+}
+
 afterAll(async () => {
   for (const pid of workerPids) {
     try {
@@ -122,6 +136,7 @@ export default function fakeExtension(pi) {
       writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
+      writeFileSync(join(ctx.cwd, 'outer-driver-ready.txt'), 'ready')
       return text('spawned supervisor ' + id)
     },
   })
@@ -140,10 +155,11 @@ export default function fakeExtension(pi) {
 `)
 
     const cancelTimeoutMs = 1_500
-    const result = await run(process.execPath, [
+    const controller = new AbortController()
+    const resultPromise = run(process.execPath, [
       '--import', 'tsx', runSupervisorShim, extension, workspace, params,
     ], {
-      timeoutMs: 1_000,
+      signal: controller.signal,
       killGraceMs: supervisorDriverKillGraceMs({
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       }),
@@ -154,9 +170,19 @@ export default function fakeExtension(pi) {
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       },
     })
+    try {
+      await waitForFile(join(workspace, 'outer-driver-ready.txt'))
+    } catch (error) {
+      controller.abort()
+      await resultPromise
+      throw error
+    }
+    controller.abort()
+    const result = await resultPromise
 
-    expect(result.code, result.stdout + result.stderr).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
+    expect(result.code, result.stdout + result.stderr).toBe(ABORT_RC)
+    expect(result.aborted).toBe(true)
+    expect(result.timedOut).toBe(false)
     expect(await readFile(join(workspace, 'outer-cancel-settled.txt'), 'utf8')).toBe('settled')
     expect(JSON.parse(await readFile(join(workspace, '.loops', 'supervisor', 'sup-2-fake34', 'state.json'), 'utf8')))
       .toMatchObject({ status: 'cancelled' })
@@ -170,10 +196,10 @@ export default function fakeExtension(pi) {
     const extension = join(dir, 'fake-extension.mjs')
     await mkdir(workspace)
     await writeFile(params, '{}')
-    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); setInterval(()=>{},1000)`
+    const workerScript = `const fs=require('node:fs'); const marker=process.argv[1]; const ready=process.argv[2]; process.on('SIGTERM',()=>setTimeout(()=>{fs.writeFileSync(marker,'clean'); process.exit(0)},30)); fs.writeFileSync(ready,'ready'); setInterval(()=>{},1000)`
     await writeFile(extension, `
 import { spawn } from 'node:child_process'
-import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'
+import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
 
 const id = 'sup-3-signal56'
@@ -188,11 +214,13 @@ export default function fakeExtension(pi) {
       writeFileSync(join(runDir, 'state.json'), JSON.stringify({ status: 'running' }))
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id, label: 'root' }) + '\\n')
       appendFileSync(join(runDir, 'journal.jsonl'), JSON.stringify({ kind: 'spawned', id: id + ':s1', parent: id, label: 'w-0' }) + '\\n')
-      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'signal-worker-cleaned.txt')], {
+      const readyPath = join(ctx.cwd, 'signal-worker-ready.txt')
+      child = spawn(process.execPath, ['-e', ${JSON.stringify(workerScript)}, join(ctx.cwd, 'signal-worker-cleaned.txt'), readyPath], {
         detached: true,
         stdio: 'ignore',
       })
       writeFileSync(join(ctx.cwd, 'signal-worker.pid'), String(child.pid))
+      while (!existsSync(readyPath)) await new Promise((resolve) => setTimeout(resolve, 5))
       writeFileSync(join(ctx.cwd, 'spawn-entered.txt'), 'entered')
       await new Promise((resolve) => setTimeout(resolve, 30_000))
       return text('spawned supervisor ' + id)
@@ -214,10 +242,11 @@ export default function fakeExtension(pi) {
 `)
 
     const cancelTimeoutMs = 2_000
-    const result = await run(process.execPath, [
+    const controller = new AbortController()
+    const resultPromise = run(process.execPath, [
       '--import', 'tsx', runSupervisorShim, extension, workspace, params,
     ], {
-      timeoutMs: 1_000,
+      signal: controller.signal,
       killGraceMs: supervisorDriverKillGraceMs({
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       }),
@@ -228,11 +257,21 @@ export default function fakeExtension(pi) {
         DRIVER_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs),
       },
     })
+    try {
+      await waitForFile(join(workspace, 'spawn-entered.txt'))
+    } catch (error) {
+      controller.abort()
+      await resultPromise
+      throw error
+    }
+    controller.abort()
+    const result = await resultPromise
     const workerPid = Number(await readFile(join(workspace, 'signal-worker.pid'), 'utf8'))
     workerPids.push(workerPid)
 
-    expect(result.code, result.stdout + result.stderr).toBe(TIMEOUT_RC)
-    expect(result.timedOut).toBe(true)
+    expect(result.code, result.stdout + result.stderr).toBe(ABORT_RC)
+    expect(result.aborted).toBe(true)
+    expect(result.timedOut).toBe(false)
     expect(await readFile(join(workspace, 'spawn-entered.txt'), 'utf8')).toBe('entered')
     expect(await readFile(join(workspace, 'signal-worker-cleaned.txt'), 'utf8')).toBe('clean')
     expect(await readFile(join(workspace, 'signal-cancel-settled.txt'), 'utf8')).toBe('settled')

package/src/swe-arena/scratch-worktree-serialization.test.mts

@@ -0,0 +1,72 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const gitCalls = vi.hoisted(() => ({
+  activeByRepository: new Map<string, number>(),
+  maximumByRepository: new Map<string, number>(),
+  active: 0,
+  maximum: 0,
+}))
+
+vi.mock('./proc', () => {
+  const recordGitCall = async (_binary: string, arguments_: string[]) => {
+    const repository = arguments_[1] ?? ''
+    const activeForRepository = (gitCalls.activeByRepository.get(repository) ?? 0) + 1
+    gitCalls.activeByRepository.set(repository, activeForRepository)
+    gitCalls.maximumByRepository.set(
+      repository,
+      Math.max(gitCalls.maximumByRepository.get(repository) ?? 0, activeForRepository),
+    )
+    gitCalls.active += 1
+    gitCalls.maximum = Math.max(gitCalls.maximum, gitCalls.active)
+    await new Promise((resolve) => setTimeout(resolve, 5))
+    gitCalls.active -= 1
+    gitCalls.activeByRepository.set(repository, activeForRepository - 1)
+    return { code: 0, stdout: '', stderr: '' }
+  }
+  return {
+    run: vi.fn(recordGitCall),
+    runOk: vi.fn(recordGitCall),
+  }
+})
+
+import {
+  createDetachedWorktree,
+  pruneDetachedWorktrees,
+  removeDetachedWorktree,
+} from './scratch-worktree.ts'
+
+describe('scratch worktree mutation serialization', () => {
+  const roots: string[] = []
+
+  afterEach(async () => {
+    gitCalls.activeByRepository.clear()
+    gitCalls.maximumByRepository.clear()
+    gitCalls.active = 0
+    gitCalls.maximum = 0
+    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
+  })
+
+  it('serializes metadata changes per repository without blocking separate repositories', async () => {
+    const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-serialization-'))
+    roots.push(output)
+    const operations = ['repository-a', 'repository-b'].flatMap((repository) =>
+      Array.from({ length: 4 }, (_, index) => {
+        const destination = join(output, `${repository}-candidate-${index}`)
+        return [
+          createDetachedWorktree(repository, 'commit', destination),
+          removeDetachedWorktree(repository, destination),
+          pruneDetachedWorktrees(repository),
+        ]
+      }),
+    )
+
+    await Promise.all(operations.flat())
+
+    expect(gitCalls.maximumByRepository.get('repository-a')).toBe(1)
+    expect(gitCalls.maximumByRepository.get('repository-b')).toBe(1)
+    expect(gitCalls.maximum).toBe(2)
+  })
+})

package/src/swe-arena/scratch-worktree.ts

@@ -1,34 +1,64 @@
 import { rm } from 'node:fs/promises'
 import { run, runOk } from './proc'
 
+// Git worktree subcommands mutate shared metadata under .git/worktrees and are
+// not safe to overlap, even when each command targets a different destination.
+const repositoryMutationQueues = new Map<string, Promise<void>>()
+
+async function mutateWorktrees<T>(
+  repository: string,
+  operation: () => Promise<T>,
+): Promise<T> {
+  const predecessor = repositoryMutationQueues.get(repository) ?? Promise.resolve()
+  const result = predecessor.then(operation, operation)
+  const settled = result.then(
+    () => undefined,
+    () => undefined,
+  )
+  repositoryMutationQueues.set(repository, settled)
+  try {
+    return await result
+  } finally {
+    if (repositoryMutationQueues.get(repository) === settled) {
+      repositoryMutationQueues.delete(repository)
+    }
+  }
+}
+
 export async function createDetachedWorktree(
   repository: string,
   commit: string,
   destination: string,
 ): Promise<void> {
-  await run('git', ['-C', repository, 'worktree', 'remove', '--force', '--', destination])
-  await rm(destination, { recursive: true, force: true })
-  await runOk('git', ['-C', repository, 'worktree', 'add', '--detach', destination, commit])
+  await mutateWorktrees(repository, async () => {
+    await run('git', ['-C', repository, 'worktree', 'remove', '--force', '--', destination])
+    await rm(destination, { recursive: true, force: true })
+    await runOk('git', ['-C', repository, 'worktree', 'add', '--detach', destination, commit])
+  })
 }
 
 export async function pruneDetachedWorktrees(repository: string): Promise<void> {
-  await runOk('git', ['-C', repository, 'worktree', 'prune'])
+  await mutateWorktrees(repository, async () => {
+    await runOk('git', ['-C', repository, 'worktree', 'prune'])
+  })
 }
 
 export async function removeDetachedWorktree(
   repository: string,
   destination: string,
 ): Promise<void> {
-  const result = await run('git', [
-    '-C',
-    repository,
-    'worktree',
-    'remove',
-    '--force',
-    '--',
-    destination,
-  ])
-  if (result.code !== 0) {
-    await rm(destination, { recursive: true, force: true })
-  }
+  await mutateWorktrees(repository, async () => {
+    const result = await run('git', [
+      '-C',
+      repository,
+      'worktree',
+      'remove',
+      '--force',
+      '--',
+      destination,
+    ])
+    if (result.code !== 0) {
+      await rm(destination, { recursive: true, force: true })
+    }
+  })
 }

package/src/swe-code-improve.mts

@@ -33,6 +33,7 @@ import { spawn, spawnSync } from 'node:child_process'
 import { existsSync, mkdirSync, symlinkSync } from 'node:fs'
 import { join } from 'node:path'
 import { improve, agenticGenerator } from '@tangle-network/agent-runtime'
+import type { ProposalFinding } from '@tangle-network/agent-eval'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
 import { createSweBenchAdapter } from './benchmarks/swe-bench'
 import type { BenchTask } from './benchmarks/types'
@@ -238,7 +239,7 @@ async function main(): Promise<void> {
 
   // Domain prompt: name the EDIT BOUNDARY (scaffold logic only) + keep the raw-trace evidence discipline
   // (agenticGenerator discards a raw-trace candidate that doesn't inspect a trace + write the diagnosis).
-  const buildPrompt = (args: { report: unknown; findings: Array<{ severity?: string; claim?: string; recommended_action?: string }> }): string => {
+  const buildPrompt = (args: { findings: ReadonlyArray<ProposalFinding> }): string => {
     const lines: string[] = [
       'You are improving a SWE-bench coding SCAFFOLD: a harness that drives a FIXED worker model to fix real GitHub bugs via list_files/read_file/edit_file tools. Your job is to rewrite the SCAFFOLD LOGIC so the SAME worker model resolves MORE instances on a held-out split.',
       '',