@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/manifest.mts

@@ -0,0 +1,293 @@
+/**
+ * Rollout manifest — a pure READER/JOIN over what the round already writes.
+ * No new capture pipeline: the lib's improve() run emits loop-provenance.json,
+ * per-cell cached results, and the durable cost log; the swe-arena dispatch
+ * emits arm result/judge files, candidate patches, and proposer-shot receipts.
+ * This module joins them into one versioned per-generation manifest of
+ * (state, action, outcome, cost, artifactPaths) triples.
+ *
+ *   tsx src/swe-arena/manifest.mts <outDir>   # writes <outDir>/rollout-manifest.json
+ *
+ * Sources (all optional — a missing file is a labeled gap, never a crash):
+ *   - <outDir>/improve-run/loop-provenance.json            (lib)
+ *   - <outDir>/improve-run/cost-ledger.jsonl               (lib, durable receipts)
+ *   - <outDir>/improve-run/baseline and gen-N/candidate-K  (lib, per-cell caches)
+ *   - <outDir>/candidates/<commit10>.patch                 (ours, diff writing)
+ *   - <outDir>/proposer-shots/genN-candK-shotS.json        (ours, shot receipts)
+ *   - per-cell artifact.runDir result.json + judge.json    (ours, arm + judge)
+ */
+
+import { readdir, readFile, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import {
+  loadCampaignCells,
+  loadCandidateCellGroups,
+  perInstanceFromCells,
+  replicateRunsFromCells,
+  resolvedInstanceCount,
+  sumWallSFromCells,
+  type EvidenceCell,
+  type StaircasePerInstance,
+} from './cell-evidence.mts'
+
+export const ROLLOUT_SCHEMA = 'swe-arena.rollout.v1'
+
+export interface RolloutCost {
+  /** Σ campaign-cell CostLedger spend (worker receipts) across the cells. */
+  cellCostUsd: number
+  cellTokensIn: number
+  cellTokensOut: number
+  /** Σ runtime spend-tree usd/tokens off the artifacts (telemetry-gap nulls skipped). */
+  armSpentUsd: number
+  armSpentTokens: number
+  wallS: number
+  judgeWallS: number
+  /** Durable cost-ledger receipts whose tags name this campaign dir. */
+  receiptCallIds: string[]
+}
+
+export interface RolloutEntry {
+  state: {
+    generation: number
+    /** -1 candidateIndex = the baseline campaign. */
+    candidateIndex: number
+    campaignDir: string
+    commit: string | null
+  }
+  action: {
+    /** Candidate diff written at dispatch time (null for the baseline / a
+     *  resumed-only run whose diff was not rewritten). */
+    diffPath: string | null
+    /** Persisted proposer-shot receipts for this (generation, candidate). */
+    shotReceiptPaths: string[]
+  }
+  outcome: {
+    perInstance: StaircasePerInstance[]
+    resolvedCount: number
+    cellsCached: number
+  }
+  cost: RolloutCost
+  artifactPaths: {
+    cells: string[]
+    armResults: string[]
+    judgeVerdicts: string[]
+    patches: string[]
+  }
+}
+
+export interface RolloutGeneration {
+  generation: number
+  rollouts: RolloutEntry[]
+}
+
+export interface RolloutManifest {
+  schema: typeof ROLLOUT_SCHEMA
+  outDir: string
+  at: string
+  /** The lib's durable provenance record, verbatim (null = not emitted). */
+  provenance: unknown | null
+  /** Improvement-set ids inferred from the baseline cells. */
+  instances: string[]
+  generations: RolloutGeneration[]
+}
+
+interface LedgerReceipt {
+  callId: string
+  tags?: Record<string, string>
+  [k: string]: unknown
+}
+
+/** Settled receipts out of the lib's append-only cost-ledger.jsonl. */
+export async function loadLedgerReceipts(improveRunDir: string): Promise<LedgerReceipt[]> {
+  const raw = await readFile(join(improveRunDir, 'cost-ledger.jsonl'), 'utf8').catch(() => '')
+  const receipts: LedgerReceipt[] = []
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue
+    let event: Record<string, unknown>
+    try {
+      event = JSON.parse(line) as Record<string, unknown>
+    } catch {
+      continue
+    }
+    const record = event.record as Record<string, unknown> | undefined
+    if (record && record.status === 'settled' && typeof record.callId === 'string') {
+      receipts.push(record as unknown as LedgerReceipt)
+    }
+  }
+  return receipts
+}
+
+function costFromCells(
+  cells: EvidenceCell[],
+  campaignDir: string,
+  receipts: LedgerReceipt[],
+): RolloutCost {
+  let cellCostUsd = 0
+  let cellTokensIn = 0
+  let cellTokensOut = 0
+  let armSpentUsd = 0
+  let armSpentTokens = 0
+  let judgeWallS = 0
+  for (const cell of cells) {
+    cellCostUsd += cell.costUsd ?? 0
+    cellTokensIn += cell.tokenUsage?.input ?? 0
+    cellTokensOut += cell.tokenUsage?.output ?? 0
+    const a = cell.artifact
+    if (a !== null && a.kind === 'swe-arm') {
+      armSpentUsd += a.spentUsd ?? 0
+      armSpentTokens += a.spentTokens ?? 0
+      judgeWallS += a.judgeWallS ?? 0
+    }
+  }
+  return {
+    cellCostUsd: Number(cellCostUsd.toFixed(6)),
+    cellTokensIn,
+    cellTokensOut,
+    armSpentUsd: Number(armSpentUsd.toFixed(6)),
+    armSpentTokens,
+    wallS: sumWallSFromCells(cells),
+    judgeWallS,
+    receiptCallIds: receipts
+      .filter((r) => r.tags?.runDir === campaignDir)
+      .map((r) => r.callId)
+      .sort(),
+  }
+}
+
+function artifactPathsFromCells(cells: EvidenceCell[], campaignDir: string): RolloutEntry['artifactPaths'] {
+  const armResults: string[] = []
+  const judgeVerdicts: string[] = []
+  const patches: string[] = []
+  for (const cell of cells) {
+    const a = cell.artifact
+    if (a === null || a.kind !== 'swe-arm') continue
+    if (a.runDir) {
+      const resultPath = join(a.runDir, 'result.json')
+      const judgePath = join(a.runDir, 'judge.json')
+      if (existsSync(resultPath)) armResults.push(resultPath)
+      if (existsSync(judgePath)) judgeVerdicts.push(judgePath)
+    }
+    if (a.patchPath && existsSync(a.patchPath)) patches.push(a.patchPath)
+  }
+  return {
+    // The campaign dir holds the per-cell cached-result.json score records.
+    cells: cells.length > 0 ? [campaignDir] : [],
+    armResults: [...new Set(armResults)].sort(),
+    judgeVerdicts: [...new Set(judgeVerdicts)].sort(),
+    patches: [...new Set(patches)].sort(),
+  }
+}
+
+async function shotReceiptPathsFor(
+  outDir: string,
+  generation: number,
+  candidateIndex: number,
+): Promise<string[]> {
+  const shotDir = join(outDir, 'proposer-shots')
+  const names = await readdir(shotDir).catch(() => [])
+  return names
+    .filter((n) => n.startsWith(`gen${generation}-cand${candidateIndex}-`) && n.endsWith('.json'))
+    .sort()
+    .map((n) => join(shotDir, n))
+}
+
+/** Build the rollout manifest for one round outDir. Pure join — reads only. */
+export async function buildRolloutManifest(outDir: string): Promise<RolloutManifest> {
+  const improveRunDir = join(outDir, 'improve-run')
+  const provenanceRaw = await readFile(join(improveRunDir, 'loop-provenance.json'), 'utf8').catch(() => null)
+  const provenance = provenanceRaw === null ? null : (JSON.parse(provenanceRaw) as unknown)
+  const receipts = await loadLedgerReceipts(improveRunDir)
+
+  const baselineDir = join(improveRunDir, 'baseline')
+  const baselineCells = await loadCampaignCells(baselineDir)
+  const instances = [...new Set(baselineCells.map((c) => c.scenarioId))].sort()
+
+  const generations = new Map<number, RolloutGeneration>()
+  const genOf = (g: number): RolloutGeneration => {
+    const existing = generations.get(g)
+    if (existing) return existing
+    const fresh: RolloutGeneration = { generation: g, rollouts: [] }
+    generations.set(g, fresh)
+    return fresh
+  }
+
+  if (baselineCells.length > 0) {
+    genOf(-1).rollouts.push({
+      state: { generation: -1, candidateIndex: -1, campaignDir: baselineDir, commit: baselineCells[0]?.artifact?.commit ?? null },
+      action: { diffPath: null, shotReceiptPaths: [] },
+      outcome: {
+        perInstance: perInstanceFromCells(baselineCells),
+        resolvedCount: resolvedInstanceCount(
+          replicateRunsFromCells(baselineCells),
+          instances,
+          maxRep(baselineCells) + 1,
+        ),
+        cellsCached: baselineCells.length,
+      },
+      cost: costFromCells(baselineCells, baselineDir, receipts),
+      artifactPaths: artifactPathsFromCells(baselineCells, baselineDir),
+    })
+  }
+
+  for (const group of await loadCandidateCellGroups(improveRunDir)) {
+    const diffPath = group.commit ? join(outDir, 'candidates', `${group.commit.slice(0, 10)}.patch`) : null
+    genOf(group.generation).rollouts.push({
+      state: {
+        generation: group.generation,
+        candidateIndex: group.candidateIndex,
+        campaignDir: group.dir,
+        commit: group.commit,
+      },
+      action: {
+        diffPath: diffPath !== null && existsSync(diffPath) ? diffPath : null,
+        shotReceiptPaths: await shotReceiptPathsFor(outDir, group.generation, group.candidateIndex),
+      },
+      outcome: {
+        perInstance: perInstanceFromCells(group.cells),
+        resolvedCount: resolvedInstanceCount(
+          replicateRunsFromCells(group.cells),
+          instances,
+          maxRep(group.cells) + 1,
+        ),
+        cellsCached: group.cells.length,
+      },
+      cost: costFromCells(group.cells, group.dir, receipts),
+      artifactPaths: artifactPathsFromCells(group.cells, group.dir),
+    })
+  }
+
+  return {
+    schema: ROLLOUT_SCHEMA,
+    outDir,
+    at: new Date().toISOString(),
+    provenance,
+    instances,
+    generations: [...generations.values()].sort((a, b) => a.generation - b.generation),
+  }
+}
+
+/** Replicate count inferred from the cells themselves (reps = max rep index
+ *  + 1). The manifest is a reader with no config in scope; an incomplete
+ *  candidate under-infers reps and its resolvedCount stays fail-closed. */
+const maxRep = (cells: EvidenceCell[]): number => cells.reduce((m, c) => Math.max(m, c.rep), 0)
+
+export async function writeRolloutManifest(outDir: string): Promise<string> {
+  const manifest = await buildRolloutManifest(outDir)
+  const path = join(outDir, 'rollout-manifest.json')
+  await writeFile(path, JSON.stringify(manifest, null, 2))
+  return path
+}
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const outDir = process.argv[2]
+  if (!outDir) {
+    console.error('usage: tsx src/swe-arena/manifest.mts <outDir>')
+    process.exit(2)
+  }
+  const path = await writeRolloutManifest(outDir)
+  console.log(`rollout manifest → ${path}`)
+}

package/src/swe-arena/manifest.test.mts

@@ -0,0 +1,169 @@
+/**
+ * Rollout-manifest reader: a synthetic round outDir with the lib's files
+ * (loop-provenance.json, cost-ledger.jsonl, per-cell caches) and the
+ * swe-arena files (arm result/judge json, patches, shot receipts) joins into
+ * one versioned manifest. Pure reads — nothing is dispatched.
+ */
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, describe, expect, it } from 'vitest'
+import type { R4Artifact } from './cell-evidence.mts'
+import { buildRolloutManifest, loadLedgerReceipts, ROLLOUT_SCHEMA, writeRolloutManifest } from './manifest.mts'
+
+const IID = 'django__django-11532'
+
+describe('rollout manifest (synthetic outDir)', () => {
+  let outDir: string
+  let armRunDir: string
+  let candDir: string
+
+  afterAll(async () => {
+    await rm(outDir, { recursive: true, force: true })
+  })
+
+  async function writeCell(campaignDir: string, rep: number, artifact: R4Artifact, costUsd: number): Promise<void> {
+    const cellDir = join(campaignDir, `cell-${IID}-r${rep}`)
+    await mkdir(cellDir, { recursive: true })
+    await writeFile(
+      join(cellDir, 'cached-result.json'),
+      JSON.stringify({
+        cellId: `cell-${IID}-r${rep}`,
+        scenarioId: IID,
+        rep,
+        artifact,
+        costUsd,
+        tokenUsage: { input: 400, output: 100 },
+        judgeScores: {},
+        durationMs: 5,
+        seed: 42,
+        cached: false,
+      }),
+    )
+  }
+
+  async function makeOutDir(): Promise<void> {
+    outDir = await mkdtemp(join(tmpdir(), 'r4-manifest-'))
+    const improveRun = join(outDir, 'improve-run')
+
+    // Arm run dir the artifacts point at (ours: result.json + judge.json).
+    armRunDir = join(outDir, 'arm-runs', 'cafebabe00', 'rep-0', 'runs', IID, 'R4')
+    await mkdir(armRunDir, { recursive: true })
+    await writeFile(join(armRunDir, 'result.json'), JSON.stringify({ arm: 'R4', iid: IID }))
+    await writeFile(join(armRunDir, 'judge.json'), JSON.stringify({ iid: IID, resolved: true, wallS: 31 }))
+    const patchPath = join(outDir, 'patches', `${IID}.r4.patch`)
+    await mkdir(join(outDir, 'patches'), { recursive: true })
+    await writeFile(patchPath, 'diff --git a b\n')
+
+    const artifact = (resolved: boolean): R4Artifact => ({
+      kind: 'swe-arm',
+      iid: IID,
+      commit: 'cafebabe0012345678',
+      resolved,
+      verifyPass: resolved,
+      patchLines: 4,
+      wallS: 120,
+      spentTokens: 2000,
+      spentUsd: 0.02,
+      recoveredTokens: 2500,
+      workerTokIn: 400,
+      workerTokOut: 100,
+      judgeAttempts: 1,
+      judgeWallS: 31,
+      runDir: armRunDir,
+      patchPath,
+    })
+
+    // Lib: baseline campaign cells (different commit).
+    const baselineDir = join(improveRun, 'baseline')
+    await writeCell(baselineDir, 0, { ...artifact(true), commit: 'basecommit12345678' }, 0.01)
+    await writeCell(baselineDir, 1, { ...artifact(true), commit: 'basecommit12345678' }, 0.01)
+
+    // Lib: gen-0 candidate campaign cells.
+    candDir = join(improveRun, 'gen-0', 'candidate-0')
+    await writeCell(candDir, 0, artifact(false), 0.02)
+    await writeCell(candDir, 1, artifact(false), 0.02)
+
+    // Lib: durable provenance + cost log.
+    await writeFile(
+      join(improveRun, 'loop-provenance.json'),
+      JSON.stringify({ schema: 'tangle.loop-provenance', runId: 'r4-test', candidates: [] }),
+    )
+    await writeFile(
+      join(improveRun, 'cost-ledger.jsonl'),
+      [
+        JSON.stringify({ version: 1, record: { status: 'pending', callId: 'p1', channel: 'agent', phase: 'x', actor: 'a', model: 'm', timestamp: 1 } }),
+        JSON.stringify({ version: 1, record: { status: 'settled', callId: 'c1', channel: 'agent', phase: 'search.candidate', actor: 'a', model: 'm', timestamp: 2, inputTokens: 400, outputTokens: 100, costUsd: 0.02, costUnknown: false, tags: { runDir: candDir, cellId: `cell-${IID}-r0` } } }),
+        JSON.stringify({ version: 1, record: { status: 'settled', callId: 'c2', channel: 'judge', phase: 'search.candidate', actor: 'j', model: 'swe-bench-official-judge', timestamp: 3, inputTokens: 0, outputTokens: 0, costUsd: 0, costUnknown: false, tags: { runDir: candDir, cellId: `cell-${IID}-r1` } } }),
+        'garbage line',
+      ].join('\n'),
+    )
+
+    // Ours: candidate diff + proposer shot receipt.
+    await mkdir(join(outDir, 'candidates'), { recursive: true })
+    await writeFile(join(outDir, 'candidates', 'cafebabe00.patch'), 'diff --git x y\n')
+    await mkdir(join(outDir, 'proposer-shots'), { recursive: true })
+    await writeFile(join(outDir, 'proposer-shots', 'gen0-cand0-shot1.json'), JSON.stringify({ receipt: { shot: 1 } }))
+  }
+
+  it('joins lib + swe-arena files into versioned per-generation rollout triples', async () => {
+    await makeOutDir()
+    const manifest = await buildRolloutManifest(outDir)
+
+    expect(manifest.schema).toBe(ROLLOUT_SCHEMA)
+    expect(manifest.instances).toEqual([IID])
+    expect((manifest.provenance as { schema: string }).schema).toBe('tangle.loop-provenance')
+    expect(manifest.generations.map((g) => g.generation)).toEqual([-1, 0])
+
+    const baseline = manifest.generations[0]!.rollouts[0]!
+    expect(baseline.state).toMatchObject({ generation: -1, candidateIndex: -1, commit: 'basecommit12345678' })
+    expect(baseline.outcome.resolvedCount).toBe(1)
+    expect(baseline.outcome.perInstance).toHaveLength(2)
+    expect(baseline.action.shotReceiptPaths).toEqual([])
+
+    const cand = manifest.generations[1]!.rollouts[0]!
+    expect(cand.state).toMatchObject({ generation: 0, candidateIndex: 0, commit: 'cafebabe0012345678', campaignDir: candDir })
+    expect(cand.outcome.resolvedCount).toBe(0)
+    expect(cand.outcome.cellsCached).toBe(2)
+    // Cost joins: cell ledger spend + artifact spend-tree + judge wall.
+    expect(cand.cost.cellCostUsd).toBeCloseTo(0.04, 6)
+    expect(cand.cost.cellTokensIn).toBe(800)
+    expect(cand.cost.armSpentUsd).toBeCloseTo(0.04, 6)
+    expect(cand.cost.armSpentTokens).toBe(4000)
+    expect(cand.cost.wallS).toBe(240)
+    expect(cand.cost.judgeWallS).toBe(62)
+    expect(cand.cost.receiptCallIds).toEqual(['c1', 'c2'])
+    // Artifact joins: our arm/judge/patch files + the shot receipt.
+    expect(cand.artifactPaths.armResults).toEqual([join(armRunDir, 'result.json')])
+    expect(cand.artifactPaths.judgeVerdicts).toEqual([join(armRunDir, 'judge.json')])
+    expect(cand.artifactPaths.patches).toHaveLength(1)
+    expect(cand.action.diffPath).toBe(join(outDir, 'candidates', 'cafebabe00.patch'))
+    expect(cand.action.shotReceiptPaths).toEqual([join(outDir, 'proposer-shots', 'gen0-cand0-shot1.json')])
+  })
+
+  it('loadLedgerReceipts keeps settled records only and survives garbage lines', async () => {
+    const receipts = await loadLedgerReceipts(join(outDir, 'improve-run'))
+    expect(receipts.map((r) => r.callId)).toEqual(['c1', 'c2'])
+  })
+
+  it('writeRolloutManifest emits <outDir>/rollout-manifest.json (the CLI body)', async () => {
+    const path = await writeRolloutManifest(outDir)
+    expect(path).toBe(join(outDir, 'rollout-manifest.json'))
+    const parsed = JSON.parse(await readFile(path, 'utf8')) as { schema: string }
+    expect(parsed.schema).toBe(ROLLOUT_SCHEMA)
+  })
+
+  it('a bare outDir (no improve-run yet) yields an empty, well-formed manifest', async () => {
+    const empty = await mkdtemp(join(tmpdir(), 'r4-manifest-empty-'))
+    try {
+      const manifest = await buildRolloutManifest(empty)
+      expect(manifest.schema).toBe(ROLLOUT_SCHEMA)
+      expect(manifest.provenance).toBeNull()
+      expect(manifest.instances).toEqual([])
+      expect(manifest.generations).toEqual([])
+    } finally {
+      await rm(empty, { recursive: true, force: true })
+    }
+  })
+})

package/src/swe-arena/materialize.ts

@@ -0,0 +1,142 @@
+/**
+ * Workspace materialization from SWE-bench instance images — the typed port of
+ * the experiment's `setup-ws.sh`, stage for stage:
+ *
+ *   docker create <image> → docker cp <container>:/testbed <dest> → docker rm
+ *   → git reset --hard <base_commit> → git clean -fd → work branch checkout
+ *
+ * IMAGE materialization is the DEFAULT and must stay so. A plain `git clone`
+ * at `base_commit` is insufficient: the instance's runnable environment (conda
+ * env, compiled C extensions, version pins) is built by the official harness at
+ * `environment_setup_commit` and lives only in the image's /testbed. Round-2 of
+ * the supervisor evolution proved this live — clone-materialized worker
+ * sandboxes dropped those build artifacts and every verify run died on imports
+ * (astropy/matplotlib), which is exactly the failure mode the image path
+ * removes. Clone materialization may only ever be an explicit opt-in for
+ * pure-python instances, never the default.
+ */
+
+import { randomUUID } from 'node:crypto'
+import { mkdir, rm, stat } from 'node:fs/promises'
+import { dirname } from 'node:path'
+import { run, runOk } from './proc'
+
+export interface MaterializeOptions {
+  instanceId: string
+  /** Instance image, e.g. swebench/sweb.eval.x86_64.pallets_1776_flask-5014:latest. */
+  image: string
+  baseCommit: string
+  /** Destination directory; removed and recreated (setup-ws.sh `rm -rf`). */
+  dest: string
+  /** Work branch forced to base_commit and checked out. Default matches the bash. */
+  workBranch?: string
+  /** Cancels Docker/Git setup before a cell begins model execution. */
+  signal?: AbortSignal
+}
+
+export interface MaterializedWorkspace {
+  head: string
+  /** `git status --porcelain` empty after materialization. */
+  clean: boolean
+}
+
+/** setup-ws.sh container-name sanitization: `tr '_/' '--'` + unique suffix. */
+export function containerName(instanceId: string): string {
+  const sanitized = instanceId.replace(/[_/]/g, '-')
+  return `ext-${sanitized}-${process.pid}-${randomUUID()}`
+}
+
+const DOCKER_CREATE_TIMEOUT_MS = 5 * 60_000
+const DOCKER_COPY_TIMEOUT_MS = 10 * 60_000
+const DOCKER_CLEANUP_TIMEOUT_MS = 60_000
+const GIT_COMMAND_TIMEOUT_MS = 2 * 60_000
+
+/**
+ * Materialize a pristine workspace for `instanceId` at `dest`. Throws with the
+ * setup-ws.sh stage label (create-fail / cp-fail / no-git / reset-fail) so
+ * failures stay greppable against the bash experiment's logs.
+ */
+export async function materializeWorkspace(opts: MaterializeOptions): Promise<MaterializedWorkspace> {
+  const { instanceId, image, baseCommit, dest } = opts
+  const workBranch = opts.workBranch ?? 'hh-work'
+  const container = containerName(instanceId)
+
+  opts.signal?.throwIfAborted()
+  await rm(dest, { recursive: true, force: true })
+  await mkdir(dirname(dest), { recursive: true })
+  opts.signal?.throwIfAborted()
+
+  const create = await run('docker', ['create', '--name', container, image], {
+    signal: opts.signal,
+    timeoutMs: DOCKER_CREATE_TIMEOUT_MS,
+  })
+  let stageError: unknown
+  try {
+    opts.signal?.throwIfAborted()
+    if (create.code !== 0) {
+      throw new Error(`materialize ${instanceId}: create-fail (${image}): ${create.stderr.slice(0, 500)}`)
+    }
+    const cp = await run('docker', ['cp', `${container}:/testbed`, dest], {
+      signal: opts.signal,
+      timeoutMs: DOCKER_COPY_TIMEOUT_MS,
+    })
+    opts.signal?.throwIfAborted()
+    if (cp.code !== 0) {
+      throw new Error(`materialize ${instanceId}: cp-fail: ${cp.stderr.slice(0, 500)}`)
+    }
+  } catch (cause) {
+    stageError = cause
+  }
+
+  // Cleanup must still run after the caller signal fires, so it gets its own
+  // short deadline instead of the already-aborted cell signal.
+  let cleanupError: unknown
+  try {
+    const cleanup = await run('docker', ['rm', '-f', container], {
+      timeoutMs: DOCKER_CLEANUP_TIMEOUT_MS,
+    })
+    const detail = cleanup.stderr || cleanup.stdout
+    const absent = /no such container/iu.test(detail)
+    if (cleanup.code !== 0 && !absent) {
+      cleanupError = new Error(
+        `materialize ${instanceId}: cleanup-fail container=${container} exit=${cleanup.code}: ${detail.slice(0, 500)}`,
+      )
+    }
+  } catch (cause) {
+    cleanupError = cause
+  }
+  if (stageError !== undefined && cleanupError !== undefined) {
+    throw new AggregateError(
+      [stageError, cleanupError],
+      `materialize ${instanceId}: workspace extraction and container cleanup both failed`,
+    )
+  }
+  if (stageError !== undefined) throw stageError
+  if (cleanupError !== undefined) throw cleanupError
+
+  opts.signal?.throwIfAborted()
+  const gitDir = await stat(`${dest}/.git`).catch(() => null)
+  if (!gitDir?.isDirectory()) {
+    throw new Error(`materialize ${instanceId}: no-git (image ${image} has no /testbed git repo)`)
+  }
+
+  const gitOpts = { signal: opts.signal, timeoutMs: GIT_COMMAND_TIMEOUT_MS }
+  const reset = await run('git', ['-C', dest, 'reset', '-q', '--hard', baseCommit], gitOpts)
+  opts.signal?.throwIfAborted()
+  if (reset.code !== 0) {
+    throw new Error(`materialize ${instanceId}: reset-fail base=${baseCommit}: ${reset.stderr.slice(0, 500)}`)
+  }
+  // Tolerated failure in the bash (`|| true`): a clean error must not kill setup.
+  const clean = await run('git', ['-C', dest, 'clean', '-qfd'], gitOpts)
+  opts.signal?.throwIfAborted()
+  if (clean.timedOut) {
+    throw new Error(`materialize ${instanceId}: clean-timeout: ${(clean.stderr || clean.stdout).slice(0, 500)}`)
+  }
+
+  await runOk('git', ['-C', dest, 'branch', '-f', workBranch, baseCommit], gitOpts)
+  await runOk('git', ['-C', dest, 'checkout', '-q', workBranch], gitOpts)
+
+  const head = (await runOk('git', ['-C', dest, 'rev-parse', 'HEAD'], gitOpts)).stdout.trim()
+  const status = (await runOk('git', ['-C', dest, 'status', '--porcelain'], gitOpts)).stdout.trim()
+  return { head, clean: status.length === 0 }
+}