@tangle-network/agent-bench 0.3.6 → 0.3.7

package/src/swe-arena/holdout-certify.test.mts

@@ -0,0 +1,160 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import type { ReplicateRun } from './cell-evidence.mts'
+import {
+  assertHoldoutBaselineMap,
+  decideHoldoutCertification,
+  DEFAULT_HOLDOUT_REPS,
+  holdoutReps,
+  loadHoldoutCells,
+  type HoldoutCellRow,
+} from './holdout-certify.mts'
+
+const IIDS = ['a', 'b', 'c']
+
+const runs = (spec: Record<string, Array<boolean | null>>): ReplicateRun[] =>
+  Object.entries(spec).flatMap(([iid, reps]) => reps.map((resolved) => ({ iid, resolved })))
+
+describe('holdoutReps', () => {
+  it('defaults to 2 — the gen-2 postmortem protocol', () => {
+    expect(DEFAULT_HOLDOUT_REPS).toBe(2)
+    expect(holdoutReps({})).toBe(2)
+    expect(holdoutReps({ holdoutRepsPerInstance: 3 })).toBe(3)
+  })
+
+  it('rejects non-positive/non-integer reps', () => {
+    expect(() => holdoutReps({ holdoutRepsPerInstance: 0 })).toThrow(/positive integer/)
+    expect(() => holdoutReps({ holdoutRepsPerInstance: 1.5 })).toThrow(/positive integer/)
+  })
+})
+
+describe('assertHoldoutBaselineMap', () => {
+  it('requires a same-protocol verdict for EVERY holdout instance', () => {
+    expect(() => assertHoldoutBaselineMap({ a: true }, IIDS)).toThrow(/missing same-protocol verdicts.*b, c/)
+  })
+
+  it('rejects verdicts for non-holdout instances', () => {
+    expect(() => assertHoldoutBaselineMap({ a: true, b: false, c: true, z: true }, IIDS)).toThrow(/non-holdout/)
+  })
+
+  it('accepts a complete map', () => {
+    expect(() => assertHoldoutBaselineMap({ a: true, b: false, c: true }, IIDS)).not.toThrow()
+  })
+})
+
+describe('decideHoldoutCertification — 2-rep fail-closed aggregation', () => {
+  const parent = { a: true, b: false, c: true } // parent 2/3 same-protocol
+
+  it('an instance resolves only when BOTH reps resolve (T/F counts as unresolved)', () => {
+    const d = decideHoldoutCertification({
+      candidateRuns: runs({ a: [true, true], b: [true, false], c: [true, true] }),
+      parentVerdicts: parent,
+      iids: IIDS,
+      reps: 2,
+    })
+    expect(d.candResolved).toBe(2) // a + c; b's discordant pair is fail-closed
+    expect(d.parentResolved).toBe(2)
+    expect(d.coverageComplete).toBe(true)
+    expect(d.certified).toBe(true) // non-regression: 2 >= 2
+    expect(d.perInstance.find((p) => p.iid === 'b')).toMatchObject({ candidate: false, parent: false, discordant: false })
+  })
+
+  it('regression fails certification even with complete coverage', () => {
+    const d = decideHoldoutCertification({
+      candidateRuns: runs({ a: [true, true], b: [false, false], c: [false, true] }),
+      parentVerdicts: parent,
+      iids: IIDS,
+      reps: 2,
+    })
+    expect(d.candResolved).toBe(1)
+    expect(d.certified).toBe(false)
+    expect(d.reasons.join(' ')).toMatch(/regression: candidate 1 < parent 2/)
+    expect(d.perInstance.find((p) => p.iid === 'c')).toMatchObject({ candidate: false, parent: true, discordant: true })
+  })
+
+  it('an inconclusive replicate (errored cell) blocks coverage and certification — fail-closed', () => {
+    const d = decideHoldoutCertification({
+      candidateRuns: runs({ a: [true, true], b: [false, false], c: [true, null] }),
+      parentVerdicts: parent,
+      iids: IIDS,
+      reps: 2,
+    })
+    expect(d.coverageComplete).toBe(false)
+    expect(d.certified).toBe(false)
+    expect(d.reasons.join(' ')).toMatch(/coverage incomplete/)
+    expect(d.perInstance.find((p) => p.iid === 'c')).toMatchObject({ candidate: null, discordant: false })
+  })
+
+  it('a missing replicate (only 1 of 2 reps ran) blocks coverage', () => {
+    const d = decideHoldoutCertification({
+      candidateRuns: runs({ a: [true, true], b: [true, true], c: [true] }),
+      parentVerdicts: parent,
+      iids: IIDS,
+      reps: 2,
+    })
+    expect(d.coverageComplete).toBe(false)
+    expect(d.certified).toBe(false)
+  })
+
+  it('gen-2 shape: one discordant instance under 1-rep would have failed; the same data at 2 reps needs both reps', () => {
+    // At 1 rep a single flipped cell decided the gen-2 verdict. The 2-rep
+    // protocol requires the flip to REPLICATE before it counts either way.
+    const oneRep = decideHoldoutCertification({
+      candidateRuns: runs({ a: [true], b: [false], c: [false] }),
+      parentVerdicts: parent,
+      iids: IIDS,
+      reps: 1,
+    })
+    expect(oneRep.certified).toBe(false) // 1 < 2 — the gen-2 outcome
+    const twoRep = decideHoldoutCertification({
+      candidateRuns: runs({ a: [true, true], b: [false, false], c: [true, true] }),
+      parentVerdicts: parent,
+      iids: IIDS,
+      reps: 2,
+    })
+    expect(twoRep.certified).toBe(true) // c held up under replication
+  })
+
+  it('validates the parent map before deciding', () => {
+    expect(() =>
+      decideHoldoutCertification({
+        candidateRuns: runs({ a: [true, true] }),
+        parentVerdicts: { a: true },
+        iids: IIDS,
+        reps: 2,
+      }),
+    ).toThrow(/missing same-protocol verdicts/)
+  })
+})
+
+describe('loadHoldoutCells', () => {
+  let dir: string
+  beforeEach(async () => {
+    dir = await mkdtemp(join(tmpdir(), 'holdout-cells-'))
+  })
+  afterEach(async () => {
+    await rm(dir, { recursive: true, force: true })
+  })
+
+  it('returns [] for a missing file and round-trips schema-valid rows', async () => {
+    const path = join(dir, 'cells.jsonl')
+    expect(await loadHoldoutCells(path)).toEqual([])
+    const row: HoldoutCellRow = {
+      schema: 'swe-arena.holdout-cell.v1',
+      at: new Date(0).toISOString(),
+      commit: 'abc123',
+      side: 'candidate',
+      iid: 'a',
+      rep: 0,
+      resolved: true,
+      verifyPass: true,
+      wallS: 100,
+      runDir: '/tmp/x',
+    }
+    await writeFile(path, `${JSON.stringify(row)}\n${JSON.stringify({ schema: 'other' })}\n`)
+    const rows = await loadHoldoutCells(path)
+    expect(rows).toEqual([row]) // foreign schemas are dropped
+  })
+})

package/src/swe-arena/judge-child.mts

@@ -0,0 +1,37 @@
+/**
+ * Judge child entrypoint — the experiment's `judge.mts`, tracked. One patch,
+ * one official swebench verdict, printed as a single `JUDGE_RESULT {...}` line
+ * that serialized-judge.ts parses. Runs as a CHILD process (not in-process) so
+ * the judge ceiling can SIGKILL a hung swebench run without taking the
+ * experiment down, exactly like the bash `timeout N node judge.mts`.
+ *
+ * usage: node --import tsx src/swe-arena/judge-child.mts <instance_id> <patchPath>
+ */
+
+import { readFileSync } from 'node:fs'
+import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
+
+const [, , iid, patchPath] = process.argv
+if (!iid || !patchPath) {
+  console.error('usage: node --import tsx judge-child.mts <instance_id> <patchPath>')
+  process.exit(2)
+}
+const patch = readFileSync(patchPath, 'utf8')
+const adapter = createSweBenchAdapter()
+const [task] = await adapter.loadTasks({ ids: [iid], split: 'test' })
+if (!task) {
+  console.log(JSON.stringify({ iid, error: 'no-task' }))
+  process.exit(2)
+}
+const t0 = Date.now()
+const score = await adapter.judge(task, patch)
+console.log(
+  'JUDGE_RESULT ' +
+    JSON.stringify({
+      iid,
+      resolved: score.resolved,
+      score: score.score,
+      secs: Math.round((Date.now() - t0) / 1000),
+      patch_bytes: patch.length,
+    }),
+)

package/src/swe-arena/manifest.mts

@@ -0,0 +1,293 @@
+/**
+ * Rollout manifest — a pure READER/JOIN over what the round already writes.
+ * No new capture pipeline: the lib's improve() run emits loop-provenance.json,
+ * per-cell cached results, and the durable cost log; the swe-arena dispatch
+ * emits arm result/judge files, candidate patches, and proposer-shot receipts.
+ * This module joins them into one versioned per-generation manifest of
+ * (state, action, outcome, cost, artifactPaths) triples.
+ *
+ *   tsx src/swe-arena/manifest.mts <outDir>   # writes <outDir>/rollout-manifest.json
+ *
+ * Sources (all optional — a missing file is a labeled gap, never a crash):
+ *   - <outDir>/improve-run/loop-provenance.json            (lib)
+ *   - <outDir>/improve-run/cost-ledger.jsonl               (lib, durable receipts)
+ *   - <outDir>/improve-run/baseline and gen-N/candidate-K  (lib, per-cell caches)
+ *   - <outDir>/candidates/<commit10>.patch                 (ours, diff writing)
+ *   - <outDir>/proposer-shots/genN-candK-shotS.json        (ours, shot receipts)
+ *   - per-cell artifact.runDir result.json + judge.json    (ours, arm + judge)
+ */
+
+import { readdir, readFile, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import {
+  loadCampaignCells,
+  loadCandidateCellGroups,
+  perInstanceFromCells,
+  replicateRunsFromCells,
+  resolvedInstanceCount,
+  sumWallSFromCells,
+  type EvidenceCell,
+  type StaircasePerInstance,
+} from './cell-evidence.mts'
+
+export const ROLLOUT_SCHEMA = 'swe-arena.rollout.v1'
+
+export interface RolloutCost {
+  /** Σ campaign-cell CostLedger spend (worker receipts) across the cells. */
+  cellCostUsd: number
+  cellTokensIn: number
+  cellTokensOut: number
+  /** Σ runtime spend-tree usd/tokens off the artifacts (telemetry-gap nulls skipped). */
+  armSpentUsd: number
+  armSpentTokens: number
+  wallS: number
+  judgeWallS: number
+  /** Durable cost-ledger receipts whose tags name this campaign dir. */
+  receiptCallIds: string[]
+}
+
+export interface RolloutEntry {
+  state: {
+    generation: number
+    /** -1 candidateIndex = the baseline campaign. */
+    candidateIndex: number
+    campaignDir: string
+    commit: string | null
+  }
+  action: {
+    /** Candidate diff written at dispatch time (null for the baseline / a
+     *  resumed-only run whose diff was not rewritten). */
+    diffPath: string | null
+    /** Persisted proposer-shot receipts for this (generation, candidate). */
+    shotReceiptPaths: string[]
+  }
+  outcome: {
+    perInstance: StaircasePerInstance[]
+    resolvedCount: number
+    cellsCached: number
+  }
+  cost: RolloutCost
+  artifactPaths: {
+    cells: string[]
+    armResults: string[]
+    judgeVerdicts: string[]
+    patches: string[]
+  }
+}
+
+export interface RolloutGeneration {
+  generation: number
+  rollouts: RolloutEntry[]
+}
+
+export interface RolloutManifest {
+  schema: typeof ROLLOUT_SCHEMA
+  outDir: string
+  at: string
+  /** The lib's durable provenance record, verbatim (null = not emitted). */
+  provenance: unknown | null
+  /** Improvement-set ids inferred from the baseline cells. */
+  instances: string[]
+  generations: RolloutGeneration[]
+}
+
+interface LedgerReceipt {
+  callId: string
+  tags?: Record<string, string>
+  [k: string]: unknown
+}
+
+/** Settled receipts out of the lib's append-only cost-ledger.jsonl. */
+export async function loadLedgerReceipts(improveRunDir: string): Promise<LedgerReceipt[]> {
+  const raw = await readFile(join(improveRunDir, 'cost-ledger.jsonl'), 'utf8').catch(() => '')
+  const receipts: LedgerReceipt[] = []
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue
+    let event: Record<string, unknown>
+    try {
+      event = JSON.parse(line) as Record<string, unknown>
+    } catch {
+      continue
+    }
+    const record = event.record as Record<string, unknown> | undefined
+    if (record && record.status === 'settled' && typeof record.callId === 'string') {
+      receipts.push(record as unknown as LedgerReceipt)
+    }
+  }
+  return receipts
+}
+
+function costFromCells(
+  cells: EvidenceCell[],
+  campaignDir: string,
+  receipts: LedgerReceipt[],
+): RolloutCost {
+  let cellCostUsd = 0
+  let cellTokensIn = 0
+  let cellTokensOut = 0
+  let armSpentUsd = 0
+  let armSpentTokens = 0
+  let judgeWallS = 0
+  for (const cell of cells) {
+    cellCostUsd += cell.costUsd ?? 0
+    cellTokensIn += cell.tokenUsage?.input ?? 0
+    cellTokensOut += cell.tokenUsage?.output ?? 0
+    const a = cell.artifact
+    if (a !== null && a.kind === 'swe-arm') {
+      armSpentUsd += a.spentUsd ?? 0
+      armSpentTokens += a.spentTokens ?? 0
+      judgeWallS += a.judgeWallS ?? 0
+    }
+  }
+  return {
+    cellCostUsd: Number(cellCostUsd.toFixed(6)),
+    cellTokensIn,
+    cellTokensOut,
+    armSpentUsd: Number(armSpentUsd.toFixed(6)),
+    armSpentTokens,
+    wallS: sumWallSFromCells(cells),
+    judgeWallS,
+    receiptCallIds: receipts
+      .filter((r) => r.tags?.runDir === campaignDir)
+      .map((r) => r.callId)
+      .sort(),
+  }
+}
+
+function artifactPathsFromCells(cells: EvidenceCell[], campaignDir: string): RolloutEntry['artifactPaths'] {
+  const armResults: string[] = []
+  const judgeVerdicts: string[] = []
+  const patches: string[] = []
+  for (const cell of cells) {
+    const a = cell.artifact
+    if (a === null || a.kind !== 'swe-arm') continue
+    if (a.runDir) {
+      const resultPath = join(a.runDir, 'result.json')
+      const judgePath = join(a.runDir, 'judge.json')
+      if (existsSync(resultPath)) armResults.push(resultPath)
+      if (existsSync(judgePath)) judgeVerdicts.push(judgePath)
+    }
+    if (a.patchPath && existsSync(a.patchPath)) patches.push(a.patchPath)
+  }
+  return {
+    // The campaign dir holds the per-cell cached-result.json score records.
+    cells: cells.length > 0 ? [campaignDir] : [],
+    armResults: [...new Set(armResults)].sort(),
+    judgeVerdicts: [...new Set(judgeVerdicts)].sort(),
+    patches: [...new Set(patches)].sort(),
+  }
+}
+
+async function shotReceiptPathsFor(
+  outDir: string,
+  generation: number,
+  candidateIndex: number,
+): Promise<string[]> {
+  const shotDir = join(outDir, 'proposer-shots')
+  const names = await readdir(shotDir).catch(() => [])
+  return names
+    .filter((n) => n.startsWith(`gen${generation}-cand${candidateIndex}-`) && n.endsWith('.json'))
+    .sort()
+    .map((n) => join(shotDir, n))
+}
+
+/** Build the rollout manifest for one round outDir. Pure join — reads only. */
+export async function buildRolloutManifest(outDir: string): Promise<RolloutManifest> {
+  const improveRunDir = join(outDir, 'improve-run')
+  const provenanceRaw = await readFile(join(improveRunDir, 'loop-provenance.json'), 'utf8').catch(() => null)
+  const provenance = provenanceRaw === null ? null : (JSON.parse(provenanceRaw) as unknown)
+  const receipts = await loadLedgerReceipts(improveRunDir)
+
+  const baselineDir = join(improveRunDir, 'baseline')
+  const baselineCells = await loadCampaignCells(baselineDir)
+  const instances = [...new Set(baselineCells.map((c) => c.scenarioId))].sort()
+
+  const generations = new Map<number, RolloutGeneration>()
+  const genOf = (g: number): RolloutGeneration => {
+    const existing = generations.get(g)
+    if (existing) return existing
+    const fresh: RolloutGeneration = { generation: g, rollouts: [] }
+    generations.set(g, fresh)
+    return fresh
+  }
+
+  if (baselineCells.length > 0) {
+    genOf(-1).rollouts.push({
+      state: { generation: -1, candidateIndex: -1, campaignDir: baselineDir, commit: baselineCells[0]?.artifact?.commit ?? null },
+      action: { diffPath: null, shotReceiptPaths: [] },
+      outcome: {
+        perInstance: perInstanceFromCells(baselineCells),
+        resolvedCount: resolvedInstanceCount(
+          replicateRunsFromCells(baselineCells),
+          instances,
+          maxRep(baselineCells) + 1,
+        ),
+        cellsCached: baselineCells.length,
+      },
+      cost: costFromCells(baselineCells, baselineDir, receipts),
+      artifactPaths: artifactPathsFromCells(baselineCells, baselineDir),
+    })
+  }
+
+  for (const group of await loadCandidateCellGroups(improveRunDir)) {
+    const diffPath = group.commit ? join(outDir, 'candidates', `${group.commit.slice(0, 10)}.patch`) : null
+    genOf(group.generation).rollouts.push({
+      state: {
+        generation: group.generation,
+        candidateIndex: group.candidateIndex,
+        campaignDir: group.dir,
+        commit: group.commit,
+      },
+      action: {
+        diffPath: diffPath !== null && existsSync(diffPath) ? diffPath : null,
+        shotReceiptPaths: await shotReceiptPathsFor(outDir, group.generation, group.candidateIndex),
+      },
+      outcome: {
+        perInstance: perInstanceFromCells(group.cells),
+        resolvedCount: resolvedInstanceCount(
+          replicateRunsFromCells(group.cells),
+          instances,
+          maxRep(group.cells) + 1,
+        ),
+        cellsCached: group.cells.length,
+      },
+      cost: costFromCells(group.cells, group.dir, receipts),
+      artifactPaths: artifactPathsFromCells(group.cells, group.dir),
+    })
+  }
+
+  return {
+    schema: ROLLOUT_SCHEMA,
+    outDir,
+    at: new Date().toISOString(),
+    provenance,
+    instances,
+    generations: [...generations.values()].sort((a, b) => a.generation - b.generation),
+  }
+}
+
+/** Replicate count inferred from the cells themselves (reps = max rep index
+ *  + 1). The manifest is a reader with no config in scope; an incomplete
+ *  candidate under-infers reps and its resolvedCount stays fail-closed. */
+const maxRep = (cells: EvidenceCell[]): number => cells.reduce((m, c) => Math.max(m, c.rep), 0)
+
+export async function writeRolloutManifest(outDir: string): Promise<string> {
+  const manifest = await buildRolloutManifest(outDir)
+  const path = join(outDir, 'rollout-manifest.json')
+  await writeFile(path, JSON.stringify(manifest, null, 2))
+  return path
+}
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const outDir = process.argv[2]
+  if (!outDir) {
+    console.error('usage: tsx src/swe-arena/manifest.mts <outDir>')
+    process.exit(2)
+  }
+  const path = await writeRolloutManifest(outDir)
+  console.log(`rollout manifest → ${path}`)
+}

package/src/swe-arena/manifest.test.mts

@@ -0,0 +1,169 @@
+/**
+ * Rollout-manifest reader: a synthetic round outDir with the lib's files
+ * (loop-provenance.json, cost-ledger.jsonl, per-cell caches) and the
+ * swe-arena files (arm result/judge json, patches, shot receipts) joins into
+ * one versioned manifest. Pure reads — nothing is dispatched.
+ */
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, describe, expect, it } from 'vitest'
+import type { R4Artifact } from './cell-evidence.mts'
+import { buildRolloutManifest, loadLedgerReceipts, ROLLOUT_SCHEMA, writeRolloutManifest } from './manifest.mts'
+
+const IID = 'django__django-11532'
+
+describe('rollout manifest (synthetic outDir)', () => {
+  let outDir: string
+  let armRunDir: string
+  let candDir: string
+
+  afterAll(async () => {
+    await rm(outDir, { recursive: true, force: true })
+  })
+
+  async function writeCell(campaignDir: string, rep: number, artifact: R4Artifact, costUsd: number): Promise<void> {
+    const cellDir = join(campaignDir, `cell-${IID}-r${rep}`)
+    await mkdir(cellDir, { recursive: true })
+    await writeFile(
+      join(cellDir, 'cached-result.json'),
+      JSON.stringify({
+        cellId: `cell-${IID}-r${rep}`,
+        scenarioId: IID,
+        rep,
+        artifact,
+        costUsd,
+        tokenUsage: { input: 400, output: 100 },
+        judgeScores: {},
+        durationMs: 5,
+        seed: 42,
+        cached: false,
+      }),
+    )
+  }
+
+  async function makeOutDir(): Promise<void> {
+    outDir = await mkdtemp(join(tmpdir(), 'r4-manifest-'))
+    const improveRun = join(outDir, 'improve-run')
+
+    // Arm run dir the artifacts point at (ours: result.json + judge.json).
+    armRunDir = join(outDir, 'arm-runs', 'cafebabe00', 'rep-0', 'runs', IID, 'R4')
+    await mkdir(armRunDir, { recursive: true })
+    await writeFile(join(armRunDir, 'result.json'), JSON.stringify({ arm: 'R4', iid: IID }))
+    await writeFile(join(armRunDir, 'judge.json'), JSON.stringify({ iid: IID, resolved: true, wallS: 31 }))
+    const patchPath = join(outDir, 'patches', `${IID}.r4.patch`)
+    await mkdir(join(outDir, 'patches'), { recursive: true })
+    await writeFile(patchPath, 'diff --git a b\n')
+
+    const artifact = (resolved: boolean): R4Artifact => ({
+      kind: 'swe-arm',
+      iid: IID,
+      commit: 'cafebabe0012345678',
+      resolved,
+      verifyPass: resolved,
+      patchLines: 4,
+      wallS: 120,
+      spentTokens: 2000,
+      spentUsd: 0.02,
+      recoveredTokens: 2500,
+      workerTokIn: 400,
+      workerTokOut: 100,
+      judgeAttempts: 1,
+      judgeWallS: 31,
+      runDir: armRunDir,
+      patchPath,
+    })
+
+    // Lib: baseline campaign cells (different commit).
+    const baselineDir = join(improveRun, 'baseline')
+    await writeCell(baselineDir, 0, { ...artifact(true), commit: 'basecommit12345678' }, 0.01)
+    await writeCell(baselineDir, 1, { ...artifact(true), commit: 'basecommit12345678' }, 0.01)
+
+    // Lib: gen-0 candidate campaign cells.
+    candDir = join(improveRun, 'gen-0', 'candidate-0')
+    await writeCell(candDir, 0, artifact(false), 0.02)
+    await writeCell(candDir, 1, artifact(false), 0.02)
+
+    // Lib: durable provenance + cost log.
+    await writeFile(
+      join(improveRun, 'loop-provenance.json'),
+      JSON.stringify({ schema: 'tangle.loop-provenance', runId: 'r4-test', candidates: [] }),
+    )
+    await writeFile(
+      join(improveRun, 'cost-ledger.jsonl'),
+      [
+        JSON.stringify({ version: 1, record: { status: 'pending', callId: 'p1', channel: 'agent', phase: 'x', actor: 'a', model: 'm', timestamp: 1 } }),
+        JSON.stringify({ version: 1, record: { status: 'settled', callId: 'c1', channel: 'agent', phase: 'search.candidate', actor: 'a', model: 'm', timestamp: 2, inputTokens: 400, outputTokens: 100, costUsd: 0.02, costUnknown: false, tags: { runDir: candDir, cellId: `cell-${IID}-r0` } } }),
+        JSON.stringify({ version: 1, record: { status: 'settled', callId: 'c2', channel: 'judge', phase: 'search.candidate', actor: 'j', model: 'swe-bench-official-judge', timestamp: 3, inputTokens: 0, outputTokens: 0, costUsd: 0, costUnknown: false, tags: { runDir: candDir, cellId: `cell-${IID}-r1` } } }),
+        'garbage line',
+      ].join('\n'),
+    )
+
+    // Ours: candidate diff + proposer shot receipt.
+    await mkdir(join(outDir, 'candidates'), { recursive: true })
+    await writeFile(join(outDir, 'candidates', 'cafebabe00.patch'), 'diff --git x y\n')
+    await mkdir(join(outDir, 'proposer-shots'), { recursive: true })
+    await writeFile(join(outDir, 'proposer-shots', 'gen0-cand0-shot1.json'), JSON.stringify({ receipt: { shot: 1 } }))
+  }
+
+  it('joins lib + swe-arena files into versioned per-generation rollout triples', async () => {
+    await makeOutDir()
+    const manifest = await buildRolloutManifest(outDir)
+
+    expect(manifest.schema).toBe(ROLLOUT_SCHEMA)
+    expect(manifest.instances).toEqual([IID])
+    expect((manifest.provenance as { schema: string }).schema).toBe('tangle.loop-provenance')
+    expect(manifest.generations.map((g) => g.generation)).toEqual([-1, 0])
+
+    const baseline = manifest.generations[0]!.rollouts[0]!
+    expect(baseline.state).toMatchObject({ generation: -1, candidateIndex: -1, commit: 'basecommit12345678' })
+    expect(baseline.outcome.resolvedCount).toBe(1)
+    expect(baseline.outcome.perInstance).toHaveLength(2)
+    expect(baseline.action.shotReceiptPaths).toEqual([])
+
+    const cand = manifest.generations[1]!.rollouts[0]!
+    expect(cand.state).toMatchObject({ generation: 0, candidateIndex: 0, commit: 'cafebabe0012345678', campaignDir: candDir })
+    expect(cand.outcome.resolvedCount).toBe(0)
+    expect(cand.outcome.cellsCached).toBe(2)
+    // Cost joins: cell ledger spend + artifact spend-tree + judge wall.
+    expect(cand.cost.cellCostUsd).toBeCloseTo(0.04, 6)
+    expect(cand.cost.cellTokensIn).toBe(800)
+    expect(cand.cost.armSpentUsd).toBeCloseTo(0.04, 6)
+    expect(cand.cost.armSpentTokens).toBe(4000)
+    expect(cand.cost.wallS).toBe(240)
+    expect(cand.cost.judgeWallS).toBe(62)
+    expect(cand.cost.receiptCallIds).toEqual(['c1', 'c2'])
+    // Artifact joins: our arm/judge/patch files + the shot receipt.
+    expect(cand.artifactPaths.armResults).toEqual([join(armRunDir, 'result.json')])
+    expect(cand.artifactPaths.judgeVerdicts).toEqual([join(armRunDir, 'judge.json')])
+    expect(cand.artifactPaths.patches).toHaveLength(1)
+    expect(cand.action.diffPath).toBe(join(outDir, 'candidates', 'cafebabe00.patch'))
+    expect(cand.action.shotReceiptPaths).toEqual([join(outDir, 'proposer-shots', 'gen0-cand0-shot1.json')])
+  })
+
+  it('loadLedgerReceipts keeps settled records only and survives garbage lines', async () => {
+    const receipts = await loadLedgerReceipts(join(outDir, 'improve-run'))
+    expect(receipts.map((r) => r.callId)).toEqual(['c1', 'c2'])
+  })
+
+  it('writeRolloutManifest emits <outDir>/rollout-manifest.json (the CLI body)', async () => {
+    const path = await writeRolloutManifest(outDir)
+    expect(path).toBe(join(outDir, 'rollout-manifest.json'))
+    const parsed = JSON.parse(await readFile(path, 'utf8')) as { schema: string }
+    expect(parsed.schema).toBe(ROLLOUT_SCHEMA)
+  })
+
+  it('a bare outDir (no improve-run yet) yields an empty, well-formed manifest', async () => {
+    const empty = await mkdtemp(join(tmpdir(), 'r4-manifest-empty-'))
+    try {
+      const manifest = await buildRolloutManifest(empty)
+      expect(manifest.schema).toBe(ROLLOUT_SCHEMA)
+      expect(manifest.provenance).toBeNull()
+      expect(manifest.instances).toEqual([])
+      expect(manifest.generations).toEqual([])
+    } finally {
+      await rm(empty, { recursive: true, force: true })
+    }
+  })
+})