@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/bootstrap-meta.mts

@@ -0,0 +1,188 @@
+/**
+ * Metadata bootstrap — regenerates every per-instance input the experiment
+ * used to hand-generate into its scratchpad (`task-meta.json` via
+ * `load_meta.py`) straight from the SWE-bench Verified dataset through the
+ * bench adapter. The HF dataset cache under ~/.cache survives host reboots;
+ * the scratchpad does not — this module removes that scratch dependency
+ * permanently (proven necessary: a reboot wiped /tmp and took task-meta.json,
+ * the verify scripts, and the round configs with it).
+ *
+ *   tsx src/swe-arena/bootstrap-meta.mts --task-meta <out.json> [iid...]
+ *   tsx src/swe-arena/bootstrap-meta.mts --problems <outDir> [iid...]
+ *   tsx src/swe-arena/bootstrap-meta.mts --calibrate <verifyDir> <workDir> [iid...]
+ *
+ * `iid...` defaults to the round-4 improvement set (defaultRound4Config).
+ *
+ * HONESTY FIREWALL: `--problems` writes ONLY problem statements — the mode a
+ * verify-script author is allowed to read. Gold patches surface ONLY on the
+ * `--calibrate` path, where calibrate.ts applies them MECHANICALLY (git apply
+ * / patch --fuzz=3) and the official judge grades them; no author-facing
+ * output ever contains patch/test_patch/FAIL_TO_PASS content.
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
+import { calibrateInstance, type CalibrationResult } from './calibrate.ts'
+import { defaultRound4Config } from './outer-loop.mts'
+import { loadInstanceImages } from './run-experiment.mts'
+import { createSerializedJudge } from './serialized-judge.ts'
+import type { SweInstance } from './types.ts'
+
+/** Validate one adapter task-metadata record into the SweInstance shape the
+ *  harness consumes. Throws on any missing/empty load-bearing field — a blank
+ *  problem statement or gold patch must never flow silently into a run. */
+export function assertSweInstance(id: string, metadata: Record<string, unknown> | undefined): SweInstance {
+  if (!metadata) throw new Error(`bootstrap-meta: ${id} has no metadata on its adapter task`)
+  const str = (key: keyof SweInstance): string => {
+    const v = metadata[key]
+    if (typeof v !== 'string' || v.length === 0) {
+      throw new Error(`bootstrap-meta: ${id} has a missing/empty ${String(key)}`)
+    }
+    return v
+  }
+  const optStr = (key: keyof SweInstance): string | null => {
+    const v = metadata[key]
+    return typeof v === 'string' && v.length > 0 ? v : null
+  }
+  const instance: SweInstance = {
+    instance_id: str('instance_id'),
+    repo: str('repo'),
+    base_commit: str('base_commit'),
+    problem_statement: str('problem_statement'),
+    patch: str('patch'),
+    test_patch: str('test_patch'),
+    FAIL_TO_PASS: str('FAIL_TO_PASS'),
+    PASS_TO_PASS: str('PASS_TO_PASS'),
+    version: optStr('version'),
+    environment_setup_commit: optStr('environment_setup_commit'),
+  }
+  if (instance.instance_id !== id) {
+    throw new Error(`bootstrap-meta: metadata instance_id ${instance.instance_id} != requested ${id}`)
+  }
+  return instance
+}
+
+/** One dataset load for all ids, via the adapter (single source of truth). */
+export async function loadSweInstances(ids: string[]): Promise<Map<string, SweInstance>> {
+  if (ids.length === 0) throw new Error('bootstrap-meta: empty instance list')
+  const adapter = createSweBenchAdapter()
+  const tasks = await adapter.loadTasks({ ids, split: 'test' })
+  const byId = new Map(tasks.map((t) => [t.id, t]))
+  const out = new Map<string, SweInstance>()
+  for (const id of ids) {
+    const task = byId.get(id)
+    if (!task) throw new Error(`bootstrap-meta: ${id} not found in SWE-bench_Verified`)
+    out.set(id, assertSweInstance(id, task.metadata as Record<string, unknown> | undefined))
+  }
+  return out
+}
+
+/** task-meta.json — the experiment's exact shape: `{ [iid]: SweInstance }`. */
+export async function writeTaskMeta(ids: string[], outPath: string): Promise<void> {
+  const instances = await loadSweInstances(ids)
+  await writeFile(outPath, JSON.stringify(Object.fromEntries(instances), null, 1) + '\n')
+}
+
+/** <outDir>/<iid>.problem.md — problem statements ONLY (author-safe). */
+export async function writeProblemStatements(ids: string[], outDir: string): Promise<string[]> {
+  const instances = await loadSweInstances(ids)
+  await mkdir(outDir, { recursive: true })
+  const written: string[] = []
+  for (const [id, inst] of instances) {
+    const path = join(outDir, `${id}.problem.md`)
+    await writeFile(path, inst.problem_statement)
+    written.push(path)
+  }
+  return written
+}
+
+/** Dual-calibrate each instance through the M2 path: repro base-fail/gold-pass
+ *  plus the official-judge gold gate. Gold patches come from the adapter and
+ *  are only ever applied mechanically. */
+export async function calibrateIds(
+  ids: string[],
+  opts: { verifyDir: string; workDir: string; instanceImagesPath?: string; keepWorkspaces?: boolean },
+): Promise<CalibrationResult[]> {
+  const instances = await loadSweInstances(ids)
+  const images = await loadInstanceImages(opts.instanceImagesPath)
+  const judge = createSerializedJudge()
+  await mkdir(opts.workDir, { recursive: true })
+  const results: CalibrationResult[] = []
+  for (const id of ids) {
+    const inst = instances.get(id)!
+    const entry = images[id]
+    if (!entry) throw new Error(`bootstrap-meta: ${id} has no image mapping (fixtures/instances.json)`)
+    // verifyCmd runs with cwd = the materialized WORKSPACE (and, downstream,
+    // worker clones) — the script path must be absolute or bash exits 127.
+    const result = await calibrateInstance({
+      instanceId: id,
+      image: entry.image,
+      baseCommit: entry.base_commit,
+      goldPatch: inst.patch,
+      verifyCmd: `bash ${resolve(opts.verifyDir, `${id}.sh`)}`,
+      workDir: opts.workDir,
+      judge,
+      ...(opts.keepWorkspaces !== undefined ? { keepWorkspaces: opts.keepWorkspaces } : {}),
+    })
+    results.push(result)
+    console.log(
+      `CALIBRATION ${id}: baseRc=${result.baseRc} goldApplyRc=${result.goldApplyRc} goldRc=${result.goldRc} ` +
+        `verifyCalibrated=${result.verifyCalibrated} goldOfficialResolved=${result.goldOfficialResolved} ` +
+        `experimentValid=${result.experimentValid}`,
+    )
+  }
+  return results
+}
+
+// ---------------------------------------------------------------------------
+// CLI.
+// ---------------------------------------------------------------------------
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  const [mode, ...rest] = process.argv.slice(2)
+  const defaultIds = defaultRound4Config().instances
+  if (mode === '--task-meta') {
+    const [outPath, ...ids] = rest
+    if (!outPath) {
+      console.error('usage: bootstrap-meta.mts --task-meta <out.json> [iid...]')
+      process.exit(2)
+    }
+    await writeTaskMeta(ids.length > 0 ? ids : defaultIds, outPath)
+    console.log(`task-meta → ${outPath}`)
+  } else if (mode === '--problems') {
+    const [outDir, ...ids] = rest
+    if (!outDir) {
+      console.error('usage: bootstrap-meta.mts --problems <outDir> [iid...]')
+      process.exit(2)
+    }
+    for (const p of await writeProblemStatements(ids.length > 0 ? ids : defaultIds, outDir)) {
+      console.log(`problem statement → ${p}`)
+    }
+  } else if (mode === '--calibrate') {
+    const [verifyDir, workDir, ...ids] = rest
+    if (!verifyDir || !workDir) {
+      console.error('usage: bootstrap-meta.mts --calibrate <verifyDir> <workDir> [iid...]')
+      process.exit(2)
+    }
+    const results = await calibrateIds(ids.length > 0 ? ids : defaultIds, { verifyDir, workDir })
+    const summaryPath = join(workDir, 'calibration.json')
+    await writeFile(summaryPath, JSON.stringify(results, null, 2) + '\n')
+    console.log(`calibration summary → ${summaryPath}`)
+    const invalid = results.filter((r) => !r.experimentValid)
+    if (invalid.length > 0) {
+      console.error(`NOT experiment-valid: ${invalid.map((r) => r.iid).join(', ')}`)
+      process.exit(1)
+    }
+  } else {
+    console.error(
+      'usage: tsx src/swe-arena/bootstrap-meta.mts --task-meta <out.json> [iid...]\n' +
+        '       tsx src/swe-arena/bootstrap-meta.mts --problems <outDir> [iid...]\n' +
+        '       tsx src/swe-arena/bootstrap-meta.mts --calibrate <verifyDir> <workDir> [iid...]',
+    )
+    process.exit(2)
+  }
+}

package/src/swe-arena/bootstrap-meta.test.mts

@@ -0,0 +1,51 @@
+/**
+ * Unit tests for the metadata bootstrap's validation layer — pure, no dataset
+ * load. The adapter-backed loaders are exercised by the calibration CLI.
+ */
+
+import { describe, expect, it } from 'vitest'
+import { assertSweInstance } from './bootstrap-meta.mts'
+
+const full = {
+  instance_id: 'django__django-11532',
+  repo: 'django/django',
+  base_commit: 'a5308514fb4bc5086c9a16a8a24a945eeebb073c',
+  problem_statement: 'Email messages crash on non-ASCII domain…',
+  patch: 'diff --git a/django/core/mail/utils.py …',
+  test_patch: 'diff --git a/tests/mail/tests.py …',
+  FAIL_TO_PASS: '["test_unicode_dns (mail.tests.MailTests)"]',
+  PASS_TO_PASS: '["test_ascii (mail.tests.MailTests)"]',
+  version: '3.0',
+  environment_setup_commit: '419a78300f7cd27611196e1e464d50fd0385ff27',
+}
+
+describe('assertSweInstance', () => {
+  it('accepts a complete metadata record and returns the typed instance', () => {
+    const inst = assertSweInstance('django__django-11532', full)
+    expect(inst.instance_id).toBe('django__django-11532')
+    expect(inst.patch).toContain('diff --git')
+    expect(inst.version).toBe('3.0')
+  })
+
+  it('normalizes absent optional fields to null', () => {
+    const inst = assertSweInstance('django__django-11532', {
+      ...full,
+      version: undefined,
+      environment_setup_commit: '',
+    })
+    expect(inst.version).toBeNull()
+    expect(inst.environment_setup_commit).toBeNull()
+  })
+
+  it('throws on a missing/empty load-bearing field (never a silent blank)', () => {
+    expect(() => assertSweInstance('django__django-11532', { ...full, problem_statement: '' })).toThrow(
+      /problem_statement/,
+    )
+    expect(() => assertSweInstance('django__django-11532', { ...full, patch: undefined })).toThrow(/patch/)
+    expect(() => assertSweInstance('django__django-11532', undefined)).toThrow(/no metadata/)
+  })
+
+  it('throws on an instance_id mismatch', () => {
+    expect(() => assertSweInstance('astropy__astropy-13033', full)).toThrow(/instance_id/)
+  })
+})

package/src/swe-arena/briefing.mts

@@ -0,0 +1,217 @@
+/**
+ * Gen-5 author briefing — MAP + TOOLBOX + PERMISSION (gen5_design).
+ *
+ * Principle: fully agentic evidence access, NO pre-digestion machinery. The
+ * dossier-compiler / ranked-briefing pipeline is dead; instead each run writes
+ * an INDEX FILE (a map: one line per evidence path) and the author prompt
+ * names the pre-existing TOOLS the author may drive itself, plus explicit
+ * permission to spawn its own subagents under a research budget. The
+ * 3-analyst diagnosis ensemble stays as ONE input among these — no longer the
+ * sole channel.
+ *
+ * SELF-IMPROVING BRIEFING: the briefing text itself lives in the declared
+ * change-space — `extensions/pi/author-briefing.md` in the loops repo. When
+ * that file exists at the incumbent ref it REPLACES the default text below,
+ * so a future generation's author can rewrite its own research instructions
+ * and the gate decides whether that rewrite earns its keep. The default here
+ * is the versioned fallback (`AUTHOR_BRIEFING_VERSION`).
+ *
+ * PUBLIC/PRIVATE INTERACTION: the index is proposer-visible text, so
+ * per-instance evidence rows for PRIVATE instances (score-split.mts) are
+ * excluded and the writer fails loud if a private id leaks into the rendered
+ * index.
+ */
+
+import { existsSync } from 'node:fs'
+import { readdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { leaksPrivateInstance, type ScoreSplit } from './score-split.mts'
+import { run } from './proc.ts'
+
+export const AUTHOR_BRIEFING_VERSION = 'map-toolbox-v1'
+
+/** Loops-repo path (inside the declared change-space) whose content, when
+ *  present at the incumbent ref, replaces the default briefing text. */
+export const AUTHOR_BRIEFING_RELPATH = 'extensions/pi/author-briefing.md'
+
+export const EVIDENCE_INDEX_FILENAME = 'evidence-index.md'
+
+/** The default TOOLBOX + PERMISSION briefing. Every tool named here already
+ *  exists — nothing bespoke is built for the author. */
+export function defaultAuthorBriefing(): string {
+  return [
+    `<!-- author-briefing ${AUTHOR_BRIEFING_VERSION} (default; override by committing ${AUTHOR_BRIEFING_RELPATH}) -->`,
+    'RESEARCH BRIEFING — you have hands. Read the evidence yourself before editing.',
+    '',
+    'TOOLBOX (all pre-existing — drive them yourself):',
+    '- traces CLI (published): `npx --yes @tangle-network/traces@latest analyze --help` — analyze any',
+    '  harness run trace (worker/supervisor sessions) for failure patterns, tool-call churn, dead ends.',
+    '- agent-eval trace analysts (`@tangle-network/agent-eval/analyst`): structured extraction over run',
+    '  artifacts when you want findings objects rather than prose.',
+    '- AxLLM (`@ax-llm/ax`): question-answering over a corpus too large to read — point it at a run dir',
+    '  and ask targeted questions instead of paging through megabytes.',
+    '- Plain grep/jq over the evidence map below — often the fastest tool.',
+    '',
+    'PERMISSION: you may spawn your own subagents (research fan-out) before writing any code.',
+    'Research budget: up to ~15 minutes / ~200k tokens of reading+subagents before your first edit;',
+    'spend it on the evidence paths in the map, not on re-deriving what they already record.',
+    '',
+    'The diagnosis findings in this prompt come from a 3-analyst ensemble. Treat them as ONE input',
+    'among the evidence sources above — verify any finding you build on against the raw artifacts.',
+    '',
+    `SELF-IMPROVEMENT: this briefing text is part of the change-space (${AUTHOR_BRIEFING_RELPATH}).`,
+    'If your research process was hampered by these instructions, you may edit that file in your',
+    'candidate alongside your main change; future generations will read your version.',
+  ].join('\n')
+}
+
+/** Resolve the briefing text: the change-space override at `ref` when present,
+ *  else the versioned default. */
+export async function resolveAuthorBriefing(
+  loopsRepo: string,
+  ref: string,
+): Promise<{ text: string; source: 'change-space' | 'default' }> {
+  const show = await run('git', ['-C', loopsRepo, 'show', `${ref}:${AUTHOR_BRIEFING_RELPATH}`])
+  if (show.code === 0 && show.stdout.trim().length > 0) {
+    return { text: show.stdout, source: 'change-space' }
+  }
+  return { text: defaultAuthorBriefing(), source: 'default' }
+}
+
+// ---------------------------------------------------------------------------
+// The evidence index — a MAP, not a briefing: one line per evidence path.
+// ---------------------------------------------------------------------------
+
+export interface EvidenceIndexRow {
+  path: string
+  note: string
+  /** Instance the row is specific to (private rows are dropped); null = run-level. */
+  iid: string | null
+}
+
+export interface EvidenceIndexArgs {
+  outDir: string
+  /** Staircase + round-summary home. */
+  roundsDir: string
+  /** Prior-round failure artifacts (dir + optional patch), with instance ids. */
+  seedArtifactRuns: Array<{ iid: string; arm: string; dir: string; patchPath?: string }>
+  /** Prior run outDirs (e.g. the gen-4 outDir) whose arm-runs/judge evidence
+   *  the author may mine. */
+  priorEvidenceDirs?: string[]
+  /** Pareto parent diffs materialized to disk (written by the outer loop). */
+  paretoParentPatches?: Array<{ label: string; path: string }>
+  /** Public/private split; null = everything is public. */
+  split: ScoreSplit | null
+}
+
+/** Collect the index rows. Pure over the filesystem — reads directory names
+ *  only, never file contents. Private-instance rows are excluded. */
+export async function collectEvidenceIndexRows(args: EvidenceIndexArgs): Promise<EvidenceIndexRow[]> {
+  const priv = new Set(args.split?.privateInstances ?? [])
+  const rows: EvidenceIndexRow[] = []
+  const push = (path: string, note: string, iid: string | null = null): void => {
+    if (iid !== null && priv.has(iid)) return
+    rows.push({ path, note, iid })
+  }
+
+  // Staircase generations + round summaries (the improvement run's history).
+  for (const name of (await readdir(args.roundsDir).catch(() => [])).sort()) {
+    if (/^gen-\d+\.jsonl$/.test(name)) {
+      push(join(args.roundsDir, name), 'staircase rows — every prior candidate: diff, per-instance verdicts, kill reasons')
+    } else if (/^round\d+-summary-.*\.json$/.test(name)) {
+      push(join(args.roundsDir, name), 'round summary — baseline, winner, gate reasons, cost rollup')
+    }
+  }
+
+  // Prior-round seed artifacts (worker evidence, judge output, patches).
+  for (const seed of args.seedArtifactRuns) {
+    if (existsSync(seed.dir)) {
+      push(seed.dir, `prior ${seed.arm} run for ${seed.iid} — worker evidence, journal, driver log`, seed.iid)
+    }
+    if (seed.patchPath && existsSync(seed.patchPath)) {
+      push(seed.patchPath, `prior ${seed.arm} delivered patch for ${seed.iid}`, seed.iid)
+    }
+  }
+
+  // Prior run outDirs: candidate diffs, arm runs (incl. judge.json near-miss
+  // details per instance), proposer shot receipts.
+  for (const dir of args.priorEvidenceDirs ?? []) {
+    if (!existsSync(dir)) continue
+    const candidates = join(dir, 'candidates')
+    if (existsSync(candidates)) push(candidates, 'prior-run candidate diffs (one .patch per surface)')
+    const armRuns = join(dir, 'arm-runs')
+    for (const tag of (await readdir(armRuns).catch(() => [])).sort()) {
+      for (const rep of (await readdir(join(armRuns, tag)).catch(() => [])).sort()) {
+        const runsRoot = join(armRuns, tag, rep, 'runs')
+        for (const iid of (await readdir(runsRoot).catch(() => [])).sort()) {
+          push(
+            join(runsRoot, iid),
+            `prior arm run ${tag} ${rep} on ${iid} — result.json, judge.json (near-miss detail), brain.jsonl, ws/.loops worker evidence`,
+            iid,
+          )
+        }
+      }
+    }
+    const shots = join(dir, 'proposer-shots')
+    if (existsSync(shots)) push(shots, 'prior-run proposer shot receipts (what earlier authors tried)')
+  }
+
+  // Pareto parent diffs.
+  for (const parent of args.paretoParentPatches ?? []) {
+    if (existsSync(parent.path)) push(parent.path, `pareto parent diff — ${parent.label}`)
+  }
+
+  return rows
+}
+
+/** Render + write `<outDir>/evidence-index.md`. Fails loud if a private
+ *  instance id leaks into the rendered text (never-surfaced invariant). */
+export async function writeEvidenceIndex(args: EvidenceIndexArgs): Promise<{ path: string; rows: EvidenceIndexRow[] }> {
+  const rows = await collectEvidenceIndexRows(args)
+  const lines: string[] = [
+    `# Evidence map (${AUTHOR_BRIEFING_VERSION})`,
+    '',
+    'One line per evidence path. This is a MAP, not a digest — open what you need.',
+    ...(args.split !== null
+      ? [
+          '',
+          `NOTE: ${args.split.privateInstances.length} improvement instance(s) are PRIVATE this run: their`,
+          'identities and per-instance evidence are withheld from authors, but your candidate is still',
+          'selected on the full public+private set. Do not overfit the visible instances.',
+        ]
+      : []),
+    '',
+    ...rows.map((r) => `- ${r.path} — ${r.note}`),
+    '',
+  ]
+  const text = lines.join('\n')
+  if (args.split !== null) {
+    const leaks = leaksPrivateInstance(text, args.split)
+    if (leaks.length > 0) {
+      throw new Error(`evidence-index: private instance id(s) leaked into the index: ${leaks.join(', ')}`)
+    }
+  }
+  const path = join(args.outDir, EVIDENCE_INDEX_FILENAME)
+  await writeFile(path, text)
+  return { path, rows }
+}
+
+// ---------------------------------------------------------------------------
+// The prompt section (MAP + TOOLBOX + PERMISSION), appended per author.
+// ---------------------------------------------------------------------------
+
+export interface BriefingContext {
+  indexPath: string
+  briefingText: string
+  briefingSource: 'change-space' | 'default'
+}
+
+export function briefingPromptSection(ctx: BriefingContext): string {
+  return [
+    `EVIDENCE MAP: ${ctx.indexPath}`,
+    'Read it first — one line per evidence path (staircase history, prior arm runs + judge near-miss',
+    'details, worker evidence, parent diffs). Open the paths you need; nothing is pre-digested for you.',
+    '',
+    ctx.briefingText.trimEnd(),
+  ].join('\n')
+}

package/src/swe-arena/briefing.test.mts

@@ -0,0 +1,178 @@
+/**
+ * Gen-5 MAP+TOOLBOX briefing: evidence-index generation (one line per
+ * evidence path, private instances excluded, leak guard fail-loud), the
+ * toolbox/permission prompt section, and the change-space briefing override.
+ */
+
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import {
+  AUTHOR_BRIEFING_RELPATH,
+  AUTHOR_BRIEFING_VERSION,
+  briefingPromptSection,
+  collectEvidenceIndexRows,
+  defaultAuthorBriefing,
+  EVIDENCE_INDEX_FILENAME,
+  resolveAuthorBriefing,
+  writeEvidenceIndex,
+} from './briefing.mts'
+import { splitInstances } from './score-split.mts'
+import { runOk } from './proc.ts'
+
+const SIX = [
+  'astropy__astropy-13033',
+  'django__django-11532',
+  'matplotlib__matplotlib-20826',
+  'pydata__xarray-4687',
+  'pytest-dev__pytest-6197',
+  'sphinx-doc__sphinx-9658',
+]
+
+describe('evidence index', () => {
+  let root: string
+  let outDir: string
+  let roundsDir: string
+  let priorDir: string
+  const split = splitInstances('r4-seed', SIX, 4)
+  const publicIid = split.publicInstances[0]!
+  const privateIid = split.privateInstances[0]!
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), 'briefing-'))
+    outDir = join(root, 'out')
+    roundsDir = join(root, 'rounds')
+    priorDir = join(root, 'gen4')
+    await mkdir(outDir, { recursive: true })
+    await mkdir(roundsDir, { recursive: true })
+    await writeFile(join(roundsDir, 'gen-0.jsonl'), '{}\n')
+    await writeFile(join(roundsDir, 'round4-summary-r4-x.json'), '{}\n')
+    // Seed artifact dirs: one public, one private instance.
+    for (const iid of [publicIid, privateIid]) {
+      await mkdir(join(root, 'runs', iid, 'SUP4'), { recursive: true })
+      await writeFile(join(root, 'patches', `${iid}.sup4.patch`), 'p\n').catch(async () => {
+        await mkdir(join(root, 'patches'), { recursive: true })
+        await writeFile(join(root, 'patches', `${iid}.sup4.patch`), 'p\n')
+      })
+    }
+    // Prior run outDir with arm-run judge evidence for a public + a private instance.
+    for (const iid of [publicIid, privateIid]) {
+      const runDir = join(priorDir, 'arm-runs', 'cand111111', 'rep-0', 'runs', iid)
+      await mkdir(runDir, { recursive: true })
+      await writeFile(join(runDir, 'judge.json'), '{}\n')
+    }
+    await mkdir(join(priorDir, 'candidates'), { recursive: true })
+    await mkdir(join(priorDir, 'proposer-shots'), { recursive: true })
+  })
+
+  afterEach(async () => {
+    await rm(root, { recursive: true, force: true })
+  })
+
+  const args = () => ({
+    outDir,
+    roundsDir,
+    seedArtifactRuns: [publicIid, privateIid].map((iid) => ({
+      iid,
+      arm: 'SUP4',
+      dir: join(root, 'runs', iid, 'SUP4'),
+      patchPath: join(root, 'patches', `${iid}.sup4.patch`),
+    })),
+    priorEvidenceDirs: [priorDir],
+    paretoParentPatches: [] as Array<{ label: string; path: string }>,
+    split,
+  })
+
+  it('lists one line per evidence path: staircase, summaries, seed runs, prior arm runs, shot receipts', async () => {
+    const { path, rows } = await writeEvidenceIndex(args())
+    expect(path).toBe(join(outDir, EVIDENCE_INDEX_FILENAME))
+    expect(existsSync(path)).toBe(true)
+    const text = await readFile(path, 'utf8')
+    expect(text).toContain(join(roundsDir, 'gen-0.jsonl'))
+    expect(text).toContain(join(roundsDir, 'round4-summary-r4-x.json'))
+    expect(text).toContain(join(root, 'runs', publicIid, 'SUP4'))
+    expect(text).toContain(join(root, 'patches', `${publicIid}.sup4.patch`))
+    expect(text).toContain(join(priorDir, 'candidates'))
+    expect(text).toContain(join(priorDir, 'proposer-shots'))
+    expect(text).toContain(join(priorDir, 'arm-runs', 'cand111111', 'rep-0', 'runs', publicIid))
+    // One markdown bullet per row.
+    expect(text.split('\n').filter((l) => l.startsWith('- ')).length).toBe(rows.length)
+  })
+
+  it('NEVER surfaces private instances: no private row, no private id anywhere in the rendered text', async () => {
+    const { path, rows } = await writeEvidenceIndex(args())
+    const text = await readFile(path, 'utf8')
+    for (const iid of split.privateInstances) expect(text).not.toContain(iid)
+    expect(rows.every((r) => r.iid === null || !split.privateInstances.includes(r.iid))).toBe(true)
+    // The withholding itself is disclosed (count only, no identities).
+    expect(text).toContain('2 improvement instance(s) are PRIVATE')
+  })
+
+  it('fails loud when a run-level path would leak a private id into the rendered index', async () => {
+    const leakPatch = join(outDir, `${privateIid}.parent.patch`)
+    await writeFile(leakPatch, 'x\n')
+    await expect(
+      writeEvidenceIndex({ ...args(), paretoParentPatches: [{ label: 'leaky', path: leakPatch }] }),
+    ).rejects.toThrow(/leaked/)
+  })
+
+  it('collectEvidenceIndexRows includes everything when no split is configured', async () => {
+    const rows = await collectEvidenceIndexRows({ ...args(), split: null })
+    const iids = rows.map((r) => r.iid).filter((i): i is string => i !== null)
+    expect(iids).toContain(privateIid)
+  })
+})
+
+describe('briefing text', () => {
+  it('default briefing names the toolbox (traces CLI, trace analysts, AxLLM) and grants subagent permission with a budget', () => {
+    const text = defaultAuthorBriefing()
+    expect(text).toContain('npx --yes @tangle-network/traces@latest')
+    expect(text).toContain('agent-eval trace analysts')
+    expect(text).toContain('AxLLM')
+    expect(text).toContain('spawn your own subagents')
+    expect(text).toContain('Research budget')
+    expect(text).toContain('ONE input')
+    expect(text).toContain(AUTHOR_BRIEFING_RELPATH)
+    expect(text).toContain(AUTHOR_BRIEFING_VERSION)
+  })
+
+  it('briefingPromptSection leads with the evidence map path', () => {
+    const section = briefingPromptSection({
+      indexPath: '/tmp/x/evidence-index.md',
+      briefingText: defaultAuthorBriefing(),
+      briefingSource: 'default',
+    })
+    expect(section).toContain('EVIDENCE MAP: /tmp/x/evidence-index.md')
+    expect(section.indexOf('EVIDENCE MAP')).toBeLessThan(section.indexOf('TOOLBOX'))
+  })
+
+  it('resolveAuthorBriefing prefers the change-space file at the ref (authors may rewrite their own briefing)', async () => {
+    const repo = await mkdtemp(join(tmpdir(), 'briefing-repo-'))
+    try {
+      await runOk('git', ['init', '-q', '-b', 'main', repo])
+      await runOk('git', ['-C', repo, 'config', 'user.email', 't@t.dev'])
+      await runOk('git', ['-C', repo, 'config', 'user.name', 'T'])
+      await writeFile(join(repo, 'base.txt'), 'x\n')
+      await runOk('git', ['-C', repo, 'add', '-A'])
+      await runOk('git', ['-C', repo, 'commit', '-q', '-m', 'init'])
+
+      // No override committed → the versioned default.
+      const fallback = await resolveAuthorBriefing(repo, 'main')
+      expect(fallback.source).toBe('default')
+      expect(fallback.text).toBe(defaultAuthorBriefing())
+
+      // Commit the change-space override → it wins.
+      await mkdir(join(repo, 'extensions', 'pi'), { recursive: true })
+      await writeFile(join(repo, AUTHOR_BRIEFING_RELPATH), 'MY OWN RESEARCH RULES\n')
+      await runOk('git', ['-C', repo, 'add', '-A'])
+      await runOk('git', ['-C', repo, 'commit', '-q', '-m', 'briefing override'])
+      const override = await resolveAuthorBriefing(repo, 'main')
+      expect(override.source).toBe('change-space')
+      expect(override.text).toBe('MY OWN RESEARCH RULES\n')
+    } finally {
+      await rm(repo, { recursive: true, force: true })
+    }
+  })
+})