@tangle-network/agent-bench 0.1.0

package/src/corpus.test.mts

@@ -0,0 +1,273 @@
+import assert from 'node:assert/strict'
+import { isRunRecord } from '@tangle-network/agent-eval'
+import {
+  type AttemptRecord,
+  benchRecordToCorpusRecords,
+  buildRunRecord,
+  buildRunRecordFromAttempts,
+  type RunRecord,
+} from './corpus'
+import { createRuntimeHookRecorder } from './runtime-hook-recorder'
+
+const measuredAttempt = (round: number, output: string, valid: boolean): AttemptRecord => ({
+  round,
+  prompt: `q${round}`,
+  output,
+  valid,
+  score: valid ? 1 : 0,
+  costUsd: 0.01 + round / 1000,
+  tokensIn: 100 + round,
+  tokensOut: 30 + round,
+  wallMs: 500 + round,
+  eventCount: 2,
+  eventTypes: { llm_call: 1, tool_call: 1 },
+})
+
+const baseRec = (attempts: AttemptRecord[], over: Partial<RunRecord> = {}): RunRecord => ({
+  ts: '2026-06-03T00:00:00.000Z',
+  benchmark: 'finsearchcomp',
+  instanceId: 'i1',
+  condition: 'random@3',
+  // canonical RunRecord requires a snapshot-pinned model; bench writes bare
+  // aliases, so the happy path supplies the resolved snapshot via opts.model.
+  model: 'gpt-5',
+  blindResolved: attempts[0]?.valid === true,
+  resolved: attempts.some((a) => a.valid === true),
+  attempts,
+  infraError: false,
+  ...over,
+})
+
+// --- runtime recorder snapshots decision points before persistent corpus storage ---
+{
+  const recorder = createRuntimeHookRecorder()
+  const largeContext = `Bearer abc.def.ghi ${'ctx'.repeat(10_000)}`
+  const largeDetail = `token=supersecret ${'detail'.repeat(1_000)}`
+  recorder.hooks.onDecisionPoint?.(
+    {
+      id: 'run-1:agent.turn:0:failure-recovery',
+      runId: 'run-1',
+      scenarioId: 'task-1',
+      stepIndex: 0,
+      kind: 'retry',
+      candidateActions: Array.from({ length: 75 }, (_, index) => `candidate-${index}`),
+      context: largeContext,
+      evidence: [
+        {
+          source: 'tool_result',
+          id: 'tool-1:result',
+          detail: largeDetail,
+          metadata: { authorization: 'Bearer should-not-survive', nested: { apiKey: 'also-redacted' } },
+        },
+      ],
+      metadata: { token: 'should-not-survive', safe: 'kept' },
+    },
+    {},
+  )
+
+  const [point] = recorder.decisionPoints
+  assert.ok(point, 'decision point recorded')
+  assert.notEqual(point, undefined)
+  assert.equal(point.candidateActions.length, 50, 'candidate actions are bounded')
+  assert.equal(point.context?.length, 20_000, 'context is bounded')
+  assert.equal(point.evidence[0]?.detail?.length, 2_000, 'evidence detail is bounded')
+  assert.equal(point.context?.includes('abc.def.ghi'), false, 'context secrets are redacted')
+  assert.equal(point.evidence[0]?.detail?.includes('supersecret'), false, 'evidence detail secrets are redacted')
+  assert.equal(point.metadata?.token, '[REDACTED]', 'top-level sensitive metadata is redacted')
+  assert.equal(point.metadata?.safe, 'kept', 'non-sensitive metadata is preserved')
+  assert.equal(point.evidence[0]?.metadata?.authorization, '[REDACTED]', 'evidence metadata is redacted')
+  assert.equal(
+    (point.evidence[0]?.metadata?.nested as { apiKey?: unknown } | undefined)?.apiKey,
+    '[REDACTED]',
+    'nested sensitive metadata is redacted',
+  )
+}
+
+// --- happy path: a measured run projects to one canonical CorpusRecord per attempt ---
+{
+  const rec = baseRec([
+    measuredAttempt(0, 'alpha', false),
+    measuredAttempt(1, 'beta', true),
+  ])
+  const { records, unmappable } = await benchRecordToCorpusRecords(rec, {
+    commitSha: 'abc123',
+    model: 'gpt-5-2025-08-07',
+    seed: 42,
+  })
+  assert.equal(unmappable.length, 0, 'all measured attempts map')
+  assert.equal(records.length, 2, 'one record per attempt')
+  for (const r of records) {
+    assert.ok(isRunRecord(r), 'each is a valid canonical RunRecord')
+  }
+  const [r0, r1] = records
+  assert.equal(r0?.candidateId, 'random@3', 'candidateId = condition (the gate-pairing arm)')
+  assert.equal(r0?.scenarioId, 'i1', 'scenarioId = instanceId (the pairing key)')
+  assert.equal(r0?.seed, 42, 'opts.seed is the base seed for attempt 0')
+  assert.equal(r1?.seed, 43, 'each attempt offsets the base → DISTINCT seeds')
+  assert.notEqual(r0?.seed, r1?.seed, 'no (scenarioId, seed) collision across a run\'s attempts')
+  assert.equal(r0?.model, 'gpt-5-2025-08-07', 'snapshot-pinned model override applied')
+  assert.equal(r0?.commitSha, 'abc123')
+  assert.equal(r0?.tokenUsage.input, 100, 'real tokens carried (not zeroed)')
+  assert.equal(r0?.costUsd, 0.01, 'real cost carried')
+  assert.equal(r0?.prompt, 'q0', 'verbatim prompt survives the validator')
+  assert.equal(r0?.completion, 'alpha', 'verbatim completion survives the validator')
+  assert.equal(r0?.outcome.searchScore, 0, 'search split → searchScore from attempt score')
+  assert.equal(r0?.outcome.raw.valid, 0)
+  assert.equal(r1?.outcome.raw.valid, 1)
+  assert.notEqual(r0?.runId, r1?.runId, 'per-attempt runIds are distinct')
+}
+
+// --- unmeasured economics → unmappable, never forged with phantom zeros ---
+{
+  const bare: AttemptRecord = { round: 0, prompt: 'q', output: 'x', valid: true, eventCount: 0, eventTypes: {} }
+  const { records, unmappable } = await benchRecordToCorpusRecords(baseRec([bare]), {
+    commitSha: 'abc',
+    model: 'gpt-5-2025-08-07',
+  })
+  assert.equal(records.length, 0, 'no record forged from unmeasured economics')
+  assert.equal(unmappable.length, 1)
+  assert.match(unmappable[0]!.reason, /unmeasured/, 'reason names the missing measurement')
+  assert.match(unmappable[0]!.reason, /costUsd/, 'reason lists the missing fields')
+}
+
+// --- bare-alias model (no override) → unmappable with the validator's reason ---
+{
+  const { records, unmappable } = await benchRecordToCorpusRecords(
+    baseRec([measuredAttempt(0, 'alpha', true)]),
+    { commitSha: 'abc' },
+  )
+  assert.equal(records.length, 0, 'a bare-alias model is not a reproducibility artifact')
+  assert.equal(unmappable.length, 1)
+  assert.match(unmappable[0]!.reason, /invalid RunRecord/, 'surfaces the validator rejection')
+  assert.match(unmappable[0]!.reason, /snapshot/, 'reason explains the snapshot-pin requirement')
+}
+
+// --- holdout split routes the score to holdoutScore ---
+{
+  const { records } = await benchRecordToCorpusRecords(baseRec([measuredAttempt(0, 'alpha', true)]), {
+    commitSha: 'abc',
+    model: 'gpt-5-2025-08-07',
+    splitTag: 'holdout',
+  })
+  assert.equal(records.length, 1)
+  assert.equal(records[0]?.splitTag, 'holdout')
+  assert.equal(records[0]?.outcome.holdoutScore, 1, 'holdout split → holdoutScore')
+  assert.equal(records[0]?.outcome.searchScore, undefined, 'no searchScore on a holdout record')
+}
+
+// --- bench writer preserves runtime trajectory evidence and semantic decision points ---
+{
+  const record = buildRunRecord({
+    benchmark: 'commit0',
+    instanceId: 'task-1',
+    condition: 'random@2',
+    model: 'gpt-5',
+    resolved: true,
+    infraError: false,
+    now: () => new Date('2026-06-03T00:00:00.000Z'),
+    iterations: [
+      {
+        index: 0,
+        task: 'prompt',
+        agentRunName: 'worker',
+        output: 'completion',
+        verdict: { valid: true, score: 1 },
+        events: [],
+        startedAt: 10,
+        endedAt: 20,
+        costUsd: 0.01,
+        tokenUsage: { input: 10, output: 5 },
+      },
+    ],
+    runtimeEvents: [
+      {
+        id: 'run-1:agent.run:before',
+        runId: 'run-1',
+        scenarioId: 'task-1',
+        target: 'agent.run',
+        phase: 'before',
+        timestamp: 1,
+      },
+    ],
+    runtimeDecisionPoints: [
+      {
+        id: 'run-1:agent.turn:0:failure-recovery',
+        runId: 'run-1',
+        scenarioId: 'task-1',
+        stepIndex: 0,
+        kind: 'retry',
+        candidateActions: ['retry', 'verify', 'stop'],
+        evidence: [{ source: 'tool_result', id: 'tool-1:result' }],
+        metadata: { target: 'failure-recovery' },
+      },
+    ],
+  })
+  assert.equal(record.runtimeEvents?.length, 1, 'runtime lifecycle events survive the writer')
+  assert.equal(record.runtimeDecisionPoints?.length, 1, 'runtime decision points survive the writer')
+  assert.equal(record.runtimeDecisionPoints?.[0]?.metadata?.target, 'failure-recovery')
+}
+
+// --- buildRunRecordFromAttempts: default derivations from the attempts ---
+{
+  const rec = buildRunRecordFromAttempts([measuredAttempt(0, 'a', false), measuredAttempt(1, 'b', true)], {
+    benchmark: 'aec-bench',
+    instanceId: 'i9',
+    condition: 'random@2',
+    model: 'gpt-5',
+    now: () => new Date('2026-06-06T00:00:00.000Z'),
+    runtimeEvents: [
+      {
+        id: 'run-2:agent.run:before',
+        runId: 'run-2',
+        target: 'agent.run',
+        phase: 'before',
+        timestamp: 1,
+      },
+    ],
+    runtimeDecisionPoints: [
+      {
+        id: 'run-2:agent.turn:0:failure-recovery',
+        runId: 'run-2',
+        stepIndex: 0,
+        kind: 'retry',
+        candidateActions: ['retry', 'verify', 'stop'],
+        evidence: [{ source: 'tool_result', id: 'tool-2:result' }],
+      },
+    ],
+  })
+  assert.equal(rec.ts, '2026-06-06T00:00:00.000Z', 'now() seam stamps ts')
+  assert.equal(rec.blindResolved, false, 'blindResolved = attempts[0].valid === true')
+  assert.equal(rec.resolved, true, 'resolved = any attempt valid')
+  assert.equal(rec.infraError, false, 'scored+valid attempts ⇒ not infra')
+  assert.equal(rec.attempts.length, 2)
+  assert.equal(rec.runtimeEvents?.length, 1, 'attempt writer preserves lifecycle events')
+  assert.equal(rec.runtimeDecisionPoints?.length, 1, 'attempt writer preserves decision points')
+}
+
+// --- no scored + no valid attempt ⇒ derived infraError ---
+{
+  const bare: AttemptRecord = { round: 0, prompt: 'q', output: '', eventCount: 0, eventTypes: {} }
+  const rec = buildRunRecordFromAttempts([bare], { benchmark: 'aec-bench', instanceId: 'i', condition: 'random@1', model: 'gpt-5' })
+  assert.equal(rec.infraError, true, 'no scored + no valid ⇒ infraError true')
+  assert.equal(rec.blindResolved, false)
+  assert.equal(rec.resolved, false)
+}
+
+// --- explicit overrides preserve a gate's bespoke recorded values ---
+{
+  const partial: AttemptRecord = { round: 0, prompt: 'q', output: 'x', valid: true, score: 0.5, costUsd: 0.01, tokensIn: 1, tokensOut: 1, wallMs: 1, eventCount: 1, eventTypes: {} }
+  const rec = buildRunRecordFromAttempts([partial], {
+    benchmark: 'clbench-codebase',
+    instanceId: 'i',
+    condition: 'random@1',
+    model: 'gpt-5',
+    // a partial-credit (score 0.5) first shot is valid but NOT a full blind-resolve.
+    blindResolved: false,
+    infraError: false,
+  })
+  assert.equal(rec.blindResolved, false, 'override beats the attempts[0].valid default')
+  assert.equal(rec.resolved, true, 'resolved still derives from valid when not overridden')
+  assert.equal(rec.infraError, false)
+}
+
+console.log('corpus.test.mts: all assertions passed')

package/src/corpus.ts

@@ -0,0 +1,325 @@
+/**
+ * The learning-flywheel corpus (docs/learning-flywheel.md).
+ *
+ * Every bench run persists the FULL tuple per attempt — not a boolean. Boolean
+ * scorecards delete the fuel the flywheel needs; this captures state · steer ·
+ * trace · output · verdict · cost so the accumulated corpus can drive
+ * cross-benchmark controller learning later (offline replay / GEPA / meta-harness).
+ *
+ * One JSONL line per condition-run (a controller steering a worker over k attempts).
+ * Append-only, durable, queryable. The trace is summarized (event count + types +
+ * tail) to stay bounded while keeping the failure signal — store more if a
+ * trace-aware optimizer needs it.
+ */
+
+import { appendFile, mkdir } from 'node:fs/promises'
+import { dirname } from 'node:path'
+import { hashContent, type RunSplitTag, validateRunRecord } from '@tangle-network/agent-eval'
+import type { CorpusRecord } from '@tangle-network/agent-eval/rl'
+import type { Iteration } from '@tangle-network/agent-runtime/loops'
+import type { BenchRuntimeDecisionPoint, BenchRuntimeHookEvent } from './runtime-hook-recorder'
+
+/** One attempt within a condition-run: the prompt/steer sent, the output, the
+ *  verdict, the measured economics, and a bounded trace summary.
+ *
+ *  `costUsd`/`tokensIn`/`tokensOut`/`wallMs` are OPTIONAL on purpose: they are
+ *  present only when the worker actually reported them (the `runLoop`/kernel
+ *  path). A worker that reports no usage (e.g. a raw opencode-stdout shot)
+ *  OMITS them — it never writes a fabricated `0`. Absence means "unmeasured",
+ *  which is honest and lets the canonical bridge below refuse to forge a
+ *  RunRecord with phantom economics. (no-fallback: a silent `0` cost reads as a
+ *  free run downstream, which is a lie the gate would then act on.) */
+export interface AttemptRecord {
+  round: number
+  prompt: string
+  output?: string
+  valid?: boolean
+  score?: number
+  /** Measured USD cost of this attempt. Absent ⇒ the worker reported none. */
+  costUsd?: number
+  /** Measured input/output tokens. Absent ⇒ the worker reported none. */
+  tokensIn?: number
+  tokensOut?: number
+  /** Measured wall time (endedAt − startedAt). Absent ⇒ not timed. */
+  wallMs?: number
+  eventCount: number
+  eventTypes: Record<string, number>
+  traceTail?: string
+  error?: string
+}
+
+/** One controller-run over a single benchmark instance under one condition. */
+export interface RunRecord {
+  ts: string
+  benchmark: string
+  instanceId: string
+  /** Condition / controller label (random@k, refineHand@k, refineGepa@k, …). */
+  condition: string
+  model: string
+  /** iteration[0] verdict — the blind (1-attempt) outcome. */
+  blindResolved: boolean
+  /** winner verdict — the condition's k-attempt outcome. */
+  resolved: boolean
+  attempts: AttemptRecord[]
+  infraError: boolean
+  /** Canonical-pairing provenance (optional; writers set when known). These are
+   *  what let a bench record project onto the substrate's `RunRecord` so it can
+   *  pair across sweeps and feed `analyzeRuns`/`HeldOutGate`/the RL exporters. */
+  seed?: number
+  splitTag?: RunSplitTag
+  commitSha?: string
+  /** Passive runtime hook evidence captured during the run. Optional and bounded by producers. */
+  runtimeEvents?: BenchRuntimeHookEvent[]
+  /** Semantic runtime decision points captured during the run. Optional and producer-defined. */
+  runtimeDecisionPoints?: BenchRuntimeDecisionPoint[]
+}
+
+const TRACE_TAIL_MAX = 600
+
+function summarizeAttempt<Task, Output>(iter: Iteration<Task, Output>): AttemptRecord {
+  const types: Record<string, number> = {}
+  let tail: string | undefined
+  for (const ev of iter.events) {
+    const t = String((ev as { type?: unknown }).type ?? 'unknown')
+    types[t] = (types[t] ?? 0) + 1
+    const d = (ev as { data?: Record<string, unknown> }).data
+    const txt = d?.finalText ?? d?.text ?? d?.result
+    if (typeof txt === 'string' && txt.length > 0) tail = txt
+  }
+  // The kernel measures these for every Iteration — carry them verbatim. A
+  // kernel-reported 0 (e.g. a loop that made no priced LLM call) is an honest
+  // measurement, NOT the fabricated 0 the raw-stdout path used to write.
+  return {
+    round: iter.index,
+    prompt: typeof iter.task === 'string' ? iter.task : JSON.stringify(iter.task),
+    output: iter.output !== undefined ? (typeof iter.output === 'string' ? iter.output : JSON.stringify(iter.output)) : undefined,
+    valid: iter.verdict?.valid,
+    score: iter.verdict?.score,
+    costUsd: iter.costUsd,
+    tokensIn: iter.tokenUsage.input,
+    tokensOut: iter.tokenUsage.output,
+    wallMs: Math.max(0, iter.endedAt - iter.startedAt),
+    eventCount: iter.events.length,
+    eventTypes: types,
+    traceTail: tail ? tail.slice(-TRACE_TAIL_MAX) : undefined,
+    error: iter.error?.message,
+  }
+}
+
+/** Build a RunRecord from a runLoop result. `now` injected for determinism in tests. */
+export function buildRunRecord<Task, Output>(args: {
+  benchmark: string
+  instanceId: string
+  condition: string
+  model: string
+  iterations: ReadonlyArray<Iteration<Task, Output>>
+  resolved: boolean
+  infraError: boolean
+  now?: () => Date
+  /** Canonical-pairing provenance — set when the caller knows it, so the
+   *  record projects cleanly onto the substrate without the bridge guessing. */
+  seed?: number
+  splitTag?: RunSplitTag
+  commitSha?: string
+  runtimeEvents?: BenchRuntimeHookEvent[]
+  runtimeDecisionPoints?: BenchRuntimeDecisionPoint[]
+}): RunRecord {
+  const attempts = args.iterations.map(summarizeAttempt)
+  return {
+    ts: (args.now ? args.now() : new Date()).toISOString(),
+    benchmark: args.benchmark,
+    instanceId: args.instanceId,
+    condition: args.condition,
+    model: args.model,
+    blindResolved: args.iterations[0]?.verdict?.valid === true,
+    resolved: args.resolved,
+    attempts,
+    infraError: args.infraError,
+    ...(args.seed !== undefined ? { seed: args.seed } : {}),
+    ...(args.splitTag !== undefined ? { splitTag: args.splitTag } : {}),
+    ...(args.commitSha !== undefined ? { commitSha: args.commitSha } : {}),
+    ...(args.runtimeEvents !== undefined && args.runtimeEvents.length > 0
+      ? { runtimeEvents: args.runtimeEvents }
+      : {}),
+    ...(args.runtimeDecisionPoints !== undefined && args.runtimeDecisionPoints.length > 0
+      ? { runtimeDecisionPoints: args.runtimeDecisionPoints }
+      : {}),
+  }
+}
+
+/**
+ * Build a RunRecord from a hand-assembled `AttemptRecord[]` (the gate runners'
+ * shape — they score attempts directly rather than carrying `Iteration`s).
+ *
+ * The three run-level verdicts derive from the attempts by default:
+ *   - `blindResolved = attempts[0]?.valid === true`         (the 1-attempt outcome)
+ *   - `resolved      = attempts.some(a => a.valid === true)`  (the k-attempt outcome)
+ *   - `infraError    = no attempt is scored AND none is valid` (the whole run errored)
+ * A gate whose recorded value differs from a derivation (e.g. a binary-pass gate
+ * that reads `blindResolved` off `score === 1`, or one that pins `infraError`)
+ * passes that field explicitly to preserve its exact recorded value. `now` is
+ * injected for determinism in tests (mirrors `buildRunRecord`).
+ */
+export function buildRunRecordFromAttempts(
+  attempts: AttemptRecord[],
+  meta: {
+    benchmark: string
+    instanceId: string
+    condition: string
+    model: string
+    blindResolved?: boolean
+    resolved?: boolean
+    infraError?: boolean
+    now?: () => Date
+    seed?: number
+    splitTag?: RunSplitTag
+    commitSha?: string
+    runtimeEvents?: BenchRuntimeHookEvent[]
+    runtimeDecisionPoints?: BenchRuntimeDecisionPoint[]
+  },
+): RunRecord {
+  const anyScored = attempts.some((a) => a.score !== undefined)
+  const anyValid = attempts.some((a) => a.valid !== undefined)
+  return {
+    ts: (meta.now ? meta.now() : new Date()).toISOString(),
+    benchmark: meta.benchmark,
+    instanceId: meta.instanceId,
+    condition: meta.condition,
+    model: meta.model,
+    blindResolved: meta.blindResolved ?? attempts[0]?.valid === true,
+    resolved: meta.resolved ?? attempts.some((a) => a.valid === true),
+    attempts,
+    infraError: meta.infraError ?? (!anyScored && !anyValid),
+    ...(meta.seed !== undefined ? { seed: meta.seed } : {}),
+    ...(meta.splitTag !== undefined ? { splitTag: meta.splitTag } : {}),
+    ...(meta.commitSha !== undefined ? { commitSha: meta.commitSha } : {}),
+    ...(meta.runtimeEvents !== undefined && meta.runtimeEvents.length > 0
+      ? { runtimeEvents: meta.runtimeEvents }
+      : {}),
+    ...(meta.runtimeDecisionPoints !== undefined && meta.runtimeDecisionPoints.length > 0
+      ? { runtimeDecisionPoints: meta.runtimeDecisionPoints }
+      : {}),
+  }
+}
+
+/** Run-level provenance the caller asserts when projecting a bench record onto
+ *  the substrate. `commitSha` is required: a canonical `RunRecord` is a
+ *  reproducibility artifact and must name the code that produced it — the
+ *  bridge will not invent one. */
+export interface CorpusProjectionOpts {
+  commitSha: string
+  /** Defaults to the record's `benchmark`. */
+  experimentId?: string
+  /** Overrides the record's `seed`; falls back to the attempt ordinal. */
+  seed?: number
+  /** Which split these attempts belong to. Defaults to the record's `splitTag`,
+   *  else `'search'`. */
+  splitTag?: RunSplitTag
+  /** Snapshot-pinned model id (`name@YYYY-MM-DD` / `name-YYYYMMDD`) the caller
+   *  resolved for this run. `validateRunRecord` REJECTS bare aliases (`gpt-5`),
+   *  so a record whose `model` is unpinned — and that has no override here —
+   *  lands in `unmappable` rather than being forged with a guessed snapshot.
+   *  (Bench writers should record the resolved snapshot so this is unneeded.) */
+  model?: string
+}
+
+/** One attempt that could not become a canonical record, with the reason. */
+export interface UnmappableAttempt {
+  round: number
+  reason: string
+}
+
+export interface CorpusProjection {
+  records: CorpusRecord[]
+  /** Attempts dropped because they lacked the canonical-mandatory signal
+   *  (measured economics / an output). The caller decides whether a non-empty
+   *  list is acceptable — it is surfaced, never silently swallowed. */
+  unmappable: UnmappableAttempt[]
+}
+
+/**
+ * Project a bench condition-run onto the substrate's canonical `CorpusRecord[]`
+ * (one record per ATTEMPT — the rollout granularity the RL/replay/gate layers
+ * pair on). This is the bridge that turns the bench's experiment-shaped corpus
+ * into substrate fuel WITHOUT rewriting the bench readers.
+ *
+ * Fail-loud, never fabricate: an attempt missing the canonical-mandatory signal
+ * (`costUsd`/`tokensIn`/`tokensOut`/`wallMs`/`output`) is reported in
+ * `unmappable` rather than backfilled with phantom zeros. This is WHY the local
+ * raw-stdout path (which omits economics) cannot feed the gate — only the
+ * measured `runLoop`/sandbox path can, which is the correct, honest constraint.
+ */
+export async function benchRecordToCorpusRecords(
+  rec: RunRecord,
+  opts: CorpusProjectionOpts,
+): Promise<CorpusProjection> {
+  const records: CorpusRecord[] = []
+  const unmappable: UnmappableAttempt[] = []
+  const experimentId = opts.experimentId ?? rec.benchmark
+  const splitTag: RunSplitTag = opts.splitTag ?? rec.splitTag ?? 'search'
+  const configHash = await hashContent(`${rec.condition}|${rec.model}`)
+
+  for (const a of rec.attempts) {
+    const missing: string[] = []
+    if (typeof a.output !== 'string' || a.output.length === 0) missing.push('output')
+    if (a.costUsd === undefined) missing.push('costUsd')
+    if (a.tokensIn === undefined) missing.push('tokensIn')
+    if (a.tokensOut === undefined) missing.push('tokensOut')
+    if (a.wallMs === undefined) missing.push('wallMs')
+    if (missing.length > 0) {
+      unmappable.push({ round: a.round, reason: `unmeasured: ${missing.join(', ')}` })
+      continue
+    }
+
+    const score = a.score ?? (a.valid === true ? 1 : 0)
+    const promptHash = await hashContent(a.prompt)
+    const candidate: CorpusRecord = {
+      runId: `${rec.benchmark}:${rec.instanceId}:${rec.condition}:r${a.round}`,
+      experimentId,
+      candidateId: rec.condition,
+      // Each attempt needs a DISTINCT seed: the RL/preference layer pairs by
+      // (scenarioId, seed), so identical seeds across a run's k attempts would
+      // collapse them into one cell. A run-level seed is a BASE; the attempt
+      // ordinal offsets it. (rec.seed/ordinal fallbacks keep the same property.)
+      seed: opts.seed !== undefined ? opts.seed + a.round : (rec.seed ?? a.round),
+      model: opts.model ?? rec.model,
+      promptHash,
+      configHash,
+      commitSha: opts.commitSha,
+      wallMs: a.wallMs as number,
+      costUsd: a.costUsd as number,
+      tokenUsage: { input: a.tokensIn as number, output: a.tokensOut as number },
+      outcome: {
+        ...(splitTag === 'holdout' ? { holdoutScore: score } : { searchScore: score }),
+        raw: { valid: a.valid === true ? 1 : 0, score },
+      },
+      splitTag,
+      scenarioId: rec.instanceId,
+      prompt: a.prompt,
+      completion: a.output as string,
+    }
+    try {
+      // Validate for the assertion (throws on a malformed record) but keep the
+      // CorpusRecord superset — the validator returns a RunRecord and may drop
+      // the `prompt`/`completion` extras the RL exporters want.
+      validateRunRecord(candidate)
+      records.push(candidate)
+    } catch (err) {
+      unmappable.push({
+        round: a.round,
+        reason: `invalid RunRecord: ${err instanceof Error ? err.message : String(err)}`,
+      })
+    }
+  }
+
+  return { records, unmappable }
+}
+
+/** Append one RunRecord to the durable corpus (creating the dir if needed). */
+export async function appendRunRecord(corpusPath: string, record: RunRecord): Promise<void> {
+  // Fail loud on a real mkdir failure (EACCES, disk-full): recursive:true is
+  // idempotent when the dir already exists, so this throws ONLY on genuine errors
+  // — a silent swallow just made the append fail later with a confusing message.
+  await mkdir(dirname(corpusPath), { recursive: true })
+  await appendFile(corpusPath, `${JSON.stringify(record)}\n`)
+}