@catheadowl/dsh-eval 0.2.0 → 0.3.0

package/src/experiment/review.mjs

@@ -1,118 +1,118 @@
-/**
- * Model-independent review experiment primitives.
- *
- * A review experiment separates four concerns:
- * - frozen inputs owned by the plugin;
- * - live observation/projection of those inputs;
- * - the blind prompt shown to an independent reviewer;
- * - the hidden human rubric used after the run.
- *
- * Nothing in this module knows how a model is invoked. Callers provide an
- * executor (dsh headless is one adapter) when they want to run the assembled
- * task.
- */
-
-const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
-
-/** Define and validate a human-graded review experiment. */
-export function defineReviewExperiment(definition) {
-  if (definition === null || typeof definition !== 'object') {
-    throw new TypeError('review experiment must be an object')
-  }
-  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
-    throw new TypeError('review experiment id must be a non-empty path-safe string')
-  }
-  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
-    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
-  }
-  if (typeof definition.observe !== 'function') {
-    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
-  }
-  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
-    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
-  }
-  const defaultRuns = definition.defaultRuns ?? 3
-  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
-    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
-  }
-  return Object.freeze({
-    ...definition,
-    kind: 'review',
-    defaultRuns,
-  })
-}
-
-/** Render the standard observation document consumed by a blind reviewer. */
-export function renderObservationSections(sections) {
-  if (!Array.isArray(sections) || sections.length === 0) {
-    throw new TypeError('review observations must contain at least one section')
-  }
-  const lines = []
-  for (const section of sections) {
-    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
-      throw new TypeError('every observation section needs a heading')
-    }
-    lines.push(`## ${section.heading}`)
-    if (section.introduction) lines.push(String(section.introduction), '')
-    for (const entry of section.entries ?? []) {
-      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
-        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
-      }
-      lines.push(`### ${entry.heading}`)
-      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
-      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
-      if (entry.json !== undefined) {
-        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
-      }
-      lines.push('')
-    }
-  }
-  return lines.join('\n').trimEnd()
-}
-
-/** Materialize live observations and assemble the blind reviewer task. */
-export async function materializeReviewExperiment(experiment) {
-  const sections = await experiment.observe()
-  const observations = renderObservationSections(sections)
-  return {
-    experimentId: experiment.id,
-    observations,
-    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
-  }
-}
-
-/**
- * Execute the same materialized task through N fresh executor calls.
- * The executor is intentionally generic: `(task, context) => result`.
- */
-export async function executeReviewExperiment(experiment, executor, options = {}) {
-  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
-  const runs = options.runs ?? experiment.defaultRuns
-  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
-
-  // Observe once. Every reviewer sees byte-identical evidence, so variance is
-  // attributable to interpretation rather than fixture/projection drift.
-  const materialized = await materializeReviewExperiment(experiment)
-  const attempts = []
-  for (let index = 1; index <= runs; index += 1) {
-    try {
-      const result = await executor(materialized.task, {
-        experiment,
-        experimentId: experiment.id,
-        index,
-        runs,
-      })
-      attempts.push({ index, ok: true, result })
-    } catch (error) {
-      attempts.push({
-        index,
-        ok: false,
-        error: error instanceof Error ? error.message : String(error),
-        result: error?.result,
-      })
-    }
-  }
-  return { ...materialized, runs, attempts }
-}
-
-export { OBSERVATIONS_PLACEHOLDER }
+/**
+ * Model-independent review experiment primitives.
+ *
+ * A review experiment separates four concerns:
+ * - frozen inputs owned by the plugin;
+ * - live observation/projection of those inputs;
+ * - the blind prompt shown to an independent reviewer;
+ * - the hidden human rubric used after the run.
+ *
+ * Nothing in this module knows how a model is invoked. Callers provide an
+ * executor (dsh headless is one adapter) when they want to run the assembled
+ * task.
+ */
+
+const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
+
+/** Define and validate a human-graded review experiment. */
+export function defineReviewExperiment(definition) {
+  if (definition === null || typeof definition !== 'object') {
+    throw new TypeError('review experiment must be an object')
+  }
+  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
+    throw new TypeError('review experiment id must be a non-empty path-safe string')
+  }
+  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
+    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
+  }
+  if (typeof definition.observe !== 'function') {
+    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
+  }
+  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
+    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
+  }
+  const defaultRuns = definition.defaultRuns ?? 3
+  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
+    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
+  }
+  return Object.freeze({
+    ...definition,
+    kind: 'review',
+    defaultRuns,
+  })
+}
+
+/** Render the standard observation document consumed by a blind reviewer. */
+export function renderObservationSections(sections) {
+  if (!Array.isArray(sections) || sections.length === 0) {
+    throw new TypeError('review observations must contain at least one section')
+  }
+  const lines = []
+  for (const section of sections) {
+    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
+      throw new TypeError('every observation section needs a heading')
+    }
+    lines.push(`## ${section.heading}`)
+    if (section.introduction) lines.push(String(section.introduction), '')
+    for (const entry of section.entries ?? []) {
+      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
+        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
+      }
+      lines.push(`### ${entry.heading}`)
+      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
+      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
+      if (entry.json !== undefined) {
+        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
+      }
+      lines.push('')
+    }
+  }
+  return lines.join('\n').trimEnd()
+}
+
+/** Materialize live observations and assemble the blind reviewer task. */
+export async function materializeReviewExperiment(experiment) {
+  const sections = await experiment.observe()
+  const observations = renderObservationSections(sections)
+  return {
+    experimentId: experiment.id,
+    observations,
+    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
+  }
+}
+
+/**
+ * Execute the same materialized task through N fresh executor calls.
+ * The executor is intentionally generic: `(task, context) => result`.
+ */
+export async function executeReviewExperiment(experiment, executor, options = {}) {
+  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
+  const runs = options.runs ?? experiment.defaultRuns
+  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
+
+  // Observe once. Every reviewer sees byte-identical evidence, so variance is
+  // attributable to interpretation rather than fixture/projection drift.
+  const materialized = await materializeReviewExperiment(experiment)
+  const attempts = []
+  for (let index = 1; index <= runs; index += 1) {
+    try {
+      const result = await executor(materialized.task, {
+        experiment,
+        experimentId: experiment.id,
+        index,
+        runs,
+      })
+      attempts.push({ index, ok: true, result })
+    } catch (error) {
+      attempts.push({
+        index,
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+        result: error?.result,
+      })
+    }
+  }
+  return { ...materialized, runs, attempts }
+}
+
+export { OBSERVATIONS_PLACEHOLDER }

package/src/experimental.mjs

@@ -16,7 +16,12 @@ export { stageProfileStore } from './sandbox.mjs'
 export { buildOverlayYaml, overlayDisableRows } from './overlay.mjs'
 
 // --- session-trace primitives ---
-export { parseSessionLog, buildTrace, loadTraceDir } from './trace.mjs'
+export {
+  parseSessionLog,
+  buildTrace,
+  collectSessionTrace,
+  KNOWN_SESSION_FORMAT_VERSIONS,
+} from './trace.mjs'
 
 // --- review experiment execution layer ---
 export {

package/src/index.mjs

@@ -25,6 +25,10 @@ export {
   toolMounted,
   userMessageTextIncludes,
   userMessageTextExcludes,
+  subagentDispatched,
+  subagentCompleted,
+  subagentDispatchCount,
+  subagentCompletedCount,
 } from './assertions.mjs'
 
 export { toolCallStep, textStep } from './mock/script.mjs'

package/src/mock/mock-adapter.mjs

@@ -1,73 +1,87 @@
-/**
- * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
- * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
- * provider and replays a scripted StreamChunk sequence: one script step per
- * model call, so a case controls exactly which tool calls and final text the
- * "model" produces. Pattern precedent:
- * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
- *
- * Deterministic layer purpose: verify the eval runner/trace/assertion
- * pipeline without an API key, and drive the plugin's tool execution path
- * (a scripted tool call runs through the real tool pipeline).
- */
-
-import { readFileSync } from 'node:fs'
-import { LlmAdapter } from '@deepseek-ai/dsh-llm'
-
-/** Environment variable pointing at this run's mock script JSON. */
-const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
-
-/** Adapter that yields one scripted chunk list per stream() call. */
-class EvalMockAdapter extends LlmAdapter {
-  constructor(script) {
-    super()
-    this.steps = script.steps ?? []
-    this.cursor = 0
-  }
-
-  async resolveModel(provider, model) {
-    return {
-      provider,
-      id: model,
-      name: model,
-      reasoning: {
-        efforts: [
-          { id: 'off', name: 'Off' },
-          { id: 'high', name: 'High' },
-        ],
-        defaultEffort: 'off',
-      },
-    }
-  }
-
-  async * stream(_options) {
-    const step = this.steps[this.cursor]
-    this.cursor += 1
-    if (step === undefined) {
-      // Script exhausted: finish the loop loudly-but-gracefully so the run
-      // still produces a trace a case can assert on.
-      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
-      yield { type: 'block-start', index: 0, blockType: 'text' }
-      yield { type: 'text-delta', index: 0, text }
-      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
-      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
-      yield { type: 'finish', reason: { kind: 'stop' } }
-      return
-    }
-    for (const chunk of step) yield chunk
-  }
-}
-
-export const name = 'eval-mock-llm'
-
-export const inject = ['llm']
-
-/** Register the `eval-mock` adapter from the scripted run. */
-export function apply(ctx) {
-  const scriptPath = process.env[SCRIPT_ENV]
-  if (scriptPath === undefined) {
-    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
-  }
-  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
-  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
-}
+/**
+ * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
+ * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
+ * provider and replays a scripted StreamChunk sequence: one script step per
+ * model call, so a case controls exactly which tool calls and final text the
+ * "model" produces. Pattern precedent:
+ * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
+ *
+ * Deterministic layer purpose: verify the eval runner/trace/assertion
+ * pipeline without an API key, and drive the plugin's tool execution path
+ * (a scripted tool call runs through the real tool pipeline).
+ */
+
+import { readFileSync } from 'node:fs'
+import { LlmAdapter } from '@deepseek-ai/dsh-llm'
+
+/** Environment variable pointing at this run's mock script JSON. */
+const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
+
+/** Adapter that yields one scripted chunk list per stream() call. */
+class EvalMockAdapter extends LlmAdapter {
+  constructor(script) {
+    super()
+    this.steps = script.steps ?? []
+    this.cursor = 0
+  }
+
+  async resolveModel(provider, model) {
+    return {
+      provider,
+      id: model,
+      name: model,
+      reasoning: {
+        efforts: [
+          { id: 'off', name: 'Off' },
+          { id: 'high', name: 'High' },
+        ],
+        defaultEffort: 'off',
+      },
+    }
+  }
+
+  // Carried explicitly instead of inherited: this plugin's `LlmAdapter` base
+  // resolves from THIS package's peer instance, which can lag the host
+  // runtime that drives it (host 0.1.5-rc.2 grew `prepareCall` while the
+  // local peer was 0.0.1-rc.1 — the inherited face was missing and every mock
+  // run died at `registration.adapter.prepareCall is not a function`). The
+  // shape is the host base-class default: model metadata plus a dispatch
+  // entry bound to this same adapter generation.
+  async prepareCall(provider, model, signal) {
+    return {
+      model: await this.resolveModel(provider, model, signal),
+      stream: options => this.stream(options),
+    }
+  }
+
+  async * stream(_options) {
+    const step = this.steps[this.cursor]
+    this.cursor += 1
+    if (step === undefined) {
+      // Script exhausted: finish the loop loudly-but-gracefully so the run
+      // still produces a trace a case can assert on.
+      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
+      yield { type: 'block-start', index: 0, blockType: 'text' }
+      yield { type: 'text-delta', index: 0, text }
+      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
+      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
+      yield { type: 'finish', reason: { kind: 'stop' } }
+      return
+    }
+    for (const chunk of step) yield chunk
+  }
+}
+
+export const name = 'eval-mock-llm'
+
+export const inject = ['llm']
+
+/** Register the `eval-mock` adapter from the scripted run. */
+export function apply(ctx) {
+  const scriptPath = process.env[SCRIPT_ENV]
+  if (scriptPath === undefined) {
+    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
+  }
+  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
+  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
+}

package/src/mock/script.mjs

@@ -1,49 +1,49 @@
-/**
- * Script builders for the eval mock-LLM layer. A mock script is
- * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
- * model call yields (its `finish` chunk included); the adapter consumes one
- * step per stream() call. Cases build steps with these helpers instead of
- * hand-writing chunk JSON.
- */
-
-let autoId = 0
-
-/** Next deterministic call id for generated steps. */
-function nextCallId() {
-  autoId += 1
-  return `eval-mock-call-${autoId}`
-}
-
-/**
- * One model call that requests a single tool invocation, then finishes with
- * `kind: 'tool-calls'` so the loop executes the tool and calls back.
- * @param {string} name - the model-facing tool name to call.
- * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
- * @param {{ id?: string }} [options] - override the generated call id.
- */
-export function toolCallStep(name, args = {}, options = {}) {
-  const id = options.id ?? nextCallId()
-  const json = JSON.stringify(args)
-  return [
-    { type: 'block-start', index: 0, blockType: 'tool-call' },
-    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
-    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
-    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
-    { type: 'finish', reason: { kind: 'tool-calls' } },
-  ]
-}
-
-/**
- * One model call that answers plain text and stops the loop
- * (`kind: 'stop'`).
- * @param {string} text - the assistant reply.
- */
-export function textStep(text) {
-  return [
-    { type: 'block-start', index: 0, blockType: 'text' },
-    { type: 'text-delta', index: 0, text },
-    { type: 'block-end', index: 0, block: { type: 'text', text } },
-    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
-    { type: 'finish', reason: { kind: 'stop' } },
-  ]
-}
+/**
+ * Script builders for the eval mock-LLM layer. A mock script is
+ * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
+ * model call yields (its `finish` chunk included); the adapter consumes one
+ * step per stream() call. Cases build steps with these helpers instead of
+ * hand-writing chunk JSON.
+ */
+
+let autoId = 0
+
+/** Next deterministic call id for generated steps. */
+function nextCallId() {
+  autoId += 1
+  return `eval-mock-call-${autoId}`
+}
+
+/**
+ * One model call that requests a single tool invocation, then finishes with
+ * `kind: 'tool-calls'` so the loop executes the tool and calls back.
+ * @param {string} name - the model-facing tool name to call.
+ * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
+ * @param {{ id?: string }} [options] - override the generated call id.
+ */
+export function toolCallStep(name, args = {}, options = {}) {
+  const id = options.id ?? nextCallId()
+  const json = JSON.stringify(args)
+  return [
+    { type: 'block-start', index: 0, blockType: 'tool-call' },
+    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
+    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'tool-calls' } },
+  ]
+}
+
+/**
+ * One model call that answers plain text and stops the loop
+ * (`kind: 'stop'`).
+ * @param {string} text - the assistant reply.
+ */
+export function textStep(text) {
+  return [
+    { type: 'block-start', index: 0, blockType: 'text' },
+    { type: 'text-delta', index: 0, text },
+    { type: 'block-end', index: 0, block: { type: 'text', text } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'stop' } },
+  ]
+}

package/src/overlay.mjs

@@ -19,6 +19,9 @@ const FRAMEWORK_ROOT = fileURLToPath(new URL('..', import.meta.url))
 /** The scripted mock adapter plugin, referenced from generated overlays. */
 const MOCK_ADAPTER_PATH = join(FRAMEWORK_ROOT, 'src', 'mock', 'mock-adapter.mjs')
 
+/** The multi-turn driver plugin, mounted when a case declares `followups`. */
+const MULTI_TURN_DRIVER_PATH = join(FRAMEWORK_ROOT, 'src', 'driver', 'multi-turn-driver.mjs')
+
 /** JSON double-quoted strings are valid YAML scalars — enough for this emitter. */
 function yamlScalar(value) {
   if (typeof value === 'boolean' || typeof value === 'number') return String(value)
@@ -92,5 +95,15 @@ export function buildOverlayYaml(parts) {
     lines.push('    - id: eval-mock-llm')
     lines.push(`      name: ${yamlScalar(pathToFileURL(MOCK_ADAPTER_PATH).href)}`)
   }
+  if (parts.followups !== undefined) {
+    // Cross-turn driving replaces the one-shot headless runner: it exits at
+    // the FIRST idle and aborts every in-process background subagent at
+    // teardown, so fire-and-forget children need the driver's longer lifetime.
+    lines.push('- id: headless-runner')
+    lines.push('  disabled: true')
+    lines.push('- insert:')
+    lines.push('    - id: eval-multi-turn-driver')
+    lines.push(`      name: ${yamlScalar(pathToFileURL(MULTI_TURN_DRIVER_PATH).href)}`)
+  }
   return `${lines.join('\n')}\n`
 }

package/src/report.mjs

@@ -30,6 +30,10 @@
  * @param {boolean} [parts.timedOut]
  * @param {number} [parts.durationMs] - wall time of the run, when it ran.
  * @param {string} [parts.artifactsDir] - where post-mortem artifacts landed, when written.
+ * @param {object} [parts.census] - the run trace's projection census
+ *   (`trace.census`), carried on both pass and fail records so a green case
+ *   whose evidence surface degraded is still inspectable after the fact.
+ *   Absent when no trace materialized.
  */
 export function createCaseRecord(parts) {
   const record = { id: parts.id, file: parts.file }
@@ -41,6 +45,7 @@ export function createCaseRecord(parts) {
   if (parts.timedOut !== undefined) record.timedOut = parts.timedOut
   if (parts.durationMs !== undefined) record.durationMs = parts.durationMs
   if (parts.artifactsDir !== undefined) record.artifactsDir = parts.artifactsDir
+  if (parts.census !== undefined) record.census = parts.census
   return record
 }
 

package/src/review-report.mjs

@@ -23,6 +23,29 @@ export function observationsFingerprint(observations) {
   return createHash('sha256').update(observations, 'utf8').digest('hex').slice(0, 16)
 }
 
+/**
+ * One-line statement of whether the tool boundary check RAN on the review
+ * runs. A run without a session artifact reports `status: 'not-executed'`;
+ * that must show up in the report header, because a review whose tool face was
+ * never verified is weaker evidence than one whose was (EVAL-021). Executors
+ * that do not validate at all (custom executors, dry runs) say so instead of
+ * implying a pass.
+ */
+function toolBoundarySummary(attempts) {
+  const skipped = attempts.filter(attempt => attempt.result?.toolValidation?.status === 'not-executed')
+  if (skipped.length > 0) {
+    return `NOT EXECUTED on run(s) ${skipped.map(attempt => attempt.index).join(', ')}`
+      + ' — no session artifact; the reviewer tool face was not verified (see the per-run notes)'
+  }
+  const checked = attempts.filter(attempt => attempt.result?.toolValidation?.status === 'checked')
+  if (attempts.length > 0 && checked.length === attempts.length) return 'checked on every run'
+  if (checked.length > 0) {
+    return `checked on ${checked.length} of ${attempts.length} run(s); the rest reported none`
+      + ' (failed before validation, or a non-verifying executor)'
+  }
+  return 'not reported by this executor'
+}
+
 /**
  * Render the review report markdown.
  *
@@ -54,6 +77,7 @@ export function renderReviewReport(parts) {
   lines.push(`- adapter: ${parts.adapter ?? 'none (dry run)'}`)
   if (parts.profile !== undefined) lines.push(`- profile: \`${parts.profile}\``)
   lines.push(`- runs: ${dry ? '0 (dry run — observations materialized only)' : result.runs}`)
+  if (!dry) lines.push(`- tool boundary: ${toolBoundarySummary(result.attempts)}`)
   lines.push(`- observations: \`observations.md\` (sha256:${observationsFingerprint(observations)})`)
   lines.push(`- rubric: ${rubric.includes('\n') ? '(inline string — see experiment definition)' : `\`${rubric}\``}`)
   lines.push('')
@@ -68,7 +92,27 @@ export function renderReviewReport(parts) {
       if (attempt.ok) {
         lines.push(`### run ${attempt.index} — ok`)
         lines.push('')
-        lines.push('```text', attempt.result.stdout.trimEnd(), '```')
+        // The answer, not the raw final message: adapters derive an
+        // answer that survives tail interactions (gate splices); stdout
+        // is the fallback for executors without trace-derived answers.
+        lines.push('```text', (attempt.result?.answer ?? attempt.result?.stdout ?? '').trimEnd(), '```')
+        // Traceability: the grader can always recover the full conversation
+        // from the persisted per-run artifacts. The divergence note matches
+        // the bin's write predicate exactly (both fields defined and
+        // differing) — a stdout-less executor writes no run-N.stdout.txt,
+        // so it must not be cited either.
+        const diverged = attempt.result?.answer !== undefined
+          && attempt.result?.stdout !== undefined
+          && attempt.result.answer !== attempt.result.stdout
+        lines.push('')
+        lines.push(`- transcript: \`run-${attempt.index}.stderr.txt\`${diverged ? ` (answer differs from the final message — see \`run-${attempt.index}.stdout.txt\`)` : ''}`)
+        if (attempt.result?.toolValidation?.status === 'not-executed') {
+          // The unverified-boundary fact rides the run it applies to, next to
+          // the answer it qualifies: this answer is stdout's final message
+          // (there was no trace to derive it from) and no request header was
+          // ever inspected for tool leakage.
+          lines.push(`- **tool-boundary check NOT EXECUTED**: ${attempt.result.traceGap ?? 'no session trace materialized'} — the answer above is stdout's final message, not a trace-derived answer.`)
+        }
       } else {
         lines.push(`### run ${attempt.index} — FAIL`)
         lines.push('')