@catheadowl/dsh-eval 0.2.0 → 0.2.1

package/src/experiment/review.mjs

@@ -1,118 +1,118 @@
-/**
- * Model-independent review experiment primitives.
- *
- * A review experiment separates four concerns:
- * - frozen inputs owned by the plugin;
- * - live observation/projection of those inputs;
- * - the blind prompt shown to an independent reviewer;
- * - the hidden human rubric used after the run.
- *
- * Nothing in this module knows how a model is invoked. Callers provide an
- * executor (dsh headless is one adapter) when they want to run the assembled
- * task.
- */
-
-const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
-
-/** Define and validate a human-graded review experiment. */
-export function defineReviewExperiment(definition) {
-  if (definition === null || typeof definition !== 'object') {
-    throw new TypeError('review experiment must be an object')
-  }
-  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
-    throw new TypeError('review experiment id must be a non-empty path-safe string')
-  }
-  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
-    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
-  }
-  if (typeof definition.observe !== 'function') {
-    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
-  }
-  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
-    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
-  }
-  const defaultRuns = definition.defaultRuns ?? 3
-  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
-    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
-  }
-  return Object.freeze({
-    ...definition,
-    kind: 'review',
-    defaultRuns,
-  })
-}
-
-/** Render the standard observation document consumed by a blind reviewer. */
-export function renderObservationSections(sections) {
-  if (!Array.isArray(sections) || sections.length === 0) {
-    throw new TypeError('review observations must contain at least one section')
-  }
-  const lines = []
-  for (const section of sections) {
-    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
-      throw new TypeError('every observation section needs a heading')
-    }
-    lines.push(`## ${section.heading}`)
-    if (section.introduction) lines.push(String(section.introduction), '')
-    for (const entry of section.entries ?? []) {
-      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
-        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
-      }
-      lines.push(`### ${entry.heading}`)
-      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
-      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
-      if (entry.json !== undefined) {
-        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
-      }
-      lines.push('')
-    }
-  }
-  return lines.join('\n').trimEnd()
-}
-
-/** Materialize live observations and assemble the blind reviewer task. */
-export async function materializeReviewExperiment(experiment) {
-  const sections = await experiment.observe()
-  const observations = renderObservationSections(sections)
-  return {
-    experimentId: experiment.id,
-    observations,
-    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
-  }
-}
-
-/**
- * Execute the same materialized task through N fresh executor calls.
- * The executor is intentionally generic: `(task, context) => result`.
- */
-export async function executeReviewExperiment(experiment, executor, options = {}) {
-  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
-  const runs = options.runs ?? experiment.defaultRuns
-  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
-
-  // Observe once. Every reviewer sees byte-identical evidence, so variance is
-  // attributable to interpretation rather than fixture/projection drift.
-  const materialized = await materializeReviewExperiment(experiment)
-  const attempts = []
-  for (let index = 1; index <= runs; index += 1) {
-    try {
-      const result = await executor(materialized.task, {
-        experiment,
-        experimentId: experiment.id,
-        index,
-        runs,
-      })
-      attempts.push({ index, ok: true, result })
-    } catch (error) {
-      attempts.push({
-        index,
-        ok: false,
-        error: error instanceof Error ? error.message : String(error),
-        result: error?.result,
-      })
-    }
-  }
-  return { ...materialized, runs, attempts }
-}
-
-export { OBSERVATIONS_PLACEHOLDER }
+/**
+ * Model-independent review experiment primitives.
+ *
+ * A review experiment separates four concerns:
+ * - frozen inputs owned by the plugin;
+ * - live observation/projection of those inputs;
+ * - the blind prompt shown to an independent reviewer;
+ * - the hidden human rubric used after the run.
+ *
+ * Nothing in this module knows how a model is invoked. Callers provide an
+ * executor (dsh headless is one adapter) when they want to run the assembled
+ * task.
+ */
+
+const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
+
+/** Define and validate a human-graded review experiment. */
+export function defineReviewExperiment(definition) {
+  if (definition === null || typeof definition !== 'object') {
+    throw new TypeError('review experiment must be an object')
+  }
+  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
+    throw new TypeError('review experiment id must be a non-empty path-safe string')
+  }
+  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
+    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
+  }
+  if (typeof definition.observe !== 'function') {
+    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
+  }
+  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
+    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
+  }
+  const defaultRuns = definition.defaultRuns ?? 3
+  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
+    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
+  }
+  return Object.freeze({
+    ...definition,
+    kind: 'review',
+    defaultRuns,
+  })
+}
+
+/** Render the standard observation document consumed by a blind reviewer. */
+export function renderObservationSections(sections) {
+  if (!Array.isArray(sections) || sections.length === 0) {
+    throw new TypeError('review observations must contain at least one section')
+  }
+  const lines = []
+  for (const section of sections) {
+    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
+      throw new TypeError('every observation section needs a heading')
+    }
+    lines.push(`## ${section.heading}`)
+    if (section.introduction) lines.push(String(section.introduction), '')
+    for (const entry of section.entries ?? []) {
+      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
+        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
+      }
+      lines.push(`### ${entry.heading}`)
+      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
+      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
+      if (entry.json !== undefined) {
+        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
+      }
+      lines.push('')
+    }
+  }
+  return lines.join('\n').trimEnd()
+}
+
+/** Materialize live observations and assemble the blind reviewer task. */
+export async function materializeReviewExperiment(experiment) {
+  const sections = await experiment.observe()
+  const observations = renderObservationSections(sections)
+  return {
+    experimentId: experiment.id,
+    observations,
+    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
+  }
+}
+
+/**
+ * Execute the same materialized task through N fresh executor calls.
+ * The executor is intentionally generic: `(task, context) => result`.
+ */
+export async function executeReviewExperiment(experiment, executor, options = {}) {
+  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
+  const runs = options.runs ?? experiment.defaultRuns
+  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
+
+  // Observe once. Every reviewer sees byte-identical evidence, so variance is
+  // attributable to interpretation rather than fixture/projection drift.
+  const materialized = await materializeReviewExperiment(experiment)
+  const attempts = []
+  for (let index = 1; index <= runs; index += 1) {
+    try {
+      const result = await executor(materialized.task, {
+        experiment,
+        experimentId: experiment.id,
+        index,
+        runs,
+      })
+      attempts.push({ index, ok: true, result })
+    } catch (error) {
+      attempts.push({
+        index,
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+        result: error?.result,
+      })
+    }
+  }
+  return { ...materialized, runs, attempts }
+}
+
+export { OBSERVATIONS_PLACEHOLDER }

package/src/index.mjs

@@ -25,6 +25,10 @@ export {
   toolMounted,
   userMessageTextIncludes,
   userMessageTextExcludes,
+  subagentDispatched,
+  subagentCompleted,
+  subagentDispatchCount,
+  subagentCompletedCount,
 } from './assertions.mjs'
 
 export { toolCallStep, textStep } from './mock/script.mjs'

package/src/mock/mock-adapter.mjs

@@ -1,73 +1,73 @@
-/**
- * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
- * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
- * provider and replays a scripted StreamChunk sequence: one script step per
- * model call, so a case controls exactly which tool calls and final text the
- * "model" produces. Pattern precedent:
- * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
- *
- * Deterministic layer purpose: verify the eval runner/trace/assertion
- * pipeline without an API key, and drive the plugin's tool execution path
- * (a scripted tool call runs through the real tool pipeline).
- */
-
-import { readFileSync } from 'node:fs'
-import { LlmAdapter } from '@deepseek-ai/dsh-llm'
-
-/** Environment variable pointing at this run's mock script JSON. */
-const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
-
-/** Adapter that yields one scripted chunk list per stream() call. */
-class EvalMockAdapter extends LlmAdapter {
-  constructor(script) {
-    super()
-    this.steps = script.steps ?? []
-    this.cursor = 0
-  }
-
-  async resolveModel(provider, model) {
-    return {
-      provider,
-      id: model,
-      name: model,
-      reasoning: {
-        efforts: [
-          { id: 'off', name: 'Off' },
-          { id: 'high', name: 'High' },
-        ],
-        defaultEffort: 'off',
-      },
-    }
-  }
-
-  async * stream(_options) {
-    const step = this.steps[this.cursor]
-    this.cursor += 1
-    if (step === undefined) {
-      // Script exhausted: finish the loop loudly-but-gracefully so the run
-      // still produces a trace a case can assert on.
-      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
-      yield { type: 'block-start', index: 0, blockType: 'text' }
-      yield { type: 'text-delta', index: 0, text }
-      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
-      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
-      yield { type: 'finish', reason: { kind: 'stop' } }
-      return
-    }
-    for (const chunk of step) yield chunk
-  }
-}
-
-export const name = 'eval-mock-llm'
-
-export const inject = ['llm']
-
-/** Register the `eval-mock` adapter from the scripted run. */
-export function apply(ctx) {
-  const scriptPath = process.env[SCRIPT_ENV]
-  if (scriptPath === undefined) {
-    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
-  }
-  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
-  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
-}
+/**
+ * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
+ * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
+ * provider and replays a scripted StreamChunk sequence: one script step per
+ * model call, so a case controls exactly which tool calls and final text the
+ * "model" produces. Pattern precedent:
+ * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
+ *
+ * Deterministic layer purpose: verify the eval runner/trace/assertion
+ * pipeline without an API key, and drive the plugin's tool execution path
+ * (a scripted tool call runs through the real tool pipeline).
+ */
+
+import { readFileSync } from 'node:fs'
+import { LlmAdapter } from '@deepseek-ai/dsh-llm'
+
+/** Environment variable pointing at this run's mock script JSON. */
+const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
+
+/** Adapter that yields one scripted chunk list per stream() call. */
+class EvalMockAdapter extends LlmAdapter {
+  constructor(script) {
+    super()
+    this.steps = script.steps ?? []
+    this.cursor = 0
+  }
+
+  async resolveModel(provider, model) {
+    return {
+      provider,
+      id: model,
+      name: model,
+      reasoning: {
+        efforts: [
+          { id: 'off', name: 'Off' },
+          { id: 'high', name: 'High' },
+        ],
+        defaultEffort: 'off',
+      },
+    }
+  }
+
+  async * stream(_options) {
+    const step = this.steps[this.cursor]
+    this.cursor += 1
+    if (step === undefined) {
+      // Script exhausted: finish the loop loudly-but-gracefully so the run
+      // still produces a trace a case can assert on.
+      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
+      yield { type: 'block-start', index: 0, blockType: 'text' }
+      yield { type: 'text-delta', index: 0, text }
+      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
+      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
+      yield { type: 'finish', reason: { kind: 'stop' } }
+      return
+    }
+    for (const chunk of step) yield chunk
+  }
+}
+
+export const name = 'eval-mock-llm'
+
+export const inject = ['llm']
+
+/** Register the `eval-mock` adapter from the scripted run. */
+export function apply(ctx) {
+  const scriptPath = process.env[SCRIPT_ENV]
+  if (scriptPath === undefined) {
+    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
+  }
+  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
+  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
+}

package/src/mock/script.mjs

@@ -1,49 +1,49 @@
-/**
- * Script builders for the eval mock-LLM layer. A mock script is
- * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
- * model call yields (its `finish` chunk included); the adapter consumes one
- * step per stream() call. Cases build steps with these helpers instead of
- * hand-writing chunk JSON.
- */
-
-let autoId = 0
-
-/** Next deterministic call id for generated steps. */
-function nextCallId() {
-  autoId += 1
-  return `eval-mock-call-${autoId}`
-}
-
-/**
- * One model call that requests a single tool invocation, then finishes with
- * `kind: 'tool-calls'` so the loop executes the tool and calls back.
- * @param {string} name - the model-facing tool name to call.
- * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
- * @param {{ id?: string }} [options] - override the generated call id.
- */
-export function toolCallStep(name, args = {}, options = {}) {
-  const id = options.id ?? nextCallId()
-  const json = JSON.stringify(args)
-  return [
-    { type: 'block-start', index: 0, blockType: 'tool-call' },
-    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
-    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
-    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
-    { type: 'finish', reason: { kind: 'tool-calls' } },
-  ]
-}
-
-/**
- * One model call that answers plain text and stops the loop
- * (`kind: 'stop'`).
- * @param {string} text - the assistant reply.
- */
-export function textStep(text) {
-  return [
-    { type: 'block-start', index: 0, blockType: 'text' },
-    { type: 'text-delta', index: 0, text },
-    { type: 'block-end', index: 0, block: { type: 'text', text } },
-    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
-    { type: 'finish', reason: { kind: 'stop' } },
-  ]
-}
+/**
+ * Script builders for the eval mock-LLM layer. A mock script is
+ * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
+ * model call yields (its `finish` chunk included); the adapter consumes one
+ * step per stream() call. Cases build steps with these helpers instead of
+ * hand-writing chunk JSON.
+ */
+
+let autoId = 0
+
+/** Next deterministic call id for generated steps. */
+function nextCallId() {
+  autoId += 1
+  return `eval-mock-call-${autoId}`
+}
+
+/**
+ * One model call that requests a single tool invocation, then finishes with
+ * `kind: 'tool-calls'` so the loop executes the tool and calls back.
+ * @param {string} name - the model-facing tool name to call.
+ * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
+ * @param {{ id?: string }} [options] - override the generated call id.
+ */
+export function toolCallStep(name, args = {}, options = {}) {
+  const id = options.id ?? nextCallId()
+  const json = JSON.stringify(args)
+  return [
+    { type: 'block-start', index: 0, blockType: 'tool-call' },
+    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
+    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'tool-calls' } },
+  ]
+}
+
+/**
+ * One model call that answers plain text and stops the loop
+ * (`kind: 'stop'`).
+ * @param {string} text - the assistant reply.
+ */
+export function textStep(text) {
+  return [
+    { type: 'block-start', index: 0, blockType: 'text' },
+    { type: 'text-delta', index: 0, text },
+    { type: 'block-end', index: 0, block: { type: 'text', text } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'stop' } },
+  ]
+}

package/src/overlay.mjs

@@ -19,6 +19,9 @@ const FRAMEWORK_ROOT = fileURLToPath(new URL('..', import.meta.url))
 /** The scripted mock adapter plugin, referenced from generated overlays. */
 const MOCK_ADAPTER_PATH = join(FRAMEWORK_ROOT, 'src', 'mock', 'mock-adapter.mjs')
 
+/** The multi-turn driver plugin, mounted when a case declares `followups`. */
+const MULTI_TURN_DRIVER_PATH = join(FRAMEWORK_ROOT, 'src', 'driver', 'multi-turn-driver.mjs')
+
 /** JSON double-quoted strings are valid YAML scalars — enough for this emitter. */
 function yamlScalar(value) {
   if (typeof value === 'boolean' || typeof value === 'number') return String(value)
@@ -92,5 +95,15 @@ export function buildOverlayYaml(parts) {
     lines.push('    - id: eval-mock-llm')
     lines.push(`      name: ${yamlScalar(pathToFileURL(MOCK_ADAPTER_PATH).href)}`)
   }
+  if (parts.followups !== undefined) {
+    // Cross-turn driving replaces the one-shot headless runner: it exits at
+    // the FIRST idle and aborts every in-process background subagent at
+    // teardown, so fire-and-forget children need the driver's longer lifetime.
+    lines.push('- id: headless-runner')
+    lines.push('  disabled: true')
+    lines.push('- insert:')
+    lines.push('    - id: eval-multi-turn-driver')
+    lines.push(`      name: ${yamlScalar(pathToFileURL(MULTI_TURN_DRIVER_PATH).href)}`)
+  }
   return `${lines.join('\n')}\n`
 }

package/src/review-report.mjs

@@ -68,7 +68,20 @@ export function renderReviewReport(parts) {
       if (attempt.ok) {
         lines.push(`### run ${attempt.index} — ok`)
         lines.push('')
-        lines.push('```text', attempt.result.stdout.trimEnd(), '```')
+        // The answer, not the raw final message: adapters derive an
+        // answer that survives tail interactions (gate splices); stdout
+        // is the fallback for executors without trace-derived answers.
+        lines.push('```text', (attempt.result?.answer ?? attempt.result?.stdout ?? '').trimEnd(), '```')
+        // Traceability: the grader can always recover the full conversation
+        // from the persisted per-run artifacts. The divergence note matches
+        // the bin's write predicate exactly (both fields defined and
+        // differing) — a stdout-less executor writes no run-N.stdout.txt,
+        // so it must not be cited either.
+        const diverged = attempt.result?.answer !== undefined
+          && attempt.result?.stdout !== undefined
+          && attempt.result.answer !== attempt.result.stdout
+        lines.push('')
+        lines.push(`- transcript: \`run-${attempt.index}.stderr.txt\`${diverged ? ` (answer differs from the final message — see \`run-${attempt.index}.stdout.txt\`)` : ''}`)
       } else {
         lines.push(`### run ${attempt.index} — FAIL`)
         lines.push('')

package/src/runner.mjs

@@ -34,7 +34,7 @@ import { tmpdir } from 'node:os'
 import { isAbsolute, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { loadTraceDir } from './trace.mjs'
-import { validateRowConfig, validateDisableRows } from './discovery.mjs'
+import { validateRowConfig, validateDisableRows, validateFollowups } from './discovery.mjs'
 import { CLI_RELATIVE_PATH } from './cli.mjs'
 import { buildOverlayYaml } from './overlay.mjs'
 import { resolveRealDshHome, stageSandboxHome, teardownSandbox, spawnHeadlessDsh } from './sandbox.mjs'
@@ -48,10 +48,17 @@ const FRAMEWORK_ROOT = fileURLToPath(new URL('..', import.meta.url))
  * Case shape: `{ id, task, mode?: 'real' | 'mock', expect: Matcher[],
  * script?: { steps: ChunkStep[] }, persona?: string, disableRows?: string[],
  * rowConfig?: Record<string, Record<string, unknown>>,
+ * followups?: string[], settleTimeoutMs?: number,
  * prepare?: (workspace: string) => void | Promise<void>,
  * inspect?: (workspace: string, helpers: { trace }) => void | Promise<void>,
  * timeoutMs?: number }`
  *
+ * `followups` opts into cross-turn driving: the overlay swaps the one-shot
+ * `headless-runner` row for the eval multi-turn driver, which — before each
+ * followup — waits for background subagent children to settle (bounded by
+ * `settleTimeoutMs`), keeping the process alive so fire-and-forget children
+ * (turn-close defer fixers) can run to completion between turns.
+ *
  * @param {object} evalCase - the case under test.
  * @param {object} options
  * @param {string} options.profile - the dsh profile booting the run (plugin installed there).
@@ -106,6 +113,15 @@ export async function runEvalCase(evalCase, options) {
       writeFileSync(scriptPath, JSON.stringify(evalCase.script))
       env.DSH_EVAL_MOCK_SCRIPT = scriptPath
     }
+    if (evalCase.followups !== undefined) {
+      const planPath = join(runDir, 'driver-plan.json')
+      writeFileSync(planPath, JSON.stringify({
+        task: evalCase.task,
+        followups: evalCase.followups,
+        ...(evalCase.settleTimeoutMs === undefined ? {} : { settleTimeoutMs: evalCase.settleTimeoutMs }),
+      }))
+      env.DSH_EVAL_DRIVER_PLAN = planPath
+    }
 
     if (evalCase.disableRows !== undefined) {
       validateDisableRows(evalCase.disableRows, `case '${evalCase.id}'`)
@@ -113,6 +129,9 @@ export async function runEvalCase(evalCase, options) {
     if (evalCase.rowConfig !== undefined) {
       validateRowConfig(evalCase.rowConfig, `case '${evalCase.id}'`)
     }
+    if (evalCase.followups !== undefined) {
+      validateFollowups(evalCase.followups, `case '${evalCase.id}'`)
+    }
     const overlayPath = join(runDir, 'eval-overlay.yml')
     writeFileSync(overlayPath, buildOverlayYaml({
       sessionsRoot,
@@ -120,6 +139,7 @@ export async function runEvalCase(evalCase, options) {
       disableRows: evalCase.disableRows,
       rowConfig: evalCase.rowConfig,
       mock: mode === 'mock',
+      ...(evalCase.followups === undefined ? {} : { followups: evalCase.followups }),
     }))
 
     const { stdout, stderr, exitCode, timedOut } = await spawnHeadlessDsh({