@catheadowl/dsh-eval 0.1.0

package/src/discovery.mjs

@@ -0,0 +1,115 @@
+/**
+ * Shared case / experiment discovery and validation for the eval CLIs.
+ * Both `dsh-eval` and `dsh-review` scan directories recursively; this
+ * module ensures they share identical skip rules and validation.
+ */
+
+import { readdirSync, statSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+
+/**
+ * Directories never to descend into during discovery. `.runs` holds
+ * prior-run artifacts that would be re-discovered as cases; `node_modules`
+ * would pull in dependencies that happen to match the suffix.
+ */
+const SKIP_DIRS = new Set(['.runs', 'node_modules'])
+
+/**
+ * Recursively collect files matching `suffix` under `path`.
+ * Always skips `.runs` and `node_modules`.
+ * @param {string} path - a file or directory path.
+ * @param {string} suffix - the filename suffix to match (e.g. '.eval.mjs').
+ * @param {string[]} [out] - accumulator (internal).
+ * @returns {string[]} list of discovered files (unsorted; callers sort if needed).
+ */
+export function discoverFiles(path, suffix, out = []) {
+  const absolute = resolve(path)
+  if (statSync(absolute).isFile()) {
+    if (absolute.endsWith(suffix)) out.push(absolute)
+    return out
+  }
+  for (const entry of readdirSync(absolute, { withFileTypes: true })) {
+    const full = join(absolute, entry.name)
+    if (entry.isDirectory() && !SKIP_DIRS.has(entry.name)) {
+      discoverFiles(full, suffix, out)
+    } else if (entry.isFile() && entry.name.endsWith(suffix)) {
+      out.push(full)
+    }
+  }
+  return out
+}
+
+/**
+ * Validate one eval case's shape. Throws with a descriptive message on
+ * the first violation found. Checks:
+ *
+ * - `id` is a non-empty string.
+ * - `task` is a string.
+ * - `mode` (if present) is `'real'` or `'mock'`.
+ * - `disableRows` (if present) is an array of strings (loader row ids to
+ *   disable in this run's overlay). An EMPTY array is legal and means
+ *   "disable nothing, explicitly" — it overrides a `disableRows` default
+ *   from `dsh-eval.config.mjs`, which is how gate-interaction cases opt
+ *   back in inside a package that disables the gate row by default.
+ * - `expect` is an array; every element has `describe` (string) and `check` (function).
+ * - mock mode requires a `script` with `steps` array.
+ *
+ * @param {object} evalCase - the case to validate.
+ * @param {string} file - the source file path (for error messages).
+ */
+export function validateEvalCase(evalCase, file) {
+  if (evalCase === null || typeof evalCase !== 'object') {
+    throw new Error(`${file}: case must be an object`)
+  }
+  if (typeof evalCase.id !== 'string' || evalCase.id === '') {
+    throw new Error(`${file}: case.id must be a non-empty string`)
+  }
+  if (typeof evalCase.task !== 'string') {
+    throw new Error(`${file}: case '${evalCase.id}': task must be a string`)
+  }
+  if (evalCase.mode !== undefined && evalCase.mode !== 'real' && evalCase.mode !== 'mock') {
+    throw new Error(`${file}: case '${evalCase.id}': mode must be 'real' or 'mock' (got '${evalCase.mode}')`)
+  }
+  if (evalCase.gates !== undefined) {
+    throw new Error(`${file}: case '${evalCase.id}': the 'gates' field was removed — declare disableRows: ['gates'] instead`)
+  }
+  if (evalCase.disableRows !== undefined) {
+    if (!Array.isArray(evalCase.disableRows)
+      || evalCase.disableRows.some(row => typeof row !== 'string' || row === '')) {
+      throw new Error(`${file}: case '${evalCase.id}': disableRows must be a string[] of loader row ids (empty = explicit none, overriding config; got '${JSON.stringify(evalCase.disableRows)}')`)
+    }
+  }
+  if (!Array.isArray(evalCase.expect)) {
+    throw new Error(`${file}: case '${evalCase.id}': expect must be a Matcher[]`)
+  }
+  for (let i = 0; i < evalCase.expect.length; i += 1) {
+    const matcher = evalCase.expect[i]
+    if (typeof matcher?.describe !== 'string' || typeof matcher?.check !== 'function') {
+      throw new Error(`${file}: case '${evalCase.id}': expect[${i}] must have { describe: string, check: function }`)
+    }
+  }
+  if (evalCase.mode === 'mock') {
+    if (evalCase.script === undefined || !Array.isArray(evalCase.script?.steps)) {
+      throw new Error(`${file}: case '${evalCase.id}': mock mode requires script.steps`)
+    }
+  }
+}
+
+/**
+ * Detect duplicate case ids across a flat case list. Throws on the first
+ * duplicate found, naming both source files.
+ * @param {{ id: string, __file: string }[]} cases - cases with `__file` attached.
+ */
+export function detectDuplicateIds(cases) {
+  const seen = new Map()
+  for (const evalCase of cases) {
+    const existing = seen.get(evalCase.id)
+    if (existing !== undefined) {
+      const locations = existing === evalCase.__file
+        ? existing
+        : `${existing} and ${evalCase.__file}`
+      throw new Error(`duplicate case id '${evalCase.id}' in ${locations}`)
+    }
+    seen.set(evalCase.id, evalCase.__file)
+  }
+}

package/src/experiment/review.mjs

@@ -0,0 +1,118 @@
+/**
+ * Model-independent review experiment primitives.
+ *
+ * A review experiment separates four concerns:
+ * - frozen inputs owned by the plugin;
+ * - live observation/projection of those inputs;
+ * - the blind prompt shown to an independent reviewer;
+ * - the hidden human rubric used after the run.
+ *
+ * Nothing in this module knows how a model is invoked. Callers provide an
+ * executor (dsh headless is one adapter) when they want to run the assembled
+ * task.
+ */
+
+const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
+
+/** Define and validate a human-graded review experiment. */
+export function defineReviewExperiment(definition) {
+  if (definition === null || typeof definition !== 'object') {
+    throw new TypeError('review experiment must be an object')
+  }
+  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
+    throw new TypeError('review experiment id must be a non-empty path-safe string')
+  }
+  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
+    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
+  }
+  if (typeof definition.observe !== 'function') {
+    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
+  }
+  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
+    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
+  }
+  const defaultRuns = definition.defaultRuns ?? 3
+  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
+    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
+  }
+  return Object.freeze({
+    ...definition,
+    kind: 'review',
+    defaultRuns,
+  })
+}
+
+/** Render the standard observation document consumed by a blind reviewer. */
+export function renderObservationSections(sections) {
+  if (!Array.isArray(sections) || sections.length === 0) {
+    throw new TypeError('review observations must contain at least one section')
+  }
+  const lines = []
+  for (const section of sections) {
+    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
+      throw new TypeError('every observation section needs a heading')
+    }
+    lines.push(`## ${section.heading}`)
+    if (section.introduction) lines.push(String(section.introduction), '')
+    for (const entry of section.entries ?? []) {
+      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
+        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
+      }
+      lines.push(`### ${entry.heading}`)
+      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
+      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
+      if (entry.json !== undefined) {
+        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
+      }
+      lines.push('')
+    }
+  }
+  return lines.join('\n').trimEnd()
+}
+
+/** Materialize live observations and assemble the blind reviewer task. */
+export async function materializeReviewExperiment(experiment) {
+  const sections = await experiment.observe()
+  const observations = renderObservationSections(sections)
+  return {
+    experimentId: experiment.id,
+    observations,
+    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
+  }
+}
+
+/**
+ * Execute the same materialized task through N fresh executor calls.
+ * The executor is intentionally generic: `(task, context) => result`.
+ */
+export async function executeReviewExperiment(experiment, executor, options = {}) {
+  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
+  const runs = options.runs ?? experiment.defaultRuns
+  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
+
+  // Observe once. Every reviewer sees byte-identical evidence, so variance is
+  // attributable to interpretation rather than fixture/projection drift.
+  const materialized = await materializeReviewExperiment(experiment)
+  const attempts = []
+  for (let index = 1; index <= runs; index += 1) {
+    try {
+      const result = await executor(materialized.task, {
+        experiment,
+        experimentId: experiment.id,
+        index,
+        runs,
+      })
+      attempts.push({ index, ok: true, result })
+    } catch (error) {
+      attempts.push({
+        index,
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+        result: error?.result,
+      })
+    }
+  }
+  return { ...materialized, runs, attempts }
+}
+
+export { OBSERVATIONS_PLACEHOLDER }

package/src/index.mjs

@@ -0,0 +1,49 @@
+/**
+ * @catheadowl/dsh-eval public surface — what plugin eval cases import:
+ * matchers for the `expect` list and mock-script step builders. The runner
+ * and trace parser are bin/runner internals, also exported for ad-hoc use.
+ */
+
+export {
+  toolCalled,
+  toolNotCalled,
+  firstTool,
+  toolSequence,
+  toolCallArgs,
+  toolResultFor,
+  toolResultIsError,
+  toolResultSucceeded,
+  toolResultTextIncludes,
+  finalTextIncludes,
+  finalTextMatches,
+  assistantTextIncludes,
+  systemPromptIncludes,
+  toolMounted,
+  userMessageTextIncludes,
+  userMessageTextExcludes,
+} from './assertions.mjs'
+
+export { toolCallStep, textStep } from './mock/script.mjs'
+
+export { runEvalCase, buildOverlayYaml, looksLikeDshRepo, stageProfileStore, FRAMEWORK_ROOT } from './runner.mjs'
+
+export { parseSessionLog, buildTrace, loadTraceDir } from './trace.mjs'
+
+export {
+  defineReviewExperiment,
+  executeReviewExperiment,
+  materializeReviewExperiment,
+  renderObservationSections,
+  OBSERVATIONS_PLACEHOLDER,
+} from './experiment/review.mjs'
+
+export {
+  createDshHeadlessReviewExecutor,
+  resolveDshCli,
+  runDshReviewExperiment,
+} from './adapters/dsh/review.mjs'
+
+export {
+  validateToolBoundary,
+  renderToolBoundaryEvidence,
+} from './tool-validation.mjs'

package/src/mock/mock-adapter.mjs

@@ -0,0 +1,73 @@
+/**
+ * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
+ * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
+ * provider and replays a scripted StreamChunk sequence: one script step per
+ * model call, so a case controls exactly which tool calls and final text the
+ * "model" produces. Pattern precedent:
+ * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
+ *
+ * Deterministic layer purpose: verify the eval runner/trace/assertion
+ * pipeline without an API key, and drive the plugin's tool execution path
+ * (a scripted tool call runs through the real tool pipeline).
+ */
+
+import { readFileSync } from 'node:fs'
+import { LlmAdapter } from '@deepseek-ai/dsh-llm'
+
+/** Environment variable pointing at this run's mock script JSON. */
+const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
+
+/** Adapter that yields one scripted chunk list per stream() call. */
+class EvalMockAdapter extends LlmAdapter {
+  constructor(script) {
+    super()
+    this.steps = script.steps ?? []
+    this.cursor = 0
+  }
+
+  async resolveModel(provider, model) {
+    return {
+      provider,
+      id: model,
+      name: model,
+      reasoning: {
+        efforts: [
+          { id: 'off', name: 'Off' },
+          { id: 'high', name: 'High' },
+        ],
+        defaultEffort: 'off',
+      },
+    }
+  }
+
+  async * stream(_options) {
+    const step = this.steps[this.cursor]
+    this.cursor += 1
+    if (step === undefined) {
+      // Script exhausted: finish the loop loudly-but-gracefully so the run
+      // still produces a trace a case can assert on.
+      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
+      yield { type: 'block-start', index: 0, blockType: 'text' }
+      yield { type: 'text-delta', index: 0, text }
+      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
+      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
+      yield { type: 'finish', reason: { kind: 'stop' } }
+      return
+    }
+    for (const chunk of step) yield chunk
+  }
+}
+
+export const name = 'eval-mock-llm'
+
+export const inject = ['llm']
+
+/** Register the `eval-mock` adapter from the scripted run. */
+export function apply(ctx) {
+  const scriptPath = process.env[SCRIPT_ENV]
+  if (scriptPath === undefined) {
+    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
+  }
+  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
+  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
+}

package/src/mock/script.mjs

@@ -0,0 +1,49 @@
+/**
+ * Script builders for the eval mock-LLM layer. A mock script is
+ * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
+ * model call yields (its `finish` chunk included); the adapter consumes one
+ * step per stream() call. Cases build steps with these helpers instead of
+ * hand-writing chunk JSON.
+ */
+
+let autoId = 0
+
+/** Next deterministic call id for generated steps. */
+function nextCallId() {
+  autoId += 1
+  return `eval-mock-call-${autoId}`
+}
+
+/**
+ * One model call that requests a single tool invocation, then finishes with
+ * `kind: 'tool-calls'` so the loop executes the tool and calls back.
+ * @param {string} name - the model-facing tool name to call.
+ * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
+ * @param {{ id?: string }} [options] - override the generated call id.
+ */
+export function toolCallStep(name, args = {}, options = {}) {
+  const id = options.id ?? nextCallId()
+  const json = JSON.stringify(args)
+  return [
+    { type: 'block-start', index: 0, blockType: 'tool-call' },
+    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
+    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'tool-calls' } },
+  ]
+}
+
+/**
+ * One model call that answers plain text and stops the loop
+ * (`kind: 'stop'`).
+ * @param {string} text - the assistant reply.
+ */
+export function textStep(text) {
+  return [
+    { type: 'block-start', index: 0, blockType: 'text' },
+    { type: 'text-delta', index: 0, text },
+    { type: 'block-end', index: 0, block: { type: 'text', text } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'stop' } },
+  ]
+}

package/src/report.mjs

@@ -0,0 +1,142 @@
+/**
+ * Machine-readable run reports for the behavior CLI (EVAL-007).
+ *
+ * One report covers ONE `dsh-eval run` invocation: the selection summary,
+ * per-case outcomes, and the invocation's environment anchors (profile /
+ * repo / mode filter). Reports exist so multi-plugin consumers and CI can
+ * aggregate results without scraping human output; the human `text` format
+ * is unchanged and remains the default.
+ *
+ * Case statuses:
+ * - `pass`  — ran, exit 0, every matcher satisfied, inspect clean.
+ * - `fail`  — ran but a failure was recorded (non-zero exit, matcher
+ *   failure, timeout, inspect error), or the runner threw, or the case
+ *   file failed to load / carried a duplicate id.
+ * - `skip`  — not run (mode filter or missing credential); `skipReason`
+ *   says why, so `--fail-on-skip` audits stay explainable.
+ */
+
+/**
+ * Build one per-case record. All fields are JSON-stable scalars/arrays.
+ * @param {object} parts
+ * @param {string} parts.id - case id (a case-file load failure uses the file path).
+ * @param {string} parts.file - absolute path of the case file.
+ * @param {'real' | 'mock'} [parts.mode] - the mode the case ran (or would have run) in;
+ *   absent for file-level failures that never resolved to a case.
+ * @param {'pass' | 'fail' | 'skip'} parts.status
+ * @param {string[]} [parts.failures] - human-readable failure lines (fail only).
+ * @param {string} [parts.skipReason] - why the case was skipped (skip only).
+ * @param {number} [parts.exitCode] - the headless CLI's exit code, when it ran.
+ * @param {boolean} [parts.timedOut]
+ * @param {number} [parts.durationMs] - wall time of the run, when it ran.
+ * @param {string} [parts.artifactsDir] - where post-mortem artifacts landed, when written.
+ */
+export function createCaseRecord(parts) {
+  const record = { id: parts.id, file: parts.file }
+  if (parts.mode !== undefined) record.mode = parts.mode
+  record.status = parts.status
+  if (parts.failures !== undefined) record.failures = parts.failures
+  if (parts.skipReason !== undefined) record.skipReason = parts.skipReason
+  if (parts.exitCode !== undefined) record.exitCode = parts.exitCode
+  if (parts.timedOut !== undefined) record.timedOut = parts.timedOut
+  if (parts.durationMs !== undefined) record.durationMs = parts.durationMs
+  if (parts.artifactsDir !== undefined) record.artifactsDir = parts.artifactsDir
+  return record
+}
+
+/**
+ * Aggregate case records into the selection summary. Counts are derived,
+ * never accumulated alongside, so the two can never disagree.
+ * @param {object[]} records - case records from one invocation.
+ */
+export function summarizeRecords(records) {
+  return {
+    selected: records.length,
+    passed: records.filter(r => r.status === 'pass').length,
+    failed: records.filter(r => r.status === 'fail').length,
+    skipped: records.filter(r => r.status === 'skip').length,
+  }
+}
+
+/**
+ * Build the full invocation report object.
+ * @param {object} parts
+ * @param {string} parts.profile - the dsh profile that ran the cases.
+ * @param {string} parts.repo - the dsh host location that ran the cases
+ *   (checkout dir, or the packaged CLI path when resolved via node_modules).
+ * @param {'flag' | 'node_modules' | 'config'} [parts.cliSource] - which C6
+ *   chain segment supplied the CLI (C6: host-checkout-resolution).
+ * @param {'real' | 'mock' | 'all'} parts.modeFilter - the `--mode` selection.
+ * @param {object[]} parts.records - per-case records, in execution order.
+ * @param {string} parts.startedAt - ISO timestamp of the invocation start.
+ * @param {string} parts.finishedAt - ISO timestamp of the invocation end.
+ * @param {boolean} parts.failOnSkip - whether skip-only selections were fatal.
+ */
+export function buildRunReport(parts) {
+  return {
+    tool: 'dsh-eval',
+    profile: parts.profile,
+    repo: parts.repo,
+    ...(parts.cliSource !== undefined ? { cliSource: parts.cliSource } : {}),
+    mode: parts.modeFilter,
+    failOnSkip: parts.failOnSkip,
+    startedAt: parts.startedAt,
+    finishedAt: parts.finishedAt,
+    summary: summarizeRecords(parts.records),
+    results: parts.records,
+  }
+}
+
+/**
+ * The exit code the CLI should use for a finished invocation.
+ * @param {object[]} records - case records from one invocation.
+ * @param {boolean} failOnSkip - `--fail-on-skip`: an all-skipped non-empty
+ *   selection is a failure ("never ran but reported success").
+ */
+export function reportExitCode(records, failOnSkip) {
+  const summary = summarizeRecords(records)
+  if (summary.failed > 0) return 1
+  if (failOnSkip && summary.selected > 0 && summary.passed + summary.failed === 0) return 1
+  return 0
+}
+
+/**
+ * Determinism diagnostic for failed MOCK runs (generic — no plugin named here
+ * is special to the framework).
+ *
+ * A mock run is deterministic only while the script owns every model call.
+ * Any non-host plugin injecting user-visible input after the script's
+ * terminal step (steer, turn-close gate feedback, ...) drives extra model
+ * calls: the script exhausts early and `finalText*` no longer means "the
+ * script's last step". When such injections are visible in the trace, name
+ * them so the failure explains itself instead of every consumer rediscovering
+ * the mechanism by reading raw traces.
+ *
+ * @param {object} parts
+ * @param {import('./trace.mjs').EvalTrace} [parts.trace] - the failed run's trace.
+ * @param {string[]} parts.failures - the recorded failure lines (heuristic:
+ *   the hint only fires when a terminal-text failure is among them).
+ * @returns {string | undefined} the hint line, or undefined when no non-host
+ *   plugin injection is visible.
+ */
+export function mockDeterminismHint(parts) {
+  const { trace, failures } = parts
+  if (trace === undefined) return undefined
+  if (!failures.some(f => f.includes('final text'))) return undefined
+  const injectors = new Set()
+  for (const message of trace.userMessages ?? []) {
+    const source = message.source
+    // Host injections (@deepseek-ai/* runtime context, skill catalogs) ride
+    // along every request and never drive extra script steps; only non-host
+    // plugin messages can splice between/after script steps.
+    if (source?.kind === 'plugin' && typeof source.plugin === 'string'
+      && !source.plugin.startsWith('@deepseek-ai/')) {
+      injectors.add(source.plugin)
+    }
+  }
+  if (injectors.size === 0) return undefined
+  const names = [...injectors].map(name => `'${name}'`).join(', ')
+  return `mock determinism broken: user messages injected by non-host plugin(s) ${names} drove model calls the script does not own`
+    + ` — if this case does not need that plugin, declare disableRows: [${names}];`
+    + ` otherwise account for the interaction in the script or assert with assistantTextIncludes`
+}

package/src/review-report.mjs

@@ -0,0 +1,101 @@
+/**
+ * Human-graded review report renderer (product-review P4).
+ *
+ * A review experiment's value lives in the HUMAN judgment applied after the
+ * runs: which reviewer flags are already-accepted trade-offs (intentional
+ * design), which are new red flags, and what to do next. Without a stable
+ * output shape that judgment stays ephemeral — "asking a model ad hoc" —
+ * and cannot be archived, compared across rounds, or audited by a third
+ * party.
+ *
+ * This renderer fills every machine-knowable field (experiment, adapter,
+ * profile, run count, observations fingerprint, rubric identity, each
+ * reviewer's verbatim answer) and leaves three explicit human sections as
+ * checklists. It deliberately does NOT score or summarize the answers:
+ * compressing fresh-model output into an automatic verdict would fake the
+ * very judgment this layer exists to preserve (product-review risk R3).
+ */
+
+import { createHash } from 'node:crypto'
+
+/** Short hex fingerprint of the materialized observations (byte-stable). */
+export function observationsFingerprint(observations) {
+  return createHash('sha256').update(observations, 'utf8').digest('hex').slice(0, 16)
+}
+
+/**
+ * Render the review report markdown.
+ *
+ * @param {object} parts
+ * @param {object} parts.experiment - the frozen definition (id, summary, rubric).
+ * @param {object} [parts.result] - executeReviewExperiment output; absent for a
+ *   dry run (no reviewer was invoked).
+ * @param {string} [parts.observations] - the materialized observations text,
+ *   when available outside `result` (dry run).
+ * @param {string} [parts.adapter] - adapter label (e.g. 'dsh-headless'); absent
+ *   for a dry run.
+ * @param {string} [parts.profile] - the profile the reviewers ran under.
+ * @returns {string} markdown report.
+ */
+export function renderReviewReport(parts) {
+  const { experiment } = parts
+  const result = parts.result
+  const dry = result === undefined
+  const observations = parts.observations ?? result?.observations ?? ''
+  const rubric = experiment.rubric instanceof URL
+    ? experiment.rubric.href
+    : String(experiment.rubric)
+  const lines = []
+  lines.push(`# Review report — ${experiment.id}`)
+  lines.push('')
+  if (experiment.summary) lines.push(`> ${experiment.summary}`)
+  lines.push('')
+  lines.push(`- experiment: \`${experiment.id}\``)
+  lines.push(`- adapter: ${parts.adapter ?? 'none (dry run)'}`)
+  if (parts.profile !== undefined) lines.push(`- profile: \`${parts.profile}\``)
+  lines.push(`- runs: ${dry ? '0 (dry run — observations materialized only)' : result.runs}`)
+  lines.push(`- observations: \`observations.md\` (sha256:${observationsFingerprint(observations)})`)
+  lines.push(`- rubric: ${rubric.includes('\n') ? '(inline string — see experiment definition)' : `\`${rubric}\``}`)
+  lines.push('')
+
+  lines.push('## Reviewer conclusions')
+  lines.push('')
+  if (dry) {
+    lines.push('_Dry run: no reviewer was invoked. Verify the materialized observations look right, then run for real._')
+    lines.push('')
+  } else {
+    for (const attempt of result.attempts) {
+      if (attempt.ok) {
+        lines.push(`### run ${attempt.index} — ok`)
+        lines.push('')
+        lines.push('```text', attempt.result.stdout.trimEnd(), '```')
+      } else {
+        lines.push(`### run ${attempt.index} — FAIL`)
+        lines.push('')
+        lines.push('```text', String(attempt.error).trimEnd(), '```')
+      }
+      lines.push('')
+    }
+  }
+
+  lines.push('## Intentional design hits (human judgment)')
+  lines.push('')
+  lines.push('Reviewer flags that match an already-accepted trade-off. Cite the run and quote the flag.')
+  lines.push('')
+  lines.push('- [ ] ')
+  lines.push('')
+  lines.push('## New red flags (human judgment)')
+  lines.push('')
+  lines.push('Flags NOT covered by the rubric or the intentional-design list. These are the actual findings.')
+  lines.push('')
+  lines.push('- [ ] ')
+  lines.push('')
+  lines.push('## Next step (pick one)')
+  lines.push('')
+  lines.push('- [ ] change plugin output')
+  lines.push('- [ ] change rubric')
+  lines.push('- [ ] add / adjust a behavior case')
+  lines.push('- [ ] no action (record why)')
+  lines.push('')
+  return lines.join('\n')
+}