@catheadowl/dsh-eval 0.1.0

package/src/runner.mjs

@@ -0,0 +1,373 @@
+/**
+ * The eval run driver. One eval case = one `dsh --profile <p> --patch
+ * <generated-overlay> "<task>"` headless run in an isolated `DSH_HOME`, so
+ * the session JSONL trace lands alone under a per-run persistence root and
+ * needs no teardown of shared state. The overlay is generated per run:
+ *
+ * - always: `session-persistence-jsonl` re-rooted to the run dir, plaintext
+ *   one-event-per-line layout (config override is whole-replace, so every
+ *   field the backend needs is restated);
+ * - optional case persona: `system-prompt` persona override;
+ * - optional `disableRows: ['<row-id>', ...]` case declaration: the listed
+ *   loader rows are disabled, so e.g. a turn-close blocking gate plugin
+ *   cannot splice feedback steps past the script's terminal step (the
+ *   disableRows × turn-close gate boundary contract);
+ * - mock mode: `agent-default-model` re-pointed at the `eval-mock` provider
+ *   plus an insert mounting the scripted adapter plugin by `file://` URL
+ *   (relative plugin names resolve against the PROFILE dir, not the overlay
+ *   file, so an absolute URL is the portable reference).
+ */
+
+import { spawn } from 'node:child_process'
+import {
+  chmodSync, copyFileSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, symlinkSync, unlinkSync, writeFileSync,
+} from 'node:fs'
+import { tmpdir, homedir } from 'node:os'
+import { isAbsolute, join, resolve } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { loadTraceDir } from './trace.mjs'
+
+/** This framework's root directory (the eval package dir). */
+const FRAMEWORK_ROOT = fileURLToPath(new URL('..', import.meta.url))
+
+/** The scripted mock adapter plugin, referenced from generated overlays. */
+const MOCK_ADAPTER_PATH = join(FRAMEWORK_ROOT, 'src', 'mock', 'mock-adapter.mjs')
+
+/** The profile-local module-fallback directory, rebuilt fresh by boot and never staged. */
+const MODULE_FALLBACK_DIR = '.dsh-module-fallback'
+
+/**
+ * Stage the profile store into a temporary DSH_HOME. The BOOTED profile's
+ * directory is COPIED (sans its own `node_modules` and `.dsh-module-fallback`):
+ * `prepareProfile` unconditionally rewrites the profile's root cordis.yml on
+ * every boot, and copying keeps that write inside the temporary home instead
+ * of leaking through a junction into the real store. Only the profile's own
+ * `node_modules` stays junctioned — out-of-tree plugin resolution needs it,
+ * and a task boot never writes it. The module-fallback directories (the
+ * store-level shared `profiles/node_modules` and the profile-local
+ * `.dsh-module-fallback`) are deliberately NOT staged: `healProfilesModuleFallback`
+ * rebuilds both fresh in the temporary home on every boot, and `.dsh-module-fallback`
+ * in particular is full of junctions that a naive recursive copy would follow
+ * into the real store. The copy is junction-aware — links are recreated as
+ * links, never descended (see `copyProfileEntry`). An absent profile copies
+ * nothing: boot initializes shipped templates inside the temporary home.
+ * @param {string} realHome - the real Harness home holding `profiles/`.
+ * @param {string} tmpHome - the temporary home (created up to `profiles/`).
+ * @param {string} profileName - the profile this run boots.
+ * @returns {string[]} every created junction path (unlink before rmSync).
+ */
+export function stageProfileStore(realHome, tmpHome, profileName) {
+  const junctions = []
+  const tmpProfiles = join(tmpHome, 'profiles')
+  mkdirSync(tmpProfiles, { recursive: true })
+  const realProfiles = join(realHome, 'profiles')
+  if (!existsSync(realProfiles)) return junctions
+  const realProfileDir = join(realProfiles, profileName)
+  if (!existsSync(join(realProfileDir, 'package.json'))) return junctions
+  const tmpProfileDir = join(tmpProfiles, profileName)
+  mkdirSync(tmpProfileDir, { recursive: true })
+  for (const entry of readdirSync(realProfileDir, { withFileTypes: true })) {
+    if (entry.name === 'node_modules' || entry.name === MODULE_FALLBACK_DIR) continue
+    copyProfileEntry(join(realProfileDir, entry.name), join(tmpProfileDir, entry.name), junctions)
+  }
+  const profileModules = join(realProfileDir, 'node_modules')
+  if (existsSync(profileModules)) {
+    const target = join(tmpProfileDir, 'node_modules')
+    symlinkSync(profileModules, target, 'junction')
+    junctions.push(target)
+  }
+  return junctions
+}
+
+/**
+ * Copy one profile entry (file, directory, or link) into the staged profile.
+ * A link is recreated as a link (`'junction'` on Windows) instead of being
+ * followed: Node's `cpSync` dereferences Windows junctions in recursive mode
+ * (no cycle guard), so a junction-bearing tree would recurse into the real
+ * store and overflow the native stack. Recreated junctions are appended to
+ * `junctions` so callers can unlink them before `rmSync` (which would
+ * otherwise descend through them into the real store).
+ */
+function copyProfileEntry(from, to, junctions) {
+  const stat = lstatSync(from)
+  if (stat.isSymbolicLink()) {
+    const target = readlinkSync(from)
+    symlinkSync(target, to, 'junction')
+    junctions.push(to)
+    return
+  }
+  if (stat.isDirectory()) {
+    mkdirSync(to, { recursive: true })
+    for (const child of readdirSync(from)) {
+      copyProfileEntry(join(from, child), join(to, child), junctions)
+    }
+    return
+  }
+  copyFileSync(from, to)
+}
+
+/** JSON double-quoted strings are valid YAML scalars — enough for this emitter. */
+function yamlScalar(value) {
+  if (typeof value === 'boolean' || typeof value === 'number') return String(value)
+  return JSON.stringify(String(value))
+}
+
+/**
+ * Serialize the per-run overlay patch list to YAML.
+ * @param {object} parts - overlay ingredients (see runEvalCase).
+ * @returns {string} the overlay file text.
+ */
+export function buildOverlayYaml(parts) {
+  const lines = []
+  lines.push('- id: session-persistence-jsonl')
+  lines.push('  config:')
+  lines.push(`    root: ${yamlScalar(parts.sessionsRoot)}`)
+  lines.push('    packChunks: false')
+  lines.push('    compression: none')
+  if (parts.persona !== undefined) {
+    lines.push('- id: system-prompt')
+    lines.push('  config:')
+    lines.push(`    persona: ${yamlScalar(parts.persona)}`)
+  }
+  for (const rowId of parts.disableRows ?? []) {
+    // Per-row disable uses the same cross-layer overlay mechanism as
+    // `session-title-llm`: an `- id: <row> / disabled: true` patch targets
+    // the row the plugin bundle itself inserts.
+    lines.push(`- id: ${yamlScalar(rowId)}`)
+    lines.push('  disabled: true')
+  }
+  if (parts.mock) {
+    lines.push('- id: agent-default-model')
+    lines.push('  config:')
+    lines.push('    provider: eval-mock')
+    lines.push('    model: eval-mock')
+    // The title generator also calls the default provider and would consume
+    // script steps; deterministic runs own every model call themselves.
+    lines.push('- id: session-title-llm')
+    lines.push('  disabled: true')
+    lines.push('- insert:')
+    lines.push('    - id: eval-mock-llm')
+    lines.push(`      name: ${yamlScalar(pathToFileURL(MOCK_ADAPTER_PATH).href)}`)
+  }
+  return `${lines.join('\n')}\n`
+}
+
+/**
+ * Run one eval case end to end.
+ *
+ * Case shape: `{ id, task, mode?: 'real' | 'mock', expect: Matcher[],
+ * script?: { steps: ChunkStep[] }, persona?: string, disableRows?: string[],
+ * prepare?: (workspace: string) => void | Promise<void>,
+ * inspect?: (workspace: string, helpers: { trace }) => void | Promise<void>,
+ * timeoutMs?: number }`
+ *
+ * @param {object} evalCase - the case under test.
+ * @param {object} options
+ * @param {string} options.profile - the dsh profile booting the run (plugin installed there).
+ * @param {string} [options.dshRepoDir] - the deepseek-harness checkout (legacy CLI
+ *   location; ignored when cliPath is given).
+ * @param {string} [options.cliPath] - explicit compiled CLI entry (C6 chain result;
+ *   takes precedence over dshRepoDir).
+ * @param {'real' | 'mock'} [options.mode] - force a mode over the case's own.
+ * @param {string} [options.artifactsDir] - copy stdout/stderr/trace/session logs here (created).
+ * @returns {Promise<EvalRunResult>}
+ */
+export async function runEvalCase(evalCase, options) {
+  const mode = options.mode ?? evalCase.mode ?? 'real'
+  const binPath = options.cliPath !== undefined
+    ? resolve(options.cliPath)
+    : join(resolve(options.dshRepoDir), 'apps', 'cli', 'lib', 'bin.js')
+  const timeoutMs = evalCase.timeoutMs ?? 180_000
+
+  const runDir = mkdtempSync(join(tmpdir(), 'dsh-eval-'))
+  // Everything past this point is wrapped in try/finally so the temp dir
+  // (and any junctions) are cleaned up even when `prepare`, profile
+  // staging, or spawn throw.  Previously only the normal exit path
+  // cleaned up — a `prepare` failure leaked the entire runDir.
+  try {
+    const dshHome = join(runDir, 'dsh-home')
+    const workspace = join(runDir, 'workspace')
+    const sessionsRoot = join(runDir, 'sessions')
+    mkdirSync(workspace, { recursive: true })
+    await evalCase.prepare?.(workspace)
+
+    // Profiles resolve under $DSH_HOME/profiles, and eval overwrites DSH_HOME
+    // for session/settings isolation: stage the profile store (see
+    // stageProfileStore — the booted profile is copied, so boot's unconditional
+    // cordis.yml rewrite stays inside the temporary home; only the profile's
+    // read-only node_modules stays linked, and the shared fallback is rebuilt
+    // by boot inside the temporary home). The managed credential
+    // document is copied in because `dsh-credentials-local` resolves it per
+    // request. Falls back to the default `~/.dsh` when the ambient environment
+    // sets no home of its own.
+    const realHome = (process.env.DSH_HOME ?? '').trim() !== '' ? process.env.DSH_HOME : join(homedir(), '.dsh')
+    mkdirSync(dshHome, { recursive: true })
+    stageProfileStore(realHome, dshHome, options.profile)
+    const realCredentials = join(realHome, '.credentials.yaml')
+    if (existsSync(realCredentials)) {
+      const credentialsCopy = join(dshHome, '.credentials.yaml')
+      copyFileSync(realCredentials, credentialsCopy)
+      try {
+        // Best-effort owner-only on POSIX (the harness's own e2e uses 0o600);
+        // a no-op beyond the read-only bit on Windows.
+        chmodSync(credentialsCopy, 0o600)
+      } catch { /* permission tightening is best-effort */ }
+    }
+
+    const env = {
+      ...process.env,
+      DSH_HOME: dshHome,
+      DSH_TELEMETRY_DISABLED: '1',
+    }
+    if (mode === 'mock') {
+      if (evalCase.script === undefined) {
+        throw new Error(`case '${evalCase.id}': mock mode requires a script`)
+      }
+      const scriptPath = join(runDir, 'mock-script.json')
+      writeFileSync(scriptPath, JSON.stringify(evalCase.script))
+      env.DSH_EVAL_MOCK_SCRIPT = scriptPath
+    }
+
+    const overlayPath = join(runDir, 'eval-overlay.yml')
+    if (evalCase.disableRows !== undefined
+      && (!Array.isArray(evalCase.disableRows)
+        || evalCase.disableRows.some(row => typeof row !== 'string' || row === ''))) {
+      throw new Error(`case '${evalCase.id}': disableRows must be a string[] of loader row ids`)
+    }
+    writeFileSync(overlayPath, buildOverlayYaml({
+      sessionsRoot,
+      persona: evalCase.persona,
+      disableRows: evalCase.disableRows,
+      mock: mode === 'mock',
+    }))
+
+    const cliArgs = [
+      binPath,
+      '--profile', options.profile,
+      '--patch', overlayPath,
+      evalCase.task,
+    ]
+    const child = spawn(process.execPath, cliArgs, { cwd: workspace, env })
+
+    let stdout = ''
+    let stderr = ''
+    child.stdout.on('data', chunk => { stdout += chunk })
+    child.stderr.on('data', chunk => { stderr += chunk })
+
+    let timedOut = false
+    const timer = setTimeout(() => {
+      timedOut = true
+      child.kill('SIGTERM')
+    }, timeoutMs)
+
+    const exitCode = await new Promise(resolveExit => {
+      child.on('error', error => { stderr += `\ndsh-eval: failed to spawn dsh CLI: ${error.message}\n`; resolveExit(127) })
+      child.on('exit', code => resolveExit(code ?? 1))
+    })
+    clearTimeout(timer)
+
+    const trace = loadTraceDir(sessionsRoot)
+    const sessionLogs = collectSessionLogTexts(sessionsRoot)
+
+    // Workspace assertions live HERE, before the run dir cleanup: a case's
+    // `inspect(workspace, { trace })` may throw; the failure text rides the
+    // result instead of leaking past cleanup.
+    let inspectError
+    if (typeof evalCase.inspect === 'function') {
+      try {
+        await evalCase.inspect(workspace, { trace })
+      } catch (error) {
+        inspectError = error instanceof Error ? error.message : String(error)
+      }
+    }
+
+    if (options.artifactsDir !== undefined) {
+      mkdirSync(options.artifactsDir, { recursive: true })
+      writeFileSync(join(options.artifactsDir, 'stdout.txt'), stdout)
+      writeFileSync(join(options.artifactsDir, 'stderr.txt'), stderr)
+      writeFileSync(join(options.artifactsDir, 'trace.json'), JSON.stringify({
+        caseId: evalCase.id,
+        mode,
+        task: evalCase.task,
+        exitCode,
+        timedOut,
+        trace,
+      }, undefined, 2))
+      try {
+        cpSync(sessionsRoot, join(options.artifactsDir, 'sessions'), { recursive: true })
+      } catch { /* no session materialized — nothing to copy */ }
+    }
+
+    return {
+      caseId: evalCase.id, mode, task: evalCase.task, exitCode, timedOut,
+      stdout, stderr, trace, sessionLogs, inspectError, runDir,
+    }
+  } finally {
+    if (process.env.DSH_EVAL_KEEP_TMP !== '1') {
+      // Drop every junction first so cleanup can never descend into the
+      // real profile store. Junctions may not exist when the error
+      // happened before stageProfileStore ran — readdirSync catches that.
+      try {
+        const profileJunctions = []
+        const walk = (dir) => {
+          let entries
+          try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return }
+          for (const entry of entries) {
+            const full = join(dir, entry.name)
+            if (entry.isSymbolicLink()) profileJunctions.push(full)
+            else if (entry.isDirectory()) walk(full)
+          }
+        }
+        walk(join(runDir, 'dsh-home'))
+        for (const junction of profileJunctions) {
+          try { unlinkSync(junction) } catch { /* junction absent — nothing to drop */ }
+        }
+      } catch { /* dsh-home not created yet */ }
+      rmSync(runDir, { recursive: true, force: true })
+    }
+  }
+}
+
+/** Read every session artifact under the root as text (best-effort, pre-cleanup). */
+function collectSessionLogTexts(sessionsRoot) {
+  const texts = []
+  const walk = (dir) => {
+    let entries
+    try {
+      entries = readdirSync(dir, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const entry of entries) {
+      const path = join(dir, entry.name)
+      if (entry.isDirectory()) walk(path)
+      else if (entry.name === 'session.jsonl') texts.push(readFileSync(path, 'utf8'))
+    }
+  }
+  walk(sessionsRoot)
+  return texts
+}
+
+/**
+ * @typedef {object} EvalRunResult
+ * @property {string} caseId
+ * @property {'real' | 'mock'} mode
+ * @property {string} task
+ * @property {number} exitCode - the headless CLI's exit code (0 = turn completed).
+ * @property {boolean} timedOut
+ * @property {string} stdout - printed final assistant text (plus any startup chatter).
+ * @property {string} stderr
+ * @property {import('./trace.mjs').EvalTrace | undefined} trace
+ * @property {string[]} sessionLogs - raw session artifact texts, pre-cleanup.
+ * @property {string | undefined} inspectError - the case's `inspect` failure text, when it threw.
+ * @property {string} runDir - removed unless DSH_EVAL_KEEP_TMP=1.
+ */
+
+/** Re-exported so bin can resolve the framework without guessing paths. */
+export { FRAMEWORK_ROOT }
+
+/** Whether a candidate dsh repo dir looks like one (the CLI artifact exists). */
+export function looksLikeDshRepo(dir) {
+  if (!isAbsolute(dir)) return false
+  return existsSync(join(dir, 'apps', 'cli', 'lib', 'bin.js'))
+}

package/src/tool-validation.mjs

@@ -0,0 +1,77 @@
+/**
+ * Post-run tool boundary validation for blind review.
+ *
+ * After a review run completes, this module inspects the session trace's
+ * `request/header` events to verify that no unexpected tools were mounted
+ * in the reviewer's session.  This is the detection half of the
+ * sterile-profile strategy: the profile prevents plugin tools from being
+ * installed, the overlay disables every host tool row, and this check
+ * makes any residual drift (a bundle leaking tools through a patch, a host
+ * regression, a misconfigured profile) an explicit adapter failure.
+ *
+ * The module is intentionally pure: it takes an already-parsed trace and
+ * returns a plain result object.  File I/O (reading the session log,
+ * writing evidence) stays in the calling adapter.
+ */
+
+/**
+ * Collect every distinct tool name mounted across all `request/header`
+ * events in a trace.
+ * @param {import('./trace.mjs').EvalTrace} trace
+ * @returns {string[]} sorted tool names.
+ */
+function collectMountedToolNames(trace) {
+  const names = new Set()
+  for (const header of trace.requestHeaders ?? []) {
+    for (const name of header.toolNames ?? []) {
+      names.add(name)
+    }
+  }
+  return [...names].sort()
+}
+
+/**
+ * Check a review trace for tool leakage.
+ *
+ * Every tool name appearing in any `request/header` event is compared
+ * against the allowed set.  An empty allowed set (the review default)
+ * means the reviewer must see no tools at all.
+ *
+ * @param {import('./trace.mjs').EvalTrace | undefined} trace - the parsed trace; `undefined` skips validation.
+ * @param {{ allowedTools?: Set<string> }} [options]
+ * @returns {{ ok: boolean, unexpected: string[], actual: string[], allowed: string[] }}
+ */
+export function validateToolBoundary(trace, options = {}) {
+  const allowed = options.allowedTools ?? new Set()
+  const allowedNames = [...allowed].sort()
+  if (trace === undefined || trace === null) {
+    return { ok: true, unexpected: [], actual: [], allowed: allowedNames }
+  }
+  const actual = collectMountedToolNames(trace)
+  const unexpected = actual.filter(name => !allowed.has(name))
+  return {
+    ok: unexpected.length === 0,
+    unexpected,
+    actual,
+    allowed: allowedNames,
+  }
+}
+
+/**
+ * Render a diagnostic evidence document for a tool boundary failure.
+ * Suitable for writing to `.runs/<id>/tool-boundary-evidence.json`.
+ *
+ * @param {{ ok: boolean, unexpected: string[], actual: string[], allowed: string[] }} validation
+ * @param {{ runDir: string, profile: string }} context
+ * @returns {string}
+ */
+export function renderToolBoundaryEvidence(validation, context) {
+  return JSON.stringify({
+    status: 'tool-boundary-violation',
+    runDir: context.runDir,
+    profile: context.profile,
+    unexpectedTools: validation.unexpected,
+    actualMounted: validation.actual,
+    allowedTools: validation.allowed,
+  }, null, 2) + '\n'
+}

package/src/trace.mjs

@@ -0,0 +1,218 @@
+/**
+ * Session-trace parsing for dsh agent eval. The evidence source is the JSONL
+ * session artifact written by `@deepseek-ai/dsh-session-persistence-jsonl`
+ * (configured `compression: none`, `packChunks: false` by the eval overlay):
+ * one `type: 'session'` header line, then one JSON record per `SessionEvent`.
+ * Event shapes follow `deepseek-harness/packages/core/session/src/types.ts`
+ * (`SessionEventMap`); packed `*-chunks` storage rows are tolerated and
+ * skipped — they only carry `assistant/chunk` deltas eval never asserts on.
+ */
+
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+/** Storage row types that pack `assistant/chunk` delta runs (see chunk-rows.ts). */
+const CHUNK_ROW_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks'])
+
+/**
+ * Parse one uncompressed JSONL session artifact.
+ * @param {string} text - the artifact's full text (header line first).
+ * @returns {{ header: object, events: object[] }} header plus event records in log order.
+ */
+export function parseSessionLog(text) {
+  const lines = text.split('\n').filter(line => line.trim() !== '')
+  if (lines.length === 0) throw new Error('empty session log')
+  const header = JSON.parse(lines[0])
+  if (header.type !== 'session') throw new Error('first line is not a session header')
+  const events = []
+  for (const line of lines.slice(1)) {
+    let record
+    try {
+      record = JSON.parse(line)
+    } catch {
+      continue // torn or partial tail line: keep the decodable prefix
+    }
+    if (record === null || typeof record !== 'object') continue
+    if (CHUNK_ROW_TYPES.has(record.type)) continue
+    events.push(record)
+  }
+  return { header, events }
+}
+
+/** Concatenate the text blocks of one assembled assistant message. */
+function messageText(message) {
+  const content = message?.content
+  if (!Array.isArray(content)) return ''
+  return content
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('')
+}
+
+/**
+ * Extract the visible text of one tool-result message. Real messages are
+ * user-role with a single wrapping `tool-result` block whose `content` holds
+ * the actual blocks (see `createToolResultMessage` in
+ * `deepseek-harness/packages/llm/llm/src/message.ts`); a bare block list is
+ * tolerated for hand-built fixtures.
+ */
+function toolResultText(message) {
+  const content = message?.content
+  if (!Array.isArray(content)) return ''
+  const wrapper = content.find(block => block?.type === 'tool-result')
+  const inner = wrapper !== undefined ? wrapper.content : content
+  if (!Array.isArray(inner)) return ''
+  return inner
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('')
+}
+
+/**
+ * Extract the `isError` flag from a tool-result message's wrapper block.
+ * Returns `undefined` when the flag is absent (treated as success by
+ * matchers — see `toolResultSucceeded`).
+ */
+function toolResultIsError(message) {
+  const content = message?.content
+  if (!Array.isArray(content)) return undefined
+  const wrapper = content.find(block => block?.type === 'tool-result')
+  return wrapper?.isError
+}
+
+/** Best-effort parse of a tool call's raw JSON arguments string. */
+function parseArguments(raw) {
+  try {
+    return JSON.parse(raw)
+  } catch {
+    return undefined
+  }
+}
+
+/**
+ * Build one assertable trace from parsed session logs. Child sessions surface
+ * only through the parent's tool events, so the MAIN log (no `origin:
+ * 'subagent'` header) owns the tool/final-text projections; all logs stay
+ * available under `sessions`.
+ * @param {{ header: object, events: object[] }[]} logs - parsed session logs.
+ * @returns {EvalTrace}
+ */
+export function buildTrace(logs) {
+  const mains = logs.filter(log => log.header.origin !== 'subagent')
+  const main = [...mains].sort((a, b) => b.events.length - a.events.length)[0]
+  const events = main?.events ?? []
+
+  const toolCalls = []
+  const toolResults = []
+  const assistantTexts = []
+  const userMessages = []
+  const requestHeaders = []
+  for (const event of events) {
+    if (event.type === 'request/header') {
+      // The assembled model request header: system prompt + mounted tool
+      // schemas. What the model is told it can do and how — the "did my
+      // plugin's section inject?" projection.
+      requestHeaders.push({
+        seq: event.seq,
+        reason: event.data?.reason,
+        system: event.data?.header?.system ?? '',
+        toolNames: Array.isArray(event.data?.header?.tools)
+          ? event.data.header.tools.map(tool => tool?.name).filter(name => typeof name === 'string')
+          : [],
+      })
+    } else if (event.type === 'tool/call') {
+      toolCalls.push({
+        seq: event.seq,
+        turn: event.data.turn,
+        step: event.data.step,
+        callId: event.data.callId,
+        name: event.data.name,
+        arguments: event.data.arguments,
+        parsedArguments: parseArguments(event.data.arguments),
+      })
+    } else if (event.type === 'tool/result') {
+      toolResults.push({
+        seq: event.seq,
+        turn: event.data.turn,
+        step: event.data.step,
+        callId: event.data.message?.source?.callId,
+        text: toolResultText(event.data.message),
+        error: event.data.error,
+        isError: toolResultIsError(event.data.message),
+      })
+    } else if (event.type === 'assistant/message') {
+      const text = messageText(event.data.message)
+      if (text !== '') assistantTexts.push(text)
+    } else if (event.type === 'user/message') {
+      // The user-role model-visible surface: the task prompt (kind 'user'),
+      // plugin steering, or injected context. `source` tells them apart —
+      // steer has no dedicated event type (the legacy `steering/message` was
+      // migrated to `user/message`), so the matcher side filters by `source`.
+      const text = messageText(event.data)
+      if (text !== '') {
+        userMessages.push({
+          seq: event.seq,
+          source: event.data?.source,
+          text,
+        })
+      }
+    }
+  }
+
+  return {
+    sessions: logs,
+    sessionId: main?.header.id,
+    toolCalls,
+    toolResults,
+    assistantTexts,
+    userMessages,
+    requestHeaders,
+    finalText: assistantTexts.at(-1) ?? '',
+  }
+}
+
+/** Recursively collect files named `name` under `dir`. */
+function collectFiles(dir, name, out = []) {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name)
+    if (entry.isDirectory()) collectFiles(path, name, out)
+    else if (entry.name === name) out.push(path)
+  }
+  return out
+}
+
+/**
+ * Load every session log under a persistence root and build one trace.
+ * @param {string} sessionsRoot - the run's `session-persistence-jsonl` root.
+ * @returns {EvalTrace | undefined} the trace, or `undefined` when no log materialized.
+ */
+export function loadTraceDir(sessionsRoot) {
+  let files
+  try {
+    files = collectFiles(sessionsRoot, 'session.jsonl')
+  } catch {
+    return undefined
+  }
+  if (files.length === 0) return undefined
+  const logs = files
+    .map(file => readFileSync(file, 'utf8'))
+    .map(parseSessionLog)
+  return buildTrace(logs)
+}
+
+/**
+ * @typedef {object} EvalTrace
+ * @property {{ header: object, events: object[] }[]} sessions - every parsed log.
+ * @property {string | undefined} sessionId - the main session's id.
+ * @property {{ seq: number, turn: number, step: number, callId: string, name: string, arguments: string, parsedArguments: unknown }[]} toolCalls
+ * @property {{ seq: number, turn: number, step: number, callId: string, text: string, error: object | undefined, isError: boolean | undefined }[]} toolResults
+ * @property {string[]} assistantTexts - non-empty assembled assistant messages, log order.
+ * @property {{ seq: number, source: object, text: string }[]} userMessages
+ *   - non-empty `user/message` events (task prompt, plugin steer, injected
+ *     context) with their verbatim `source` (`kind` + plugin-specific fields),
+ *     in log order. Steer has no dedicated event type; matchers filter by
+ *     `source`.
+ * @property {{ seq: number, reason: string, system: string, toolNames: string[] }[]} requestHeaders
+ *   - projected `request/header` events (assembled system prompt + mounted tools).
+ * @property {string} finalText - the last assembled assistant text ('' when none).
+ */