@catheadowl/dsh-eval 0.2.0 → 0.3.0

package/src/runner.mjs

@@ -29,12 +29,12 @@
  * module is the orchestration only.
  */
 
-import { existsSync, mkdirSync, mkdtempSync, writeFileSync, cpSync, readdirSync, readFileSync } from 'node:fs'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync, cpSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { isAbsolute, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { loadTraceDir } from './trace.mjs'
-import { validateRowConfig, validateDisableRows } from './discovery.mjs'
+import { collectSessionTrace, listSessionLogFiles } from './trace.mjs'
+import { validateRowConfig, validateDisableRows, validateFollowups } from './discovery.mjs'
 import { CLI_RELATIVE_PATH } from './cli.mjs'
 import { buildOverlayYaml } from './overlay.mjs'
 import { resolveRealDshHome, stageSandboxHome, teardownSandbox, spawnHeadlessDsh } from './sandbox.mjs'
@@ -48,28 +48,32 @@ const FRAMEWORK_ROOT = fileURLToPath(new URL('..', import.meta.url))
  * Case shape: `{ id, task, mode?: 'real' | 'mock', expect: Matcher[],
  * script?: { steps: ChunkStep[] }, persona?: string, disableRows?: string[],
  * rowConfig?: Record<string, Record<string, unknown>>,
+ * followups?: string[], settleTimeoutMs?: number,
  * prepare?: (workspace: string) => void | Promise<void>,
  * inspect?: (workspace: string, helpers: { trace }) => void | Promise<void>,
  * timeoutMs?: number }`
  *
+ * `followups` opts into cross-turn driving: the overlay swaps the one-shot
+ * `headless-runner` row for the eval multi-turn driver, which — before each
+ * followup — waits for background subagent children to settle (bounded by
+ * `settleTimeoutMs`), keeping the process alive so fire-and-forget children
+ * (turn-close defer fixers) can run to completion between turns.
+ *
  * @param {object} evalCase - the case under test.
  * @param {object} options
  * @param {string} options.profile - the dsh profile booting the run (plugin installed there).
- * @param {string} [options.dshRepoDir] - the deepseek-harness checkout (legacy CLI
- *   location; ignored when cliPath is given).
- * @deprecated options.dshRepoDir — pass the C6 chain result via cliPath
- *   instead; this legacy option is removed in the next minor release.
- * @param {string} [options.cliPath] - explicit compiled CLI entry (C6 chain result;
- *   takes precedence over dshRepoDir).
+ * @param {string} options.cliPath - the compiled dsh CLI entry (a
+ *   `resolveDshCliChain` result); required.
  * @param {'real' | 'mock'} [options.mode] - force a mode over the case's own.
  * @param {string} [options.artifactsDir] - copy stdout/stderr/trace/session logs here (created).
  * @returns {Promise<EvalRunResult>}
  */
 export async function runEvalCase(evalCase, options) {
   const mode = options.mode ?? evalCase.mode ?? 'real'
-  const binPath = options.cliPath !== undefined
-    ? resolve(options.cliPath)
-    : join(resolve(options.dshRepoDir), ...CLI_RELATIVE_PATH.split(/[\\/]/))
+  if (options.cliPath === undefined) {
+    throw new TypeError('runEvalCase needs options.cliPath (a resolveDshCliChain result)')
+  }
+  const binPath = resolve(options.cliPath)
   const timeoutMs = evalCase.timeoutMs ?? 180_000
 
   const runDir = mkdtempSync(join(tmpdir(), 'dsh-eval-'))
@@ -106,6 +110,15 @@ export async function runEvalCase(evalCase, options) {
       writeFileSync(scriptPath, JSON.stringify(evalCase.script))
       env.DSH_EVAL_MOCK_SCRIPT = scriptPath
     }
+    if (evalCase.followups !== undefined) {
+      const planPath = join(runDir, 'driver-plan.json')
+      writeFileSync(planPath, JSON.stringify({
+        task: evalCase.task,
+        followups: evalCase.followups,
+        ...(evalCase.settleTimeoutMs === undefined ? {} : { settleTimeoutMs: evalCase.settleTimeoutMs }),
+      }))
+      env.DSH_EVAL_DRIVER_PLAN = planPath
+    }
 
     if (evalCase.disableRows !== undefined) {
       validateDisableRows(evalCase.disableRows, `case '${evalCase.id}'`)
@@ -113,6 +126,9 @@ export async function runEvalCase(evalCase, options) {
     if (evalCase.rowConfig !== undefined) {
       validateRowConfig(evalCase.rowConfig, `case '${evalCase.id}'`)
     }
+    if (evalCase.followups !== undefined) {
+      validateFollowups(evalCase.followups, `case '${evalCase.id}'`)
+    }
     const overlayPath = join(runDir, 'eval-overlay.yml')
     writeFileSync(overlayPath, buildOverlayYaml({
       sessionsRoot,
@@ -120,6 +136,7 @@ export async function runEvalCase(evalCase, options) {
       disableRows: evalCase.disableRows,
       rowConfig: evalCase.rowConfig,
       mock: mode === 'mock',
+      ...(evalCase.followups === undefined ? {} : { followups: evalCase.followups }),
     }))
 
     const { stdout, stderr, exitCode, timedOut } = await spawnHeadlessDsh({
@@ -130,7 +147,7 @@ export async function runEvalCase(evalCase, options) {
       timeoutMs,
     })
 
-    const trace = loadTraceDir(sessionsRoot)
+    const { trace, gap: traceGap } = collectSessionTrace(sessionsRoot)
     const sessionLogs = collectSessionLogTexts(sessionsRoot)
 
     // Workspace assertions live HERE, before the run dir cleanup: a case's
@@ -164,7 +181,7 @@ export async function runEvalCase(evalCase, options) {
 
     return {
       caseId: evalCase.id, mode, task: evalCase.task, exitCode, timedOut,
-      stdout, stderr, trace, sessionLogs, inspectError, runDir,
+      stdout, stderr, trace, traceGap, sessionLogs, inspectError, runDir,
     }
   } finally {
     teardownSandbox(runDir, { keep: process.env.DSH_EVAL_KEEP_TMP === '1' })
@@ -173,22 +190,7 @@ export async function runEvalCase(evalCase, options) {
 
 /** Read every session artifact under the root as text (best-effort, pre-cleanup). */
 function collectSessionLogTexts(sessionsRoot) {
-  const texts = []
-  const walk = (dir) => {
-    let entries
-    try {
-      entries = readdirSync(dir, { withFileTypes: true })
-    } catch {
-      return
-    }
-    for (const entry of entries) {
-      const path = join(dir, entry.name)
-      if (entry.isDirectory()) walk(path)
-      else if (entry.name === 'session.jsonl') texts.push(readFileSync(path, 'utf8'))
-    }
-  }
-  walk(sessionsRoot)
-  return texts
+  return listSessionLogFiles(sessionsRoot).map(path => readFileSync(path, 'utf8'))
 }
 
 /**
@@ -201,6 +203,9 @@ function collectSessionLogTexts(sessionsRoot) {
  * @property {string} stdout - printed final assistant text (plus any startup chatter).
  * @property {string} stderr
  * @property {import('./trace.mjs').EvalTrace | undefined} trace
+ * @property {string | undefined} traceGap - why no trace was built (the host
+ *   session seam diagnosis from `collectSessionTrace`); `undefined` whenever
+ *   `trace` is defined. The CLI prints it as the failure text.
  * @property {string[]} sessionLogs - raw session artifact texts, pre-cleanup.
  * @property {string | undefined} inspectError - the case's `inspect` failure text, when it threw.
  * @property {string} runDir - removed unless DSH_EVAL_KEEP_TMP=1.

package/src/sandbox.mjs

@@ -17,7 +17,7 @@
 
 import { spawn } from 'node:child_process'
 import {
-  chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readdirSync, readlinkSync, rmSync, symlinkSync, unlinkSync,
+  chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, rmSync, symlinkSync, unlinkSync,
 } from 'node:fs'
 import { homedir } from 'node:os'
 import { join } from 'node:path'
@@ -104,6 +104,67 @@ function copyProfileEntry(from, to, junctions) {
   copyFileSync(from, to)
 }
 
+/**
+ * Enumerate every loader row id the STAGED profile composes BEYOND the host
+ * templates (`@deepseek-ai/*` bundles) — the data source for the review
+ * adapter's default blank environment (blank = dsh-base/dsh-headless only,
+ * no out-of-tree plugin face regardless of what the host profile carries).
+ *
+ * Composition-aware, per the host's ordered-layer model: the profile root
+ * `cordis.yml` ships as an EMPTY entry list — plugins enter either through
+ * the profile's own `cordis.patch.yml` rows or through out-of-tree bundles
+ * listed in `package.json`'s `dsh.profile.bundles`, each bundle contributing
+ * rows from its own `cordis.patch.yml`/`cordis.yml` under the profile's
+ * (junctioned) `node_modules`. All three sources live inside the staged home.
+ *
+ * A minimal token scan (`- id: <token>` at any indentation, which also
+ * covers rows nested under `- insert:`), not a YAML parse (same precedent as
+ * the file-history minimal session-log decoder). Entries without an `id` are
+ * invisible to id-targeted disables by construction and thus not collected.
+ * Unreadable sources contribute nothing — callers keep their static
+ * fallback rows.
+ * @param {string} tmpHome - the staged temporary home.
+ * @param {string} profileName - the staged profile name.
+ * @returns {string[]} deduplicated loader row ids beyond the host templates.
+ */
+export function stagedPluginRows(tmpHome, profileName) {
+  const profileDir = join(tmpHome, 'profiles', profileName)
+  const rows = []
+  const collectIds = (text) => {
+    for (const match of text.matchAll(/^[ \t]*-[ \t]+id:[ \t]*"?'?([^\s"']+)/gm)) {
+      if (!rows.includes(match[1])) rows.push(match[1])
+    }
+  }
+  const readText = (file) => {
+    try {
+      return readFileSync(file, 'utf8')
+    } catch {
+      return undefined
+    }
+  }
+  // Source 1+2: the profile's own composition files (root list is normally
+  // the shipped empty `[]`, but a non-empty one is scanned all the same).
+  for (const name of ['cordis.patch.yml', 'cordis.yml']) {
+    const text = readText(join(profileDir, name))
+    if (text !== undefined) collectIds(text)
+  }
+  // Source 3: every NON-host bundle in dsh.profile.bundles — host template
+  // bundles (`@deepseek-ai/*`) define the sterile baseline and stay.
+  let bundles
+  try {
+    bundles = JSON.parse(readFileSync(join(profileDir, 'package.json'), 'utf8'))?.dsh?.profile?.bundles
+  } catch { /* no package.json → no bundle rows to collect */ }
+  for (const bundle of Array.isArray(bundles) ? bundles : []) {
+    if (typeof bundle !== 'string' || bundle.startsWith('@deepseek-ai/')) continue
+    const bundleDir = join(profileDir, 'node_modules', ...bundle.split('/'))
+    for (const name of ['cordis.patch.yml', 'cordis.yml']) {
+      const text = readText(join(bundleDir, name))
+      if (text !== undefined) collectIds(text)
+    }
+  }
+  return rows
+}
+
 /**
  * Copy the managed credential document into the temporary home (copy, not
  * junction: single file, best-effort owner-only 0o600 — a no-op beyond the

package/src/tool-validation.mjs

@@ -1,77 +1,85 @@
-/**
- * Post-run tool boundary validation for blind review.
- *
- * After a review run completes, this module inspects the session trace's
- * `request/header` events to verify that no unexpected tools were mounted
- * in the reviewer's session.  This is the detection half of the
- * sterile-profile strategy: the profile prevents plugin tools from being
- * installed, the overlay disables every host tool row, and this check
- * makes any residual drift (a bundle leaking tools through a patch, a host
- * regression, a misconfigured profile) an explicit adapter failure.
- *
- * The module is intentionally pure: it takes an already-parsed trace and
- * returns a plain result object.  File I/O (reading the session log,
- * writing evidence) stays in the calling adapter.
- */
-
-/**
- * Collect every distinct tool name mounted across all `request/header`
- * events in a trace.
- * @param {import('./trace.mjs').EvalTrace} trace
- * @returns {string[]} sorted tool names.
- */
-function collectMountedToolNames(trace) {
-  const names = new Set()
-  for (const header of trace.requestHeaders ?? []) {
-    for (const name of header.toolNames ?? []) {
-      names.add(name)
-    }
-  }
-  return [...names].sort()
-}
-
-/**
- * Check a review trace for tool leakage.
- *
- * Every tool name appearing in any `request/header` event is compared
- * against the allowed set.  An empty allowed set (the review default)
- * means the reviewer must see no tools at all.
- *
- * @param {import('./trace.mjs').EvalTrace | undefined} trace - the parsed trace; `undefined` skips validation.
- * @param {{ allowedTools?: Set<string> }} [options]
- * @returns {{ ok: boolean, unexpected: string[], actual: string[], allowed: string[] }}
- */
-export function validateToolBoundary(trace, options = {}) {
-  const allowed = options.allowedTools ?? new Set()
-  const allowedNames = [...allowed].sort()
-  if (trace === undefined || trace === null) {
-    return { ok: true, unexpected: [], actual: [], allowed: allowedNames }
-  }
-  const actual = collectMountedToolNames(trace)
-  const unexpected = actual.filter(name => !allowed.has(name))
-  return {
-    ok: unexpected.length === 0,
-    unexpected,
-    actual,
-    allowed: allowedNames,
-  }
-}
-
-/**
- * Render a diagnostic evidence document for a tool boundary failure.
- * Suitable for writing to `.runs/<id>/tool-boundary-evidence.json`.
- *
- * @param {{ ok: boolean, unexpected: string[], actual: string[], allowed: string[] }} validation
- * @param {{ runDir: string, profile: string }} context
- * @returns {string}
- */
-export function renderToolBoundaryEvidence(validation, context) {
-  return JSON.stringify({
-    status: 'tool-boundary-violation',
-    runDir: context.runDir,
-    profile: context.profile,
-    unexpectedTools: validation.unexpected,
-    actualMounted: validation.actual,
-    allowedTools: validation.allowed,
-  }, null, 2) + '\n'
-}
+/**
+ * Post-run tool boundary validation for blind review.
+ *
+ * After a review run completes, this module inspects the session trace's
+ * `request/header` events to verify that no unexpected tools were mounted
+ * in the reviewer's session.  This is the detection half of the
+ * blank-environment strategy: the overlay disables every staged out-of-tree
+ * plugin row plus every host tool row, and this check makes any residual
+ * drift (a bundle leaking tools through a patch, a host regression, a
+ * misconfigured profile) an explicit adapter failure.
+ *
+ * The module is intentionally pure: it takes an already-parsed trace and
+ * returns a plain result object.  File I/O (reading the session log,
+ * writing evidence) stays in the calling adapter.
+ */
+
+/**
+ * Collect every distinct tool name mounted across all `request/header`
+ * events in a trace.
+ * @param {import('./trace.mjs').EvalTrace} trace
+ * @returns {string[]} sorted tool names.
+ */
+function collectMountedToolNames(trace) {
+  const names = new Set()
+  for (const header of trace.requestHeaders ?? []) {
+    for (const name of header.toolNames ?? []) {
+      names.add(name)
+    }
+  }
+  return [...names].sort()
+}
+
+/**
+ * Check a review trace for tool leakage.
+ *
+ * Every tool name appearing in any `request/header` event is compared
+ * against the allowed set.  An empty allowed set (the review default)
+ * means the reviewer must see no tools at all.
+ *
+ * The result is explicit about whether the check RAN: `status` is
+ * `'checked'` only when a trace was available, and `'not-executed'` when
+ * there was none. `ok` is true only for a checked boundary with no
+ * unexpected tool — a missing trace is NOT a pass, so callers must branch
+ * on `status` and account for the skip (see `createDshHeadlessReviewExecutor`
+ * and docs/review.md) instead of reading `ok` alone.
+ *
+ * @param {import('./trace.mjs').EvalTrace | undefined} trace - the parsed trace; `undefined` means the check could not run.
+ * @param {{ allowedTools?: Set<string> }} [options]
+ * @returns {{ status: 'checked' | 'not-executed', ok: boolean, unexpected: string[], actual: string[], allowed: string[] }}
+ */
+export function validateToolBoundary(trace, options = {}) {
+  const allowed = options.allowedTools ?? new Set()
+  const allowedNames = [...allowed].sort()
+  if (trace === undefined || trace === null) {
+    return { status: 'not-executed', ok: false, unexpected: [], actual: [], allowed: allowedNames }
+  }
+  const actual = collectMountedToolNames(trace)
+  const unexpected = actual.filter(name => !allowed.has(name))
+  return {
+    status: 'checked',
+    ok: unexpected.length === 0,
+    unexpected,
+    actual,
+    allowed: allowedNames,
+  }
+}
+
+/**
+ * Render a diagnostic evidence document for a tool boundary failure.
+ * Suitable for writing to `.runs/<id>/tool-boundary-evidence.json`.
+ *
+ * @param {{ status: 'checked' | 'not-executed', ok: boolean, unexpected: string[], actual: string[], allowed: string[] }} validation
+ * @param {{ runDir: string, profile: string }} context
+ * @returns {string}
+ */
+export function renderToolBoundaryEvidence(validation, context) {
+  return JSON.stringify({
+    status: 'tool-boundary-violation',
+    runDir: context.runDir,
+    profile: context.profile,
+    unexpectedTools: validation.unexpected,
+    actualMounted: validation.actual,
+    allowedTools: validation.allowed,
+  }, null, 2) + '\n'
+}