@catheadowl/dsh-eval 0.1.0 → 0.2.1

package/src/cli.mjs

@@ -1,16 +1,15 @@
 /**
- * Host CLI resolution chain (release-plan C6 / spec host-checkout-resolution).
+ * Host CLI resolution chain.
  *
  * Locating the compiled dsh CLI follows the same two-layer model as package
  * imports: committed files carry no real host-checkout path — the machine's
- * resolution layer (node_modules, junction-built by the relink anchor tool)
- * absorbs it. Precedence, first hit wins:
+ * resolution layer (node_modules) absorbs it. Precedence, first hit wins:
  *
  *   1. explicit `--repo <dir>` flag — the documented escape hatch;
  *   2. resolution layer — `node_modules/@deepseek-ai/dsh/lib/bin.js`
  *      (the CLI package's own bin target) reachable upward from startDir;
  *   3. config `repo` key (legacy) — kept working for existing checked-in
- *      `dsh-eval.config.mjs` files until the repo split retires them.
+ *      `dsh-eval.config.mjs` files.
  *
  * Every miss fails loud with a fingerprint and placeholder-only guidance —
  * no machine-specific example paths, no silent fallback to guessing.
@@ -29,10 +28,11 @@ const PACKAGED_CLI_PATH = join('node_modules', '@deepseek-ai', 'dsh', 'lib', 'bi
 export const NO_CLI_GUIDANCE = [
   'no dsh CLI found. In order:',
   "  1) pass --repo <host-checkout> explicitly;",
-  '  2) or make the resolution layer provide it: node_modules/@deepseek-ai/dsh/lib/bin.js',
-  '     (run the repo relink script to (re)build the junction tree from DSH_REPO,',
-  '     then build the host checkout if lib/ is missing);',
-  '  3) or set repo in dsh-eval.config.mjs (legacy, retired at repo split).',
+  '  2) or provide the CLI through the resolution layer:',
+  '     node_modules/@deepseek-ai/dsh/lib/bin.js — install the @deepseek-ai/dsh',
+  '     package (or link a built deepseek-harness checkout into node_modules',
+  '     and build it so lib/bin.js exists);',
+  '  3) or set repo in dsh-eval.config.mjs (legacy).',
 ].join('\n')
 
 /** Validate a repo-style candidate: return the CLI path or undefined. */

package/src/config.mjs

@@ -1,5 +1,5 @@
 /**
- * Shared `dsh-eval.config.mjs` discovery and loading (EVAL-008).
+ * Shared `dsh-eval.config.mjs` discovery and loading.
  *
  * Both CLIs (`dsh-eval`, `dsh-review`) repeat `--profile/--repo` wiring in
  * every consumer's package scripts. A per-package config file removes that

package/src/discovery.mjs

@@ -1,115 +1,190 @@
-/**
- * Shared case / experiment discovery and validation for the eval CLIs.
- * Both `dsh-eval` and `dsh-review` scan directories recursively; this
- * module ensures they share identical skip rules and validation.
- */
-
-import { readdirSync, statSync } from 'node:fs'
-import { join, resolve } from 'node:path'
-
-/**
- * Directories never to descend into during discovery. `.runs` holds
- * prior-run artifacts that would be re-discovered as cases; `node_modules`
- * would pull in dependencies that happen to match the suffix.
- */
-const SKIP_DIRS = new Set(['.runs', 'node_modules'])
-
-/**
- * Recursively collect files matching `suffix` under `path`.
- * Always skips `.runs` and `node_modules`.
- * @param {string} path - a file or directory path.
- * @param {string} suffix - the filename suffix to match (e.g. '.eval.mjs').
- * @param {string[]} [out] - accumulator (internal).
- * @returns {string[]} list of discovered files (unsorted; callers sort if needed).
- */
-export function discoverFiles(path, suffix, out = []) {
-  const absolute = resolve(path)
-  if (statSync(absolute).isFile()) {
-    if (absolute.endsWith(suffix)) out.push(absolute)
-    return out
-  }
-  for (const entry of readdirSync(absolute, { withFileTypes: true })) {
-    const full = join(absolute, entry.name)
-    if (entry.isDirectory() && !SKIP_DIRS.has(entry.name)) {
-      discoverFiles(full, suffix, out)
-    } else if (entry.isFile() && entry.name.endsWith(suffix)) {
-      out.push(full)
-    }
-  }
-  return out
-}
-
-/**
- * Validate one eval case's shape. Throws with a descriptive message on
- * the first violation found. Checks:
- *
- * - `id` is a non-empty string.
- * - `task` is a string.
- * - `mode` (if present) is `'real'` or `'mock'`.
- * - `disableRows` (if present) is an array of strings (loader row ids to
- *   disable in this run's overlay). An EMPTY array is legal and means
- *   "disable nothing, explicitly" — it overrides a `disableRows` default
- *   from `dsh-eval.config.mjs`, which is how gate-interaction cases opt
- *   back in inside a package that disables the gate row by default.
- * - `expect` is an array; every element has `describe` (string) and `check` (function).
- * - mock mode requires a `script` with `steps` array.
- *
- * @param {object} evalCase - the case to validate.
- * @param {string} file - the source file path (for error messages).
- */
-export function validateEvalCase(evalCase, file) {
-  if (evalCase === null || typeof evalCase !== 'object') {
-    throw new Error(`${file}: case must be an object`)
-  }
-  if (typeof evalCase.id !== 'string' || evalCase.id === '') {
-    throw new Error(`${file}: case.id must be a non-empty string`)
-  }
-  if (typeof evalCase.task !== 'string') {
-    throw new Error(`${file}: case '${evalCase.id}': task must be a string`)
-  }
-  if (evalCase.mode !== undefined && evalCase.mode !== 'real' && evalCase.mode !== 'mock') {
-    throw new Error(`${file}: case '${evalCase.id}': mode must be 'real' or 'mock' (got '${evalCase.mode}')`)
-  }
-  if (evalCase.gates !== undefined) {
-    throw new Error(`${file}: case '${evalCase.id}': the 'gates' field was removed — declare disableRows: ['gates'] instead`)
-  }
-  if (evalCase.disableRows !== undefined) {
-    if (!Array.isArray(evalCase.disableRows)
-      || evalCase.disableRows.some(row => typeof row !== 'string' || row === '')) {
-      throw new Error(`${file}: case '${evalCase.id}': disableRows must be a string[] of loader row ids (empty = explicit none, overriding config; got '${JSON.stringify(evalCase.disableRows)}')`)
-    }
-  }
-  if (!Array.isArray(evalCase.expect)) {
-    throw new Error(`${file}: case '${evalCase.id}': expect must be a Matcher[]`)
-  }
-  for (let i = 0; i < evalCase.expect.length; i += 1) {
-    const matcher = evalCase.expect[i]
-    if (typeof matcher?.describe !== 'string' || typeof matcher?.check !== 'function') {
-      throw new Error(`${file}: case '${evalCase.id}': expect[${i}] must have { describe: string, check: function }`)
-    }
-  }
-  if (evalCase.mode === 'mock') {
-    if (evalCase.script === undefined || !Array.isArray(evalCase.script?.steps)) {
-      throw new Error(`${file}: case '${evalCase.id}': mock mode requires script.steps`)
-    }
-  }
-}
-
-/**
- * Detect duplicate case ids across a flat case list. Throws on the first
- * duplicate found, naming both source files.
- * @param {{ id: string, __file: string }[]} cases - cases with `__file` attached.
- */
-export function detectDuplicateIds(cases) {
-  const seen = new Map()
-  for (const evalCase of cases) {
-    const existing = seen.get(evalCase.id)
-    if (existing !== undefined) {
-      const locations = existing === evalCase.__file
-        ? existing
-        : `${existing} and ${evalCase.__file}`
-      throw new Error(`duplicate case id '${evalCase.id}' in ${locations}`)
-    }
-    seen.set(evalCase.id, evalCase.__file)
-  }
-}
+/**
+ * Shared case / experiment discovery and validation for the eval CLIs.
+ * Both `dsh-eval` and `dsh-review` scan directories recursively; this
+ * module ensures they share identical skip rules and validation.
+ */
+
+import { readdirSync, statSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+
+/**
+ * Directories never to descend into during discovery. `.runs` holds
+ * prior-run artifacts that would be re-discovered as cases; `node_modules`
+ * would pull in dependencies that happen to match the suffix.
+ */
+const SKIP_DIRS = new Set(['.runs', 'node_modules'])
+
+/**
+ * Recursively collect files matching `suffix` under `path`.
+ * Always skips `.runs` and `node_modules`.
+ * @param {string} path - a file or directory path.
+ * @param {string} suffix - the filename suffix to match (e.g. '.eval.mjs').
+ * @param {string[]} [out] - accumulator (internal).
+ * @returns {string[]} list of discovered files (unsorted; callers sort if needed).
+ */
+export function discoverFiles(path, suffix, out = []) {
+  const absolute = resolve(path)
+  if (statSync(absolute).isFile()) {
+    if (absolute.endsWith(suffix)) out.push(absolute)
+    return out
+  }
+  for (const entry of readdirSync(absolute, { withFileTypes: true })) {
+    const full = join(absolute, entry.name)
+    if (entry.isDirectory() && !SKIP_DIRS.has(entry.name)) {
+      discoverFiles(full, suffix, out)
+    } else if (entry.isFile() && entry.name.endsWith(suffix)) {
+      out.push(full)
+    }
+  }
+  return out
+}
+
+/**
+ * Validate a `disableRows` declaration (case-level or ad-hoc): an array of
+ * non-empty loader row id strings. An EMPTY array is legal and means
+ * "disable nothing, explicitly" — it overrides a `disableRows` default
+ * from `dsh-eval.config.mjs`. Throws with `label` context. Shared by
+ * `validateEvalCase` (load time) and `runEvalCase` (execution time) so both
+ * report the identical message.
+ * @param {unknown} disableRows - the value to validate.
+ * @param {string} label - error-message context (e.g. `case '<id>'`).
+ */
+export function validateDisableRows(disableRows, label) {
+  if (!Array.isArray(disableRows)
+    || disableRows.some(row => typeof row !== 'string' || row === '')) {
+    throw new Error(`${label}: disableRows must be a string[] of loader row ids (empty = explicit none, overriding config; got '${JSON.stringify(disableRows)}')`)
+  }
+}
+
+/**
+ * Validate a `followups` declaration (case-level): an array of non-empty
+ * strings, one per additional driven turn. Throws with `label` context.
+ * @param {unknown} followups - the value to validate.
+ * @param {string} label - error-message context (e.g. `case '<id>'`).
+ */
+export function validateFollowups(followups, label) {
+  if (!Array.isArray(followups)
+    || followups.length === 0
+    || followups.some(text => typeof text !== 'string' || text === '')) {
+    throw new Error(`${label}: followups must be a non-empty string[] of followup turn texts (got '${JSON.stringify(followups)}')`)
+  }
+}
+
+/**
+ * Validate one eval case's shape. Throws with a descriptive message on
+ * the first violation found. Checks:
+ *
+ * - `id` is a non-empty string.
+ * - `task` is a string.
+ * - `mode` (if present) is `'real'` or `'mock'`.
+ * - `disableRows` (if present) is an array of strings (loader row ids to
+ *   disable in this run's overlay). An EMPTY array is legal and means
+ *   "disable nothing, explicitly" — it overrides a `disableRows` default
+ *   from `dsh-eval.config.mjs`, which is how gate-interaction cases opt
+ *   back in inside a package that disables the gate row by default.
+ * - `rowConfig` (if present) maps loader row ids to config objects whose
+ *   leaf values are scalars or arrays of scalars (see `validateRowConfig`).
+ *   The overlay REPLACES the row's whole config — restate any keys the row
+ *   needs, not just the ones being changed.
+ * - `followups` (if present) is a non-empty array of followup turn texts
+ *   (cross-turn driving; see `validateFollowups`), with optional positive
+ *   finite `settleTimeoutMs`.
+ * - `expect` is an array; every element has `describe` (string) and `check` (function).
+ * - mock mode requires a `script` with `steps` array.
+ *
+ * @param {object} evalCase - the case to validate.
+ * @param {string} file - the source file path (for error messages).
+ */
+export function validateEvalCase(evalCase, file) {
+  if (evalCase === null || typeof evalCase !== 'object') {
+    throw new Error(`${file}: case must be an object`)
+  }
+  if (typeof evalCase.id !== 'string' || evalCase.id === '') {
+    throw new Error(`${file}: case.id must be a non-empty string`)
+  }
+  if (typeof evalCase.task !== 'string') {
+    throw new Error(`${file}: case '${evalCase.id}': task must be a string`)
+  }
+  if (evalCase.mode !== undefined && evalCase.mode !== 'real' && evalCase.mode !== 'mock') {
+    throw new Error(`${file}: case '${evalCase.id}': mode must be 'real' or 'mock' (got '${evalCase.mode}')`)
+  }
+  if (evalCase.gates !== undefined) {
+    throw new Error(`${file}: case '${evalCase.id}': the 'gates' field was removed — declare disableRows: ['gates'] instead`)
+  }
+  if (evalCase.disableRows !== undefined) {
+    validateDisableRows(evalCase.disableRows, `${file}: case '${evalCase.id}'`)
+  }
+  if (evalCase.rowConfig !== undefined) {
+    validateRowConfig(evalCase.rowConfig, `case '${evalCase.id}'`)
+  }
+  if (evalCase.followups !== undefined) {
+    validateFollowups(evalCase.followups, `${file}: case '${evalCase.id}'`)
+    if (evalCase.settleTimeoutMs !== undefined
+      && (typeof evalCase.settleTimeoutMs !== 'number' || !Number.isFinite(evalCase.settleTimeoutMs) || evalCase.settleTimeoutMs <= 0)) {
+      throw new Error(`${file}: case '${evalCase.id}': settleTimeoutMs must be a positive finite number (got '${JSON.stringify(evalCase.settleTimeoutMs)}')`)
+    }
+  }
+  if (!Array.isArray(evalCase.expect)) {
+    throw new Error(`${file}: case '${evalCase.id}': expect must be a Matcher[]`)
+  }
+  for (let i = 0; i < evalCase.expect.length; i += 1) {
+    const matcher = evalCase.expect[i]
+    if (typeof matcher?.describe !== 'string' || typeof matcher?.check !== 'function') {
+      throw new Error(`${file}: case '${evalCase.id}': expect[${i}] must have { describe: string, check: function }`)
+    }
+  }
+  if (evalCase.mode === 'mock') {
+    if (evalCase.script === undefined || !Array.isArray(evalCase.script?.steps)) {
+      throw new Error(`${file}: case '${evalCase.id}': mock mode requires script.steps`)
+    }
+  }
+}
+
+/**
+ * Validate a `rowConfig` mapping (case-level or ad-hoc): keys are loader row
+ * ids, values are config objects whose leaf values must be scalars (string /
+ * number / boolean) or arrays of scalars. Nested objects are rejected — the
+ * overlay emitter only handles flat config keys. Throws with `label` context.
+ * @param {unknown} rowConfig - the value to validate.
+ * @param {string} label - error-message context (e.g. `case '<id>'`).
+ */
+export function validateRowConfig(rowConfig, label) {
+  if (rowConfig === null || typeof rowConfig !== 'object' || Array.isArray(rowConfig)) {
+    throw new Error(`${label}: rowConfig must be an object mapping row ids to config objects (got '${JSON.stringify(rowConfig)}')`)
+  }
+  for (const [rowId, config] of Object.entries(rowConfig)) {
+    if (rowId === '') throw new Error(`${label}: rowConfig row id must be a non-empty string`)
+    if (config === null || typeof config !== 'object' || Array.isArray(config)) {
+      throw new Error(`${label}: rowConfig['${rowId}'] must be a config object (got '${JSON.stringify(config)}')`)
+    }
+    for (const [key, value] of Object.entries(config)) {
+      if (key === '') throw new Error(`${label}: rowConfig['${rowId}'] has an empty config key`)
+      if (Array.isArray(value)) {
+        if (value.some(item => item === null || typeof item === 'object')) {
+          throw new Error(`${label}: rowConfig['${rowId}']['${key}'] must be an array of scalars`)
+        }
+      } else if (value === null || typeof value === 'object') {
+        throw new Error(`${label}: rowConfig['${rowId}']['${key}'] must be a scalar or scalar array (nested objects are not supported)`)
+      }
+    }
+  }
+}
+
+/**
+ * Detect duplicate case ids across a flat case list. Throws on the first
+ * duplicate found, naming both source files.
+ * @param {{ id: string, __file: string }[]} cases - cases with `__file` attached.
+ */
+export function detectDuplicateIds(cases) {
+  const seen = new Map()
+  for (const evalCase of cases) {
+    const existing = seen.get(evalCase.id)
+    if (existing !== undefined) {
+      const locations = existing === evalCase.__file
+        ? existing
+        : `${existing} and ${evalCase.__file}`
+      throw new Error(`duplicate case id '${evalCase.id}' in ${locations}`)
+    }
+    seen.set(evalCase.id, evalCase.__file)
+  }
+}

package/src/driver/multi-turn-driver.mjs

@@ -0,0 +1,163 @@
+/**
+ * The eval multi-turn driver, loaded as a Cordis plugin through the eval
+ * overlay when a case declares `followups`. It REPLACES the one-shot
+ * `headless-runner` row (the overlay disables that row and inserts this one):
+ * the headless runner exits at the main agent's first idle, which aborts every
+ * in-process background subagent at process teardown (measured: a turn-close
+ * fixer child dies between publication and its first model call). Keeping the
+ * process alive across turns is what makes fire-and-forget children observable
+ * at all — the survival window is the driver's lifetime.
+ *
+ * Plan (env `DSH_EVAL_DRIVER_PLAN`, a JSON file written by the runner):
+ * `{ followups: string[], settleTimeoutMs?: number }`. The FIRST turn's task
+ * still arrives as the CLI positional; each followup is one additional user
+ * message. Before each followup the driver waits for background subagent
+ * children to settle (tracked via the global `session/event` feed: a session
+ * whose header marks it a subagent is pending from its `turn/start` until its
+ * `turn/end`), so a case can assert "the fixer child finished, THEN the next
+ * turn re-scanned".
+ *
+ * Output contract mirrors the headless runner's stdout/exit semantics: last
+ * non-empty assistant text of the main session on stdout, exit 0 iff the LAST
+ * turn ended `completed`.
+ */
+
+import { readFileSync } from 'node:fs'
+import { createUserMessage } from '@deepseek-ai/dsh-llm'
+
+/** Environment variable pointing at this run's driver plan JSON. */
+const PLAN_ENV = 'DSH_EVAL_DRIVER_PLAN'
+
+/** Default bound on waiting for background children to settle. */
+const DEFAULT_SETTLE_TIMEOUT_MS = 60_000
+
+/**
+ * Quiescence grace after the pending set drains: a turn-close dispatch races
+ * the driver's poll loop, so an empty set is only trusted after this many ms
+ * pass without a new subagent `turn/start`.
+ */
+const SETTLE_GRACE_MS = 250
+
+/** Poll interval while waiting on pending children. */
+const SETTLE_POLL_MS = 25
+
+/** One user-role followup message. */
+function followupMessage(text) {
+  return createUserMessage({
+    content: [{ type: 'text', text }],
+    source: { kind: 'user' },
+  })
+}
+
+/** Whether a session (from the event feed) is a subagent child log. */
+function isSubagentSession(session) {
+  const header = session?.header
+  if (header === null || typeof header !== 'object') return false
+  return header.origin === 'subagent' || header.parentSession !== undefined
+}
+
+/** Aggregate the last assistant text and turn outcome over the whole log. */
+function summarize(agent) {
+  let text = ''
+  let reason
+  for (const event of agent.session.events) {
+    if (event.type === 'assistant/message') {
+      const joined = event.data.message.content
+        .filter(block => block.type === 'text')
+        .map(block => block.text)
+        .join('')
+      if (joined !== '') text = joined
+    }
+    if (event.type === 'turn/end') reason = event.data.reason
+  }
+  return { text, reason }
+}
+
+/**
+ * Run the plan: task turn, then per followup — wait for background subagent
+ * children to settle, submit the followup, wait for idle.
+ */
+async function run(ctx, plan, io) {
+  // Pending background children: subagent sessions between their turn/start
+  // and turn/end. Registered BEFORE awaiting the loader so a plugin that
+  // dispatches during mount cannot slip a turn/start past the counter; the
+  // listener is global — child sessions publish their own events on the same
+  // feed the persistence layer listens on.
+  const pending = new Set()
+  ctx.on('session/event', (session, event) => {
+    if (!isSubagentSession(session)) return
+    if (event.type === 'turn/start') pending.add(session.header.id)
+    if (event.type === 'turn/end') pending.delete(session.header.id)
+  })
+  await ctx.get('loader')?.await()
+  const agents = ctx.get('agents')
+  const defaultModel = ctx.get('agentDefaultModel')
+  const sessions = ctx.get('sessions')
+  if (agents === undefined || defaultModel === undefined || sessions === undefined) {
+    throw new Error('eval-multi-turn-driver: agents/sessions services are unavailable')
+  }
+  const settleTimeoutMs = plan.settleTimeoutMs ?? DEFAULT_SETTLE_TIMEOUT_MS
+  const waitForSettle = async () => {
+    const deadline = Date.now() + settleTimeoutMs
+    for (;;) {
+      if (pending.size === 0) {
+        await new Promise(resolve => setTimeout(resolve, SETTLE_GRACE_MS))
+        if (pending.size === 0) return
+      }
+      if (Date.now() > deadline) {
+        throw new Error(`eval-multi-turn-driver: background subagents did not settle within ${settleTimeoutMs}ms (${[...pending].join(', ')})`)
+      }
+      await new Promise(resolve => setTimeout(resolve, SETTLE_POLL_MS))
+    }
+  }
+
+  const selection = defaultModel.currentSelection()
+  const { agent } = await agents.create({
+    sessionId: `session-${crypto.randomUUID()}`,
+    meta: { cwd: process.cwd() },
+    agentOptions: { provider: selection.provider, model: selection.model },
+  })
+  await agent.whenIdle()
+  const stop = plan.task ?? ''
+  if (stop === '') throw new Error('eval-multi-turn-driver: plan.task must be the case task text')
+  agent.followup(followupMessage(stop))
+  await agent.whenIdle()
+  for (const followup of plan.followups) {
+    await waitForSettle()
+    agent.followup(followupMessage(followup))
+    await agent.whenIdle()
+  }
+  // The LAST turn's close also dispatches (a defer fixer fires on every
+  // failed stop): wait once more so close-dispatched children are not
+  // silently aborted at exit — "dispatched ⇒ observable outcome" holds for
+  // every turn, not just the ones a followup follows.
+  await waitForSettle()
+  await sessions.flush(agent.session)
+  const outcome = summarize(agent)
+  io.stdout.write(outcome.text + '\n')
+  io.exit(outcome.reason?.kind === 'completed' ? 0 : 1)
+}
+
+export const name = 'eval-multi-turn-driver'
+
+export const inject = ['agentDefaultModel', 'agents', 'sessions']
+
+/** Mount the multi-turn driver from the run plan. */
+export function apply(ctx) {
+  // Read through the global service store: appExit is an optional launcher
+  // host value, never an injected dependency.
+  const exit = ctx.get('appExit')
+  if (exit === undefined) {
+    throw new Error('eval-multi-turn-driver: the launcher must provide ctx.appExit before the tree mounts')
+  }
+  const planPath = process.env[PLAN_ENV]
+  if (planPath === undefined) {
+    throw new Error(`eval-multi-turn-driver: ${PLAN_ENV} must point at the run's driver plan JSON`)
+  }
+  const plan = JSON.parse(readFileSync(planPath, 'utf8'))
+  const io = { stdout: process.stdout, stderr: process.stderr, exit }
+  void run(ctx, plan, io).catch(error => {
+    io.stderr.write(`dsh: ${error instanceof Error ? error.message : String(error)}\n`)
+    exit(1)
+  })
+}