@catheadowl/dsh-eval 0.2.0 → 0.2.1

package/src/adapters/dsh/review.mjs

@@ -7,17 +7,18 @@ import { executeReviewExperiment } from '../../experiment/review.mjs'
 import { CLI_RELATIVE_PATH } from '../../cli.mjs'
 import { overlayDisableRows } from '../../overlay.mjs'
 import {
-  resolveRealDshHome, stageSandboxHome, teardownSandbox, spawnHeadlessDsh,
+  resolveRealDshHome, stageSandboxHome, stagedPluginRows, teardownSandbox, spawnHeadlessDsh,
 } from '../../sandbox.mjs'
 import { loadTraceDir } from '../../trace.mjs'
 import { validateToolBoundary, renderToolBoundaryEvidence } from '../../tool-validation.mjs'
 
 /**
  * Model-facing tool rows every shipped dsh profile mounts from `dsh-base`.
- * The overlay disables all host tool rows as a supplementary guard; the
- * primary isolation comes from the sterile review profile (dsh-base +
- * dsh-headless only, no out-of-tree plugins).  Post-run tool boundary
- * validation (see `validateToolBoundary`) detects any residual drift.
+ * A static supplementary guard on top of the blank-environment enumeration
+ * (host template tool rows never appear in `stagedPluginRows` — they are the
+ * sterile baseline, so their ids are enumerated here instead).  Post-run
+ * tool boundary validation (see `validateToolBoundary`) detects any residual
+ * drift.
  */
 const REVIEW_DISABLED_TOOL_ROWS = [
   'tool-bash',
@@ -39,9 +40,34 @@ const REVIEW_DISABLED_TOOL_ROWS = [
   'tool-workflow',
 ]
 
-/** Serialize the tool-less overlay: disable every host model-facing tool row. */
-function buildReviewOverlayYaml() {
-  return overlayDisableRows(REVIEW_DISABLED_TOOL_ROWS)
+/**
+ * Rows the reviewer session must KEEP even when an out-of-tree bundle or the
+ * profile patch touches them: model wiring and the session log — without
+ * these the reviewer cannot answer at all and the trace (tool boundary
+ * check, run artifacts) never materializes.  These ids are host-template
+ * rows; an out-of-tree patch row targeting one of them only overrides its
+ * config, so keeping it enabled stays safe.
+ */
+const REVIEW_REQUIRED_ROWS = new Set([
+  'agent-default-model',
+  'session-title-llm',
+  'system-prompt',
+  'session-persistence-jsonl',
+])
+
+/**
+ * Serialize the blank-environment overlay: disable every plugin row the
+ * staged profile composes beyond the host templates (`stagedPluginRows` —
+ * out-of-tree bundle rows + profile patch rows), UNION the static host tool
+ * rows, MINUS the model/session wiring the reviewer needs.  With
+ * `keepPluginRows` (explicit opt-in to study the host plugin face itself)
+ * the enumeration is skipped and only the static tool lockdown remains.
+ */
+function buildReviewOverlayYaml(pluginRows, { keepPluginRows }) {
+  const disabled = new Set(keepPluginRows ? [] : pluginRows)
+  for (const row of REVIEW_DISABLED_TOOL_ROWS) disabled.add(row)
+  for (const row of REVIEW_REQUIRED_ROWS) disabled.delete(row)
+  return overlayDisableRows([...disabled])
 }
 
 /** Resolve and validate the compiled dsh CLI entry point. */
@@ -66,16 +92,22 @@ function executorCli(options) {
 /**
  * Create an executor compatible with executeReviewExperiment.
  *
- * The executor boots a sterile review profile (default: the host's
- * `headless` template, bundles = dsh-base + dsh-headless, no out-of-tree
- * plugins) in an isolated DSH_HOME (staging/teardown mechanics shared with
- * the behavior runner via sandbox.mjs), disables every host tool row via
- * overlay, and validates the tool boundary after the run.  `options.profile`
- * must name a profile whose installed plugin set is empty or review-safe.
+ * The executor boots a blank review environment in an isolated DSH_HOME
+ * (staging/teardown mechanics shared with the behavior runner via
+ * sandbox.mjs): the staged profile's every out-of-tree plugin row and every
+ * host model-facing tool row is disabled via overlay (blank =
+ * dsh-base/dsh-headless templates + model wiring, nothing else — regardless
+ * of what the host profile carries), and the tool boundary is validated
+ * after the run.
  *
  * @param {object} options
- * @param {string} [options.profile='headless'] - the sterile review profile.
+ * @param {string} [options.profile='headless'] - the review profile (host
+ *   templates + whatever the host machine carries; plugin rows are disabled
+ *   by the overlay anyway).
  * @param {Set<string>} [options.allowedTools] - tool names permitted in the reviewer's session (default: empty).
+ * @param {boolean} [options.keepPluginRows=false] - opt back into the host
+ *   profile's plugin face (e.g. to review a plugin's own gate behavior);
+ *   only the static tool lockdown remains.
  */
 export function createDshHeadlessReviewExecutor(options) {
   const cli = executorCli(options)
@@ -85,6 +117,7 @@ export function createDshHeadlessReviewExecutor(options) {
   }
   const timeoutMs = options.timeoutMs ?? 300_000
   const allowedTools = options.allowedTools ?? new Set()
+  const keepPluginRows = options.keepPluginRows ?? false
 
   return async function executeWithDsh(task) {
     // A fresh process alone is not enough: dsh also stores settings, titles,
@@ -93,9 +126,18 @@ export function createDshHeadlessReviewExecutor(options) {
     // while retaining the selected profile's model config and plugin links.
     const runDir = mkdtempSync(join(tmpdir(), 'dsh-review-'))
     const dshHome = join(runDir, 'dsh-home')
-    const overlayPath = join(runDir, 'review-overlay.yml')
-    writeFileSync(overlayPath, buildReviewOverlayYaml(), 'utf8')
     stageSandboxHome(options.dshHome ?? resolveRealDshHome(), dshHome, profile)
+    // Blank environment by DEFAULT (review-blank-environment TODO): disable
+    // every row the staged profile composes beyond the host templates, so a
+    // host profile carrying out-of-tree plugins (gates included) can no
+    // longer steer or crash the reviewer. `keepPluginRows` opts back into
+    // the host plugin face deliberately. Enumeration happens on the STAGED
+    // copy, after staging — the staged home is what actually boots.
+    const overlayPath = join(runDir, 'review-overlay.yml')
+    writeFileSync(overlayPath, buildReviewOverlayYaml(
+      stagedPluginRows(dshHome, profile),
+      { keepPluginRows },
+    ), 'utf8')
 
     try {
       const { stdout, stderr, exitCode, timedOut } = await spawnHeadlessDsh({
@@ -141,6 +183,17 @@ export function createDshHeadlessReviewExecutor(options) {
           boundaryError.result = result
           throw boundaryError
         }
+        // The ANSWER to the task, not the last message: stdout carries the
+        // headless CLI's final assistant message — whatever the reviewer
+        // said LAST. If any tail interaction intervened (a turn-close gate
+        // splice that slipped past the blank environment, an infra
+        // complaint), stdout holds that instead of the analysis. The
+        // trace's answerText (last assistant text before the first
+        // plugin-sourced injection) IS the analysis; stdout remains the
+        // fallback for trace-less runs.
+        result.answer = trace.answerText !== '' ? trace.answerText : stdout
+      } else {
+        result.answer = stdout
       }
 
       return result