@catheadowl/dsh-eval 0.1.0 → 0.2.0

package/src/index.mjs

@@ -1,49 +1,34 @@
-/**
- * @catheadowl/dsh-eval public surface — what plugin eval cases import:
- * matchers for the `expect` list and mock-script step builders. The runner
- * and trace parser are bin/runner internals, also exported for ad-hoc use.
- */
-
-export {
-  toolCalled,
-  toolNotCalled,
-  firstTool,
-  toolSequence,
-  toolCallArgs,
-  toolResultFor,
-  toolResultIsError,
-  toolResultSucceeded,
-  toolResultTextIncludes,
-  finalTextIncludes,
-  finalTextMatches,
-  assistantTextIncludes,
-  systemPromptIncludes,
-  toolMounted,
-  userMessageTextIncludes,
-  userMessageTextExcludes,
-} from './assertions.mjs'
-
-export { toolCallStep, textStep } from './mock/script.mjs'
-
-export { runEvalCase, buildOverlayYaml, looksLikeDshRepo, stageProfileStore, FRAMEWORK_ROOT } from './runner.mjs'
-
-export { parseSessionLog, buildTrace, loadTraceDir } from './trace.mjs'
-
-export {
-  defineReviewExperiment,
-  executeReviewExperiment,
-  materializeReviewExperiment,
-  renderObservationSections,
-  OBSERVATIONS_PLACEHOLDER,
-} from './experiment/review.mjs'
-
-export {
-  createDshHeadlessReviewExecutor,
-  resolveDshCli,
-  runDshReviewExperiment,
-} from './adapters/dsh/review.mjs'
-
-export {
-  validateToolBoundary,
-  renderToolBoundaryEvidence,
-} from './tool-validation.mjs'
+/**
+ * @catheadowl/dsh-eval stable SDK surface — what eval/review case authors
+ * import: assertion matchers, mock-script step builders, the review
+ * experiment DSL, and the programmatic case runner.
+ *
+ * Mechanism primitives (sandbox/overlay/trace, review executors, CLI chain)
+ * live behind the `./experimental` subpath with no compatibility promise;
+ * everything else is bin-internal.
+ */
+
+export {
+  toolCalled,
+  toolNotCalled,
+  firstTool,
+  toolSequence,
+  toolCallArgs,
+  toolResultFor,
+  toolResultIsError,
+  toolResultSucceeded,
+  toolResultTextIncludes,
+  finalTextIncludes,
+  finalTextMatches,
+  assistantTextIncludes,
+  systemPromptIncludes,
+  toolMounted,
+  userMessageTextIncludes,
+  userMessageTextExcludes,
+} from './assertions.mjs'
+
+export { toolCallStep, textStep } from './mock/script.mjs'
+
+export { runEvalCase } from './runner.mjs'
+
+export { defineReviewExperiment } from './experiment/review.mjs'

package/src/overlay.mjs

@@ -0,0 +1,96 @@
+/**
+ * Overlay (cordis patch) serialization — the ONLY hand-rolled YAML emitter
+ * in the package. Both the behavior runner (buildOverlayYaml) and the dsh
+ * review adapter (overlayDisableRows for its tool-less overlay) generate
+ * per-run overlay files; sharing one emitter keeps quoting rules and
+ * row-patch syntax (`- id: <row>` / `disabled: true`) identical everywhere.
+ *
+ * YAML strategy: JSON double-quoted strings are valid YAML scalars, so the
+ * emitters lean on JSON.stringify — zero dependencies, identical quoting
+ * across scalar and array leaves.
+ */
+
+import { join } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+
+/** This framework's root directory (the eval package dir). */
+const FRAMEWORK_ROOT = fileURLToPath(new URL('..', import.meta.url))
+
+/** The scripted mock adapter plugin, referenced from generated overlays. */
+const MOCK_ADAPTER_PATH = join(FRAMEWORK_ROOT, 'src', 'mock', 'mock-adapter.mjs')
+
+/** JSON double-quoted strings are valid YAML scalars — enough for this emitter. */
+function yamlScalar(value) {
+  if (typeof value === 'boolean' || typeof value === 'number') return String(value)
+  return JSON.stringify(String(value))
+}
+
+/**
+ * One rowConfig leaf: a scalar, or an array of scalars emitted as a YAML flow
+ * sequence (a JSON array is valid YAML flow syntax, and keeps quoting rules
+ * identical to `yamlScalar`).
+ */
+function yamlConfigValue(value) {
+  if (Array.isArray(value)) return JSON.stringify(value.map(item => typeof item === 'string' ? item : String(item)))
+  return yamlScalar(value)
+}
+
+/**
+ * Serialize `- id: <row> / disabled: true` patches for every row id — the
+ * cross-layer row-disable mechanism shared by case `disableRows` and the
+ * review adapter's host-tool lockdown.
+ * @param {string[]} rowIds - loader row ids to disable.
+ * @returns {string} overlay entries (each line-terminated).
+ */
+export function overlayDisableRows(rowIds) {
+  return rowIds.map(rowId => `- id: ${yamlScalar(rowId)}\n  disabled: true\n`).join('')
+}
+
+/**
+ * Serialize the per-run overlay patch list for one eval case.
+ * @param {object} parts - overlay ingredients (see runEvalCase):
+ *   `sessionsRoot` (required), optional `persona`, `disableRows`,
+ *   `rowConfig`, and `mock` (mount the scripted adapter + re-point the
+ *   default model).
+ * @returns {string} the overlay file text.
+ */
+export function buildOverlayYaml(parts) {
+  const lines = []
+  lines.push('- id: session-persistence-jsonl')
+  lines.push('  config:')
+  lines.push(`    root: ${yamlScalar(parts.sessionsRoot)}`)
+  lines.push('    packChunks: false')
+  lines.push('    compression: none')
+  if (parts.persona !== undefined) {
+    lines.push('- id: system-prompt')
+    lines.push('  config:')
+    lines.push(`    persona: ${yamlScalar(parts.persona)}`)
+  }
+  if (parts.disableRows !== undefined && parts.disableRows.length > 0) {
+    lines.push(overlayDisableRows(parts.disableRows).trimEnd())
+  }
+  for (const [rowId, config] of Object.entries(parts.rowConfig ?? {})) {
+    // Whole-replace semantics: these config keys REPLACE the row's config
+    // (cordis patch layer), so the emitter adds to a fresh `- id:` entry —
+    // restating keys is the declaring case's responsibility.
+    lines.push(`- id: ${yamlScalar(rowId)}`)
+    lines.push('  config:')
+    for (const [key, value] of Object.entries(config)) {
+      lines.push(`    ${key}: ${yamlConfigValue(value)}`)
+    }
+  }
+  if (parts.mock) {
+    lines.push('- id: agent-default-model')
+    lines.push('  config:')
+    lines.push('    provider: eval-mock')
+    lines.push('    model: eval-mock')
+    // The title generator also calls the default provider and would consume
+    // script steps; deterministic runs own every model call themselves.
+    lines.push('- id: session-title-llm')
+    lines.push('  disabled: true')
+    lines.push('- insert:')
+    lines.push('    - id: eval-mock-llm')
+    lines.push(`      name: ${yamlScalar(pathToFileURL(MOCK_ADAPTER_PATH).href)}`)
+  }
+  return `${lines.join('\n')}\n`
+}

package/src/report.mjs

@@ -1,5 +1,5 @@
 /**
- * Machine-readable run reports for the behavior CLI (EVAL-007).
+ * Machine-readable run reports for the behavior CLI.
  *
  * One report covers ONE `dsh-eval run` invocation: the selection summary,
  * per-case outcomes, and the invocation's environment anchors (profile /