@catheadowl/dsh-eval 0.1.0 → 0.2.1

package/src/experiment/review.mjs

@@ -1,118 +1,118 @@
-/**
- * Model-independent review experiment primitives.
- *
- * A review experiment separates four concerns:
- * - frozen inputs owned by the plugin;
- * - live observation/projection of those inputs;
- * - the blind prompt shown to an independent reviewer;
- * - the hidden human rubric used after the run.
- *
- * Nothing in this module knows how a model is invoked. Callers provide an
- * executor (dsh headless is one adapter) when they want to run the assembled
- * task.
- */
-
-const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
-
-/** Define and validate a human-graded review experiment. */
-export function defineReviewExperiment(definition) {
-  if (definition === null || typeof definition !== 'object') {
-    throw new TypeError('review experiment must be an object')
-  }
-  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
-    throw new TypeError('review experiment id must be a non-empty path-safe string')
-  }
-  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
-    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
-  }
-  if (typeof definition.observe !== 'function') {
-    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
-  }
-  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
-    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
-  }
-  const defaultRuns = definition.defaultRuns ?? 3
-  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
-    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
-  }
-  return Object.freeze({
-    ...definition,
-    kind: 'review',
-    defaultRuns,
-  })
-}
-
-/** Render the standard observation document consumed by a blind reviewer. */
-export function renderObservationSections(sections) {
-  if (!Array.isArray(sections) || sections.length === 0) {
-    throw new TypeError('review observations must contain at least one section')
-  }
-  const lines = []
-  for (const section of sections) {
-    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
-      throw new TypeError('every observation section needs a heading')
-    }
-    lines.push(`## ${section.heading}`)
-    if (section.introduction) lines.push(String(section.introduction), '')
-    for (const entry of section.entries ?? []) {
-      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
-        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
-      }
-      lines.push(`### ${entry.heading}`)
-      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
-      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
-      if (entry.json !== undefined) {
-        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
-      }
-      lines.push('')
-    }
-  }
-  return lines.join('\n').trimEnd()
-}
-
-/** Materialize live observations and assemble the blind reviewer task. */
-export async function materializeReviewExperiment(experiment) {
-  const sections = await experiment.observe()
-  const observations = renderObservationSections(sections)
-  return {
-    experimentId: experiment.id,
-    observations,
-    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
-  }
-}
-
-/**
- * Execute the same materialized task through N fresh executor calls.
- * The executor is intentionally generic: `(task, context) => result`.
- */
-export async function executeReviewExperiment(experiment, executor, options = {}) {
-  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
-  const runs = options.runs ?? experiment.defaultRuns
-  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
-
-  // Observe once. Every reviewer sees byte-identical evidence, so variance is
-  // attributable to interpretation rather than fixture/projection drift.
-  const materialized = await materializeReviewExperiment(experiment)
-  const attempts = []
-  for (let index = 1; index <= runs; index += 1) {
-    try {
-      const result = await executor(materialized.task, {
-        experiment,
-        experimentId: experiment.id,
-        index,
-        runs,
-      })
-      attempts.push({ index, ok: true, result })
-    } catch (error) {
-      attempts.push({
-        index,
-        ok: false,
-        error: error instanceof Error ? error.message : String(error),
-        result: error?.result,
-      })
-    }
-  }
-  return { ...materialized, runs, attempts }
-}
-
-export { OBSERVATIONS_PLACEHOLDER }
+/**
+ * Model-independent review experiment primitives.
+ *
+ * A review experiment separates four concerns:
+ * - frozen inputs owned by the plugin;
+ * - live observation/projection of those inputs;
+ * - the blind prompt shown to an independent reviewer;
+ * - the hidden human rubric used after the run.
+ *
+ * Nothing in this module knows how a model is invoked. Callers provide an
+ * executor (dsh headless is one adapter) when they want to run the assembled
+ * task.
+ */
+
+const OBSERVATIONS_PLACEHOLDER = '{{EVAL_OBSERVATIONS}}'
+
+/** Define and validate a human-graded review experiment. */
+export function defineReviewExperiment(definition) {
+  if (definition === null || typeof definition !== 'object') {
+    throw new TypeError('review experiment must be an object')
+  }
+  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(definition.id ?? '')) {
+    throw new TypeError('review experiment id must be a non-empty path-safe string')
+  }
+  if (typeof definition.prompt !== 'string' || definition.prompt.split(OBSERVATIONS_PLACEHOLDER).length !== 2) {
+    throw new TypeError(`review experiment '${definition.id}': prompt must contain exactly one ${OBSERVATIONS_PLACEHOLDER}`)
+  }
+  if (typeof definition.observe !== 'function') {
+    throw new TypeError(`review experiment '${definition.id}': observe must be a function`)
+  }
+  if (!(typeof definition.rubric === 'string' || definition.rubric instanceof URL)) {
+    throw new TypeError(`review experiment '${definition.id}': rubric must identify the hidden grading standard`)
+  }
+  const defaultRuns = definition.defaultRuns ?? 3
+  if (!Number.isInteger(defaultRuns) || defaultRuns < 1) {
+    throw new TypeError(`review experiment '${definition.id}': defaultRuns must be a positive integer`)
+  }
+  return Object.freeze({
+    ...definition,
+    kind: 'review',
+    defaultRuns,
+  })
+}
+
+/** Render the standard observation document consumed by a blind reviewer. */
+export function renderObservationSections(sections) {
+  if (!Array.isArray(sections) || sections.length === 0) {
+    throw new TypeError('review observations must contain at least one section')
+  }
+  const lines = []
+  for (const section of sections) {
+    if (typeof section?.heading !== 'string' || section.heading.length === 0) {
+      throw new TypeError('every observation section needs a heading')
+    }
+    lines.push(`## ${section.heading}`)
+    if (section.introduction) lines.push(String(section.introduction), '')
+    for (const entry of section.entries ?? []) {
+      if (typeof entry?.heading !== 'string' || entry.heading.length === 0) {
+        throw new TypeError(`section '${section.heading}' contains an entry without a heading`)
+      }
+      lines.push(`### ${entry.heading}`)
+      for (const paragraph of entry.paragraphs ?? []) lines.push(String(paragraph))
+      if (entry.call !== undefined) lines.push(`Call: ${JSON.stringify(entry.call)}`)
+      if (entry.json !== undefined) {
+        lines.push('```json', JSON.stringify(entry.json, null, 2), '```')
+      }
+      lines.push('')
+    }
+  }
+  return lines.join('\n').trimEnd()
+}
+
+/** Materialize live observations and assemble the blind reviewer task. */
+export async function materializeReviewExperiment(experiment) {
+  const sections = await experiment.observe()
+  const observations = renderObservationSections(sections)
+  return {
+    experimentId: experiment.id,
+    observations,
+    task: experiment.prompt.replace(OBSERVATIONS_PLACEHOLDER, observations),
+  }
+}
+
+/**
+ * Execute the same materialized task through N fresh executor calls.
+ * The executor is intentionally generic: `(task, context) => result`.
+ */
+export async function executeReviewExperiment(experiment, executor, options = {}) {
+  if (typeof executor !== 'function') throw new TypeError('review executor must be a function')
+  const runs = options.runs ?? experiment.defaultRuns
+  if (!Number.isInteger(runs) || runs < 1) throw new TypeError('runs must be a positive integer')
+
+  // Observe once. Every reviewer sees byte-identical evidence, so variance is
+  // attributable to interpretation rather than fixture/projection drift.
+  const materialized = await materializeReviewExperiment(experiment)
+  const attempts = []
+  for (let index = 1; index <= runs; index += 1) {
+    try {
+      const result = await executor(materialized.task, {
+        experiment,
+        experimentId: experiment.id,
+        index,
+        runs,
+      })
+      attempts.push({ index, ok: true, result })
+    } catch (error) {
+      attempts.push({
+        index,
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+        result: error?.result,
+      })
+    }
+  }
+  return { ...materialized, runs, attempts }
+}
+
+export { OBSERVATIONS_PLACEHOLDER }

package/src/experimental.mjs

@@ -0,0 +1,39 @@
+/**
+ * Escape-hatch entry: mechanism primitives for ad-hoc diagnostic scripts and
+ * self-built execution faces. Reachable as `<pkg>/experimental`.
+ *
+ * Contract: NO compatibility promise — symbols here may change or move in
+ * any minor release. The stable surface for eval/review case authors is the
+ * package root entry (src/index.mjs); import from here only when you are
+ * building your own runner/driver and accept the follow-up cost.
+ */
+
+// --- host CLI resolution (modern three-segment chain) ---
+export { resolveDshCliChain } from './cli.mjs'
+
+// --- sandbox / overlay mechanism ---
+export { stageProfileStore } from './sandbox.mjs'
+export { buildOverlayYaml, overlayDisableRows } from './overlay.mjs'
+
+// --- session-trace primitives ---
+export { parseSessionLog, buildTrace, loadTraceDir } from './trace.mjs'
+
+// --- review experiment execution layer ---
+export {
+  executeReviewExperiment,
+  materializeReviewExperiment,
+  renderObservationSections,
+  OBSERVATIONS_PLACEHOLDER,
+} from './experiment/review.mjs'
+
+// --- dsh review executors ---
+export {
+  createDshHeadlessReviewExecutor,
+  runDshReviewExperiment,
+} from './adapters/dsh/review.mjs'
+
+// --- tool-boundary validation ---
+export {
+  validateToolBoundary,
+  renderToolBoundaryEvidence,
+} from './tool-validation.mjs'

package/src/index.mjs

@@ -1,49 +1,38 @@
-/**
- * @catheadowl/dsh-eval public surface — what plugin eval cases import:
- * matchers for the `expect` list and mock-script step builders. The runner
- * and trace parser are bin/runner internals, also exported for ad-hoc use.
- */
-
-export {
-  toolCalled,
-  toolNotCalled,
-  firstTool,
-  toolSequence,
-  toolCallArgs,
-  toolResultFor,
-  toolResultIsError,
-  toolResultSucceeded,
-  toolResultTextIncludes,
-  finalTextIncludes,
-  finalTextMatches,
-  assistantTextIncludes,
-  systemPromptIncludes,
-  toolMounted,
-  userMessageTextIncludes,
-  userMessageTextExcludes,
-} from './assertions.mjs'
-
-export { toolCallStep, textStep } from './mock/script.mjs'
-
-export { runEvalCase, buildOverlayYaml, looksLikeDshRepo, stageProfileStore, FRAMEWORK_ROOT } from './runner.mjs'
-
-export { parseSessionLog, buildTrace, loadTraceDir } from './trace.mjs'
-
-export {
-  defineReviewExperiment,
-  executeReviewExperiment,
-  materializeReviewExperiment,
-  renderObservationSections,
-  OBSERVATIONS_PLACEHOLDER,
-} from './experiment/review.mjs'
-
-export {
-  createDshHeadlessReviewExecutor,
-  resolveDshCli,
-  runDshReviewExperiment,
-} from './adapters/dsh/review.mjs'
-
-export {
-  validateToolBoundary,
-  renderToolBoundaryEvidence,
-} from './tool-validation.mjs'
+/**
+ * @catheadowl/dsh-eval stable SDK surface — what eval/review case authors
+ * import: assertion matchers, mock-script step builders, the review
+ * experiment DSL, and the programmatic case runner.
+ *
+ * Mechanism primitives (sandbox/overlay/trace, review executors, CLI chain)
+ * live behind the `./experimental` subpath with no compatibility promise;
+ * everything else is bin-internal.
+ */
+
+export {
+  toolCalled,
+  toolNotCalled,
+  firstTool,
+  toolSequence,
+  toolCallArgs,
+  toolResultFor,
+  toolResultIsError,
+  toolResultSucceeded,
+  toolResultTextIncludes,
+  finalTextIncludes,
+  finalTextMatches,
+  assistantTextIncludes,
+  systemPromptIncludes,
+  toolMounted,
+  userMessageTextIncludes,
+  userMessageTextExcludes,
+  subagentDispatched,
+  subagentCompleted,
+  subagentDispatchCount,
+  subagentCompletedCount,
+} from './assertions.mjs'
+
+export { toolCallStep, textStep } from './mock/script.mjs'
+
+export { runEvalCase } from './runner.mjs'
+
+export { defineReviewExperiment } from './experiment/review.mjs'

package/src/mock/mock-adapter.mjs

@@ -1,73 +1,73 @@
-/**
- * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
- * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
- * provider and replays a scripted StreamChunk sequence: one script step per
- * model call, so a case controls exactly which tool calls and final text the
- * "model" produces. Pattern precedent:
- * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
- *
- * Deterministic layer purpose: verify the eval runner/trace/assertion
- * pipeline without an API key, and drive the plugin's tool execution path
- * (a scripted tool call runs through the real tool pipeline).
- */
-
-import { readFileSync } from 'node:fs'
-import { LlmAdapter } from '@deepseek-ai/dsh-llm'
-
-/** Environment variable pointing at this run's mock script JSON. */
-const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
-
-/** Adapter that yields one scripted chunk list per stream() call. */
-class EvalMockAdapter extends LlmAdapter {
-  constructor(script) {
-    super()
-    this.steps = script.steps ?? []
-    this.cursor = 0
-  }
-
-  async resolveModel(provider, model) {
-    return {
-      provider,
-      id: model,
-      name: model,
-      reasoning: {
-        efforts: [
-          { id: 'off', name: 'Off' },
-          { id: 'high', name: 'High' },
-        ],
-        defaultEffort: 'off',
-      },
-    }
-  }
-
-  async * stream(_options) {
-    const step = this.steps[this.cursor]
-    this.cursor += 1
-    if (step === undefined) {
-      // Script exhausted: finish the loop loudly-but-gracefully so the run
-      // still produces a trace a case can assert on.
-      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
-      yield { type: 'block-start', index: 0, blockType: 'text' }
-      yield { type: 'text-delta', index: 0, text }
-      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
-      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
-      yield { type: 'finish', reason: { kind: 'stop' } }
-      return
-    }
-    for (const chunk of step) yield chunk
-  }
-}
-
-export const name = 'eval-mock-llm'
-
-export const inject = ['llm']
-
-/** Register the `eval-mock` adapter from the scripted run. */
-export function apply(ctx) {
-  const scriptPath = process.env[SCRIPT_ENV]
-  if (scriptPath === undefined) {
-    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
-  }
-  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
-  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
-}
+/**
+ * The eval mock-LLM adapter, loaded as a Cordis plugin through the eval
+ * overlay (`name: file://...` insert). It registers the keyless `eval-mock`
+ * provider and replays a scripted StreamChunk sequence: one script step per
+ * model call, so a case controls exactly which tool calls and final text the
+ * "model" produces. Pattern precedent:
+ * deepseek-harness/examples/headless-agent/tests/fixtures/cli-mock-llm.ts.
+ *
+ * Deterministic layer purpose: verify the eval runner/trace/assertion
+ * pipeline without an API key, and drive the plugin's tool execution path
+ * (a scripted tool call runs through the real tool pipeline).
+ */
+
+import { readFileSync } from 'node:fs'
+import { LlmAdapter } from '@deepseek-ai/dsh-llm'
+
+/** Environment variable pointing at this run's mock script JSON. */
+const SCRIPT_ENV = 'DSH_EVAL_MOCK_SCRIPT'
+
+/** Adapter that yields one scripted chunk list per stream() call. */
+class EvalMockAdapter extends LlmAdapter {
+  constructor(script) {
+    super()
+    this.steps = script.steps ?? []
+    this.cursor = 0
+  }
+
+  async resolveModel(provider, model) {
+    return {
+      provider,
+      id: model,
+      name: model,
+      reasoning: {
+        efforts: [
+          { id: 'off', name: 'Off' },
+          { id: 'high', name: 'High' },
+        ],
+        defaultEffort: 'off',
+      },
+    }
+  }
+
+  async * stream(_options) {
+    const step = this.steps[this.cursor]
+    this.cursor += 1
+    if (step === undefined) {
+      // Script exhausted: finish the loop loudly-but-gracefully so the run
+      // still produces a trace a case can assert on.
+      const text = `eval-mock: script exhausted after ${this.cursor - 1} step(s)`
+      yield { type: 'block-start', index: 0, blockType: 'text' }
+      yield { type: 'text-delta', index: 0, text }
+      yield { type: 'block-end', index: 0, block: { type: 'text', text } }
+      yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }
+      yield { type: 'finish', reason: { kind: 'stop' } }
+      return
+    }
+    for (const chunk of step) yield chunk
+  }
+}
+
+export const name = 'eval-mock-llm'
+
+export const inject = ['llm']
+
+/** Register the `eval-mock` adapter from the scripted run. */
+export function apply(ctx) {
+  const scriptPath = process.env[SCRIPT_ENV]
+  if (scriptPath === undefined) {
+    throw new Error(`eval-mock-llm: ${SCRIPT_ENV} must point at the run's mock script JSON`)
+  }
+  const script = JSON.parse(readFileSync(scriptPath, 'utf8'))
+  ctx.llm.registerAdapter(['eval-mock'], new EvalMockAdapter(script))
+}

package/src/mock/script.mjs

@@ -1,49 +1,49 @@
-/**
- * Script builders for the eval mock-LLM layer. A mock script is
- * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
- * model call yields (its `finish` chunk included); the adapter consumes one
- * step per stream() call. Cases build steps with these helpers instead of
- * hand-writing chunk JSON.
- */
-
-let autoId = 0
-
-/** Next deterministic call id for generated steps. */
-function nextCallId() {
-  autoId += 1
-  return `eval-mock-call-${autoId}`
-}
-
-/**
- * One model call that requests a single tool invocation, then finishes with
- * `kind: 'tool-calls'` so the loop executes the tool and calls back.
- * @param {string} name - the model-facing tool name to call.
- * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
- * @param {{ id?: string }} [options] - override the generated call id.
- */
-export function toolCallStep(name, args = {}, options = {}) {
-  const id = options.id ?? nextCallId()
-  const json = JSON.stringify(args)
-  return [
-    { type: 'block-start', index: 0, blockType: 'tool-call' },
-    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
-    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
-    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
-    { type: 'finish', reason: { kind: 'tool-calls' } },
-  ]
-}
-
-/**
- * One model call that answers plain text and stops the loop
- * (`kind: 'stop'`).
- * @param {string} text - the assistant reply.
- */
-export function textStep(text) {
-  return [
-    { type: 'block-start', index: 0, blockType: 'text' },
-    { type: 'text-delta', index: 0, text },
-    { type: 'block-end', index: 0, block: { type: 'text', text } },
-    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
-    { type: 'finish', reason: { kind: 'stop' } },
-  ]
-}
+/**
+ * Script builders for the eval mock-LLM layer. A mock script is
+ * `{ steps: ChunkStep[] }` where each step is the exact StreamChunk list one
+ * model call yields (its `finish` chunk included); the adapter consumes one
+ * step per stream() call. Cases build steps with these helpers instead of
+ * hand-writing chunk JSON.
+ */
+
+let autoId = 0
+
+/** Next deterministic call id for generated steps. */
+function nextCallId() {
+  autoId += 1
+  return `eval-mock-call-${autoId}`
+}
+
+/**
+ * One model call that requests a single tool invocation, then finishes with
+ * `kind: 'tool-calls'` so the loop executes the tool and calls back.
+ * @param {string} name - the model-facing tool name to call.
+ * @param {object} [args] - the tool arguments (serialized as the model's raw JSON).
+ * @param {{ id?: string }} [options] - override the generated call id.
+ */
+export function toolCallStep(name, args = {}, options = {}) {
+  const id = options.id ?? nextCallId()
+  const json = JSON.stringify(args)
+  return [
+    { type: 'block-start', index: 0, blockType: 'tool-call' },
+    { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: json },
+    { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: json } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'tool-calls' } },
+  ]
+}
+
+/**
+ * One model call that answers plain text and stops the loop
+ * (`kind: 'stop'`).
+ * @param {string} text - the assistant reply.
+ */
+export function textStep(text) {
+  return [
+    { type: 'block-start', index: 0, blockType: 'text' },
+    { type: 'text-delta', index: 0, text },
+    { type: 'block-end', index: 0, block: { type: 'text', text } },
+    { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
+    { type: 'finish', reason: { kind: 'stop' } },
+  ]
+}