@catheadowl/dsh-eval 0.2.0 → 0.2.1

package/src/trace.mjs

@@ -1,218 +1,293 @@
-/**
- * Session-trace parsing for dsh agent eval. The evidence source is the JSONL
- * session artifact written by `@deepseek-ai/dsh-session-persistence-jsonl`
- * (configured `compression: none`, `packChunks: false` by the eval overlay):
- * one `type: 'session'` header line, then one JSON record per `SessionEvent`.
- * Event shapes follow `deepseek-harness/packages/core/session/src/types.ts`
- * (`SessionEventMap`); packed `*-chunks` storage rows are tolerated and
- * skipped — they only carry `assistant/chunk` deltas eval never asserts on.
- */
-
-import { readdirSync, readFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-/** Storage row types that pack `assistant/chunk` delta runs (see chunk-rows.ts). */
-const CHUNK_ROW_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks'])
-
-/**
- * Parse one uncompressed JSONL session artifact.
- * @param {string} text - the artifact's full text (header line first).
- * @returns {{ header: object, events: object[] }} header plus event records in log order.
- */
-export function parseSessionLog(text) {
-  const lines = text.split('\n').filter(line => line.trim() !== '')
-  if (lines.length === 0) throw new Error('empty session log')
-  const header = JSON.parse(lines[0])
-  if (header.type !== 'session') throw new Error('first line is not a session header')
-  const events = []
-  for (const line of lines.slice(1)) {
-    let record
-    try {
-      record = JSON.parse(line)
-    } catch {
-      continue // torn or partial tail line: keep the decodable prefix
-    }
-    if (record === null || typeof record !== 'object') continue
-    if (CHUNK_ROW_TYPES.has(record.type)) continue
-    events.push(record)
-  }
-  return { header, events }
-}
-
-/** Concatenate the text blocks of one assembled assistant message. */
-function messageText(message) {
-  const content = message?.content
-  if (!Array.isArray(content)) return ''
-  return content
-    .filter(block => block?.type === 'text' && typeof block.text === 'string')
-    .map(block => block.text)
-    .join('')
-}
-
-/**
- * Extract the visible text of one tool-result message. Real messages are
- * user-role with a single wrapping `tool-result` block whose `content` holds
- * the actual blocks (see `createToolResultMessage` in
- * `deepseek-harness/packages/llm/llm/src/message.ts`); a bare block list is
- * tolerated for hand-built fixtures.
- */
-function toolResultText(message) {
-  const content = message?.content
-  if (!Array.isArray(content)) return ''
-  const wrapper = content.find(block => block?.type === 'tool-result')
-  const inner = wrapper !== undefined ? wrapper.content : content
-  if (!Array.isArray(inner)) return ''
-  return inner
-    .filter(block => block?.type === 'text' && typeof block.text === 'string')
-    .map(block => block.text)
-    .join('')
-}
-
-/**
- * Extract the `isError` flag from a tool-result message's wrapper block.
- * Returns `undefined` when the flag is absent (treated as success by
- * matchers — see `toolResultSucceeded`).
- */
-function toolResultIsError(message) {
-  const content = message?.content
-  if (!Array.isArray(content)) return undefined
-  const wrapper = content.find(block => block?.type === 'tool-result')
-  return wrapper?.isError
-}
-
-/** Best-effort parse of a tool call's raw JSON arguments string. */
-function parseArguments(raw) {
-  try {
-    return JSON.parse(raw)
-  } catch {
-    return undefined
-  }
-}
-
-/**
- * Build one assertable trace from parsed session logs. Child sessions surface
- * only through the parent's tool events, so the MAIN log (no `origin:
- * 'subagent'` header) owns the tool/final-text projections; all logs stay
- * available under `sessions`.
- * @param {{ header: object, events: object[] }[]} logs - parsed session logs.
- * @returns {EvalTrace}
- */
-export function buildTrace(logs) {
-  const mains = logs.filter(log => log.header.origin !== 'subagent')
-  const main = [...mains].sort((a, b) => b.events.length - a.events.length)[0]
-  const events = main?.events ?? []
-
-  const toolCalls = []
-  const toolResults = []
-  const assistantTexts = []
-  const userMessages = []
-  const requestHeaders = []
-  for (const event of events) {
-    if (event.type === 'request/header') {
-      // The assembled model request header: system prompt + mounted tool
-      // schemas. What the model is told it can do and how — the "did my
-      // plugin's section inject?" projection.
-      requestHeaders.push({
-        seq: event.seq,
-        reason: event.data?.reason,
-        system: event.data?.header?.system ?? '',
-        toolNames: Array.isArray(event.data?.header?.tools)
-          ? event.data.header.tools.map(tool => tool?.name).filter(name => typeof name === 'string')
-          : [],
-      })
-    } else if (event.type === 'tool/call') {
-      toolCalls.push({
-        seq: event.seq,
-        turn: event.data.turn,
-        step: event.data.step,
-        callId: event.data.callId,
-        name: event.data.name,
-        arguments: event.data.arguments,
-        parsedArguments: parseArguments(event.data.arguments),
-      })
-    } else if (event.type === 'tool/result') {
-      toolResults.push({
-        seq: event.seq,
-        turn: event.data.turn,
-        step: event.data.step,
-        callId: event.data.message?.source?.callId,
-        text: toolResultText(event.data.message),
-        error: event.data.error,
-        isError: toolResultIsError(event.data.message),
-      })
-    } else if (event.type === 'assistant/message') {
-      const text = messageText(event.data.message)
-      if (text !== '') assistantTexts.push(text)
-    } else if (event.type === 'user/message') {
-      // The user-role model-visible surface: the task prompt (kind 'user'),
-      // plugin steering, or injected context. `source` tells them apart —
-      // steer has no dedicated event type (the legacy `steering/message` was
-      // migrated to `user/message`), so the matcher side filters by `source`.
-      const text = messageText(event.data)
-      if (text !== '') {
-        userMessages.push({
-          seq: event.seq,
-          source: event.data?.source,
-          text,
-        })
-      }
-    }
-  }
-
-  return {
-    sessions: logs,
-    sessionId: main?.header.id,
-    toolCalls,
-    toolResults,
-    assistantTexts,
-    userMessages,
-    requestHeaders,
-    finalText: assistantTexts.at(-1) ?? '',
-  }
-}
-
-/** Recursively collect files named `name` under `dir`. */
-function collectFiles(dir, name, out = []) {
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    const path = join(dir, entry.name)
-    if (entry.isDirectory()) collectFiles(path, name, out)
-    else if (entry.name === name) out.push(path)
-  }
-  return out
-}
-
-/**
- * Load every session log under a persistence root and build one trace.
- * @param {string} sessionsRoot - the run's `session-persistence-jsonl` root.
- * @returns {EvalTrace | undefined} the trace, or `undefined` when no log materialized.
- */
-export function loadTraceDir(sessionsRoot) {
-  let files
-  try {
-    files = collectFiles(sessionsRoot, 'session.jsonl')
-  } catch {
-    return undefined
-  }
-  if (files.length === 0) return undefined
-  const logs = files
-    .map(file => readFileSync(file, 'utf8'))
-    .map(parseSessionLog)
-  return buildTrace(logs)
-}
-
-/**
- * @typedef {object} EvalTrace
- * @property {{ header: object, events: object[] }[]} sessions - every parsed log.
- * @property {string | undefined} sessionId - the main session's id.
- * @property {{ seq: number, turn: number, step: number, callId: string, name: string, arguments: string, parsedArguments: unknown }[]} toolCalls
- * @property {{ seq: number, turn: number, step: number, callId: string, text: string, error: object | undefined, isError: boolean | undefined }[]} toolResults
- * @property {string[]} assistantTexts - non-empty assembled assistant messages, log order.
- * @property {{ seq: number, source: object, text: string }[]} userMessages
- *   - non-empty `user/message` events (task prompt, plugin steer, injected
- *     context) with their verbatim `source` (`kind` + plugin-specific fields),
- *     in log order. Steer has no dedicated event type; matchers filter by
- *     `source`.
- * @property {{ seq: number, reason: string, system: string, toolNames: string[] }[]} requestHeaders
- *   - projected `request/header` events (assembled system prompt + mounted tools).
- * @property {string} finalText - the last assembled assistant text ('' when none).
- */
+/**
+ * Session-trace parsing for dsh agent eval. The evidence source is the JSONL
+ * session artifact written by `@deepseek-ai/dsh-session-persistence-jsonl`
+ * (configured `compression: none`, `packChunks: false` by the eval overlay):
+ * one `type: 'session'` header line, then one JSON record per `SessionEvent`.
+ * Event shapes follow `deepseek-harness/packages/core/session/src/types.ts`
+ * (`SessionEventMap`); packed `*-chunks` storage rows are tolerated and
+ * skipped — they only carry `assistant/chunk` deltas eval never asserts on.
+ */
+
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+/** Storage row types that pack `assistant/chunk` delta runs (see chunk-rows.ts). */
+const CHUNK_ROW_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks'])
+
+/**
+ * Parse one uncompressed JSONL session artifact.
+ * @param {string} text - the artifact's full text (header line first).
+ * @returns {{ header: object, events: object[] }} header plus event records in log order.
+ */
+export function parseSessionLog(text) {
+  const lines = text.split('\n').filter(line => line.trim() !== '')
+  if (lines.length === 0) throw new Error('empty session log')
+  const header = JSON.parse(lines[0])
+  if (header.type !== 'session') throw new Error('first line is not a session header')
+  const events = []
+  for (const line of lines.slice(1)) {
+    let record
+    try {
+      record = JSON.parse(line)
+    } catch {
+      continue // torn or partial tail line: keep the decodable prefix
+    }
+    if (record === null || typeof record !== 'object') continue
+    if (CHUNK_ROW_TYPES.has(record.type)) continue
+    events.push(record)
+  }
+  return { header, events }
+}
+
+/** Concatenate the text blocks of one assembled assistant message. */
+function messageText(message) {
+  const content = message?.content
+  if (!Array.isArray(content)) return ''
+  return content
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('')
+}
+
+/**
+ * Extract the visible text of one tool-result message. Real messages are
+ * user-role with a single wrapping `tool-result` block whose `content` holds
+ * the actual blocks (see `createToolResultMessage` in
+ * `deepseek-harness/packages/llm/llm/src/message.ts`); a bare block list is
+ * tolerated for hand-built fixtures.
+ */
+function toolResultText(message) {
+  const content = message?.content
+  if (!Array.isArray(content)) return ''
+  const wrapper = content.find(block => block?.type === 'tool-result')
+  const inner = wrapper !== undefined ? wrapper.content : content
+  if (!Array.isArray(inner)) return ''
+  return inner
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('')
+}
+
+/**
+ * Extract the `isError` flag from a tool-result message's wrapper block.
+ * Returns `undefined` when the flag is absent (treated as success by
+ * matchers — see `toolResultSucceeded`).
+ */
+function toolResultIsError(message) {
+  const content = message?.content
+  if (!Array.isArray(content)) return undefined
+  const wrapper = content.find(block => block?.type === 'tool-result')
+  return wrapper?.isError
+}
+
+/** Best-effort parse of a tool call's raw JSON arguments string. */
+function parseArguments(raw) {
+  try {
+    return JSON.parse(raw)
+  } catch {
+    return undefined
+  }
+}
+
+/**
+ * Project one subagent child log into an assertable record. The durable
+ * identity (`label` / `mode` / `provider`) comes from the FIRST
+ * `subagent/descriptor` event whose payload carries the descriptor version
+ * this projection supports (3) — mirroring `foldSubagentDescriptor` in
+ * `deepseek-harness/packages/subagent/subagent/src/descriptor.ts`, where the
+ * establishing provider appends exactly one authoritative descriptor and
+ * later events cannot rewrite it. Completion is the child's own last
+ * assistant text — a child that produced none may have been dispatched but
+ * never ran to an answer (turn/end reasons are not consulted).
+ */
+function projectChild(log) {
+  let label
+  let mode
+  let provider
+  for (const event of log.events) {
+    if (event.type !== 'subagent/descriptor') continue
+    const data = event.data
+    if (data === null || typeof data !== 'object') continue
+    if (label !== undefined || mode !== undefined || provider !== undefined) break
+    if (data.version !== 3) continue
+    if (typeof data.label === 'string') label = data.label
+    if (typeof data.mode === 'string') mode = data.mode
+    if (typeof data.provider === 'string') provider = data.provider
+  }
+  const assistantTexts = log.events
+    .filter(event => event.type === 'assistant/message')
+    .map(event => messageText(event.data.message))
+    .filter(text => text !== '')
+  return {
+    sessionId: log.header.id,
+    parentSession: log.header.parentSession,
+    delegationDepth: log.header.delegationDepth,
+    label,
+    mode,
+    provider,
+    assistantTexts,
+    finalText: assistantTexts.at(-1) ?? '',
+  }
+}
+
+/**
+ * Build one assertable trace from parsed session logs. Child sessions surface
+ * only through the parent's tool events, so the MAIN log (no `origin:
+ * 'subagent'` header) owns the tool/final-text projections; subagent children
+ * project separately under `subagentChildren`; all logs stay available under
+ * `sessions`. A log carrying `parentSession` without `origin: 'subagent'`
+ * (non-subagent fork/resume shape) counts as a child record here but remains
+ * a main candidate too — the host-side ownership check walks the agent chain,
+ * which the log alone cannot reproduce.
+ * @param {{ header: object, events: object[] }[]} logs - parsed session logs.
+ * @returns {EvalTrace}
+ */
+export function buildTrace(logs) {
+  const mains = logs.filter(log => log.header.origin !== 'subagent')
+  const main = [...mains].sort((a, b) => b.events.length - a.events.length)[0]
+  const events = main?.events ?? []
+  const subagentChildren = logs
+    .filter(log => log.header.origin === 'subagent' || log.header.parentSession !== undefined)
+    .map(projectChild)
+
+  const toolCalls = []
+  const toolResults = []
+  const assistantEntries = []
+  const userMessages = []
+  const requestHeaders = []
+  for (const event of events) {
+    if (event.type === 'request/header') {
+      // The assembled model request header: system prompt + mounted tool
+      // schemas. What the model is told it can do and how — the "did my
+      // plugin's section inject?" projection.
+      requestHeaders.push({
+        seq: event.seq,
+        reason: event.data?.reason,
+        system: event.data?.header?.system ?? '',
+        toolNames: Array.isArray(event.data?.header?.tools)
+          ? event.data.header.tools.map(tool => tool?.name).filter(name => typeof name === 'string')
+          : [],
+      })
+    } else if (event.type === 'tool/call') {
+      toolCalls.push({
+        seq: event.seq,
+        turn: event.data.turn,
+        step: event.data.step,
+        callId: event.data.callId,
+        name: event.data.name,
+        arguments: event.data.arguments,
+        parsedArguments: parseArguments(event.data.arguments),
+      })
+    } else if (event.type === 'tool/result') {
+      toolResults.push({
+        seq: event.seq,
+        turn: event.data.turn,
+        step: event.data.step,
+        callId: event.data.message?.source?.callId,
+        text: toolResultText(event.data.message),
+        error: event.data.error,
+        isError: toolResultIsError(event.data.message),
+      })
+    } else if (event.type === 'assistant/message') {
+      const text = messageText(event.data.message)
+      if (text !== '') assistantEntries.push({ seq: event.seq, text })
+    } else if (event.type === 'user/message') {
+      // The user-role model-visible surface: the task prompt (kind 'user'),
+      // plugin steering, or injected context. `source` tells them apart —
+      // steer has no dedicated event type (the legacy `steering/message` was
+      // migrated to `user/message`), so the matcher side filters by `source`.
+      const text = messageText(event.data)
+      if (text !== '') {
+        userMessages.push({
+          seq: event.seq,
+          source: event.data?.source,
+          text,
+        })
+      }
+    }
+  }
+
+  const assistantTexts = assistantEntries.map(entry => entry.text)
+  // The answer to the task, as opposed to the last message: once a
+  // plugin-sourced injection (kind 'plugin' — a turn-close gate splice, an
+  // infra complaint) enters the conversation, every assistant message after
+  // it responds to the injection, not to the task. The answer is therefore
+  // the last assistant text BEFORE the first plugin injection; without one
+  // it degenerates to finalText (the task was the reviewer's last business).
+  const firstInjectionSeq = userMessages.find(
+    message => message.source?.kind === 'plugin',
+  )?.seq
+  const answerEntries = firstInjectionSeq === undefined
+    ? assistantEntries
+    : assistantEntries.filter(entry => entry.seq < firstInjectionSeq)
+  const answerText = answerEntries.at(-1)?.text ?? ''
+
+  return {
+    sessions: logs,
+    sessionId: main?.header.id,
+    toolCalls,
+    toolResults,
+    assistantTexts,
+    answerText,
+    userMessages,
+    requestHeaders,
+    subagentChildren,
+    finalText: assistantTexts.at(-1) ?? '',
+  }
+}
+
+/** Recursively collect files named `name` under `dir`. */
+function collectFiles(dir, name, out = []) {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name)
+    if (entry.isDirectory()) collectFiles(path, name, out)
+    else if (entry.name === name) out.push(path)
+  }
+  return out
+}
+
+/**
+ * Load every session log under a persistence root and build one trace.
+ * @param {string} sessionsRoot - the run's `session-persistence-jsonl` root.
+ * @returns {EvalTrace | undefined} the trace, or `undefined` when no log materialized.
+ */
+export function loadTraceDir(sessionsRoot) {
+  let files
+  try {
+    files = collectFiles(sessionsRoot, 'session.jsonl')
+  } catch {
+    return undefined
+  }
+  if (files.length === 0) return undefined
+  const logs = files
+    .map(file => readFileSync(file, 'utf8'))
+    .map(parseSessionLog)
+  return buildTrace(logs)
+}
+
+/**
+ * @typedef {object} EvalTrace
+ * @property {{ header: object, events: object[] }[]} sessions - every parsed log.
+ * @property {string | undefined} sessionId - the main session's id.
+ * @property {{ seq: number, turn: number, step: number, callId: string, name: string, arguments: string, parsedArguments: unknown }[]} toolCalls
+ * @property {{ seq: number, turn: number, step: number, callId: string, text: string, error: object | undefined, isError: boolean | undefined }[]} toolResults
+ * @property {string[]} assistantTexts - non-empty assembled assistant messages, log order.
+ * @property {string} answerText - the last assistant text BEFORE the first
+ *   plugin-sourced user message (gate splice / injected complaint); equals
+ *   finalText when no plugin injection intervened ('' when none at all).
+ *   The "answer to the task", as opposed to the possibly-hijacked last message.
+ * @property {{ seq: number, source: object, text: string }[]} userMessages
+ *   - non-empty `user/message` events (task prompt, plugin steer, injected
+ *     context) with their verbatim `source` (`kind` + plugin-specific fields),
+ *     in log order. Steer has no dedicated event type; matchers filter by
+ *     `source`.
+ * @property {{ seq: number, reason: string, system: string, toolNames: string[] }[]} requestHeaders
+ *   - projected `request/header` events (assembled system prompt + mounted tools).
+ * @property {{ sessionId: string | undefined, parentSession: string | undefined, delegationDepth: number | undefined, label: string | undefined, mode: string | undefined, provider: string | undefined, assistantTexts: string[], finalText: string }[]} subagentChildren
+ *   - one record per subagent child log (`origin: 'subagent'` header, or a
+ *     header carrying `parentSession`). Identity comes from the first
+ *     version-3 `subagent/descriptor` event; `finalText` is the child's own
+ *     last assistant text ('' when it produced none — dispatched but not
+ *     answered).
+ * @property {string} finalText - the last assembled assistant text ('' when none).
+ */