@plaited/agent-eval-harness 0.5.0

package/src/core/loading.ts

@@ -0,0 +1,96 @@
+/**
+ * Shared loading utilities for JSONL files.
+ *
+ * @remarks
+ * Provides consistent loading and parsing of prompts and results files.
+ * Used by capture, trials, summarize, calibrate, and pipeline commands.
+ *
+ * @packageDocumentation
+ */
+
+import type { CaptureResult, PromptCase } from '../schemas.ts'
+import { CaptureResultSchema, PromptCaseSchema } from '../schemas.ts'
+
+/**
+ * Load prompts from a JSONL file.
+ *
+ * @remarks
+ * Each line in the file should be a valid JSON object matching PromptCaseSchema.
+ * Supports both single-turn (string input) and multi-turn (string[] input) formats.
+ *
+ * @param path - Path to the prompts.jsonl file
+ * @returns Parsed and validated prompt cases
+ * @throws Error if file cannot be read or any line is invalid
+ *
+ * @public
+ */
+export const loadPrompts = async (path: string): Promise<PromptCase[]> => {
+  const content = await Bun.file(path).text()
+  return content
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map((line, index) => {
+      try {
+        return PromptCaseSchema.parse(JSON.parse(line))
+      } catch (error) {
+        throw new Error(`Invalid prompt at line ${index + 1}: ${error instanceof Error ? error.message : error}`)
+      }
+    })
+}
+
+/**
+ * Load capture results from a JSONL file.
+ *
+ * @remarks
+ * Each line should be a valid JSON object matching CaptureResultSchema.
+ * Used by summarize, calibrate, and compare commands.
+ *
+ * @param path - Path to the results.jsonl file
+ * @returns Parsed and validated capture results
+ * @throws Error if file cannot be read or any line is invalid
+ *
+ * @public
+ */
+export const loadResults = async (path: string): Promise<CaptureResult[]> => {
+  const content = await Bun.file(path).text()
+  return content
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map((line, index) => {
+      try {
+        return CaptureResultSchema.parse(JSON.parse(line))
+      } catch (error) {
+        throw new Error(`Invalid result at line ${index + 1}: ${error instanceof Error ? error.message : error}`)
+      }
+    })
+}
+
+/**
+ * Load raw JSONL file as parsed JSON objects.
+ *
+ * @remarks
+ * Lower-level loading without schema validation.
+ * Useful for pipeline commands that need flexible input handling.
+ *
+ * @param path - Path to JSONL file
+ * @returns Array of parsed JSON objects
+ * @throws Error if file cannot be read or any line is invalid JSON
+ *
+ * @public
+ */
+export const loadJsonl = async <T = unknown>(path: string): Promise<T[]> => {
+  const content = await Bun.file(path).text()
+  return content
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map((line, index) => {
+      try {
+        return JSON.parse(line) as T
+      } catch (error) {
+        throw new Error(`Invalid JSON at line ${index + 1}: ${error instanceof Error ? error.message : error}`)
+      }
+    })
+}

package/src/core/output.ts

@@ -0,0 +1,121 @@
+/**
+ * Shared output utilities for writing results and logging.
+ *
+ * @remarks
+ * Provides consistent output handling across all commands:
+ * - Writing to stdout or files
+ * - Progress logging to stderr
+ * - Path resolution
+ * - Content preview (head/tail)
+ *
+ * @packageDocumentation
+ */
+
+import { appendFile } from 'node:fs/promises'
+import { HEAD_LINES, TAIL_LINES } from '../schemas/constants.ts'
+
+/**
+ * Write output line to stdout or file.
+ *
+ * @remarks
+ * When writing to a file, supports both overwrite and append modes.
+ * When writing to stdout, uses console.log.
+ *
+ * @param line - Content to write (without trailing newline)
+ * @param outputPath - Optional file path (stdout if undefined)
+ * @param append - If true, append to file instead of overwrite
+ *
+ * @public
+ */
+export const writeOutput = async (line: string, outputPath?: string, append?: boolean): Promise<void> => {
+  if (outputPath) {
+    if (append) {
+      await appendFile(outputPath, `${line}\n`)
+    } else {
+      await Bun.write(outputPath, `${line}\n`)
+    }
+  } else {
+    // biome-ignore lint/suspicious/noConsole: CLI stdout output
+    console.log(line)
+  }
+}
+
+/**
+ * Log progress message to stderr.
+ *
+ * @remarks
+ * Progress output goes to stderr to avoid polluting stdout
+ * when piping command output.
+ *
+ * @param message - Progress message to display
+ * @param showProgress - If false, message is suppressed
+ *
+ * @public
+ */
+export const logProgress = (message: string, showProgress: boolean): void => {
+  if (showProgress) {
+    console.error(message)
+  }
+}
+
+/**
+ * Resolve path relative to process.cwd().
+ *
+ * @remarks
+ * Absolute paths (starting with /) are returned as-is.
+ * Relative paths are joined with current working directory.
+ *
+ * @param path - Path to resolve
+ * @returns Absolute path
+ *
+ * @public
+ */
+export const resolvePath = (path: string): string => {
+  if (path.startsWith('/')) return path
+  return `${process.cwd()}/${path}`
+}
+
+/**
+ * Create head/tail preview of content.
+ *
+ * @remarks
+ * Shows first N and last M lines with omission indicator in between.
+ * Useful for large files/content in markdown output.
+ *
+ * @param content - Full content string
+ * @param headLines - Number of lines from start (default from constants)
+ * @param tailLines - Number of lines from end (default from constants)
+ * @returns Truncated content with omission indicator
+ *
+ * @public
+ */
+export const headTailPreview = (content: string, headLines = HEAD_LINES, tailLines = TAIL_LINES): string => {
+  const lines = content.split('\n')
+  if (lines.length <= headLines + tailLines) {
+    return content
+  }
+  const head = lines.slice(0, headLines).join('\n')
+  const tail = lines.slice(-tailLines).join('\n')
+  const omitted = lines.length - headLines - tailLines
+  return `${head}\n\n// ... ${omitted} lines omitted ...\n\n${tail}`
+}
+
+/**
+ * Get preview text for input (handles string or array).
+ *
+ * @remarks
+ * For arrays (multi-turn), shows turn count and preview of first turn.
+ * For strings, shows first 50 characters.
+ *
+ * @param input - String or array input
+ * @returns Preview text suitable for progress display
+ *
+ * @public
+ */
+export const getInputPreview = (input: string | string[]): string => {
+  if (Array.isArray(input)) {
+    const first = input[0] ?? ''
+    return `[${input.length} turns] ${first.slice(0, 40)}...`
+  }
+  return input.slice(0, 50)
+}

package/src/core/tests/core.spec.ts

@@ -0,0 +1,309 @@
+/**
+ * Unit tests for core utilities.
+ *
+ * @remarks
+ * Tests for shared utility functions in the core module:
+ * - loading: loadPrompts, loadResults, loadJsonl
+ * - trajectory: extractTrajectory, extractOutput, hasToolErrors
+ * - output: writeOutput, logProgress, headTailPreview
+ *
+ * @packageDocumentation
+ */
+
+import { afterEach, describe, expect, test } from 'bun:test'
+import { unlink, writeFile } from 'node:fs/promises'
+import type { ParsedUpdate } from '../../headless/headless-output-parser.ts'
+import { loadJsonl, loadPrompts, loadResults } from '../loading.ts'
+import { headTailPreview, resolvePath } from '../output.ts'
+import { detectTrajectoryRichness, extractOutput, extractTrajectory, hasToolErrors } from '../trajectory.ts'
+
+// ============================================================================
+// Loading Tests
+// ============================================================================
+
+describe('loadJsonl', () => {
+  const testFile = '/tmp/core-test-jsonl.jsonl'
+
+  afterEach(async () => {
+    try {
+      await unlink(testFile)
+    } catch {
+      // Ignore if file doesn't exist
+    }
+  })
+
+  test('loads and parses JSONL file', async () => {
+    await writeFile(testFile, '{"a":1}\n{"a":2}\n{"a":3}')
+    const results = await loadJsonl<{ a: number }>(testFile)
+    expect(results.length).toBe(3)
+    expect(results[0]?.a).toBe(1)
+    expect(results[2]?.a).toBe(3)
+  })
+
+  test('skips empty lines', async () => {
+    await writeFile(testFile, '{"a":1}\n\n{"a":2}\n')
+    const results = await loadJsonl<{ a: number }>(testFile)
+    expect(results.length).toBe(2)
+  })
+
+  test('handles empty file', async () => {
+    await writeFile(testFile, '')
+    const results = await loadJsonl(testFile)
+    expect(results.length).toBe(0)
+  })
+})
+
+describe('loadPrompts', () => {
+  const testFile = '/tmp/core-test-prompts.jsonl'
+
+  afterEach(async () => {
+    try {
+      await unlink(testFile)
+    } catch {
+      // Ignore
+    }
+  })
+
+  test('loads valid prompts', async () => {
+    await writeFile(testFile, '{"id":"p1","input":"hello"}\n{"id":"p2","input":"world"}')
+    const prompts = await loadPrompts(testFile)
+    expect(prompts.length).toBe(2)
+    expect(prompts[0]?.id).toBe('p1')
+    expect(prompts[0]?.input).toBe('hello')
+  })
+
+  test('loads multi-turn prompts', async () => {
+    await writeFile(testFile, '{"id":"m1","input":["turn1","turn2"]}')
+    const prompts = await loadPrompts(testFile)
+    expect(prompts.length).toBe(1)
+    expect(Array.isArray(prompts[0]?.input)).toBe(true)
+    expect((prompts[0]?.input as string[]).length).toBe(2)
+  })
+})
+
+describe('loadResults', () => {
+  const testFile = '/tmp/core-test-results.jsonl'
+
+  afterEach(async () => {
+    try {
+      await unlink(testFile)
+    } catch {
+      // Ignore
+    }
+  })
+
+  test('loads capture results with full schema', async () => {
+    const result = {
+      id: 'r1',
+      input: 'test',
+      output: 'result',
+      trajectory: [],
+      metadata: {},
+      toolErrors: false,
+      timing: {
+        start: 0,
+        end: 100,
+        total: 100,
+        sessionCreation: 10,
+      },
+    }
+    await writeFile(testFile, JSON.stringify(result))
+    const results = await loadResults(testFile)
+    expect(results.length).toBe(1)
+    expect(results[0]?.id).toBe('r1')
+    expect(results[0]?.output).toBe('result')
+  })
+})
+
+// ============================================================================
+// Trajectory Tests
+// ============================================================================
+
+describe('extractTrajectory', () => {
+  const startTime = 1000
+
+  test('extracts message updates', () => {
+    const updates: ParsedUpdate[] = [{ type: 'message', content: 'Hello', raw: {} }]
+    const trajectory = extractTrajectory(updates, startTime)
+    expect(trajectory.length).toBe(1)
+    expect(trajectory[0]?.type).toBe('message')
+    expect(trajectory[0]?.type === 'message' && trajectory[0]?.content).toBe('Hello')
+  })
+
+  test('extracts thought updates', () => {
+    const updates: ParsedUpdate[] = [{ type: 'thought', content: 'Thinking...', raw: {} }]
+    const trajectory = extractTrajectory(updates, startTime)
+    expect(trajectory.length).toBe(1)
+    expect(trajectory[0]?.type).toBe('thought')
+  })
+
+  test('extracts tool_call with title', () => {
+    const updates: ParsedUpdate[] = [
+      {
+        type: 'tool_call',
+        title: 'Read',
+        status: 'completed',
+        raw: {},
+      },
+    ]
+    const trajectory = extractTrajectory(updates, startTime)
+    expect(trajectory.length).toBe(1)
+    expect(trajectory[0]?.type).toBe('tool_call')
+    const step = trajectory[0]
+    if (step?.type === 'tool_call') {
+      expect(step.name).toBe('Read')
+    }
+  })
+
+  test('handles empty updates', () => {
+    const trajectory = extractTrajectory([], startTime)
+    expect(trajectory.length).toBe(0)
+  })
+})
+
+describe('extractOutput', () => {
+  test('concatenates all message content', () => {
+    const trajectory = [
+      { type: 'thought' as const, content: 'Thinking', timestamp: 50 },
+      { type: 'message' as const, content: 'First message', timestamp: 100 },
+      { type: 'message' as const, content: 'Final answer', timestamp: 150 },
+    ]
+    const output = extractOutput(trajectory)
+    // extractOutput joins all messages with newline
+    expect(output).toBe('First message\nFinal answer')
+  })
+
+  test('returns empty string when no messages', () => {
+    const trajectory = [{ type: 'thought' as const, content: 'Thinking only', timestamp: 50 }]
+    const output = extractOutput(trajectory)
+    expect(output).toBe('')
+  })
+
+  test('handles empty trajectory', () => {
+    const output = extractOutput([])
+    expect(output).toBe('')
+  })
+})
+
+describe('hasToolErrors', () => {
+  test('returns false for successful tool calls', () => {
+    const trajectory = [
+      {
+        type: 'tool_call' as const,
+        name: 'Read',
+        status: 'completed',
+        timestamp: 100,
+      },
+    ]
+    expect(hasToolErrors(trajectory)).toBe(false)
+  })
+
+  test('returns true for failed status', () => {
+    const trajectory = [
+      {
+        type: 'tool_call' as const,
+        name: 'Read',
+        status: 'failed',
+        timestamp: 100,
+      },
+    ]
+    // hasToolErrors checks for status === 'failed'
+    expect(hasToolErrors(trajectory)).toBe(true)
+  })
+
+  test('returns false for error status (not failed)', () => {
+    // The implementation checks for 'failed', not 'error'
+    const trajectory = [
+      {
+        type: 'tool_call' as const,
+        name: 'Read',
+        status: 'error',
+        timestamp: 100,
+      },
+    ]
+    expect(hasToolErrors(trajectory)).toBe(false)
+  })
+
+  test('returns false for empty trajectory', () => {
+    expect(hasToolErrors([])).toBe(false)
+  })
+})
+
+describe('detectTrajectoryRichness', () => {
+  test('returns full when has thoughts', () => {
+    const trajectory = [
+      { type: 'thought' as const, content: 'Let me think', timestamp: 50 },
+      { type: 'message' as const, content: 'Done', timestamp: 150 },
+    ]
+    expect(detectTrajectoryRichness(trajectory)).toBe('full')
+  })
+
+  test('returns full when has tool_calls', () => {
+    const trajectory = [
+      {
+        type: 'tool_call' as const,
+        name: 'Read',
+        status: 'completed',
+        timestamp: 100,
+      },
+      { type: 'message' as const, content: 'Done', timestamp: 150 },
+    ]
+    // Any tool_call means 'full'
+    expect(detectTrajectoryRichness(trajectory)).toBe('full')
+  })
+
+  test('returns messages-only when only messages', () => {
+    const trajectory = [{ type: 'message' as const, content: 'Just a message', timestamp: 100 }]
+    expect(detectTrajectoryRichness(trajectory)).toBe('messages-only')
+  })
+
+  test('returns minimal for empty trajectory', () => {
+    // Empty trajectory returns 'minimal', not 'messages-only'
+    expect(detectTrajectoryRichness([])).toBe('minimal')
+  })
+})
+
+// ============================================================================
+// Output Tests
+// ============================================================================
+
+describe('headTailPreview', () => {
+  test('returns full content when short', () => {
+    const content = 'line1\nline2\nline3'
+    const preview = headTailPreview(content, 5, 3)
+    expect(preview).toBe(content)
+  })
+
+  test('truncates long content with omission indicator', () => {
+    const lines = Array.from({ length: 20 }, (_, i) => `line${i + 1}`).join('\n')
+    const preview = headTailPreview(lines, 3, 2)
+
+    expect(preview).toContain('line1')
+    expect(preview).toContain('line2')
+    expect(preview).toContain('line3')
+    // Actual format uses "// ... N lines omitted ..."
+    expect(preview).toContain('// ... 15 lines omitted ...')
+    expect(preview).toContain('line19')
+    expect(preview).toContain('line20')
+  })
+
+  test('handles exact boundary', () => {
+    const lines = 'line1\nline2\nline3\nline4\nline5'
+    const preview = headTailPreview(lines, 3, 2)
+    // 5 lines is exactly head(3) + tail(2), no truncation needed
+    expect(preview).toBe(lines)
+  })
+})
+
+describe('resolvePath', () => {
+  test('resolves relative path from cwd', () => {
+    const resolved = resolvePath('./test.txt')
+    expect(resolved.endsWith('test.txt')).toBe(true)
+    expect(resolved.startsWith('/')).toBe(true)
+  })
+
+  test('returns absolute path unchanged', () => {
+    const path = '/absolute/path/file.txt'
+    expect(resolvePath(path)).toBe(path)
+  })
+})

package/src/core/trajectory.ts

@@ -0,0 +1,166 @@
+/**
+ * Shared trajectory utilities for extraction and analysis.
+ *
+ * @remarks
+ * Provides functions for extracting trajectory data from parsed updates,
+ * detecting richness levels, and checking for tool errors.
+ *
+ * @packageDocumentation
+ */
+
+import type { ParsedUpdate } from '../headless/headless-output-parser.ts'
+import type { TrajectoryRichness, TrajectoryStep } from '../schemas.ts'
+import { ToolInputSchema } from '../schemas.ts'
+
+/**
+ * Extract trajectory from parsed updates.
+ *
+ * @remarks
+ * Converts ParsedUpdate stream into TrajectoryStep array.
+ * Handles tool call deduplication (start/completion events).
+ *
+ * @param updates - Parsed updates from output parser
+ * @param startTime - Reference time for timestamp calculation
+ * @returns Array of trajectory steps with relative timestamps
+ *
+ * @public
+ */
+export const extractTrajectory = (updates: ParsedUpdate[], startTime: number): TrajectoryStep[] => {
+  const trajectory: TrajectoryStep[] = []
+  const toolCallMap = new Map<string, { start: number; step: TrajectoryStep & { type: 'tool_call' } }>()
+
+  for (const update of updates) {
+    const timestamp = Date.now() - startTime
+
+    if (update.type === 'thought') {
+      trajectory.push({
+        type: 'thought',
+        content: update.content ?? '',
+        timestamp,
+      })
+    } else if (update.type === 'message') {
+      trajectory.push({
+        type: 'message',
+        content: update.content ?? '',
+        timestamp,
+      })
+    } else if (update.type === 'tool_call') {
+      const toolCallId = update.title ?? `tool_${Date.now()}`
+      const existing = toolCallMap.get(toolCallId)
+
+      if (existing && update.status === 'completed') {
+        // Update existing tool call with completion info
+        existing.step.status = update.status
+        existing.step.duration = timestamp - existing.start
+      } else if (!existing) {
+        // New tool call
+        const step: TrajectoryStep & { type: 'tool_call' } = {
+          type: 'tool_call',
+          name: update.title ?? 'unknown',
+          status: update.status ?? 'pending',
+          timestamp,
+        }
+        toolCallMap.set(toolCallId, { start: timestamp, step })
+        trajectory.push(step)
+      }
+    } else if (update.type === 'plan') {
+      trajectory.push({
+        type: 'plan',
+        entries: [],
+        timestamp,
+      })
+    }
+  }
+
+  return trajectory
+}
+
+/**
+ * Extract final text output from trajectory.
+ *
+ * @remarks
+ * Concatenates all message step content to produce final output string.
+ *
+ * @param trajectory - Trajectory steps from capture
+ * @returns Concatenated message content
+ *
+ * @public
+ */
+export const extractOutput = (trajectory: TrajectoryStep[]): string => {
+  return trajectory
+    .filter((step): step is TrajectoryStep & { type: 'message' } => step.type === 'message')
+    .map((step) => step.content)
+    .join('\n')
+}
+
+/**
+ * Check if any tool calls failed in trajectory.
+ *
+ * @param trajectory - Trajectory steps from capture
+ * @returns True if any tool call has 'failed' status
+ *
+ * @public
+ */
+export const hasToolErrors = (trajectory: TrajectoryStep[]): boolean => {
+  return trajectory.some((step) => step.type === 'tool_call' && step.status === 'failed')
+}
+
+/**
+ * Detect trajectory richness level from captured steps.
+ *
+ * @remarks
+ * Different adapters provide varying levels of detail:
+ * - `full`: Has thoughts, tool calls, or plans (e.g., Claude Code)
+ * - `messages-only`: Only message steps present
+ * - `minimal`: Empty or unknown content
+ *
+ * Uses single-pass iteration with early exit for efficiency.
+ *
+ * @param trajectory - Trajectory steps from capture
+ * @returns Detected richness level
+ *
+ * @public
+ */
+export const detectTrajectoryRichness = (trajectory: TrajectoryStep[]): TrajectoryRichness => {
+  let hasMessages = false
+
+  for (const step of trajectory) {
+    // Early exit: any of these means 'full' richness
+    if (step.type === 'thought' || step.type === 'tool_call' || step.type === 'plan') {
+      return 'full'
+    }
+    if (step.type === 'message') {
+      hasMessages = true
+    }
+  }
+
+  return hasMessages ? 'messages-only' : 'minimal'
+}
+
+/**
+ * Extract file path from tool input if present.
+ *
+ * @param input - Tool call input object
+ * @returns File path string or undefined
+ *
+ * @public
+ */
+export const extractFilePath = (input: unknown): string | undefined => {
+  const result = ToolInputSchema.safeParse(input)
+  if (!result.success) return undefined
+  return result.data.file_path ?? result.data.path
+}
+
+/**
+ * Extract content from tool input if present.
+ *
+ * @param input - Tool call input object
+ * @returns Content string or undefined
+ *
+ * @public
+ */
+export const extractContent = (input: unknown): string | undefined => {
+  const result = ToolInputSchema.safeParse(input)
+  if (!result.success) return undefined
+  return result.data.content ?? result.data.new_string
+}