@plaited/agent-eval-harness 0.5.0

package/src/headless.ts

@@ -0,0 +1,72 @@
+/**
+ * Headless adapter factory - schema-driven adapter for any CLI agent.
+ *
+ * @remarks
+ * Re-exports public API from the headless module. The headless adapter enables
+ * capturing trajectories from ANY headless CLI agent by defining a schema
+ * that describes how to interact with the CLI.
+ *
+ * **CLI Usage:**
+ * ```bash
+ * agent-eval-harness headless --schema ./my-agent.json
+ * ```
+ *
+ * **Programmatic Usage:**
+ * ```typescript
+ * import { parseHeadlessConfig, createSessionManager } from '@plaited/agent-eval-harness/headless'
+ *
+ * const schema = parseHeadlessConfig(jsonConfig)
+ * const sessions = createSessionManager({ schema })
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+// Schema definitions and parsing
+export {
+  HeadlessAdapterSchema,
+  OutputConfigSchema,
+  OutputEventExtractSchema,
+  OutputEventMappingSchema,
+  OutputEventMatchSchema,
+  PromptConfigSchema,
+  parseHeadlessConfig,
+  ResultConfigSchema,
+  ResumeConfigSchema,
+  safeParseHeadlessConfig,
+} from './headless/headless.schemas.ts'
+// Types
+export type {
+  HeadlessAdapterConfig,
+  OutputConfig,
+  OutputEventExtract,
+  OutputEventMapping,
+  OutputEventMatch,
+  PromptConfig,
+  ResultConfig,
+  ResumeConfig,
+} from './headless/headless.types.ts'
+// CLI entry point
+export { headless } from './headless/headless-cli.ts'
+export type { HistoryBuilder, HistoryBuilderConfig, HistoryTurn } from './headless/headless-history-builder.ts'
+// History builder
+export { createHistoryBuilder } from './headless/headless-history-builder.ts'
+export type {
+  OutputParser,
+  ParsedResult,
+  ParsedUpdate,
+  ResultParseResult,
+  SessionUpdateType,
+} from './headless/headless-output-parser.ts'
+// Output parser
+export { createOutputParser, jsonPath, jsonPathString } from './headless/headless-output-parser.ts'
+export type {
+  ProcessExitInfo,
+  PromptResult,
+  Session,
+  SessionManager,
+  SessionManagerConfig,
+  UpdateCallback,
+} from './headless/headless-session-manager.ts'
+// Session manager
+export { createSessionManager } from './headless/headless-session-manager.ts'

package/src/integration_tests/claude.spec.ts

@@ -0,0 +1,157 @@
+/**
+ * Integration tests for Claude Code headless adapter.
+ *
+ * @remarks
+ * Tests verify the headless session manager works correctly with Claude Code CLI
+ * using the schema-driven approach from `.claude/skills/headless-adapters/schemas/`.
+ *
+ * Run locally with API key:
+ * ```bash
+ * ANTHROPIC_API_KEY=sk-... bun test ./src/integration_tests/claude.spec.ts
+ * ```
+ *
+ * Prerequisites:
+ * 1. Claude CLI installed (`curl -fsSL https://claude.ai/install.sh | bash`)
+ * 2. API key: `ANTHROPIC_API_KEY` environment variable
+ *
+ * These tests make real API calls and consume credits.
+ */
+
+import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from 'bun:test'
+import { join } from 'node:path'
+import { parseHeadlessConfig } from '../headless/headless.schemas.ts'
+import { createSessionManager } from '../headless/headless-session-manager.ts'
+
+// Long timeout for real agent interactions (2 minutes)
+setDefaultTimeout(120000)
+
+// Use project root as cwd - agents discover MCP servers from config files
+const PROJECT_ROOT = process.cwd()
+
+// Schema path for Claude headless adapter
+const SCHEMA_PATH = join(PROJECT_ROOT, '.claude/skills/headless-adapters/schemas/claude-headless.json')
+
+// Get API key from environment
+const API_KEY = process.env.ANTHROPIC_API_KEY ?? ''
+
+// Skip all tests if no API key is available
+const describeWithApiKey = API_KEY ? describe : describe.skip
+
+describeWithApiKey('Claude Code Integration', () => {
+  let sessionManager: ReturnType<typeof createSessionManager>
+  let schemaConfig: ReturnType<typeof parseHeadlessConfig>
+
+  beforeAll(async () => {
+    // Load JSON from file, then parse with Zod schema
+    const schemaJson = await Bun.file(SCHEMA_PATH).json()
+    schemaConfig = parseHeadlessConfig(schemaJson)
+
+    // Create session manager with the schema
+    sessionManager = createSessionManager({
+      schema: schemaConfig,
+      timeout: 120000,
+      debug: false,
+    })
+  })
+
+  afterAll(async () => {
+    // Cleanup handled automatically by session manager
+  })
+
+  test('creates session successfully', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    expect(session).toBeDefined()
+    expect(session.id).toBeDefined()
+    expect(typeof session.id).toBe('string')
+    expect(session.active).toBe(true)
+    expect(session.cwd).toBe(PROJECT_ROOT)
+  })
+
+  test('sends prompt and receives response', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    // Simple prompt that doesn't require tools
+    const result = await sessionManager.prompt(session.id, 'What is 2 + 2? Reply with just the number.')
+
+    expect(result).toBeDefined()
+    expect(result.output).toBeDefined()
+    expect(result.output.length).toBeGreaterThan(0)
+    expect(result.updates).toBeInstanceOf(Array)
+
+    // Should contain "4" somewhere in the response
+    expect(result.output).toMatch(/4/)
+  })
+
+  test('collects trajectory updates during execution', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+    const collectedUpdates: unknown[] = []
+
+    const result = await sessionManager.prompt(session.id, 'Say "hello" and nothing else.', (update) => {
+      collectedUpdates.push(update)
+    })
+
+    expect(result.updates.length).toBeGreaterThan(0)
+
+    // Should have at least one message update
+    const messageUpdates = result.updates.filter((u) => u.type === 'message')
+    expect(messageUpdates.length).toBeGreaterThan(0)
+  })
+
+  test('uses MCP server from project config', async () => {
+    // This test verifies that Claude discovers MCP servers from .mcp.json
+    // The bun-docs MCP server is configured at project root
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    // Query the bun-docs MCP server (configured in .mcp.json)
+    const result = await sessionManager.prompt(
+      session.id,
+      'Use the bun-docs MCP server to search for information about Bun.serve(). ' +
+        'What are the key options for creating an HTTP server with Bun?',
+    )
+
+    // Response should contain Bun server-related information
+    expect(result.output.length).toBeGreaterThan(0)
+    // Should mention server/HTTP-related concepts from Bun docs
+    expect(result.output.toLowerCase()).toMatch(/serve|server|http|port|fetch|handler/)
+  })
+
+  test('multi-turn conversation maintains context (stream mode)', async () => {
+    // Multi-turn: multiple prompts to same session
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    // Turn 1: Establish context
+    const turn1Result = await sessionManager.prompt(session.id, 'Remember this number: 42. Just confirm you have it.')
+    expect(turn1Result.output).toMatch(/42|forty.?two|remember/i)
+
+    // Turn 2: Reference previous context
+    const turn2Result = await sessionManager.prompt(
+      session.id,
+      'What number did I ask you to remember? Reply with just the number.',
+    )
+    expect(turn2Result.output).toMatch(/42/)
+  })
+
+  test('receives valid trajectory updates', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    // Prompt that generates a response with trajectory updates
+    const result = await sessionManager.prompt(
+      session.id,
+      'What programming language is this project written in? Look at the file extensions.',
+    )
+
+    // Result should have output
+    expect(result.output).toBeDefined()
+    expect(result.output.length).toBeGreaterThan(0)
+
+    // Should have collected updates during execution
+    expect(result.updates).toBeInstanceOf(Array)
+    expect(result.updates.length).toBeGreaterThan(0)
+
+    // All updates should have valid types
+    const validTypes = ['thought', 'tool_call', 'message', 'plan']
+    const allValidTypes = result.updates.every((u) => validTypes.includes(u.type))
+    expect(allValidTypes).toBe(true)
+  })
+})

package/src/integration_tests/gemini.spec.ts

@@ -0,0 +1,139 @@
+/**
+ * Integration tests for Gemini CLI headless adapter.
+ *
+ * @remarks
+ * Tests verify the headless session manager works correctly with Gemini CLI
+ * using the schema-driven approach from `.claude/skills/headless-adapters/schemas/`.
+ *
+ * Run locally with API key:
+ * ```bash
+ * GEMINI_API_KEY=... bun test ./src/integration_tests/gemini.spec.ts
+ * ```
+ *
+ * Prerequisites:
+ * 1. Gemini CLI installed (`npm install -g @google/gemini-cli`)
+ * 2. API key: `GEMINI_API_KEY` environment variable
+ *
+ * These tests make real API calls and consume credits.
+ */
+
+import { afterAll, beforeAll, describe, expect, setDefaultTimeout, test } from 'bun:test'
+import { join } from 'node:path'
+import { parseHeadlessConfig } from '../headless/headless.schemas.ts'
+import { createSessionManager } from '../headless/headless-session-manager.ts'
+
+// Long timeout for real agent interactions (2 minutes)
+setDefaultTimeout(120000)
+
+// Use project root as cwd - agents discover MCP servers from config files
+const PROJECT_ROOT = process.cwd()
+
+// Schema path for Gemini headless adapter
+const SCHEMA_PATH = join(PROJECT_ROOT, '.claude/skills/headless-adapters/schemas/gemini-headless.json')
+
+// Get API key from environment
+const GEMINI_API_KEY = process.env.GEMINI_API_KEY ?? ''
+
+// Skip all tests if no API key is available
+const describeWithApiKey = GEMINI_API_KEY ? describe : describe.skip
+
+describeWithApiKey('Gemini CLI Integration', () => {
+  let sessionManager: ReturnType<typeof createSessionManager>
+  let schemaConfig: ReturnType<typeof parseHeadlessConfig>
+
+  beforeAll(async () => {
+    // Load JSON from file, then parse with Zod schema
+    const schemaJson = await Bun.file(SCHEMA_PATH).json()
+    schemaConfig = parseHeadlessConfig(schemaJson)
+
+    // Create session manager with the schema
+    sessionManager = createSessionManager({
+      schema: schemaConfig,
+      timeout: 120000,
+      debug: false,
+    })
+  })
+
+  afterAll(async () => {
+    // Cleanup handled automatically by session manager
+  })
+
+  test('creates session successfully', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    expect(session).toBeDefined()
+    expect(session.id).toBeDefined()
+    expect(typeof session.id).toBe('string')
+    expect(session.active).toBe(true)
+    expect(session.cwd).toBe(PROJECT_ROOT)
+  })
+
+  test('sends prompt and receives response', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    // Simple prompt that doesn't require tools
+    const result = await sessionManager.prompt(session.id, 'What is 2 + 2? Reply with just the number.')
+
+    expect(result).toBeDefined()
+    expect(result.output).toBeDefined()
+    expect(result.output.length).toBeGreaterThan(0)
+    expect(result.updates).toBeInstanceOf(Array)
+
+    // Should contain "4" somewhere in the response
+    expect(result.output).toMatch(/4/)
+  })
+
+  test('collects trajectory updates during execution', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+    const collectedUpdates: unknown[] = []
+
+    const result = await sessionManager.prompt(session.id, 'Say "hello" and nothing else.', (update) => {
+      collectedUpdates.push(update)
+    })
+
+    expect(result.updates.length).toBeGreaterThan(0)
+
+    // Should have at least one message update
+    const messageUpdates = result.updates.filter((u) => u.type === 'message')
+    expect(messageUpdates.length).toBeGreaterThan(0)
+  })
+
+  test('multi-turn conversation maintains context (iterative mode)', async () => {
+    // Multi-turn via headless adapter in iterative mode (history accumulation)
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    // Turn 1: Establish context
+    const turn1Result = await sessionManager.prompt(session.id, 'Remember this number: 42. Just confirm you have it.')
+    expect(turn1Result.output).toMatch(/42|forty.?two|remember/i)
+
+    // Turn 2: Reference previous context
+    const turn2Result = await sessionManager.prompt(
+      session.id,
+      'What number did I ask you to remember? Reply with just the number.',
+    )
+    expect(turn2Result.output).toMatch(/42/)
+  })
+
+  test('handles simple math question correctly', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    const result = await sessionManager.prompt(session.id, 'Calculate 15 * 7. Reply with just the number.')
+
+    // Gemini CLI may include formatting variations (newlines, spaces)
+    // Strip whitespace to verify the correct answer is present
+    expect(result.output.replace(/\s/g, '')).toContain('105')
+  })
+
+  test('processes longer response without timeout', async () => {
+    const session = await sessionManager.create(PROJECT_ROOT)
+
+    const result = await sessionManager.prompt(
+      session.id,
+      'List 5 programming languages and one key feature of each. Be brief.',
+    )
+
+    expect(result.output.length).toBeGreaterThan(50)
+    // Should mention at least some programming languages
+    expect(result.output.toLowerCase()).toMatch(/python|javascript|java|rust|go|typescript|c\+\+|ruby/)
+  })
+})

package/src/pipeline/compare.ts

@@ -0,0 +1,325 @@
+/**
+ * Pipeline compare command - compare multiple runs of the same prompts.
+ *
+ * @remarks
+ * Compares results from different configurations (agents, MCP servers, models)
+ * using a user-provided comparison grader that ranks the runs.
+ *
+ * Terminology: "runs" (not "agents") because comparisons can be:
+ * - Same agent, different MCP servers
+ * - Same agent, different skills enabled
+ * - Same agent, different system prompts
+ * - Same agent, different model versions
+ * - Different agents entirely
+ *
+ * @packageDocumentation
+ */
+
+import { basename, extname } from 'node:path'
+import { parseArgs } from 'node:util'
+import { loadResults, logProgress, writeOutput } from '../core.ts'
+import type { CaptureResult } from '../schemas.ts'
+import type {
+  CompareConfig,
+  ComparisonGrader,
+  ComparisonGraderInput,
+  ComparisonResult,
+  LabeledRun,
+} from './pipeline.types.ts'
+
+/**
+ * Load comparison grader from file.
+ *
+ * @remarks
+ * Similar to loadGrader but expects ComparisonGrader interface.
+ *
+ * @param path - Path to grader module
+ * @returns Loaded comparison grader function
+ */
+const loadComparisonGrader = async (path: string): Promise<ComparisonGrader> => {
+  const module = await import(path)
+
+  if (typeof module.grade === 'function') {
+    return module.grade as ComparisonGrader
+  }
+  if (typeof module.default === 'function') {
+    return module.default as ComparisonGrader
+  }
+  if (typeof module.compare === 'function') {
+    return module.compare as ComparisonGrader
+  }
+
+  throw new Error(`Comparison grader must export 'grade', 'compare', or 'default' function`)
+}
+
+/**
+ * Derive label from file path.
+ *
+ * @param path - File path
+ * @returns Label derived from filename without extension
+ */
+const labelFromPath = (path: string): string => {
+  const base = basename(path)
+  const ext = extname(base)
+  return base.slice(0, -ext.length)
+}
+
+/**
+ * Parse labeled run argument.
+ *
+ * @remarks
+ * Supports formats:
+ * - "path.jsonl" - label derived from filename
+ * - "label:path.jsonl" - explicit label
+ *
+ * @param arg - Run argument string
+ * @returns Labeled run object
+ */
+const parseLabeledRun = (arg: string): LabeledRun => {
+  const colonIndex = arg.indexOf(':')
+
+  // Check if this looks like a label:path format (not a Windows drive letter)
+  if (colonIndex > 0 && colonIndex !== 1) {
+    return {
+      label: arg.slice(0, colonIndex),
+      path: arg.slice(colonIndex + 1),
+    }
+  }
+
+  return {
+    label: labelFromPath(arg),
+    path: arg,
+  }
+}
+
+/**
+ * Execute pipeline compare with configuration.
+ *
+ * @param config - Compare configuration
+ */
+export const runCompare = async (config: CompareConfig): Promise<void> => {
+  const { runs, graderPath, outputPath, progress = false } = config
+
+  if (runs.length < 2) {
+    throw new Error('At least 2 runs required for comparison')
+  }
+
+  // Load comparison grader
+  const grader = await loadComparisonGrader(graderPath)
+
+  logProgress(`Comparing ${runs.length} runs with: ${graderPath}`, progress)
+  for (const run of runs) {
+    logProgress(`  - ${run.label}: ${run.path}`, progress)
+  }
+
+  // Load all runs
+  const runResults: Record<string, CaptureResult[]> = {}
+  for (const run of runs) {
+    logProgress(`Loading ${run.label}...`, progress)
+    runResults[run.label] = await loadResults(run.path)
+  }
+
+  // Build map of prompt IDs to runs
+  const promptIds = new Set<string>()
+  for (const results of Object.values(runResults)) {
+    for (const result of results) {
+      promptIds.add(result.id)
+    }
+  }
+
+  logProgress(`Comparing ${promptIds.size} prompts...`, progress)
+
+  let isFirstOutput = true
+
+  // Clear output file if specified
+  if (outputPath) {
+    await Bun.write(outputPath, '')
+  }
+
+  const results: ComparisonResult[] = []
+
+  for (const promptId of promptIds) {
+    logProgress(`  ${promptId}`, progress)
+
+    // Build comparison input
+    const runsData: ComparisonGraderInput['runs'] = {}
+    let input: string | string[] = ''
+    let hint: string | undefined
+
+    for (const [label, labelResults] of Object.entries(runResults)) {
+      const result = labelResults.find((r) => r.id === promptId)
+      if (result) {
+        runsData[label] = {
+          output: result.output,
+          trajectory: result.trajectory,
+        }
+        // Use first found input/hint as the reference
+        if (!input) {
+          input = result.input
+          hint = result.hint
+        }
+      }
+    }
+
+    // Skip if not present in at least 2 runs
+    if (Object.keys(runsData).length < 2) {
+      logProgress(`    Skipped (only in ${Object.keys(runsData).length} run)`, progress)
+      continue
+    }
+
+    // Apply comparison grader
+    const graderInput: ComparisonGraderInput = {
+      id: promptId,
+      input,
+      hint,
+      runs: runsData,
+    }
+
+    const graderResult = await grader(graderInput)
+
+    const comparisonResult: ComparisonResult = {
+      id: promptId,
+      input,
+      hint,
+      rankings: graderResult.rankings,
+      reasoning: graderResult.reasoning,
+    }
+
+    results.push(comparisonResult)
+
+    // Log winner
+    const winner = graderResult.rankings.find((r) => r.rank === 1)
+    if (winner) {
+      logProgress(`    Winner: ${winner.run} (${winner.score.toFixed(2)})`, progress)
+    }
+
+    await writeOutput(JSON.stringify(comparisonResult), outputPath, !isFirstOutput)
+    isFirstOutput = false
+  }
+
+  // Summary statistics
+  logProgress('', progress)
+  logProgress('=== Summary ===', progress)
+
+  const winCounts: Record<string, number> = {}
+  for (const run of runs) {
+    winCounts[run.label] = 0
+  }
+
+  for (const result of results) {
+    const winner = result.rankings.find((r) => r.rank === 1)
+    if (winner && winner.run in winCounts) {
+      const currentCount = winCounts[winner.run] ?? 0
+      winCounts[winner.run] = currentCount + 1
+    }
+  }
+
+  for (const [label, wins] of Object.entries(winCounts)) {
+    const pct = ((wins / results.length) * 100).toFixed(1)
+    logProgress(`  ${label}: ${wins} wins (${pct}%)`, progress)
+  }
+
+  logProgress('Done!', progress)
+}
+
+/**
+ * Pipeline compare command CLI handler.
+ *
+ * @param args - Command line arguments (after 'compare')
+ */
+export const compare = async (args: string[]): Promise<void> => {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      run: { type: 'string', multiple: true },
+      grader: { type: 'string', short: 'g' },
+      output: { type: 'string', short: 'o' },
+      progress: { type: 'boolean', default: false },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: true,
+  })
+
+  if (values.help) {
+    // biome-ignore lint/suspicious/noConsole: CLI help output
+    console.log(`
+Usage: agent-eval-harness compare [files...] --grader <grader> [options]
+
+Compare multiple runs of the same prompts.
+
+Arguments:
+  files...          Result files to compare (positional, unlimited)
+
+Options:
+  --run             Labeled run format: "label:path.jsonl" (alternative to positional)
+  -g, --grader      Path to comparison grader (.ts/.js module) (required)
+  -o, --output      Output file (default: stdout)
+  --progress        Show progress to stderr
+  -h, --help        Show this help message
+
+Comparison Grader:
+  Must export 'grade' or 'compare' function with signature:
+    (params: ComparisonGraderInput) => Promise<ComparisonGraderResult>
+
+  Input includes all runs' results for a single prompt.
+  Output should rank runs from best to worst.
+
+Examples:
+  # Compare multiple result files (positional)
+  agent-eval-harness compare run1.jsonl run2.jsonl run3.jsonl -g ./compare-grader.ts
+
+  # With explicit labels
+  agent-eval-harness compare \\
+    --run "with-bun-mcp:results-bun.jsonl" \\
+    --run "vanilla:results-vanilla.jsonl" \\
+    -g ./compare-grader.ts
+
+  # Mix positional and labeled
+  agent-eval-harness compare results-*.jsonl \\
+    --run "baseline:baseline.jsonl" \\
+    -g ./compare-grader.ts -o comparison.jsonl
+
+  # Typical workflow
+  # 1. Capture with different configs
+  agent-eval-harness capture prompts.jsonl -s claude.json -o vanilla.jsonl
+  agent-eval-harness capture prompts.jsonl -s claude-with-mcp.json -o with-mcp.jsonl
+
+  # 2. Compare results
+  agent-eval-harness compare vanilla.jsonl with-mcp.jsonl -g ./compare-grader.ts
+`)
+    return
+  }
+
+  if (!values.grader) {
+    console.error('Error: --grader is required')
+    process.exit(1)
+  }
+
+  // Collect runs from positional args and --run flags
+  const runs: LabeledRun[] = []
+
+  // Positional arguments (file paths)
+  for (const arg of positionals) {
+    runs.push(parseLabeledRun(arg))
+  }
+
+  // --run flags
+  if (values.run) {
+    for (const arg of values.run) {
+      runs.push(parseLabeledRun(arg))
+    }
+  }
+
+  if (runs.length < 2) {
+    console.error('Error: At least 2 result files required for comparison')
+    console.error('Example: agent-eval-harness compare run1.jsonl run2.jsonl -g ./grader.ts')
+    process.exit(1)
+  }
+
+  await runCompare({
+    runs,
+    graderPath: values.grader,
+    outputPath: values.output,
+    progress: values.progress,
+  })
+}