@plaited/agent-eval-harness 0.5.0

package/src/pipeline/extract.ts

@@ -0,0 +1,241 @@
+/**
+ * Pipeline extract command - parse raw output into trajectories.
+ *
+ * @remarks
+ * Converts RawOutput from `run` command into ExtractedResult with
+ * parsed trajectory and final output. Uses the same schema-driven
+ * parsing as the capture command.
+ *
+ * @packageDocumentation
+ */
+
+import { parseArgs } from 'node:util'
+import { loadJsonl, logProgress, writeOutput } from '../core.ts'
+import { parseHeadlessConfig } from '../headless/headless.schemas.ts'
+import { createOutputParser } from '../headless/headless-output-parser.ts'
+import type { TrajectoryStep } from '../schemas.ts'
+import type { ExtractedResult, RawOutput } from './pipeline.types.ts'
+
+/**
+ * Extract trajectory from raw output using schema parser.
+ *
+ * @param rawOutput - Raw output from run command
+ * @param parser - Output parser created from schema
+ * @returns Extracted result with trajectory
+ */
+const extractFromRaw = (rawOutput: RawOutput, parser: ReturnType<typeof createOutputParser>): ExtractedResult => {
+  const trajectory: TrajectoryStep[] = []
+  let finalOutput = ''
+  let toolErrors = false
+
+  // Parse each raw line
+  for (const line of rawOutput.rawLines) {
+    // Try to parse as trajectory update
+    const parsed = parser.parseLine(line)
+    if (parsed) {
+      const updates = Array.isArray(parsed) ? parsed : [parsed]
+      for (const update of updates) {
+        const timestamp = Date.now() - rawOutput.timing.start
+
+        if (update.type === 'thought') {
+          trajectory.push({
+            type: 'thought',
+            content: update.content ?? '',
+            timestamp,
+          })
+        } else if (update.type === 'message') {
+          trajectory.push({
+            type: 'message',
+            content: update.content ?? '',
+            timestamp,
+          })
+        } else if (update.type === 'tool_call') {
+          trajectory.push({
+            type: 'tool_call',
+            name: update.title ?? 'unknown',
+            status: update.status ?? 'pending',
+            timestamp,
+          })
+          if (update.status === 'failed') {
+            toolErrors = true
+          }
+        } else if (update.type === 'plan') {
+          trajectory.push({
+            type: 'plan',
+            entries: [],
+            timestamp,
+          })
+        }
+      }
+    }
+
+    // Try to parse as result
+    const result = parser.parseResult(line)
+    if (result.isResult) {
+      finalOutput = result.content
+    }
+  }
+
+  // If no explicit result, extract from messages
+  if (!finalOutput) {
+    finalOutput = trajectory
+      .filter((step): step is TrajectoryStep & { type: 'message' } => step.type === 'message')
+      .map((step) => step.content)
+      .join('\n')
+  }
+
+  return {
+    id: rawOutput.id,
+    input: rawOutput.input,
+    hint: rawOutput.hint,
+    output: finalOutput,
+    trajectory,
+    toolErrors: toolErrors || !!rawOutput.error,
+    timing: rawOutput.timing,
+    ...(rawOutput.error && { error: rawOutput.error }),
+  }
+}
+
+/**
+ * Execute pipeline extract with configuration.
+ *
+ * @param schemaPath - Path to headless adapter schema
+ * @param rawOutputs - Raw outputs from run command
+ * @param outputPath - Optional output file path
+ * @param progress - Show progress to stderr
+ */
+export const runExtract = async (
+  schemaPath: string,
+  rawOutputs: RawOutput[],
+  outputPath?: string,
+  progress = false,
+): Promise<void> => {
+  // Load and validate schema
+  const schemaFile = Bun.file(schemaPath)
+  if (!(await schemaFile.exists())) {
+    throw new Error(`Schema file not found: ${schemaPath}`)
+  }
+
+  const rawSchema = await schemaFile.json()
+  const schema = parseHeadlessConfig(rawSchema)
+  const parser = createOutputParser(schema)
+
+  logProgress(`Extracting with schema: ${schema.name}`, progress)
+
+  let isFirstOutput = true
+
+  // Clear output file if specified
+  if (outputPath) {
+    await Bun.write(outputPath, '')
+  }
+
+  for (let i = 0; i < rawOutputs.length; i++) {
+    const rawOutput = rawOutputs[i]
+    if (!rawOutput) continue
+
+    logProgress(`[${i + 1}/${rawOutputs.length}] ${rawOutput.id}`, progress)
+
+    const extracted = extractFromRaw(rawOutput, parser)
+
+    await writeOutput(JSON.stringify(extracted), outputPath, !isFirstOutput)
+    isFirstOutput = false
+  }
+
+  logProgress('Done!', progress)
+}
+
+/**
+ * Read raw outputs from stdin.
+ *
+ * @returns Array of parsed raw outputs or null if stdin is empty
+ */
+const readStdinRawOutputs = async (): Promise<RawOutput[] | null> => {
+  if (process.stdin.isTTY) {
+    return null
+  }
+
+  const chunks: Buffer[] = []
+  for await (const chunk of process.stdin) {
+    chunks.push(chunk)
+  }
+
+  const content = Buffer.concat(chunks).toString('utf-8').trim()
+  if (!content) return null
+
+  return content
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as RawOutput)
+}
+
+/**
+ * Pipeline extract command CLI handler.
+ *
+ * @param args - Command line arguments (after 'extract')
+ */
+export const extract = async (args: string[]): Promise<void> => {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      schema: { type: 'string', short: 's' },
+      output: { type: 'string', short: 'o' },
+      progress: { type: 'boolean', default: false },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: true,
+  })
+
+  if (values.help) {
+    // biome-ignore lint/suspicious/noConsole: CLI help output
+    console.log(`
+Usage: agent-eval-harness extract [raw.jsonl] --schema <schema.json> [options]
+
+Parse raw output into trajectories and final output.
+
+Arguments:
+  raw.jsonl         Input file from 'run' command (or pipe from stdin)
+
+Options:
+  -s, --schema      Path to headless adapter schema (required)
+  -o, --output      Output file (default: stdout)
+  --progress        Show progress to stderr
+  -h, --help        Show this help message
+
+Examples:
+  # From file
+  agent-eval-harness extract raw.jsonl --schema claude.json -o extracted.jsonl
+
+  # Piped from run
+  agent-eval-harness run prompts.jsonl -s claude.json | agent-eval-harness extract -s claude.json
+
+  # Full pipeline
+  cat prompts.jsonl | \\
+    agent-eval-harness run -s claude.json | \\
+    agent-eval-harness extract -s claude.json | \\
+    agent-eval-harness grade --grader ./grader.ts
+`)
+    return
+  }
+
+  if (!values.schema) {
+    console.error('Error: --schema is required')
+    process.exit(1)
+  }
+
+  // Load raw outputs from file or stdin
+  const inputPath = positionals[0]
+  let rawOutputs: RawOutput[]
+
+  if (inputPath) {
+    rawOutputs = await loadJsonl<RawOutput>(inputPath)
+  } else {
+    const stdinOutputs = await readStdinRawOutputs()
+    if (!stdinOutputs || stdinOutputs.length === 0) {
+      console.error('Error: No raw output provided (use file argument or pipe to stdin)')
+      process.exit(1)
+    }
+    rawOutputs = stdinOutputs
+  }
+
+  await runExtract(values.schema, rawOutputs, values.output, values.progress)
+}

package/src/pipeline/format.ts

@@ -0,0 +1,292 @@
+/**
+ * Pipeline format command - convert results to different output formats.
+ *
+ * @remarks
+ * Transforms graded or extracted results into various formats:
+ * - jsonl: Pass-through JSONL (default)
+ * - markdown: Human-readable report
+ * - csv: Comma-separated values for spreadsheets
+ *
+ * @packageDocumentation
+ */
+
+import { parseArgs } from 'node:util'
+import { loadJsonl, logProgress, writeOutput } from '../core.ts'
+import type { CaptureResult } from '../schemas.ts'
+import type { ExtractedResult, FormatStyle, GradedResult } from './pipeline.types.ts'
+
+/** Union of all formattable result types */
+type FormattableResult = ExtractedResult | GradedResult | CaptureResult
+
+/**
+ * Check if result has a score (graded).
+ */
+const isGraded = (
+  result: FormattableResult,
+): result is GradedResult | (CaptureResult & { score: NonNullable<CaptureResult['score']> }) => {
+  return 'score' in result && result.score !== undefined
+}
+
+/**
+ * Format results as markdown report.
+ *
+ * @param results - Results to format
+ * @returns Markdown string
+ */
+const formatMarkdown = (results: FormattableResult[]): string => {
+  const lines: string[] = [
+    '# Evaluation Results',
+    '',
+    `Generated: ${new Date().toISOString()}`,
+    `Total: ${results.length} test cases`,
+    '',
+  ]
+
+  // Summary statistics if graded
+  const gradedResults = results.filter(isGraded)
+  if (gradedResults.length > 0) {
+    const passed = gradedResults.filter((r) => r.score.pass).length
+    const avgScore = gradedResults.reduce((sum, r) => sum + r.score.score, 0) / gradedResults.length
+
+    lines.push('## Summary')
+    lines.push('')
+    lines.push(
+      `- **Pass rate**: ${passed}/${gradedResults.length} (${((passed / gradedResults.length) * 100).toFixed(1)}%)`,
+    )
+    lines.push(`- **Average score**: ${avgScore.toFixed(3)}`)
+    lines.push('')
+  }
+
+  lines.push('## Results')
+  lines.push('')
+
+  for (const result of results) {
+    const input = Array.isArray(result.input) ? result.input.join(' → ') : result.input
+    const inputPreview = input.length > 100 ? `${input.slice(0, 100)}...` : input
+
+    lines.push(`### ${result.id}`)
+    lines.push('')
+    lines.push(`**Input**: ${inputPreview}`)
+    lines.push('')
+
+    if (result.hint) {
+      lines.push(`**Hint**: ${result.hint}`)
+      lines.push('')
+    }
+
+    const outputPreview = result.output.length > 500 ? `${result.output.slice(0, 500)}...` : result.output
+    lines.push(`**Output**:`)
+    lines.push('```')
+    lines.push(outputPreview)
+    lines.push('```')
+    lines.push('')
+
+    if (isGraded(result)) {
+      const icon = result.score.pass ? '✅' : '❌'
+      lines.push(`**Score**: ${icon} ${result.score.score.toFixed(3)} (${result.score.pass ? 'PASS' : 'FAIL'})`)
+      if (result.score.reasoning) {
+        lines.push(`**Reasoning**: ${result.score.reasoning}`)
+      }
+      lines.push('')
+    }
+
+    if (result.toolErrors) {
+      lines.push('⚠️ **Tool errors detected**')
+      lines.push('')
+    }
+
+    if ('error' in result && result.error) {
+      lines.push(`❌ **Error**: ${result.error}`)
+      lines.push('')
+    }
+
+    lines.push('---')
+    lines.push('')
+  }
+
+  return lines.join('\n')
+}
+
+/**
+ * Format results as CSV.
+ *
+ * @param results - Results to format
+ * @returns CSV string
+ */
+const formatCsv = (results: FormattableResult[]): string => {
+  const lines: string[] = []
+
+  // Header
+  const hasScores = results.some(isGraded)
+  const headers = ['id', 'input', 'hint', 'output', 'tool_errors', 'duration_ms']
+  if (hasScores) {
+    headers.push('pass', 'score', 'reasoning')
+  }
+  lines.push(headers.join(','))
+
+  // Data rows
+  for (const result of results) {
+    const input = Array.isArray(result.input) ? result.input.join(' | ') : result.input
+    const escapeCsv = (str: string) => `"${str.replace(/"/g, '""').replace(/\n/g, '\\n')}"`
+
+    const row = [
+      escapeCsv(result.id),
+      escapeCsv(input),
+      escapeCsv(result.hint ?? ''),
+      escapeCsv(result.output),
+      result.toolErrors ? 'true' : 'false',
+      String(result.timing.total),
+    ]
+
+    if (hasScores) {
+      if (isGraded(result)) {
+        row.push(
+          result.score.pass ? 'true' : 'false',
+          result.score.score.toFixed(3),
+          escapeCsv(result.score.reasoning ?? ''),
+        )
+      } else {
+        row.push('', '', '')
+      }
+    }
+
+    lines.push(row.join(','))
+  }
+
+  return lines.join('\n')
+}
+
+/**
+ * Execute pipeline format with configuration.
+ *
+ * @param style - Output format style
+ * @param results - Results to format
+ * @param outputPath - Optional output file path
+ * @param progress - Show progress to stderr
+ */
+export const runFormat = async (
+  style: FormatStyle,
+  results: FormattableResult[],
+  outputPath?: string,
+  progress = false,
+): Promise<void> => {
+  logProgress(`Formatting ${results.length} results as ${style}`, progress)
+
+  let output: string
+
+  switch (style) {
+    case 'jsonl':
+      // Pass-through as JSONL
+      output = results.map((r) => JSON.stringify(r)).join('\n')
+      break
+
+    case 'markdown':
+      output = formatMarkdown(results)
+      break
+
+    case 'csv':
+      output = formatCsv(results)
+      break
+  }
+
+  await writeOutput(output, outputPath, false)
+  logProgress('Done!', progress)
+}
+
+/**
+ * Read results from stdin.
+ *
+ * @returns Array of parsed results or null if stdin is empty
+ */
+const readStdinResults = async (): Promise<FormattableResult[] | null> => {
+  if (process.stdin.isTTY) {
+    return null
+  }
+
+  const chunks: Buffer[] = []
+  for await (const chunk of process.stdin) {
+    chunks.push(chunk)
+  }
+
+  const content = Buffer.concat(chunks).toString('utf-8').trim()
+  if (!content) return null
+
+  return content
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as FormattableResult)
+}
+
+/**
+ * Pipeline format command CLI handler.
+ *
+ * @param args - Command line arguments (after 'format')
+ */
+export const format = async (args: string[]): Promise<void> => {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      style: { type: 'string', short: 'f', default: 'jsonl' },
+      output: { type: 'string', short: 'o' },
+      progress: { type: 'boolean', default: false },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: true,
+  })
+
+  if (values.help) {
+    // biome-ignore lint/suspicious/noConsole: CLI help output
+    console.log(`
+Usage: agent-eval-harness format [results.jsonl] [options]
+
+Convert results to different output formats.
+
+Arguments:
+  results.jsonl     Input file (or pipe from stdin)
+
+Options:
+  -f, --style       Output format: jsonl, markdown, csv (default: jsonl)
+  -o, --output      Output file (default: stdout)
+  --progress        Show progress to stderr
+  -h, --help        Show this help message
+
+Examples:
+  # Convert to markdown report
+  agent-eval-harness format graded.jsonl --style markdown -o report.md
+
+  # Piped from grade
+  agent-eval-harness grade extracted.jsonl -g ./grader.ts | agent-eval-harness format -f csv
+
+  # Full pipeline to markdown
+  cat prompts.jsonl | \\
+    agent-eval-harness run -s claude.json | \\
+    agent-eval-harness extract -s claude.json | \\
+    agent-eval-harness grade -g ./grader.ts | \\
+    agent-eval-harness format -f markdown > report.md
+`)
+    return
+  }
+
+  const style = values.style as FormatStyle
+  if (!['jsonl', 'markdown', 'csv'].includes(style)) {
+    console.error(`Error: Invalid format style '${style}'. Must be: jsonl, markdown, csv`)
+    process.exit(1)
+  }
+
+  // Load results from file or stdin
+  const inputPath = positionals[0]
+  let results: FormattableResult[]
+
+  if (inputPath) {
+    results = await loadJsonl<FormattableResult>(inputPath)
+  } else {
+    const stdinResults = await readStdinResults()
+    if (!stdinResults || stdinResults.length === 0) {
+      console.error('Error: No results provided (use file argument or pipe to stdin)')
+      process.exit(1)
+    }
+    results = stdinResults
+  }
+
+  await runFormat(style, results, values.output, values.progress)
+}

package/src/pipeline/grade.ts

@@ -0,0 +1,169 @@
+/**
+ * Pipeline grade command - apply grader to extracted results.
+ *
+ * @remarks
+ * Takes ExtractedResult from `extract` command and adds grader scores.
+ * Uses the same grader loading mechanism as the capture command.
+ *
+ * @packageDocumentation
+ */
+
+import { parseArgs } from 'node:util'
+import { loadJsonl, logProgress, writeOutput } from '../core.ts'
+import { loadGrader } from '../schemas/grader-loader.ts'
+import type { ExtractedResult, GradedResult } from './pipeline.types.ts'
+
+/**
+ * Execute pipeline grade with configuration.
+ *
+ * @param graderPath - Path to grader module or executable
+ * @param extractedResults - Extracted results from extract command
+ * @param outputPath - Optional output file path
+ * @param progress - Show progress to stderr
+ */
+export const runGrade = async (
+  graderPath: string,
+  extractedResults: ExtractedResult[],
+  outputPath?: string,
+  progress = false,
+): Promise<void> => {
+  // Load grader
+  const grader = await loadGrader(graderPath)
+
+  logProgress(`Grading with: ${graderPath}`, progress)
+
+  let isFirstOutput = true
+
+  // Clear output file if specified
+  if (outputPath) {
+    await Bun.write(outputPath, '')
+  }
+
+  for (let i = 0; i < extractedResults.length; i++) {
+    const extracted = extractedResults[i]
+    if (!extracted) continue
+
+    logProgress(`[${i + 1}/${extractedResults.length}] ${extracted.id}`, progress)
+
+    // Apply grader
+    const score = await grader({
+      input: extracted.input,
+      output: extracted.output,
+      hint: extracted.hint,
+      trajectory: extracted.trajectory,
+    })
+
+    const graded: GradedResult = {
+      ...extracted,
+      score,
+    }
+
+    const icon = score.pass ? '✓' : '✗'
+    logProgress(`  ${icon} score=${score.score.toFixed(2)}`, progress)
+
+    await writeOutput(JSON.stringify(graded), outputPath, !isFirstOutput)
+    isFirstOutput = false
+  }
+
+  logProgress('Done!', progress)
+}
+
+/**
+ * Read extracted results from stdin.
+ *
+ * @returns Array of parsed extracted results or null if stdin is empty
+ */
+const readStdinExtracted = async (): Promise<ExtractedResult[] | null> => {
+  if (process.stdin.isTTY) {
+    return null
+  }
+
+  const chunks: Buffer[] = []
+  for await (const chunk of process.stdin) {
+    chunks.push(chunk)
+  }
+
+  const content = Buffer.concat(chunks).toString('utf-8').trim()
+  if (!content) return null
+
+  return content
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as ExtractedResult)
+}
+
+/**
+ * Pipeline grade command CLI handler.
+ *
+ * @param args - Command line arguments (after 'grade')
+ */
+export const grade = async (args: string[]): Promise<void> => {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      grader: { type: 'string', short: 'g' },
+      output: { type: 'string', short: 'o' },
+      progress: { type: 'boolean', default: false },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: true,
+  })
+
+  if (values.help) {
+    // biome-ignore lint/suspicious/noConsole: CLI help output
+    console.log(`
+Usage: agent-eval-harness grade [extracted.jsonl] --grader <grader> [options]
+
+Apply grader to extracted results.
+
+Arguments:
+  extracted.jsonl   Input file from 'extract' command (or pipe from stdin)
+
+Options:
+  -g, --grader      Path to grader (.ts/.js module or executable script) (required)
+  -o, --output      Output file (default: stdout)
+  --progress        Show progress to stderr
+  -h, --help        Show this help message
+
+Graders:
+  TS/JS modules must export a 'grade' function.
+  Executable scripts (Python, etc.) use stdin/stdout JSON protocol.
+
+Examples:
+  # From file
+  agent-eval-harness grade extracted.jsonl --grader ./grader.ts -o graded.jsonl
+
+  # Piped from extract
+  agent-eval-harness extract raw.jsonl -s claude.json | agent-eval-harness grade -g ./grader.ts
+
+  # Full pipeline
+  cat prompts.jsonl | \\
+    agent-eval-harness run -s claude.json | \\
+    agent-eval-harness extract -s claude.json | \\
+    agent-eval-harness grade -g ./grader.ts > results.jsonl
+`)
+    return
+  }
+
+  if (!values.grader) {
+    console.error('Error: --grader is required')
+    process.exit(1)
+  }
+
+  // Load extracted results from file or stdin
+  const inputPath = positionals[0]
+  let extractedResults: ExtractedResult[]
+
+  if (inputPath) {
+    extractedResults = await loadJsonl<ExtractedResult>(inputPath)
+  } else {
+    const stdinResults = await readStdinExtracted()
+    if (!stdinResults || stdinResults.length === 0) {
+      console.error('Error: No extracted results provided (use file argument or pipe to stdin)')
+      process.exit(1)
+    }
+    extractedResults = stdinResults
+  }
+
+  await runGrade(values.grader, extractedResults, values.output, values.progress)
+}