@plaited/acp-harness 0.2.5 → 0.3.1

package/src/summarize.ts

@@ -0,0 +1,245 @@
+/**
+ * Summarize command - derive compact views from full trajectory results.
+ *
+ * @remarks
+ * Transforms full trajectory JSONL into:
+ * - Summary JSONL: Compact format for jq analysis
+ * - Markdown: Human-readable format for LLM-as-judge workflows
+ *
+ * @packageDocumentation
+ */
+
+import { parseArgs } from 'node:util'
+import { extractContent, extractFilePath, headTailPreview } from './capture.ts'
+import { HEAD_LINES, MAX_CONTENT_LENGTH, TAIL_LINES } from './constants.ts'
+import type { CaptureResult, SummaryResult } from './schemas.ts'
+import { CaptureResultSchema } from './schemas.ts'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/** Configuration for summarize command */
+export type SummarizeConfig = {
+  /** Path to results.jsonl file */
+  resultsPath: string
+  /** Output file path */
+  outputPath?: string
+  /** Output as markdown instead of JSONL */
+  markdown?: boolean
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/** Resolve path relative to process.cwd() */
+const resolvePath = (path: string): string => {
+  if (path.startsWith('/')) return path
+  return `${process.cwd()}/${path}`
+}
+
+/** Load capture results from JSONL file */
+const loadResults = async (path: string): Promise<CaptureResult[]> => {
+  const content = await Bun.file(path).text()
+  return content
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map((line, index) => {
+      try {
+        return CaptureResultSchema.parse(JSON.parse(line))
+      } catch (error) {
+        throw new Error(`Invalid result at line ${index + 1}: ${error instanceof Error ? error.message : error}`)
+      }
+    })
+}
+
+/** Format result as summary JSONL */
+const formatSummary = (result: CaptureResult): SummaryResult => {
+  return {
+    id: result.id,
+    input: result.input,
+    output: result.output,
+    toolCalls: result.trajectory.filter((s) => s.type === 'tool_call').map((s) => (s as { name: string }).name),
+    duration: result.timing.end - result.timing.start,
+  }
+}
+
+/** Format result as markdown with step IDs */
+const formatMarkdown = (result: CaptureResult): string => {
+  const lines: string[] = [
+    `## Evaluation Record: ${result.id}`,
+    '',
+    `**Input:** ${result.input}`,
+    '',
+    '**Trajectory:**',
+  ]
+
+  let stepNum = 1
+  for (const step of result.trajectory) {
+    const stepId = `${result.id}-step-${stepNum}`
+
+    if (step.type === 'thought') {
+      const preview = step.content.slice(0, 100)
+      const truncated = step.content.length > 100 ? '...' : ''
+      lines.push(`${stepNum}. [THOUGHT] ${preview}${truncated} [→${stepId}]`)
+      stepNum++
+    } else if (step.type === 'tool_call') {
+      const duration = step.duration ? ` (${step.duration}ms)` : ''
+      const filePath = extractFilePath(step.input)
+      const content = extractContent(step.input)
+
+      lines.push(`${stepNum}. [TOOL:${step.name}] -> ${step.status}${duration} [→${stepId}]`)
+
+      // Add file path if present
+      if (filePath) {
+        const charCount = content?.length ?? 0
+        lines.push(`   File: ${filePath}${charCount > 0 ? ` (${charCount} chars)` : ''}`)
+      }
+
+      // Add head/tail preview for content-producing tools
+      if (content && content.length > 0) {
+        const preview = content.length > MAX_CONTENT_LENGTH ? headTailPreview(content, HEAD_LINES, TAIL_LINES) : content
+        // Detect file extension for syntax highlighting
+        const ext = filePath?.split('.').pop() ?? 'typescript'
+        lines.push(`   \`\`\`${ext}`)
+        lines.push(`   ${preview.split('\n').join('\n   ')}`)
+        lines.push('   ```')
+      }
+      stepNum++
+    } else if (step.type === 'plan') {
+      const entries = step.entries as Array<{ content: string; status: string }>
+      const planSummary = entries.map((e) => `${e.content}: ${e.status}`).join(', ')
+      const truncated = planSummary.length > 80 ? '...' : ''
+      lines.push(`${stepNum}. [PLAN] ${planSummary.slice(0, 80)}${truncated} [→${stepId}]`)
+      stepNum++
+    } else if (step.type === 'message') {
+      const preview = step.content.slice(0, 100)
+      const truncated = step.content.length > 100 ? '...' : ''
+      lines.push(`${stepNum}. [MESSAGE] ${preview}${truncated} [→${stepId}]`)
+      stepNum++
+    }
+  }
+
+  lines.push('')
+  const outputPreview = result.output.slice(0, 200)
+  const outputTruncated = result.output.length > 200 ? '...' : ''
+  lines.push(`**Output:** ${outputPreview}${outputTruncated}`)
+  lines.push('')
+
+  const metadataStr = Object.entries(result.metadata)
+    .map(([k, v]) => `${k}=${v}`)
+    .join(', ')
+  lines.push(`**Metadata:** ${metadataStr}`)
+  lines.push(`**Tool Errors:** ${result.toolErrors}`)
+  lines.push(`**Duration:** ${result.timing.end - result.timing.start}ms`)
+
+  if (result.score) {
+    lines.push(`**Score:** ${result.score.pass ? 'PASS' : 'FAIL'} (${result.score.score})`)
+    if (result.score.reasoning) {
+      lines.push(`**Reasoning:** ${result.score.reasoning}`)
+    }
+  }
+
+  lines.push('')
+  lines.push('---')
+  lines.push('')
+
+  return lines.join('\n')
+}
+
+// ============================================================================
+// Summarize Implementation
+// ============================================================================
+
+/**
+ * Execute summarize with configuration object.
+ *
+ * @param config - Summarize configuration
+ * @returns Formatted output string
+ */
+export const runSummarize = async (config: SummarizeConfig): Promise<string> => {
+  const { resultsPath, outputPath, markdown = false } = config
+
+  // Load results
+  const results = await loadResults(resultsPath)
+
+  // Format output
+  let output: string
+  if (markdown) {
+    output = results.map(formatMarkdown).join('\n')
+  } else {
+    output = results.map((r) => JSON.stringify(formatSummary(r))).join('\n')
+  }
+
+  // Write output
+  if (outputPath) {
+    await Bun.write(resolvePath(outputPath), output)
+  } else {
+    // biome-ignore lint/suspicious/noConsole: CLI stdout output
+    console.log(output)
+  }
+
+  return output
+}
+
+// ============================================================================
+// CLI Entry Point
+// ============================================================================
+
+/**
+ * Summarize command CLI handler.
+ *
+ * @param args - Command line arguments (after 'summarize')
+ */
+export const summarize = async (args: string[]): Promise<void> => {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      output: { type: 'string', short: 'o' },
+      markdown: { type: 'boolean', short: 'm', default: false },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: true,
+  })
+
+  if (values.help) {
+    // biome-ignore lint/suspicious/noConsole: CLI help output
+    console.log(`
+Usage: acp-harness summarize <results.jsonl> [options]
+
+Arguments:
+  results.jsonl     Input file with capture results
+
+Options:
+  -o, --output      Output file (default: stdout)
+  -m, --markdown    Output as markdown instead of JSONL
+  -h, --help        Show this help message
+
+Output Formats:
+  JSONL (default): Compact summary with id, input, output, toolCalls, duration
+  Markdown (-m):   Human-readable format with step IDs for LLM-as-judge
+
+Examples:
+  # Summary JSONL for jq analysis
+  acp-harness summarize results.jsonl -o summary.jsonl
+
+  # Markdown for LLM evaluation
+  acp-harness summarize results.jsonl --markdown -o results.md
+`)
+    return
+  }
+
+  const resultsPath = positionals[0]
+  if (!resultsPath) {
+    console.error('Error: results.jsonl path is required')
+    process.exit(1)
+  }
+
+  await runSummarize({
+    resultsPath,
+    outputPath: values.output,
+    markdown: values.markdown ?? false,
+  })
+}

package/src/tests/adapter-check.spec.ts

@@ -0,0 +1,70 @@
+/**
+ * Tests for adapter compliance checking functionality.
+ */
+
+import { describe, expect, test } from 'bun:test'
+import { type CheckConfig, runCheck } from '../adapter-check.ts'
+
+describe('runCheck', () => {
+  test('fails spawn check for non-existent command', async () => {
+    const config: CheckConfig = {
+      command: ['nonexistent-command-xyz'],
+      timeout: 1000,
+      verbose: false,
+    }
+
+    const result = await runCheck(config)
+
+    expect(result.passed).toBe(false)
+    expect(result.checks.length).toBeGreaterThanOrEqual(1)
+    expect(result.checks[0]?.name).toBe('spawn')
+    expect(result.checks[0]?.passed).toBe(false)
+  })
+
+  test('fails spawn check for command that exits immediately', async () => {
+    const config: CheckConfig = {
+      command: ['false'], // Unix command that exits with code 1
+      timeout: 1000,
+      verbose: false,
+    }
+
+    const result = await runCheck(config)
+
+    expect(result.passed).toBe(false)
+    expect(result.summary.failed).toBeGreaterThanOrEqual(1)
+  })
+
+  test('returns structured result with summary', async () => {
+    const config: CheckConfig = {
+      command: ['echo', 'test'],
+      timeout: 1000,
+      verbose: false,
+    }
+
+    const result = await runCheck(config)
+
+    expect(result).toHaveProperty('passed')
+    expect(result).toHaveProperty('checks')
+    expect(result).toHaveProperty('summary')
+    expect(result.summary).toHaveProperty('total')
+    expect(result.summary).toHaveProperty('passed')
+    expect(result.summary).toHaveProperty('failed')
+    expect(typeof result.passed).toBe('boolean')
+    expect(Array.isArray(result.checks)).toBe(true)
+  })
+
+  test('includes verbose details when enabled', async () => {
+    const config: CheckConfig = {
+      command: ['echo', 'test'],
+      timeout: 1000,
+      verbose: true,
+    }
+
+    const result = await runCheck(config)
+
+    // At least the spawn check should have details in verbose mode
+    const spawnCheck = result.checks.find((c) => c.name === 'spawn')
+    expect(spawnCheck).toBeDefined()
+    // Note: details may or may not be present depending on check outcome
+  })
+})

package/src/tests/adapter-scaffold.spec.ts

@@ -0,0 +1,112 @@
+/**
+ * Tests for adapter scaffolding functionality.
+ */
+
+import { afterEach, describe, expect, test } from 'bun:test'
+import { rm } from 'node:fs/promises'
+import { join } from 'node:path'
+import { runScaffold, type ScaffoldConfig } from '../adapter-scaffold.ts'
+
+const testDir = join(import.meta.dir, 'fixtures', 'scaffold-output')
+
+describe('runScaffold', () => {
+  afterEach(async () => {
+    // Clean up test output
+    await rm(testDir, { recursive: true, force: true })
+  })
+
+  test('generates TypeScript adapter structure', async () => {
+    const config: ScaffoldConfig = {
+      name: 'test-agent',
+      outputDir: testDir,
+      lang: 'ts',
+      minimal: false,
+    }
+
+    const result = await runScaffold(config)
+
+    expect(result.outputDir).toBe(testDir)
+    expect(result.lang).toBe('ts')
+    expect(result.files).toContain('package.json')
+    expect(result.files).toContain('tsconfig.json')
+    expect(result.files).toContain('src/index.ts')
+    expect(result.files).toContain('src/types.ts')
+    expect(result.files).toContain('src/session-manager.ts')
+    expect(result.files).toContain('src/handlers/initialize.ts')
+    expect(result.files).toContain('src/handlers/session-new.ts')
+    expect(result.files).toContain('src/handlers/session-prompt.ts')
+    expect(result.files).toContain('src/handlers/session-cancel.ts')
+    expect(result.files).toContain('README.md')
+
+    // Verify files actually exist
+    const packageJson = await Bun.file(join(testDir, 'package.json')).text()
+    expect(packageJson).toContain('"test-agent-acp"')
+
+    const indexTs = await Bun.file(join(testDir, 'src', 'index.ts')).text()
+    expect(indexTs).toContain('#!/usr/bin/env bun')
+    expect(indexTs).toContain('handleInitialize')
+  })
+
+  test('generates minimal TypeScript structure without README', async () => {
+    const config: ScaffoldConfig = {
+      name: 'minimal-agent',
+      outputDir: testDir,
+      lang: 'ts',
+      minimal: true,
+    }
+
+    const result = await runScaffold(config)
+
+    expect(result.files).not.toContain('README.md')
+    expect(result.files).toContain('package.json')
+    expect(result.files).toContain('src/index.ts')
+  })
+
+  test('generates Python adapter structure', async () => {
+    const config: ScaffoldConfig = {
+      name: 'python-agent',
+      outputDir: testDir,
+      lang: 'python',
+      minimal: false,
+    }
+
+    const result = await runScaffold(config)
+
+    expect(result.lang).toBe('python')
+    expect(result.files).toContain('adapter.py')
+    expect(result.files).toContain('README.md')
+
+    const adapterPy = await Bun.file(join(testDir, 'adapter.py')).text()
+    expect(adapterPy).toContain('#!/usr/bin/env python3')
+    expect(adapterPy).toContain('python-agent')
+    expect(adapterPy).toContain('def handle_initialize')
+  })
+
+  test('generates minimal Python structure without README', async () => {
+    const config: ScaffoldConfig = {
+      name: 'minimal-python',
+      outputDir: testDir,
+      lang: 'python',
+      minimal: true,
+    }
+
+    const result = await runScaffold(config)
+
+    expect(result.files).toContain('adapter.py')
+    expect(result.files).not.toContain('README.md')
+  })
+
+  test('package.json contains correct name', async () => {
+    const config: ScaffoldConfig = {
+      name: 'my-special-agent',
+      outputDir: testDir,
+      lang: 'ts',
+      minimal: true,
+    }
+
+    await runScaffold(config)
+
+    const packageJson = JSON.parse(await Bun.file(join(testDir, 'package.json')).text())
+    expect(packageJson.name).toBe('my-special-agent-acp')
+  })
+})

package/src/tests/fixtures/grader-bad-module.ts

@@ -0,0 +1,5 @@
+/**
+ * Test fixture: Invalid TypeScript grader (no 'grade' export).
+ */
+
+export const evaluate = () => ({ pass: true, score: 1.0 })

package/src/tests/fixtures/grader-exec-fail.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""
+Test fixture: Python grader that exits with non-zero code.
+"""
+
+import sys
+
+sys.stderr.write("Intentional failure")
+sys.exit(1)

package/src/tests/fixtures/grader-exec-invalid.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+"""
+Test fixture: Python grader that outputs invalid JSON.
+"""
+
+print("not valid json")

package/src/tests/fixtures/grader-exec.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""
+Test fixture: Python grader script using stdin/stdout JSON protocol.
+"""
+
+import json
+import sys
+
+def main():
+    data = json.load(sys.stdin)
+
+    output = data.get("output", "").lower()
+    expected = (data.get("expected") or "").lower()
+
+    if expected:
+        pass_result = expected in output
+    else:
+        pass_result = True
+
+    result = {
+        "pass": pass_result,
+        "score": 1.0 if pass_result else 0.0,
+        "reasoning": "Contains expected" if pass_result else "Missing expected"
+    }
+
+    print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()

package/src/tests/fixtures/grader-module.ts

@@ -0,0 +1,14 @@
+/**
+ * Test fixture: TypeScript grader module.
+ */
+
+import type { Grader } from '../../schemas.ts'
+
+export const grade: Grader = async ({ input: _input, output, expected }) => {
+  const pass = expected ? output.toLowerCase().includes(expected.toLowerCase()) : true
+  return {
+    pass,
+    score: pass ? 1.0 : 0.0,
+    reasoning: pass ? 'Contains expected text' : 'Missing expected text',
+  }
+}

package/src/tests/grader-loader.spec.ts

@@ -0,0 +1,153 @@
+import { describe, expect, test } from 'bun:test'
+import { join } from 'node:path'
+import { loadGrader } from '../grader-loader.ts'
+
+const fixturesDir = join(import.meta.dir, 'fixtures')
+
+// ============================================================================
+// Module Graders (TypeScript/JavaScript)
+// ============================================================================
+
+describe('loadGrader - module graders', () => {
+  test('loads TypeScript grader module', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-module.ts'))
+
+    const result = await grader({
+      input: 'What is 2+2?',
+      output: 'The answer is 4',
+      expected: '4',
+    })
+
+    expect(result.pass).toBe(true)
+    expect(result.score).toBe(1.0)
+    expect(result.reasoning).toBe('Contains expected text')
+  })
+
+  test('fails when module does not export grade function', async () => {
+    await expect(loadGrader(join(fixturesDir, 'grader-bad-module.ts'))).rejects.toThrow(
+      "Grader module must export a 'grade' function",
+    )
+  })
+
+  test('fails when module does not exist', async () => {
+    await expect(loadGrader(join(fixturesDir, 'nonexistent.ts'))).rejects.toThrow('Grader not found')
+  })
+})
+
+// ============================================================================
+// Executable Graders (Python, etc.)
+// ============================================================================
+
+describe('loadGrader - executable graders', () => {
+  test('loads and executes Python grader', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+
+    const result = await grader({
+      input: 'What is 2+2?',
+      output: 'The answer is 4',
+      expected: '4',
+    })
+
+    expect(result.pass).toBe(true)
+    expect(result.score).toBe(1.0)
+    expect(result.reasoning).toBe('Contains expected')
+  })
+
+  test('Python grader returns pass=false when expected not in output', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+
+    const result = await grader({
+      input: 'What is 2+2?',
+      output: 'I do not know',
+      expected: '4',
+    })
+
+    expect(result.pass).toBe(false)
+    expect(result.score).toBe(0.0)
+  })
+
+  test('throws when executable exits with non-zero code', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec-fail.py'))
+
+    await expect(
+      grader({
+        input: 'test',
+        output: 'test',
+      }),
+    ).rejects.toThrow('Grader exited with code 1')
+  })
+
+  test('throws when executable outputs invalid JSON', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec-invalid.py'))
+
+    await expect(
+      grader({
+        input: 'test',
+        output: 'test',
+      }),
+    ).rejects.toThrow('Grader output is not valid JSON')
+  })
+
+  test('fails when executable does not exist', async () => {
+    await expect(loadGrader(join(fixturesDir, 'nonexistent.py'))).rejects.toThrow('Grader not found')
+  })
+})
+
+// ============================================================================
+// Extension Detection
+// ============================================================================
+
+describe('loadGrader - extension detection', () => {
+  test('detects .ts as module', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-module.ts'))
+    // If this doesn't throw, it was loaded as a module (not executed)
+    expect(grader).toBeInstanceOf(Function)
+  })
+
+  test('detects .py as executable', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+    expect(grader).toBeInstanceOf(Function)
+  })
+})
+
+// ============================================================================
+// Trajectory Support
+// ============================================================================
+
+describe('loadGrader - trajectory support', () => {
+  test('passes trajectory to module grader', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-module.ts'))
+
+    const trajectory = [
+      { type: 'message' as const, content: 'Hello', timestamp: 0 },
+      { type: 'tool_call' as const, name: 'read', status: 'completed', timestamp: 100 },
+    ]
+
+    const result = await grader({
+      input: 'test',
+      output: 'The answer is 4',
+      expected: '4',
+      trajectory,
+    })
+
+    expect(result.pass).toBe(true)
+  })
+
+  test('passes trajectory to executable grader', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+
+    const trajectory = [
+      { type: 'message' as const, content: 'Hello', timestamp: 0 },
+      { type: 'tool_call' as const, name: 'read', status: 'completed', timestamp: 100 },
+    ]
+
+    const result = await grader({
+      input: 'test',
+      output: 'The answer is 4',
+      expected: '4',
+      trajectory,
+    })
+
+    expect(result.pass).toBe(true)
+  })
+})