@plaited/acp-harness 0.2.5

package/.claude/skills/acp-harness/references/downstream.md

@@ -0,0 +1,288 @@
+# Downstream Integration
+
+Patterns for piping harness output to analysis tools.
+
+## Loading Results
+
+Both output formats use JSONL (newline-delimited JSON):
+
+```typescript
+// TypeScript pattern (validated in tests)
+const parseResults = (jsonl: string) =>
+  jsonl.trim().split('\n').map((line) => JSON.parse(line))
+
+// Load from file
+const results = parseResults(await Bun.file('results.jsonl').text())
+```
+
+## jq Analysis
+
+Summary JSONL is designed for quick analysis with `jq`:
+
+```bash
+# Calculate average duration
+cat results.jsonl | jq -s 'map(.duration) | add / length'
+
+# Count tool usage
+cat results.jsonl | jq -s 'map(.toolCalls) | flatten | group_by(.) | map({tool: .[0], count: length})'
+
+# Filter by status
+cat results.jsonl | jq 'select(.status == "failed")'
+
+# Pass rate
+cat results.jsonl | jq -s 'map(select(.status == "passed")) | length as $p | length as $t | "\($p)/\($t) passed"'
+
+# Group by category
+cat results.jsonl | jq -s 'group_by(.metadata.category) | map({category: .[0].metadata.category, count: length})'
+
+# Find slowest runs
+cat results.jsonl | jq -s 'sort_by(-.duration) | .[0:5] | map({id, duration})'
+```
+
+## TypeScript Analysis Patterns
+
+These patterns are validated by tests in `bin/tests/cli.spec.ts`:
+
+### Filter by Status
+
+```typescript
+const failed = results.filter((r) => r.status === 'failed')
+const passed = results.filter((r) => r.status === 'passed')
+const passRate = passed.length / results.length
+```
+
+### Filter by Tool Usage
+
+```typescript
+// Find runs that used Write tool
+const withWrite = results.filter((r) => r.toolCalls.includes('Write'))
+
+// Find runs that used multiple tools
+const multiTool = results.filter((r) => r.toolCalls.length > 1)
+```
+
+### Filter by Duration
+
+```typescript
+// Slow runs (> 2 seconds)
+const slow = results.filter((r) => r.duration > 2000)
+
+// Find top 5 slowest
+const slowest = [...results].sort((a, b) => b.duration - a.duration).slice(0, 5)
+```
+
+### Filter by Metadata
+
+```typescript
+// Filter by category
+const uiResults = results.filter((r) => r.metadata.category === 'ui')
+
+// Group and count by category
+const grouped = results.reduce<Record<string, number>>((acc, r) => {
+  const cat = r.metadata.category as string
+  acc[cat] = (acc[cat] ?? 0) + 1
+  return acc
+}, {})
+```
+
+### Count Tool Usage
+
+```typescript
+const allTools = results.flatMap((r) => r.toolCalls)
+const toolCounts = allTools.reduce<Record<string, number>>((acc, tool) => {
+  acc[tool] = (acc[tool] ?? 0) + 1
+  return acc
+}, {})
+```
+
+### Deduplicate by ID
+
+```typescript
+// Keep latest occurrence when merging multiple runs
+const byId = new Map<string, unknown>()
+for (const result of results) {
+  byId.set(result.id, result)
+}
+const deduped = Array.from(byId.values())
+```
+
+## Step-Level Retrieval
+
+For judge format, correlate markdown step IDs with full JSONL:
+
+```typescript
+// Load both files
+const markdown = await Bun.file('results.md').text()
+const fullResults = parseResults(await Bun.file('results.full.jsonl').text())
+
+// Build step index
+const stepIndex = new Map<string, unknown>()
+for (const result of fullResults) {
+  for (const step of result.trajectory) {
+    stepIndex.set(step.stepId, step)
+  }
+}
+
+// Retrieve full step by ID (from markdown [→stepId])
+const stepId = 'test-001-step-2'
+const fullStep = stepIndex.get(stepId) as { name: string; input: unknown }
+console.log('Tool name:', fullStep.name)
+console.log('Full input:', fullStep.input)
+```
+
+## Extract Tool Calls from Trajectory
+
+```typescript
+const toolCalls = result.trajectory.filter((s) => s.type === 'tool_call')
+const toolNames = toolCalls.map((t) => t.name)
+```
+
+## Timing Information
+
+```typescript
+const result = results[0]
+const duration = result.timing.end - result.timing.start
+const timeToFirstResponse = result.timing.firstResponse // ms after start
+```
+
+## LLM-as-Judge
+
+### Large Context Models (Gemini 1M+)
+
+Feed full trajectory directly:
+
+```typescript
+import { GoogleGenerativeAI } from '@google/generative-ai'
+
+const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY!)
+const model = genAI.getGenerativeModel({ model: 'gemini-2.5-pro' })
+
+const results = parseResults(await Bun.file('results.full.jsonl').text())
+
+const prompt = `
+Evaluate these agent trajectories for code quality and reasoning.
+
+${JSON.stringify(results, null, 2)}
+
+For each evaluation, score 1-3:
+- 1: Major issues (wrong tools, broken logic, incorrect output)
+- 2: Minor issues (inefficient but correct)
+- 3: Excellent (efficient trajectory, correct output)
+
+Respond as JSON array: [{"id": "...", "score": N, "reasoning": "..."}]
+`
+
+const response = await model.generateContent(prompt)
+console.log(response.response.text())
+```
+
+### Medium Context Models (Claude 200k)
+
+Use full trajectory for most runs:
+
+```typescript
+import Anthropic from '@anthropic-ai/sdk'
+
+const client = new Anthropic()
+const markdown = await Bun.file('results.md').text()
+
+const response = await client.messages.create({
+  model: 'claude-sonnet-4-20250514',
+  max_tokens: 4096,
+  messages: [{
+    role: 'user',
+    content: `Evaluate these agent trajectories:\n\n${markdown}\n\nScore each 1-3 and explain.`
+  }]
+})
+
+console.log(response.content[0].text)
+```
+
+## Braintrust Integration
+
+Upload results programmatically:
+
+```typescript
+import { initLogger } from 'braintrust'
+
+const logger = initLogger({
+  projectName: 'agent-eval',
+  apiKey: process.env.BRAINTRUST_API_KEY,
+})
+
+const results = parseResults(await Bun.file('results.jsonl').text())
+
+for (const result of results) {
+  logger.log({
+    input: result.input,
+    output: result.output,
+    expected: result.expected,
+    scores: {
+      passed: result.status === 'passed' ? 1 : 0,
+      duration_ms: result.duration,
+    },
+    metadata: {
+      ...result.metadata,
+      toolCalls: result.toolCalls,
+    },
+  })
+}
+
+await logger.flush()
+```
+
+## CI Integration
+
+### GitHub Actions
+
+```yaml
+name: Agent Eval
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Weekly
+
+jobs:
+  eval:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install ACP adapter
+        run: npm install -g @zed-industries/claude-code-acp
+
+      - name: Install dependencies
+        run: bun add @plaited/acp-harness
+
+      - name: Run harness
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          bunx @plaited/acp-harness prompts.jsonl \
+            --format judge \
+            --progress \
+            -o eval-results
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results
+          path: |
+            eval-results.md
+            eval-results.full.jsonl
+```
+
+## Output Aggregation
+
+Combine multiple runs:
+
+```bash
+# Append mode during runs
+bunx @plaited/acp-harness prompts-1.jsonl --append -o combined.jsonl
+bunx @plaited/acp-harness prompts-2.jsonl --append -o combined.jsonl
+
+# Merge separate files
+cat run1.jsonl run2.jsonl run3.jsonl > combined.jsonl
+
+# Dedupe by ID (keep latest) - use TypeScript pattern above
+```

package/.claude/skills/acp-harness/references/output-formats.md

@@ -0,0 +1,221 @@
+# Output Formats
+
+The harness supports two output formats optimized for different use cases.
+
+## Format Selection
+
+```bash
+bunx @plaited/acp-harness prompts.jsonl --format <format> -o <output>
+```
+
+| Format | Files Created | Use Case |
+|--------|---------------|----------|
+| `summary` | Single JSONL | Quick metrics, dashboards, jq analysis |
+| `judge` | `.md` + `.full.jsonl` | Downstream LLM-as-judge scoring |
+
+## Summary Format (Default)
+
+Minimal JSONL for quick metrics and analysis.
+
+### Schema
+
+```typescript
+type SummaryResult = {
+  id: string                    // Prompt identifier
+  input: string                 // Original prompt text
+  output: string                // Final agent response
+  toolCalls: string[]           // List of tool names used
+  status: 'passed' | 'failed' | 'error' | 'timeout'
+  duration: number              // Total execution time (ms)
+}
+```
+
+### Example Output
+
+```jsonl
+{"id":"test-001","input":"Create a primary button","output":"I created the button in src/button.tsx","toolCalls":["Write"],"status":"passed","duration":1234}
+{"id":"test-002","input":"Fix the TypeScript error","output":"I fixed the type error...","toolCalls":["Read","Edit"],"status":"passed","duration":2567}
+```
+
+### Analysis with jq
+
+```bash
+# Calculate average duration
+cat results.jsonl | jq -s 'map(.duration) | add / length'
+
+# Count tool usage
+cat results.jsonl | jq -s 'map(.toolCalls) | flatten | group_by(.) | map({tool: .[0], count: length})'
+
+# Filter by status
+cat results.jsonl | jq 'select(.status == "failed")'
+
+# Pass rate
+cat results.jsonl | jq -s 'map(select(.status == "passed")) | length as $p | length as $t | "\($p)/\($t) passed"'
+```
+
+## Judge Format (Two-Tier)
+
+Creates two files for downstream LLM-as-judge scoring with step-level correlation.
+
+```bash
+bunx @plaited/acp-harness prompts.jsonl --format judge -o results
+# Creates: results.md + results.full.jsonl
+```
+
+### Markdown File (`<output>.md`)
+
+Human-readable summary with step IDs and code previews.
+
+**Structure:**
+
+```markdown
+## Capture Record: <id>
+
+**Input:** <original prompt>
+
+**Trajectory:**
+1. [THOUGHT] <truncated content> [->stepId]
+2. [TOOL:<name>] -> <status> (<duration>ms) [->stepId]
+   File: <path> (<size> chars)
+   ```<ext>
+   <head lines>
+
+   // ... N lines omitted ...
+
+   <tail lines>
+   ```
+3. [PLAN] <plan summary> [->stepId]
+4. [MESSAGE] <truncated content> [->stepId]
+
+**Output:** <truncated final output>
+**Metadata:** category=ui, agent=claude-code-acp, ...
+**Status:** passed|failed|error|timeout
+**Duration:** <ms>ms
+
+---
+```
+
+**Step ID Format:** `<prompt-id>-step-<N>` (e.g., `test-001-step-2`)
+
+**Truncation Rules:**
+- Thought/message content: First 100 characters
+- Output: First 200 characters
+- Code preview: Head (8 lines) + tail (4 lines) for files > 12 lines
+
+### Full JSONL File (`<output>.full.jsonl`)
+
+Complete trajectory with step IDs for correlation.
+
+**Schema:**
+
+```typescript
+type FullResult = {
+  id: string
+  input: string
+  output: string
+  expected?: string
+  trajectory: IndexedStep[]     // Steps with stepId
+  metadata: Record<string, unknown>
+  timing: {
+    start: number               // Unix timestamp (ms)
+    end: number                 // Unix timestamp (ms)
+    firstResponse?: number      // Time to first response (ms)
+  }
+  status: 'passed' | 'failed' | 'error' | 'timeout'
+  errors?: string[]
+}
+
+type IndexedStep = TrajectoryStep & { stepId: string }
+
+type TrajectoryStep =
+  | { type: 'thought'; content: string; timestamp: number }
+  | { type: 'message'; content: string; timestamp: number }
+  | {
+      type: 'tool_call'
+      name: string              // Tool title from ACP SDK
+      status: string            // pending, in_progress, completed, failed
+      input?: unknown           // Raw input parameters
+      output?: unknown          // Raw output
+      duration?: number         // Execution time (ms)
+      timestamp: number
+    }
+  | { type: 'plan'; entries: PlanEntry[]; timestamp: number }
+```
+
+**Example:**
+
+```jsonl
+{"id":"test-001","input":"Create a primary button","output":"I created the button...","trajectory":[{"type":"thought","content":"I'll create a styled button template with createStyles","timestamp":100,"stepId":"test-001-step-1"},{"type":"tool_call","name":"Write","status":"completed","input":{"file_path":"src/button.tsx","content":"import { createStyles }..."},"output":"File written successfully","duration":234,"timestamp":150,"stepId":"test-001-step-2"},{"type":"message","content":"I created the button template","timestamp":500,"stepId":"test-001-step-3"}],"metadata":{"category":"ui","agent":"claude"},"timing":{"start":1704067200000,"end":1704067201234,"firstResponse":100},"status":"passed"}
+```
+
+## Two-Tier Scoring Workflow
+
+### Direct Scoring (Large Context)
+
+For judges with large context windows (Gemini 1M+, Claude 200k):
+
+```bash
+# Feed full JSONL directly
+cat results.full.jsonl | your-gemini-judge.ts
+```
+
+### Step-Level Retrieval (Small Context)
+
+For smaller models or step-specific analysis:
+
+```typescript
+// Load both files
+const markdown = await Bun.file('results.md').text()
+const fullLines = (await Bun.file('results.full.jsonl').text()).trim().split('\n')
+
+// Parse full results indexed by step ID
+const stepIndex = new Map<string, unknown>()
+for (const line of fullLines) {
+  const result = JSON.parse(line)
+  for (const step of result.trajectory) {
+    stepIndex.set(step.stepId, step)
+  }
+}
+
+// Judge requests full content for specific step
+const stepId = 'test-001-step-2'  // From markdown [->stepId]
+const fullStep = stepIndex.get(stepId)
+console.log(fullStep.input)  // Complete tool input
+```
+
+## Status Values
+
+| Status | Meaning |
+|--------|---------|
+| `passed` | Completed without tool errors |
+| `failed` | Completed but one or more tool calls failed |
+| `error` | Unhandled exception during execution |
+| `timeout` | Request exceeded timeout limit |
+
+## Input Format
+
+Both formats accept the same JSONL input:
+
+```jsonl
+{"id":"test-001","input":"Create a primary button","expected":"should contain <button>","metadata":{"category":"ui"}}
+```
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `id` | Yes | Unique identifier |
+| `input` | Yes | Prompt text for the agent |
+| `expected` | No | Expected output (for downstream scoring) |
+| `metadata` | No | Tags, category, difficulty for filtering |
+| `timeout` | No | Override default timeout for this prompt |
+
+## Streaming Behavior
+
+Both formats stream output line-by-line as results complete:
+
+```bash
+# Watch results in real-time
+bunx @plaited/acp-harness prompts.jsonl --progress -o results.jsonl &
+tail -f results.jsonl
+```
+
+Use `--append` to continue interrupted runs without overwriting previous results.

package/.claude-plugin/marketplace.json

@@ -0,0 +1,15 @@
+{
+  "name": "plaited-acp-harness",
+  "metadata": {
+    "description": "Claude Code plugin for Agent Client Protocol testing and evaluation"
+  },
+  "owner": {
+    "name": "Plaited Labs"
+  },
+  "plugins": [
+    {
+      "name": "acp-harness",
+      "source": "./"
+    }
+  ]
+}

package/.claude-plugin/plugin.json

@@ -0,0 +1,16 @@
+{
+  "name": "acp-harness",
+  "version": "0.2.5",
+  "description": "Agent Client Protocol client and evaluation harness for TypeScript/Bun projects. Includes: createACPClient for programmatic agent access, run-harness.ts for trajectory capture, and LLM-as-judge evaluation templates. Requires Bun runtime.",
+  "author": {
+    "name": "Plaited Labs"
+  },
+  "license": "ISC",
+  "mcpServers": {
+    "agent-client-protocol": {
+      "type": "http",
+      "url": "https://agentclientprotocol.com/mcp"
+    }
+  },
+  "skills": "./.claude/skills"
+}

package/.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# Code owners for acp-harness repository
+# These users will be automatically requested for review when someone opens a pull request.
+# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+
+# Organization admins own all files
+* @EdwardIrby @alisonailea

package/.github/workflows/ci.yml

@@ -0,0 +1,63 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  # Detect which paths changed to conditionally run expensive jobs
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      acp: ${{ steps.filter.outputs.acp }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            acp:
+              - 'src/**'
+
+  test-pr:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+      - name: Install dependencies
+        run: bun install
+      - name: Run check
+        run: bun run check
+      - name: Run test
+        run: bun run test
+
+  # ACP integration tests run in Docker container for consistent environment
+  # Only runs when src/ files change to reduce API costs
+  test-acp-integration:
+    needs: changes
+    if: ${{ needs.changes.outputs.acp == 'true' }}
+    runs-on: ubuntu-latest
+    container:
+      image: oven/bun:1.2.9
+      options: --user root
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Run ACP integration tests
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: bun test ./src/tests/acp-integration.docker.ts