@plaited/acp-harness 0.3.2 → 0.4.0

package/README.md

@@ -11,21 +11,25 @@ CLI tool for capturing agent trajectories from ACP-compatible agents. Execute pr
 Use these tools directly via the CLI without installation:
 
 ```bash
-# Run without installing
-bunx @plaited/acp-harness capture prompts.jsonl bunx claude-code-acp -o results.jsonl
+# Using built-in headless adapter (recommended - no extra install needed)
+export ANTHROPIC_API_KEY=sk-...
+bunx @plaited/acp-harness capture prompts.jsonl \
+  bunx @plaited/acp-harness headless --schema ./schemas/claude-headless.json \
+  -o results.jsonl
 
-# Or install globally
-bun add -g @plaited/acp-harness
-acp-harness capture prompts.jsonl bunx claude-code-acp -o results.jsonl
+# Or with an external ACP adapter
+bunx @plaited/acp-harness capture prompts.jsonl bunx claude-code-acp -o results.jsonl
 ```
 
-**Prerequisite:** Install an ACP adapter and set your API key:
+**Prerequisite:** Set your API key. The `headless` command works with any CLI agent that supports JSON output - no adapter installation required:
 
 ```bash
-npm install -g @anthropic-ai/claude-code-acp
-export ANTHROPIC_API_KEY=sk-...
+export ANTHROPIC_API_KEY=sk-...   # For Claude
+export GEMINI_API_KEY=...         # For Gemini
 ```
 
+Pre-built schemas are available in `.claude/skills/acp-adapters/schemas/` for Claude and Gemini.
+
 ### Commands
 
 | Command | Description |
@@ -37,17 +41,21 @@ export ANTHROPIC_API_KEY=sk-...
 | `validate-refs <prompts>` | Check reference solutions |
 | `balance <prompts>` | Analyze test set coverage |
 | `schemas [name]` | Export JSON schemas |
-| `adapter:scaffold [name]` | Scaffold new ACP adapter project |
+| `headless --schema <path>` | Schema-driven adapter for any CLI agent |
 | `adapter:check <cmd>` | Validate adapter ACP compliance |
 
 ### Examples
 
 ```bash
-# Capture trajectories
-bunx @plaited/acp-harness capture prompts.jsonl bunx claude-code-acp -o results.jsonl
+# Capture trajectories using headless adapter (recommended)
+bunx @plaited/acp-harness capture prompts.jsonl \
+  bunx @plaited/acp-harness headless --schema ./schemas/claude-headless.json \
+  -o results.jsonl
 
 # Run trials for pass@k analysis
-bunx @plaited/acp-harness trials prompts.jsonl bunx claude-code-acp -k 5 --grader ./grader.ts
+bunx @plaited/acp-harness trials prompts.jsonl \
+  bunx @plaited/acp-harness headless --schema ./schemas/claude-headless.json \
+  -k 5 --grader ./grader.ts
 
 # Summarize results
 bunx @plaited/acp-harness summarize results.jsonl -o summary.jsonl
@@ -55,11 +63,9 @@ bunx @plaited/acp-harness summarize results.jsonl -o summary.jsonl
 # Export schemas
 bunx @plaited/acp-harness schemas CaptureResult --json
 
-# Scaffold a new adapter
-bunx @plaited/acp-harness adapter:scaffold my-agent -o ./my-agent-acp
-
 # Validate adapter compliance
-bunx @plaited/acp-harness adapter:check bun ./my-adapter/src/main.ts
+bunx @plaited/acp-harness adapter:check \
+  bunx @plaited/acp-harness headless --schema ./schemas/claude-headless.json
 ```
 
 ## Skills for AI Agents
@@ -110,10 +116,12 @@ Discover, create, and validate ACP adapters for agent integration.
 
 | Command | Description |
 |---------|-------------|
+| `headless` | Schema-driven adapter for any CLI agent |
 | `adapter:scaffold` | Generate new adapter project with handlers |
 | `adapter:check` | Validate ACP protocol compliance |
 
 **Use cases:**
+- Wrapping headless CLI agents with schema-driven adapter
 - Finding existing adapters for your agent
 - Building custom ACP adapters from scratch
 - Validating adapter implementations
@@ -121,10 +129,18 @@ Discover, create, and validate ACP adapters for agent integration.
 ## Input Format
 
 ```jsonl
-{"id":"test-001","input":"Create a primary button","expected":"should contain <button>","metadata":{"category":"ui"}}
-{"id":"test-002","input":"Fix the TypeScript error","metadata":{"category":"bugfix"}}
+{"id":"test-001","input":"Create a primary button","hint":"should contain <button>","metadata":{"category":"ui"}}
+{"id":"test-002","input":["Create a component","Now add tests"],"metadata":{"category":"multi-turn"}}
 ```
 
+| Field | Required | Description |
+|-------|----------|-------------|
+| `id` | Yes | Unique identifier |
+| `input` | Yes | Single prompt (string) or conversation turns (string[]) |
+| `hint` | No | Grader context - what to look for |
+| `reference` | No | Reference solution (for validate-refs) |
+| `metadata` | No | Tags, category, difficulty for filtering |
+
 ## Output Format
 
 The harness outputs full trajectory JSONL (`CaptureResult` schema):
@@ -134,12 +150,12 @@ The harness outputs full trajectory JSONL (`CaptureResult` schema):
   "id": "test-001",
   "input": "Create a primary button",
   "output": "Here's a button component...",
-  "expected": "should contain <button>",
+  "hint": "should contain <button>",
   "trajectory": [...],
-  "metadata": {"category": "ui", "agent": "bunx claude-code-acp"},
-  "timing": {"start": 1234567890, "end": 1234567900},
+  "metadata": {"category": "ui", "agent": "bunx claude-code-acp", "trajectoryRichness": "full", "turnCount": 1},
+  "timing": {"start": 1234567890, "end": 1234567900, "sessionCreation": 234, "total": 10},
   "toolErrors": false,
-  "score": {"pass": true, "score": 1.0, "reasoning": "Contains expected"}
+  "score": {"pass": true, "score": 1.0, "reasoning": "Contains hint"}
 }
 ```
 
@@ -147,6 +163,9 @@ Key fields:
 - `toolErrors`: Boolean indicating if any tool calls failed
 - `score`: Grader result (only if `--grader` provided)
 - `trajectory`: Full execution trace (thoughts, messages, tool calls, plans)
+- `metadata.trajectoryRichness`: `"full"` | `"messages-only"` | `"minimal"`
+- `timing.sessionCreation`: Time to initialize session (ms)
+- `timing.total`: End-to-end duration (ms)
 
 ## Graders
 
@@ -159,12 +178,12 @@ Export a `grade` function:
 ```typescript
 import type { Grader } from '@plaited/acp-harness/schemas'
 
-export const grade: Grader = async ({ input, output, expected, trajectory }) => {
-  const pass = output.toLowerCase().includes(expected?.toLowerCase() ?? '')
+export const grade: Grader = async ({ input, output, hint, trajectory }) => {
+  const pass = output.toLowerCase().includes(hint?.toLowerCase() ?? '')
   return {
     pass,
     score: pass ? 1.0 : 0.0,
-    reasoning: pass ? 'Contains expected answer' : 'Missing expected answer'
+    reasoning: pass ? 'Contains hint content' : 'Missing hint content'
   }
 }
 ```
@@ -184,13 +203,13 @@ import sys
 
 data = json.load(sys.stdin)
 output = data["output"].lower()
-expected = (data.get("expected") or "").lower()
+hint = (data.get("hint") or "").lower()
 
-pass_result = expected in output if expected else True
+pass_result = hint in output if hint else True
 print(json.dumps({
     "pass": pass_result,
     "score": 1.0 if pass_result else 0.0,
-    "reasoning": "Contains expected" if pass_result else "Missing expected"
+    "reasoning": "Contains hint" if pass_result else "Missing hint"
 }))
 ```
 
@@ -200,7 +219,7 @@ acp-harness capture prompts.jsonl bunx claude-code-acp --grader ./grader.py
 ```
 
 **Protocol:**
-- Input (stdin): `{"input": "...", "output": "...", "expected": "...", "trajectory": [...]}`
+- Input (stdin): `{"input": "...", "output": "...", "hint": "...", "trajectory": [...]}`
 - Output (stdout): `{"pass": true, "score": 1.0, "reasoning": "..."}`
 
 ## Downstream Integration
@@ -222,13 +241,16 @@ cat results.jsonl | your-scoring-script.ts
 bun install          # Install dependencies
 bun run check        # Type check + lint + format
 bun test             # Run unit tests
+
+# Run integration tests in Docker (requires API keys)
+ANTHROPIC_API_KEY=sk-... docker compose -f docker-compose.test.yml run --rm acp-test
 ```
 
 ## Requirements
 
 - **Runtime:** Bun >= 1.2.9
-- **ACP Adapter:** `@anthropic-ai/claude-code-acp` or compatible
-- **API Key:** `ANTHROPIC_API_KEY` environment variable
+- **ACP Adapter:** Built-in `headless` command (recommended) or external adapter
+- **API Key:** `ANTHROPIC_API_KEY` for Claude, `GEMINI_API_KEY` for Gemini
 
 ## License
 

package/bin/cli.ts

@@ -14,6 +14,7 @@
  * - validate-refs: Check reference solutions
  * - balance: Analyze test set coverage
  * - schemas: Export JSON schemas for non-TS users
+ * - headless: Schema-driven adapter for any headless CLI agent
  * - adapter:scaffold: Scaffold new ACP adapter project
  * - adapter:check: Validate adapter ACP compliance
  */
@@ -23,6 +24,7 @@ import { adapterScaffold } from '../src/adapter-scaffold.ts'
 import { balance } from '../src/balance.ts'
 import { calibrate } from '../src/calibrate.ts'
 import { capture } from '../src/capture.ts'
+import { headless } from '../src/headless.ts'
 import { schemasCli } from '../src/schemas-cli.ts'
 import { summarize } from '../src/summarize.ts'
 import { trials } from '../src/trials.ts'
@@ -43,6 +45,7 @@ Commands:
   validate-refs    Check reference solutions against grader
   balance          Analyze test set coverage
   schemas          Export JSON schemas for non-TypeScript users
+  headless         Schema-driven adapter for any headless CLI agent
   adapter:scaffold Scaffold a new ACP adapter project
   adapter:check    Validate adapter ACP compliance
 
@@ -70,6 +73,14 @@ Examples:
   # Validate adapter compliance
   acp-harness adapter:check bun ./my-adapter/src/main.ts
 
+  # Run headless adapter with schema
+  acp-harness headless --schema ./claude-headless.json
+
+  # Capture with headless adapter
+  acp-harness capture prompts.jsonl \\
+    acp-harness headless --schema ./claude-headless.json \\
+    -o results.jsonl
+
 Documentation: https://github.com/plaited/acp-harness
 `)
 }
@@ -104,6 +115,10 @@ const main = async () => {
       await schemasCli(args)
       break
 
+    case 'headless':
+      await headless(args)
+      break
+
     case 'adapter:scaffold':
       await adapterScaffold(args)
       break

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plaited/acp-harness",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "CLI tool for capturing agent trajectories from ACP-compatible agents",
   "license": "ISC",
   "engines": {
@@ -28,10 +28,8 @@
     "./bin/**",
     "!./src/**/tests/*",
     "!./src/**/*.spec.ts",
-    "!./src/**/*.docker.ts",
     "!./bin/**/tests/*",
-    "!./bin/**/*.spec.ts",
-    "!./bin/**/*.docker.ts"
+    "!./bin/**/*.spec.ts"
   ],
   "publishConfig": {
     "access": "public"
@@ -43,8 +41,8 @@
     "check:types": "tsc --noEmit",
     "check:write": "biome check --write && format-package --write",
     "prepare": "git rev-parse --git-dir > /dev/null 2>&1 && git config core.hooksPath .hooks || true",
-    "test": "bun test ./src/ ./bin/ ./.claude",
-    "test:docker": "docker compose -f docker-compose.test.yml run --rm acp-test"
+    "test": "bun test ./**/tests/*.spec.ts",
+    "test:integration": "bun test ./**/integration_tests/*.spec.ts"
   },
   "lint-staged": {
     "*.{js,cjs,jsx,tsx,ts}": [
@@ -56,7 +54,7 @@
   },
   "dependencies": {
     "zod": "^4.3.5",
-    "@plaited/development-skills": "0.6.2"
+    "@plaited/development-skills": "0.6.3"
   },
   "peerDependencies": {
     "typescript-language-server": "^5.1.3",

package/src/acp-client.ts

@@ -22,7 +22,6 @@ import type {
   Implementation,
   InitializeRequest,
   InitializeResponse,
-  McpServer,
   PromptRequest,
   PromptResponse,
   RequestPermissionRequest,
@@ -277,18 +276,22 @@ export const createACPClient = (config: ACPClientConfig) => {
   /**
    * Creates a new conversation session.
    *
-   * @param params - Session parameters with working directory and optional MCP servers
+   * @remarks
+   * MCP servers are auto-discovered by the agent from configuration files
+   * in the working directory (e.g., `.mcp.json`, `.gemini/settings.json`).
+   *
+   * @param params - Session parameters with working directory
    * @returns The created session
    * @throws {ACPClientError} If not connected
    */
-  const createSession = async (params: { cwd: string; mcpServers?: McpServer[] }): Promise<Session> => {
+  const createSession = async (params: { cwd: string }): Promise<Session> => {
     if (!transport?.isConnected()) {
       throw new ACPClientError('Not connected')
     }
 
     const response = await transport.request<{ sessionId: string }>(ACP_METHODS.CREATE_SESSION, {
       cwd: params.cwd,
-      mcpServers: params.mcpServers ?? [],
+      mcpServers: [], // Required field - empty array lets agents auto-discover from cwd
     })
     return { id: response.sessionId }
   }

package/src/adapter-check.ts

@@ -185,7 +185,6 @@ const checkSessionNew = async (
   try {
     const response = await transport.request<{ sessionId: string }>(ACP_METHODS.CREATE_SESSION, {
       cwd: process.cwd(),
-      mcpServers: [],
     })
 
     if (!response || !response.sessionId) {

package/src/adapter-scaffold.ts

@@ -296,7 +296,6 @@ import { sessionManager } from '../session-manager.ts'
 
 type SessionNewParams = {
   cwd: string
-  mcpServers?: unknown[]
 }
 
 type SessionNewResult = {
@@ -304,12 +303,11 @@ type SessionNewResult = {
 }
 
 export const handleSessionNew = async (params: unknown): Promise<SessionNewResult> => {
-  const { cwd, mcpServers = [] } = params as SessionNewParams
+  const { cwd } = params as SessionNewParams
 
-  const sessionId = sessionManager.createSession({
-    cwd,
-    mcpServers,
-  })
+  // MCP servers are discovered from cwd configuration files
+  // (e.g., .mcp.json, .gemini/settings.json)
+  const sessionId = sessionManager.createSession({ cwd })
 
   return { sessionId }
 }
@@ -438,19 +436,17 @@ import { randomUUID } from 'node:crypto'
 type Session = {
   id: string
   cwd: string
-  mcpServers: unknown[]
   createdAt: Date
 }
 
 class SessionManager {
   #sessions = new Map<string, Session>()
 
-  createSession(params: { cwd: string; mcpServers: unknown[] }): string {
+  createSession(params: { cwd: string }): string {
     const id = \`sess_\${randomUUID().slice(0, 8)}\`
     this.#sessions.set(id, {
       id,
       cwd: params.cwd,
-      mcpServers: params.mcpServers,
       createdAt: new Date(),
     })
     return id
@@ -550,13 +546,15 @@ from typing import Any, Dict, Optional
 sessions: Dict[str, Dict[str, Any]] = {}
 
 
-def create_session(cwd: str, mcp_servers: list) -> str:
-    """Create a new session."""
+def create_session(cwd: str) -> str:
+    """Create a new session.
+
+    MCP servers are discovered from cwd configuration files.
+    """
     session_id = f"sess_{uuid.uuid4().hex[:8]}"
     sessions[session_id] = {
         "id": session_id,
         "cwd": cwd,
-        "mcp_servers": mcp_servers,
     }
     return session_id
 
@@ -597,10 +595,13 @@ def handle_initialize(params: Dict[str, Any]) -> Dict[str, Any]:
 
 
 def handle_session_new(params: Dict[str, Any]) -> Dict[str, Any]:
-    """Handle session/new request."""
+    """Handle session/new request.
+
+    MCP servers are discovered from cwd configuration files
+    (e.g., .mcp.json, .gemini/settings.json).
+    """
     cwd = params.get("cwd", ".")
-    mcp_servers = params.get("mcpServers", [])
-    session_id = create_session(cwd, mcp_servers)
+    session_id = create_session(cwd)
     return {"sessionId": session_id}
 
 

package/src/calibrate.ts

@@ -57,17 +57,37 @@ const loadResults = async (path: string): Promise<CaptureResult[]> => {
 }
 
 /**
- * Random sample from array.
+ * Randomly sample n elements from an array using Fisher-Yates shuffle.
  *
  * @param arr - Array to sample from
  * @param n - Number of samples to take
- * @returns Array of sampled elements
+ * @returns Array of sampled elements in random order
+ *
+ * @remarks
+ * Uses Fisher-Yates (Knuth) shuffle for uniform distribution.
+ * Creates a copy to avoid mutating the input array.
+ * O(n) time complexity with O(n) space for the copy.
+ * Not cryptographically secure (uses Math.random).
  *
  * @public
  */
 export const sampleArray = <T>(arr: T[], n: number): T[] => {
-  const shuffled = [...arr].sort(() => 0.5 - Math.random())
-  return shuffled.slice(0, n)
+  if (n <= 0) return []
+  if (n >= arr.length) return [...arr]
+
+  const copy = [...arr]
+
+  // Fisher-Yates shuffle working backwards through array
+  // Only shuffle enough elements to get n samples
+  const limit = copy.length - n
+  for (let i = copy.length - 1; i >= limit && i > 0; i--) {
+    // Random index from 0 to i (inclusive)
+    const j = Math.floor(Math.random() * (i + 1))
+    // Swap elements
+    ;[copy[i], copy[j]] = [copy[j]!, copy[i]!]
+  }
+
+  return copy.slice(-n)
 }
 
 /**
@@ -129,8 +149,8 @@ const formatCalibrationMarkdown = (samples: CalibrationSample[]): string => {
     lines.push(`**Input:** ${sample.input}`)
     lines.push('')
 
-    if (sample.expected) {
-      lines.push(`**Expected:** ${sample.expected}`)
+    if (sample.hint) {
+      lines.push(`**Hint:** ${sample.hint}`)
       lines.push('')
     }
 
@@ -212,7 +232,7 @@ export const runCalibrate = async (config: CalibrateConfig): Promise<Calibration
       id: result.id,
       input: result.input,
       output: result.output,
-      expected: result.expected,
+      hint: result.hint,
       originalScore: result.score as GraderResult,
       trajectorySnippet: getTrajectorySnippet(result.trajectory),
     }
@@ -222,7 +242,7 @@ export const runCalibrate = async (config: CalibrateConfig): Promise<Calibration
       calibrationSample.rescoredResult = await grader({
         input: result.input,
         output: result.output,
-        expected: result.expected,
+        hint: result.hint,
         trajectory: result.trajectory,
       })
     }