@plaited/acp-harness 0.2.5

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@plaited/acp-harness",
+  "version": "0.2.5",
+  "description": "CLI tool for capturing agent trajectories from ACP-compatible agents",
+  "license": "ISC",
+  "engines": {
+    "bun": ">= v1.2.9"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/plaited/acp-harness.git"
+  },
+  "bugs": {
+    "url": "https://github.com/plaited/acp-harness/issues"
+  },
+  "homepage": "https://github.com/plaited/acp-harness/tree/main#readme",
+  "bin": {
+    "acp-harness": "./bin/cli.ts"
+  },
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "check": "bun run check:biome && bun run check:types && bun run check:package",
+    "check:biome": "biome check",
+    "check:package": "format-package --check",
+    "check:types": "tsc --noEmit",
+    "check:write": "biome check --write && format-package --write",
+    "prepare": "git rev-parse --git-dir > /dev/null 2>&1 && git config core.hooksPath .hooks || true",
+    "test": "bun test ./src/ ./bin/ ./.claude",
+    "test:docker": "docker compose -f docker-compose.test.yml run --rm acp-test"
+  },
+  "lint-staged": {
+    "*.{js,cjs,jsx,tsx,ts}": [
+      "bunx biome check --write --files-ignore-unknown"
+    ],
+    "package.json": [
+      "format-package -w"
+    ]
+  },
+  "dependencies": {
+    "@agentclientprotocol/sdk": "^0.13.0",
+    "zod": "^4.3.5"
+  },
+  "peerDependencies": {
+    "bun": ">=1.2.9"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.3.11",
+    "@types/bun": "1.3.6",
+    "@zed-industries/claude-code-acp": "0.13.0",
+    "format-package": "7.0.0",
+    "lint-staged": "16.2.7",
+    "typescript": "5.9.3"
+  }
+}

package/scripts/bun-test-wrapper.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Wrapper for bun test that handles Bun's post-test cleanup crash
+# See: https://github.com/oven-sh/bun/issues/23643
+#
+# Bun 1.3.x has a known bug where the test runner crashes during cleanup
+# after all tests complete successfully. This wrapper catches that crash
+# (exit code 133 = SIGTRAP) and exits cleanly if tests actually passed.
+#
+# Usage:
+#   ./bun-test-wrapper.sh [args...]     # Run bun test with provided args
+#   ./bun-test-wrapper.sh               # (No args) Find and run all *.docker.ts files
+
+# Determine test files to run
+if [ $# -eq 0 ]; then
+  # No arguments: find all *.docker.ts files for Docker integration tests
+  docker_tests=$(find ./src ./bin -name "*.docker.ts" -type f 2>/dev/null)
+  if [ -z "$docker_tests" ]; then
+    echo "No *.docker.ts files found in ./src or ./bin"
+    exit 0
+  fi
+  echo "Found Docker integration tests:"
+  echo "$docker_tests" | sed 's/^/  /'
+  echo ""
+  # Convert newlines to arguments
+  set -- $docker_tests
+fi
+
+# Create temp file for output
+tmpfile=$(mktemp)
+trap "rm -f $tmpfile" EXIT
+
+# Run tests with output to both terminal and file
+bun test "$@" 2>&1 | tee "$tmpfile"
+exit_code=${PIPESTATUS[0]}
+
+# Check if tests passed (look for "X pass" and "0 fail" in output)
+if grep -q " pass" "$tmpfile" && grep -q "0 fail" "$tmpfile"; then
+  # Tests passed - exit 0 even if Bun crashed during cleanup
+  if [ $exit_code -eq 133 ]; then
+    echo ""
+    echo "Note: Bun crashed during cleanup (known bug), but all tests passed."
+    exit 0
+  fi
+fi
+
+exit $exit_code

package/src/acp-client.ts

@@ -0,0 +1,503 @@
+/**
+ * Headless ACP client for programmatic agent interaction.
+ *
+ * @remarks
+ * This client enables automated evaluation of ACP-compatible agents like
+ * Claude Code, Droid, Gemini CLI, and others. It provides:
+ *
+ * - **Subprocess management**: Spawn and control agent processes
+ * - **Session handling**: Create and manage conversation sessions
+ * - **Streaming prompts**: AsyncGenerator for real-time updates
+ * - **Sync prompts**: Simple request/response for basic evals
+ * - **Auto-permissions**: Automatically approves all permissions for headless use
+ *
+ * Designed for testing and evaluation, not for user-facing applications.
+ */
+
+import type {
+  AgentCapabilities,
+  CancelNotification,
+  ClientCapabilities,
+  ContentBlock,
+  Implementation,
+  InitializeRequest,
+  InitializeResponse,
+  McpServer,
+  PromptRequest,
+  PromptResponse,
+  RequestPermissionRequest,
+  RequestPermissionResponse,
+  SessionNotification,
+} from '@agentclientprotocol/sdk'
+import { version } from '../package.json' with { type: 'json' }
+import { ACP_METHODS, ACP_PROTOCOL_VERSION, DEFAULT_ACP_CLIENT_NAME } from './acp.constants.ts'
+import { RequestPermissionRequestSchema, SessionNotificationSchema } from './acp.schemas.ts'
+import type { Session } from './acp.types.ts'
+import { createACPTransport } from './acp-transport.ts'
+// ============================================================================
+// Types
+// ============================================================================
+
+/** Configuration for the ACP client */
+export type ACPClientConfig = {
+  /** Command to spawn agent (e.g., ['claude', 'code'] or ['droid']) */
+  command: string[]
+  /** Working directory for agent process */
+  cwd?: string
+  /** Environment variables for agent process */
+  env?: Record<string, string>
+  /** Client info for initialization */
+  clientInfo?: Implementation
+  /** Client capabilities to advertise */
+  capabilities?: ClientCapabilities
+  /** Timeout for operations in milliseconds (default: 30000) */
+  timeout?: number
+  /**
+   * Polling interval for streaming updates in milliseconds (default: 50).
+   * Lower values provide more responsive updates but increase CPU usage.
+   * Consider increasing for testing to reduce timing-related flakiness.
+   */
+  pollingInterval?: number
+  /**
+   * Permission handler for agent requests.
+   * Default: auto-approve all permissions (headless mode)
+   */
+  onPermissionRequest?: (params: RequestPermissionRequest) => Promise<RequestPermissionResponse>
+}
+
+/** Session update emitted during prompt streaming */
+export type SessionUpdate = {
+  type: 'update'
+  params: SessionNotification
+}
+
+/** Prompt completion emitted when prompt finishes */
+export type PromptComplete = {
+  type: 'complete'
+  result: PromptResponse
+}
+
+/** Events emitted during prompt streaming */
+export type PromptEvent = SessionUpdate | PromptComplete
+
+/** Error thrown by ACP client operations */
+export class ACPClientError extends Error {
+  constructor(
+    message: string,
+    public readonly code?: string,
+  ) {
+    super(message)
+    this.name = 'ACPClientError'
+  }
+}
+
+// ============================================================================
+// Client Implementation
+// ============================================================================
+
+/**
+ * Creates a headless ACP client for agent evaluation.
+ *
+ * @param config - Client configuration including command, cwd, and permission handling
+ * @returns Client object with lifecycle, session, and prompt methods
+ *
+ * @remarks
+ * The client manages:
+ * - Agent subprocess lifecycle (connect/disconnect)
+ * - Protocol initialization and capability negotiation
+ * - Session creation and management
+ * - Prompt streaming with real-time updates
+ * - Automatic permission approval for headless evaluation
+ *
+ * See module-level documentation in `src/acp.ts` for usage guidance.
+ * See client tests for usage patterns.
+ */
+export const createACPClient = (config: ACPClientConfig) => {
+  const {
+    command,
+    cwd,
+    env,
+    clientInfo = { name: DEFAULT_ACP_CLIENT_NAME, version },
+    capabilities = {},
+    timeout = 30000,
+    pollingInterval = 50,
+    onPermissionRequest,
+  } = config
+
+  let transport: ReturnType<typeof createACPTransport> | undefined
+  let agentCapabilities: AgentCapabilities | undefined
+  let initializeResult: InitializeResponse | undefined
+
+  // Track active prompt sessions for update routing
+  const activePrompts = new Map<
+    string,
+    {
+      updates: SessionNotification[]
+      resolve: (result: PromptResponse) => void
+      reject: (error: Error) => void
+    }
+  >()
+
+  // --------------------------------------------------------------------------
+  // Permission Handling
+  // --------------------------------------------------------------------------
+
+  /**
+   * Default permission handler: auto-approve all requests.
+   * For headless evaluation in trusted environments.
+   *
+   * @remarks
+   * Validates params with Zod before processing.
+   * Prioritizes `allow_always` for faster headless evaluation with fewer
+   * permission round-trips. Cancels if validation fails or no allow option
+   * is available.
+   */
+  const autoApprovePermission = async (params: RequestPermissionRequest): Promise<RequestPermissionResponse> => {
+    const result = RequestPermissionRequestSchema.safeParse(params)
+    if (!result.success) {
+      return { outcome: { outcome: 'cancelled' } }
+    }
+
+    const { options } = result.data
+
+    // Priority: allow_always (fewer round-trips) > allow_once
+    const allowAlways = options.find((opt) => opt.kind === 'allow_always')
+    if (allowAlways) {
+      return { outcome: { outcome: 'selected', optionId: allowAlways.optionId } }
+    }
+
+    const allowOnce = options.find((opt) => opt.kind === 'allow_once')
+    if (allowOnce) {
+      return { outcome: { outcome: 'selected', optionId: allowOnce.optionId } }
+    }
+
+    // No allow option available - cancel
+    return { outcome: { outcome: 'cancelled' } }
+  }
+
+  const handlePermissionRequest = onPermissionRequest ?? autoApprovePermission
+
+  // --------------------------------------------------------------------------
+  // Transport Callbacks
+  // --------------------------------------------------------------------------
+
+  const onNotification = (method: string, params: unknown) => {
+    if (method === ACP_METHODS.UPDATE) {
+      const updateParams = SessionNotificationSchema.parse(params)
+      const activePrompt = activePrompts.get(updateParams.sessionId)
+      if (activePrompt) {
+        activePrompt.updates.push(updateParams)
+      }
+    }
+  }
+
+  const onRequest = async (method: string, params: unknown): Promise<unknown> => {
+    if (method === ACP_METHODS.REQUEST_PERMISSION) {
+      return handlePermissionRequest(RequestPermissionRequestSchema.parse(params))
+    }
+
+    throw new ACPClientError(`Unknown request method: ${method}`)
+  }
+
+  // --------------------------------------------------------------------------
+  // Lifecycle Methods
+  // --------------------------------------------------------------------------
+
+  /**
+   * Connects to the agent by spawning the subprocess and initializing the protocol.
+   *
+   * @returns Initialize result with agent capabilities
+   * @throws {ACPClientError} If already connected or connection fails
+   */
+  const connect = async (): Promise<InitializeResponse> => {
+    if (transport?.isConnected()) {
+      throw new ACPClientError('Already connected')
+    }
+
+    transport = createACPTransport({
+      command,
+      cwd,
+      env,
+      timeout,
+      onNotification,
+      onRequest,
+      onError: (error) => {
+        console.error('[ACP Client Error]:', error.message)
+      },
+      onClose: (code) => {
+        // Reject all active prompts on unexpected close
+        for (const [sessionId, prompt] of activePrompts) {
+          prompt.reject(new ACPClientError(`Agent process exited with code ${code}`))
+          activePrompts.delete(sessionId)
+        }
+      },
+    })
+
+    await transport.start()
+
+    // Initialize protocol
+    const initParams: InitializeRequest = {
+      protocolVersion: ACP_PROTOCOL_VERSION,
+      clientInfo,
+      clientCapabilities: capabilities,
+    }
+
+    initializeResult = await transport.request<InitializeResponse>(ACP_METHODS.INITIALIZE, initParams)
+
+    agentCapabilities = initializeResult?.agentCapabilities
+
+    return initializeResult
+  }
+
+  /**
+   * Disconnects from the agent, closing the subprocess.
+   *
+   * @param graceful - If true, sends shutdown request first (default: true)
+   */
+  const disconnect = async (graceful = true): Promise<void> => {
+    if (!transport) return
+
+    // Cancel all active prompts
+    for (const [sessionId, prompt] of activePrompts) {
+      prompt.reject(new ACPClientError('Client disconnected'))
+      activePrompts.delete(sessionId)
+    }
+
+    await transport.close(graceful)
+    transport = undefined
+    agentCapabilities = undefined
+    initializeResult = undefined
+  }
+
+  // --------------------------------------------------------------------------
+  // Session Methods
+  // --------------------------------------------------------------------------
+
+  /**
+   * Creates a new conversation session.
+   *
+   * @param params - Session parameters with working directory and optional MCP servers
+   * @returns The created session
+   * @throws {ACPClientError} If not connected
+   */
+  const createSession = async (params: { cwd: string; mcpServers?: McpServer[] }): Promise<Session> => {
+    if (!transport?.isConnected()) {
+      throw new ACPClientError('Not connected')
+    }
+
+    const response = await transport.request<{ sessionId: string }>(ACP_METHODS.CREATE_SESSION, {
+      cwd: params.cwd,
+      mcpServers: params.mcpServers ?? [],
+    })
+    return { id: response.sessionId }
+  }
+
+  /**
+   * Sets the model for a session.
+   *
+   * @experimental This is an unstable ACP feature and may change.
+   * @param sessionId - The session ID to set the model for
+   * @param modelId - The model ID (e.g., 'claude-3-5-haiku-20241022', 'claude-sonnet-4-20250514')
+   * @throws {ACPClientError} If not connected
+   */
+  const setModel = async (sessionId: string, modelId: string): Promise<void> => {
+    if (!transport?.isConnected()) {
+      throw new ACPClientError('Not connected')
+    }
+
+    await transport.request(ACP_METHODS.SET_MODEL, { sessionId, modelId })
+  }
+
+  // --------------------------------------------------------------------------
+  // Prompt Methods
+  // --------------------------------------------------------------------------
+
+  /**
+   * Sends a prompt and streams updates as they arrive.
+   *
+   * @param sessionId - The session ID to send the prompt to
+   * @param content - Content blocks for the prompt
+   * @yields Session updates and final completion
+   * @throws {ACPClientError} If not connected
+   *
+   * @remarks
+   * Use this for evaluation scenarios where you need access to
+   * intermediate updates (tool calls, plan changes, etc).
+   */
+  async function* prompt(sessionId: string, content: ContentBlock[]): AsyncGenerator<PromptEvent> {
+    if (!transport?.isConnected()) {
+      throw new ACPClientError('Not connected')
+    }
+
+    const { promise, resolve, reject } = Promise.withResolvers<PromptResponse>()
+    const updates: SessionNotification[] = []
+    const promptState = {
+      updates,
+      resolve,
+      reject,
+    }
+
+    activePrompts.set(sessionId, promptState)
+
+    // Send prompt request
+    const promptParams: PromptRequest = {
+      sessionId,
+      prompt: content,
+    }
+
+    // Start the prompt request (don't await - we'll poll for updates)
+    const promptPromise = transport
+      .request<PromptResponse>(ACP_METHODS.PROMPT, promptParams)
+      .then(resolve)
+      .catch(reject)
+
+    try {
+      // Poll for updates until prompt completes
+      let lastYieldedIndex = 0
+
+      while (true) {
+        // Yield any new updates
+        while (lastYieldedIndex < promptState.updates.length) {
+          const update = promptState.updates[lastYieldedIndex]
+          if (update) {
+            yield { type: 'update', params: update }
+          }
+          lastYieldedIndex++
+        }
+
+        // Check if prompt completed
+        const raceResult = await Promise.race([
+          promise.then((result) => ({ done: true as const, result })),
+          new Promise<{ done: false }>((res) => setTimeout(() => res({ done: false }), pollingInterval)),
+        ])
+
+        if (raceResult.done) {
+          // Yield any remaining updates
+          while (lastYieldedIndex < promptState.updates.length) {
+            const update = promptState.updates[lastYieldedIndex]
+            if (update) {
+              yield { type: 'update', params: update }
+            }
+            lastYieldedIndex++
+          }
+
+          // Yield completion
+          yield {
+            type: 'complete',
+            result: raceResult.result,
+          }
+          break
+        }
+      }
+
+      await promptPromise
+    } finally {
+      activePrompts.delete(sessionId)
+    }
+  }
+
+  /**
+   * Sends a prompt and waits for the final result.
+   *
+   * @param sessionId - The session ID to send the prompt to
+   * @param content - Content blocks for the prompt
+   * @returns The prompt result with all accumulated updates
+   * @throws {ACPClientError} If not connected
+   *
+   * @remarks
+   * Use this for simple evaluation scenarios where you only need
+   * the final result. All intermediate updates are collected but
+   * returned together at the end.
+   */
+  const promptSync = async (
+    sessionId: string,
+    content: ContentBlock[],
+  ): Promise<{
+    result: PromptResponse
+    updates: SessionNotification[]
+  }> => {
+    const updates: SessionNotification[] = []
+    let result: PromptResponse | undefined
+
+    for await (const event of prompt(sessionId, content)) {
+      if (event.type === 'update') {
+        updates.push(event.params)
+      } else if (event.type === 'complete') {
+        result = event.result
+      }
+    }
+
+    if (!result) {
+      throw new ACPClientError('Prompt completed without result')
+    }
+
+    return { result, updates }
+  }
+
+  /**
+   * Cancels an ongoing prompt.
+   *
+   * @param sessionId - The session ID to cancel
+   * @throws {ACPClientError} If not connected
+   */
+  const cancelPrompt = async (sessionId: string): Promise<void> => {
+    if (!transport?.isConnected()) {
+      throw new ACPClientError('Not connected')
+    }
+
+    const cancelParams: CancelNotification = { sessionId }
+    await transport.notify(ACP_METHODS.CANCEL, cancelParams)
+  }
+
+  // --------------------------------------------------------------------------
+  // State Methods
+  // --------------------------------------------------------------------------
+
+  /**
+   * Gets the agent capabilities negotiated during initialization.
+   *
+   * @returns Agent capabilities or undefined if not connected
+   */
+  const getCapabilities = (): AgentCapabilities | undefined => {
+    return agentCapabilities
+  }
+
+  /**
+   * Gets the full initialization result.
+   *
+   * @returns Initialize result or undefined if not connected
+   */
+  const getInitializeResult = (): InitializeResponse | undefined => {
+    return initializeResult
+  }
+
+  /**
+   * Checks if the client is connected to an agent.
+   */
+  const isConnected = (): boolean => {
+    return transport?.isConnected() ?? false
+  }
+
+  return {
+    // Lifecycle
+    connect,
+    disconnect,
+
+    // Sessions
+    createSession,
+    setModel,
+
+    // Prompts
+    prompt,
+    promptSync,
+    cancelPrompt,
+
+    // State
+    getCapabilities,
+    getInitializeResult,
+    isConnected,
+  }
+}
+
+/** Client instance type */
+export type ACPClient = ReturnType<typeof createACPClient>