@plaited/acp-harness 0.2.5

package/.github/workflows/publish.yml

@@ -0,0 +1,146 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "New version tag (e.g., 1.0.0)"
+        required: true
+      next:
+        description: "Next prerelease number (optional, will create a version like x.y.z-next.N)"
+        required: false
+jobs:
+  publish:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - name: Validate inputs
+        env:
+          INPUT_VERSION: ${{ github.event.inputs.version }}
+          INPUT_NEXT: ${{ github.event.inputs.next }}
+        run: |
+          # SECURITY: Validate all inputs before use to prevent injection
+          # Using env vars instead of direct interpolation to prevent shell injection
+
+          # Validate version format (semver)
+          if ! echo "$INPUT_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "::error::Invalid version format: $INPUT_VERSION"
+            echo "::error::Expected: X.Y.Z (e.g., 1.0.0)"
+            exit 1
+          fi
+
+          # Validate next is a number if provided
+          if [[ -n "$INPUT_NEXT" ]] && ! echo "$INPUT_NEXT" | grep -qE '^[0-9]+$'; then
+            echo "::error::Invalid next value: $INPUT_NEXT"
+            echo "::error::Expected: number (e.g., 1, 2, 3)"
+            exit 1
+          fi
+
+          echo "✓ Input validation passed"
+
+      - name: Set version
+        id: set_version
+        env:
+          INPUT_VERSION: ${{ github.event.inputs.version }}
+          INPUT_NEXT: ${{ github.event.inputs.next }}
+        run: |
+          base_version="$INPUT_VERSION"
+          next="$INPUT_NEXT"
+
+          if [[ -n "$next" ]]; then
+            echo "version=${base_version}-next.${next}" >> $GITHUB_OUTPUT
+            echo "is_prerelease=true" >> $GITHUB_OUTPUT
+          else
+            echo "version=${base_version}" >> $GITHUB_OUTPUT
+            echo "is_prerelease=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Validate version format
+        env:
+          VERSION: ${{ steps.set_version.outputs.version }}
+        run: |
+          version="$VERSION"
+
+          # Check for 'v' prefix (common mistake)
+          if [[ "$version" =~ ^v ]]; then
+            echo "::error::Version should not include 'v' prefix"
+            echo "::error::Enter '1.3.4' not 'v1.3.4'"
+            echo "::error::The workflow will automatically add 'v' for Git tags"
+            exit 1
+          fi
+
+          # Validate semver format (allows dots in prerelease suffix for formats like 1.0.0-next.1)
+          if [[ ! "$version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(-([a-zA-Z0-9._-]+))?$ ]]; then
+            echo "::error::Version must have the format 'x.y.z' or 'x.y.z-<string>', where x, y, and z are numbers."
+            exit 1
+          fi
+
+          echo "✅ Version format validated: $version"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_PAT }}
+
+      - name: Setup Node.js (for NPM OIDC)
+        uses: actions/setup-node@v4
+        with:
+          node-version: "lts/*"
+          registry-url: https://registry.npmjs.org
+          check-latest: true
+
+      - uses: oven-sh/setup-bun@v1
+
+      - name: Setup
+        run: |
+          bun install
+
+      - name: Configure Git
+        env:
+          GIT_AUTHOR_NAME: ${{ github.actor }}
+          GIT_AUTHOR_EMAIL: ${{ github.actor }}@users.noreply.github.com
+        run: |
+          git config user.name "$GIT_AUTHOR_NAME"
+          git config user.email "$GIT_AUTHOR_EMAIL"
+
+      - name: Version
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+          VERSION: ${{ steps.set_version.outputs.version }}
+          IS_PRERELEASE: ${{ steps.set_version.outputs.is_prerelease }}
+        run: |
+          npm version --no-git-tag-version "$VERSION"
+
+          # Update plugin.json version
+          jq --arg v "$VERSION" '.version = $v' .claude-plugin/plugin.json > .claude-plugin/plugin.json.tmp
+          mv .claude-plugin/plugin.json.tmp .claude-plugin/plugin.json
+
+          git add -A
+          git commit -m "ci: publish [skip ci]"
+          git push
+
+          # Create GitHub release
+          if [[ "$IS_PRERELEASE" == "true" ]]; then
+            gh release create "v$VERSION" --generate-notes --prerelease
+          else
+            gh release create "v$VERSION" --generate-notes
+          fi
+
+      - name: Publish to NPM (Trusted Publishing)
+        env:
+          IS_PRERELEASE: ${{ steps.set_version.outputs.is_prerelease }}
+        run: |
+          # NPM trusted publishing uses OIDC token automatically
+          # Provenance attestations are generated automatically (npm >=11.5.1)
+          # No NPM_TOKEN secret needed - authentication via GitHub OIDC
+
+          if [[ "$IS_PRERELEASE" == "true" ]]; then
+            echo "Publishing as next (prerelease)"
+            npm publish --access public --tag next
+          else
+            echo "Publishing as latest (stable)"
+            npm publish --access public
+          fi

package/.mcp.json

@@ -0,0 +1,20 @@
+{
+  "mcpServers": {
+    "agent-skills-spec": {
+      "type": "http",
+      "url": "https://agentskills.io/mcp"
+    },
+    "agent-client-protocol": {
+      "type": "http",
+      "url": "https://agentclientprotocol.com/mcp"
+    },
+    "model-context-protocol-docs":{
+      "type": "http",
+      "url": "https://modelcontextprotocol.io/mcp"
+    },
+    "braintrust-docs": {
+      "type": "http",
+      "url": "https://www.braintrust.dev/docs/mcp"
+    }
+  }
+}

package/CLAUDE.md

@@ -0,0 +1,92 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Essential Commands
+
+### Development Setup
+```bash
+# Install dependencies (requires bun >= v1.2.9)
+bun install
+
+# Type, lint, and format check (check only, no fixes)
+bun run check
+
+# Lint and format fix (auto-fix issues)
+bun run check:write
+
+# Run unit tests
+bun test
+
+# Run Docker integration tests (requires ANTHROPIC_API_KEY)
+ANTHROPIC_API_KEY=sk-... bun run test:docker
+```
+
+## Project Organization
+
+This project uses `.claude/rules/` for project-specific guidance:
+
+- **Testing**: @.claude/rules/testing.md - Test commands and workflow
+- **Code Review**: @.claude/rules/code-review.md - Review standards
+- **Accuracy**: @.claude/rules/accuracy.md - Confidence thresholds
+- **Bun APIs**: @.claude/rules/bun-apis.md - Bun platform API preferences
+- **Git Workflow**: @.claude/rules/git-workflow.md - Commit conventions
+- **GitHub**: @.claude/rules/github.md - GitHub CLI integration
+
+## Quick Reference
+
+### Package Overview
+
+`@plaited/acp-harness` is a CLI tool for capturing agent trajectories from ACP-compatible agents. It executes prompts, captures full trajectories (tools, thoughts, plans), and outputs structured JSONL for downstream scoring.
+
+**CLI usage:**
+```bash
+bunx @plaited/acp-harness prompts.jsonl -o results.jsonl
+```
+
+### Code Style Essentials
+
+- Prefer arrow functions and `type` over `interface`
+- Use `test` instead of `it` in test files
+- Prefer Bun native APIs over Node.js equivalents
+- Object parameters for functions with 2+ parameters
+- JSON imports require `with { type: 'json' }` attribute
+
+For complete conventions, see `.claude/rules/code-review.md`
+
+### Plugin Development
+
+This project is a Claude Code plugin distributed via the plaited/marketplace aggregator. Structure:
+- `.claude/.claude-plugin/plugin.json` - Plugin manifest
+- `.claude/` - Plugin source (skills, rules, settings)
+
+When working on plugins:
+- Clear cache after changes: `rm -rf ~/.claude/plugins-cache`
+- Restart Claude Code to see updates
+- Skills are auto-invoked (won't show in `/plugins` UI)
+- Test installation locally: `claude plugins add github:plaited/marketplace`
+
+### Documentation
+
+- Public APIs require comprehensive TSDoc documentation
+- No `@example` sections - tests are living examples
+- Use `@internal` marker for non-public APIs
+- Always use `type` over `interface`
+- Use Mermaid diagrams only (not ASCII art)
+
+## Important Constraints
+
+1. **No Open Contribution**: This is open-source but not open-contribution
+2. **Bun Required**: Development requires bun >= v1.2.9
+3. **ES2024 Features**: Uses Promise.withResolvers() and other modern APIs
+
+## Plugin
+
+The bundled **acp-harness** skill (`.claude/skills/acp-harness/`) provides:
+- CLI usage and examples
+- Output format specifications
+- Downstream integration patterns
+
+Install via Claude Code: `/plugin marketplace add plaited/acp-harness`
+
+See `.claude/skills/acp-harness/SKILL.md` for complete documentation.

package/Dockerfile.test

@@ -0,0 +1,23 @@
+# Dockerfile for integration tests requiring Docker environment
+# Uses same bun version as CI for consistency
+#
+# Tests use *.docker.ts naming to:
+# 1. Avoid bun test pattern matching in normal test runs
+# 2. Signal these tests require Docker (external APIs, etc.)
+
+FROM oven/bun:1.2.9
+
+# Install git (required for some operations)
+RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy source
+COPY . .
+
+# Install dependencies
+RUN bun install --frozen-lockfile
+
+# Run all Docker integration tests
+# The wrapper script handles finding and running *.docker.ts files
+CMD ["bash", "scripts/bun-test-wrapper.sh"]

package/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2025 Plaited
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -0,0 +1,94 @@
+# @plaited/acp-harness
+
+[![npm version](https://img.shields.io/npm/v/@plaited/acp-harness.svg)](https://www.npmjs.com/package/@plaited/acp-harness)
+[![CI](https://github.com/plaited/acp-harness/actions/workflows/ci.yml/badge.svg)](https://github.com/plaited/acp-harness/actions/workflows/ci.yml)
+[![License: ISC](https://img.shields.io/badge/License-ISC-blue.svg)](https://opensource.org/licenses/ISC)
+
+CLI tool for capturing agent trajectories from ACP-compatible agents. Execute prompts, capture full trajectories (tools, thoughts, plans), and output structured JSONL for downstream scoring.
+
+## Quick Start
+
+```bash
+# Run without installing
+bunx @plaited/acp-harness prompts.jsonl -o results.jsonl
+
+# Or install globally
+bun add -g @plaited/acp-harness
+acp-harness prompts.jsonl -o results.jsonl
+```
+
+**Prerequisite:** Install an ACP adapter and set your API key:
+
+```bash
+npm install -g @zed-industries/claude-code-acp
+export ANTHROPIC_API_KEY=sk-...
+```
+
+## Usage
+
+```bash
+acp-harness <prompts.jsonl> [options]
+
+Options:
+  --cmd, --command  ACP agent command (default: "claude-code-acp")
+  -o, --output      Output file (default: stdout)
+  -c, --cwd         Working directory for agent
+  -t, --timeout     Request timeout in ms (default: 60000)
+  -f, --format      Output format: summary, judge (default: summary)
+  --progress        Show progress to stderr
+  --append          Append to output file
+  --mcp-server      MCP server config JSON (repeatable)
+  -h, --help        Show help
+```
+
+## Input Format
+
+```jsonl
+{"id":"test-001","input":"Create a primary button","expected":"should contain <button>","metadata":{"category":"ui"}}
+{"id":"test-002","input":"Fix the TypeScript error","metadata":{"category":"bugfix"}}
+```
+
+## Output
+
+The harness captures trajectories and outputs structured JSONL. **You provide the scoring logic.**
+
+```bash
+# Capture trajectories
+acp-harness prompts.jsonl -o results.jsonl
+
+# Score with your tools
+cat results.jsonl | jq 'select(.status == "failed")'
+cat results.jsonl | your-scoring-script.ts
+```
+
+## Plugin
+
+This package includes an **acp-harness skill** for AI coding agents with complete documentation:
+
+- CLI usage and examples
+- Output format schemas
+- Integration patterns (Braintrust, jq, custom scorers)
+
+**Install via Claude Code:**
+
+```bash
+/plugin marketplace add plaited/marketplace
+```
+
+## Development
+
+```bash
+bun install          # Install dependencies
+bun run check        # Type check + lint + format
+bun test             # Run unit tests
+```
+
+## Requirements
+
+- **Runtime:** Bun >= 1.2.9
+- **ACP Adapter:** `@zed-industries/claude-code-acp` or compatible
+- **API Key:** `ANTHROPIC_API_KEY` environment variable
+
+## License
+
+ISC © [Plaited Labs](https://github.com/plaited)