agent-relay 2.3.4 → 2.3.6

package/packages/benchmark/src/types.ts

@@ -1,205 +0,0 @@
-/**
- * Benchmark Types
- *
- * Type definitions for the agent swarm performance benchmark system.
- */
-
-/**
- * Configuration type for benchmark runs
- */
-export type ConfigurationType = 'single' | 'subagent' | 'swarm';
-
-/**
- * Task complexity level
- */
-export type TaskComplexity = 'low' | 'medium' | 'high';
-
-/**
- * A benchmark task definition
- */
-export interface Task {
-  /** Unique task identifier */
-  id: string;
-  /** Human-readable task description */
-  description: string;
-  /** Files the task operates on */
-  files: string[];
-  /** Success criteria for the task */
-  expectedOutcome: string;
-  /** Optional Harbor-style success criteria key for compatibility */
-  success_criteria?: string;
-  /** Task complexity level */
-  complexity: TaskComplexity;
-  /** Optional timeout in milliseconds (default: 300000 = 5 min) */
-  timeoutMs?: number;
-  /** Optional tags for categorization */
-  tags?: string[];
-}
-
-/**
- * Result of a single benchmark run
- */
-export interface RunResult {
-  /** Task identifier */
-  taskId: string;
-  /** Configuration used for this run */
-  configuration: ConfigurationType;
-
-  // Performance metrics
-  /** Total time from start to completion in milliseconds */
-  totalTimeMs: number;
-  /** Time to first agent action in milliseconds */
-  timeToFirstActionMs: number;
-
-  // Communication metrics (multi-agent only)
-  /** Total number of inter-agent messages */
-  messageCount: number;
-  /** Average message latency in milliseconds */
-  avgLatencyMs: number;
-  /** P50 latency in milliseconds */
-  latencyP50Ms: number;
-  /** P99 latency in milliseconds */
-  latencyP99Ms: number;
-  /** Number of coordination rounds */
-  coordinationRounds: number;
-
-  // Resource metrics
-  /** Number of agents used */
-  agentCount: number;
-  /** Total tokens consumed (if available) */
-  totalTokensUsed: number;
-  /** Peak memory usage in MB */
-  peakMemoryMb: number;
-
-  // Outcome metrics
-  /** Whether the task completed successfully */
-  success: boolean;
-  /** Completion rate (0-1) for partial success */
-  completionRate: number;
-  /** Error messages if any */
-  errors: string[];
-
-  // Metadata
-  /** Timestamp when the run started */
-  startedAt: number;
-  /** Timestamp when the run completed */
-  completedAt: number;
-}
-
-/**
- * Comparison result across all configurations
- */
-export interface ComparisonResult {
-  /** Task identifier */
-  taskId: string;
-  /** Results for each configuration */
-  results: Map<ConfigurationType, RunResult>;
-  /** The winning configuration based on scoring */
-  winner: ConfigurationType;
-  /** Score breakdown for each configuration */
-  scores: Map<ConfigurationType, ScoreBreakdown>;
-}
-
-/**
- * Score breakdown for a configuration
- */
-export interface ScoreBreakdown {
-  /** Total score (0-100) */
-  total: number;
-  /** Success component (0-50) */
-  successScore: number;
-  /** Time efficiency component (0-30) */
-  timeScore: number;
-  /** Resource efficiency component (0-20) */
-  efficiencyScore: number;
-}
-
-/**
- * Benchmark configuration options
- */
-export interface BenchmarkConfig {
-  /** Which configurations to run */
-  configurations: ConfigurationType[];
-  /** CLI to use for agents (default: 'claude') */
-  cli: string;
-  /** Working directory for tasks */
-  cwd?: string;
-  /** Suppress console output */
-  quiet: boolean;
-  /** Cool-down time between runs in milliseconds */
-  cooldownMs: number;
-  /** Maximum concurrent agents for swarm */
-  maxSwarmSize: number;
-  /** Custom socket path for relay */
-  socketPath?: string;
-}
-
-/**
- * Default benchmark configuration
- */
-export const DEFAULT_BENCHMARK_CONFIG: BenchmarkConfig = {
-  configurations: ['single', 'subagent', 'swarm'],
-  cli: 'claude',
-  quiet: false,
-  cooldownMs: 5000,
-  maxSwarmSize: 10,
-};
-
-/**
- * Metrics collected during a run
- */
-export interface RunMetrics {
-  /** Number of messages sent */
-  messages: number;
-  /** Message latencies in milliseconds */
-  latencies: number[];
-  /** Run start timestamp */
-  startTime: number;
-  /** Spawned agent names */
-  spawnedAgents: string[];
-  /** Error events */
-  errors: string[];
-}
-
-/**
- * Task dataset definition
- */
-export interface TaskDataset {
-  /** Dataset name */
-  name: string;
-  /** Dataset description */
-  description?: string;
-  /** Version identifier */
-  version?: string;
-  /** Tasks in the dataset */
-  tasks: Task[];
-}
-
-/**
- * Harbor-compatible evaluation input
- */
-export interface HarborTaskInput {
-  id: string;
-  description: string;
-  files?: string[];
-  success_criteria?: string;
-  complexity?: TaskComplexity;
-  agents_required?: number;
-  [key: string]: unknown;
-}
-
-/**
- * Harbor-compatible evaluation output
- */
-export interface HarborEvaluationOutput {
-  task_id: string;
-  configurations: Record<ConfigurationType, RunResult>;
-  winner: ConfigurationType;
-  scores: Record<ConfigurationType, ScoreBreakdown>;
-  metadata: {
-    benchmark_version: string;
-    started_at: number;
-    completed_at: number;
-    total_duration_ms: number;
-  };
-}

package/packages/benchmark/tsconfig.json

@@ -1,20 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "lib": ["ES2022"],
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "resolveJsonModule": true
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "**/*.test.ts"]
-}

package/packages/cli-tester/README.md

@@ -1,277 +0,0 @@
-# @agent-relay/cli-tester
-
-Manual interactive testing environment for CLI authentication flows.
-
-## Purpose
-
-This package provides a Docker-based environment for testing CLI authentication with real OAuth providers. It's designed for:
-
-- **Debugging auth issues** - Isolate problems with specific CLIs (e.g., "Cursor doesn't work")
-- **Testing auth flows** - Verify OAuth flows work end-to-end
-- **Message injection** - Test relay-pty message delivery
-- **Credential verification** - Check that credentials are saved correctly
-
-## Quick Start
-
-From the relay repo root:
-
-```bash
-# Start the test environment (drops into container shell)
-npm run cli-tester:start
-
-# Start with clean credentials (removes any cached auth)
-npm run cli-tester:start:clean
-
-# Start with daemon for full integration testing
-npm run cli-tester:start:daemon
-```
-
-## Inside the Container
-
-### Test a CLI
-
-```bash
-# Test Claude CLI with relay-pty
-./scripts/test-cli.sh claude
-
-# Test Codex with device auth
-./scripts/test-cli.sh codex --device-auth
-
-# Test with debug output
-DEBUG=1 ./scripts/test-cli.sh cursor
-```
-
-### Verify Credentials
-
-```bash
-# Check if credentials exist (after authenticating)
-./scripts/verify-auth.sh claude
-./scripts/verify-auth.sh codex
-./scripts/verify-auth.sh gemini
-```
-
-### Inject Messages
-
-In a second terminal (while CLI is running):
-
-```bash
-# Send a message via relay-pty socket
-./scripts/inject-message.sh test-claude "What is 2+2?"
-```
-
-### Clear Credentials
-
-```bash
-# Clear credentials for fresh testing
-./scripts/clear-auth.sh claude
-./scripts/clear-auth.sh all  # Clear all CLIs
-```
-
-## Advanced: Testing Spawn Flow
-
-The simple `test-cli.sh` tests the CLI in isolation. For debugging issues where the CLI works in isolation but fails when spawned via the application (e.g., registration timeout), use these advanced tests:
-
-### Test Spawn Behavior
-
-Simulates what `AgentSpawner.spawn()` does, including CLI-specific flags:
-
-```bash
-# Test with same flags as spawner (--force for cursor, --dangerously-skip-permissions for claude)
-./scripts/test-spawn.sh cursor
-
-# Test in interactive mode (without auto-accept flags)
-./scripts/test-spawn.sh cursor --interactive
-
-# With verbose debug output
-DEBUG_SPAWN=1 ./scripts/test-spawn.sh cursor
-```
-
-### Test Registration Flow
-
-Monitors the registration files that the spawner polls. This is the step that times out:
-
-```bash
-# Watch registration with 60 second timeout
-./scripts/test-registration.sh cursor 60
-
-# With debug output
-DEBUG_SPAWN=1 ./scripts/test-registration.sh cursor
-```
-
-### Full Daemon Integration Test
-
-Starts a real daemon and tests the complete flow:
-
-```bash
-# Full end-to-end test with daemon
-./scripts/test-with-daemon.sh cursor
-
-# With debug output
-DEBUG=1 ./scripts/test-with-daemon.sh cursor
-```
-
-**Note:** Requires the daemon to be built: `cd packages/daemon && npm run build`
-
-## Debugging a Broken CLI
-
-When a CLI isn't working, use this workflow:
-
-```bash
-# 1. Start fresh
-npm run cli-tester:start:clean
-
-# 2. Test the problematic CLI
-./scripts/test-cli.sh cursor
-
-# 3. Observe the output for:
-#    - Auth URLs being printed
-#    - Error messages
-#    - Prompt patterns
-
-# 4. Check credentials
-./scripts/verify-auth.sh cursor
-ls -la ~/.cursor/
-
-# 5. Compare with a working CLI
-./scripts/test-cli.sh claude
-```
-
-## Debugging Registration Timeout
-
-If a CLI works in isolation but times out when spawned ("Agent registration timeout"), the issue is in the daemon registration flow.
-
-### Quick Test (Run This First)
-
-```bash
-# Test the EXACT setup flow - this is what TerminalProviderSetup.tsx does
-DEBUG=1 ./scripts/test-full-spawn.sh cursor true
-```
-
-This simulates:
-- `interactive: true` (no --force flag, like setup terminal)
-- 30 second registration timeout
-- Verbose logging of what's happening
-
-### Understanding the Flow
-
-**Normal spawn (non-interactive):**
-```bash
-./scripts/test-full-spawn.sh cursor       # Has --force flag
-```
-
-**Setup terminal (interactive):**
-```bash
-./scripts/test-full-spawn.sh cursor true  # NO --force flag
-```
-
-The key difference is `interactive: true` **skips auto-accept flags**. Setup terminals expect the user to respond to prompts in the browser terminal.
-
-### What the Tests Show
-
-1. **test-full-spawn.sh** - Simulates spawner's 30s registration timeout
-   - Shows poll count (like spawner logs)
-   - Shows socket status
-   - Captures CLI output to log file
-   - Tells you exactly where things fail
-
-2. **test-setup-flow.sh** - Identical to what TerminalProviderSetup.tsx does
-   - Uses `__setup__cursor-xxx` naming
-   - No CLI flags (interactive mode)
-
-### Debugging Steps
-
-```bash
-# 1. Test in isolation (verify CLI starts)
-./scripts/test-cli.sh cursor
-
-# 2. Test NON-INTERACTIVE spawn (with --force)
-DEBUG=1 ./scripts/test-full-spawn.sh cursor
-
-# 3. Test INTERACTIVE spawn (setup terminal flow)
-DEBUG=1 ./scripts/test-full-spawn.sh cursor true
-
-# 4. Watch the log file in another terminal
-tail -f /tmp/relay-spawn-*.log
-```
-
-### Common Causes
-
-| Symptom | Cause | Fix |
-|---------|-------|-----|
-| CLI exits immediately | Not installed or crash | Check `which agent` |
-| Socket never created | CLI stuck on early prompt | Check log for prompts |
-| 30s timeout | CLI waiting for user input | Respond to prompts (trust, etc.) |
-| 30s timeout | No daemon to register with | Run with daemon profile |
-
-### The Registration Flow
-
-The spawner waits for TWO conditions:
-1. Agent in `connected-agents.json` (daemon updates this when CLI connects)
-2. Agent in `agents.json` (relay-pty hook updates this)
-
-Without a running daemon, both files are empty → timeout.
-
-## Available CLIs
-
-The container includes these pre-installed CLIs:
-
-| CLI | Command | Auth Command | Credential Path |
-|-----|---------|--------------|-----------------|
-| Claude | `claude` | (auto) | `~/.claude/.credentials.json` |
-| Codex | `codex` | `login` | `~/.codex/auth.json` |
-| Gemini | `gemini` | (auto) | `~/.gemini/credentials.json` |
-| Cursor | `agent` | (auto) | `~/.cursor/auth.json` |
-| OpenCode | `opencode` | `auth login` | `~/.local/share/opencode/auth.json` |
-| Droid | `droid` | `--login` | `~/.droid/auth.json` |
-| Copilot | `copilot` | `auth login` | `~/.config/gh/hosts.yml` |
-
-**Note:** Cursor CLI installs as `agent`, not `cursor`. The test scripts handle this mapping automatically.
-
-## How It Works
-
-1. **relay-pty** wraps the CLI and provides:
-   - Unix socket for message injection
-   - Output parsing for relay commands
-   - Idle detection for message timing
-
-2. **Docker volumes** persist credentials between runs so you don't have to re-authenticate each time.
-
-3. **Shell scripts** provide simple commands for common operations.
-
-## TypeScript API
-
-For programmatic use:
-
-```typescript
-import { RelayPtyClient, checkCredentials } from '@agent-relay/cli-tester';
-
-// Check credentials
-const result = checkCredentials('claude');
-console.log(result.exists, result.valid, result.hasAccessToken);
-
-// Inject messages via socket
-const client = new RelayPtyClient('/tmp/relay-pty-test-claude.sock');
-await client.connect();
-await client.inject({ from: 'Test', body: 'Hello' });
-```
-
-## File Structure
-
-```
-packages/cli-tester/
-├── docker/
-│   ├── Dockerfile           # Test environment image
-│   └── docker-compose.yml   # Container configuration
-├── scripts/
-│   ├── start.sh             # Start container
-│   ├── test-cli.sh          # Test a CLI with relay-pty
-│   ├── verify-auth.sh       # Check credentials
-│   ├── inject-message.sh    # Send message via socket
-│   └── clear-auth.sh        # Clear credentials
-├── src/
-│   └── utils/
-│       ├── socket-client.ts     # relay-pty socket communication
-│       └── credential-check.ts  # Credential file utilities
-└── tests/
-    └── credential-check.test.ts
-```

package/packages/cli-tester/dist/index.d.ts

@@ -1,21 +0,0 @@
-/**
- * CLI Auth Tester - Manual interactive testing for CLI authentication flows
- *
- * This package provides utilities for testing CLI authentication in a Docker container.
- * Primary use case is debugging auth issues with various CLI tools (Claude, Codex, Gemini, etc.)
- *
- * @example
- * ```bash
- * # Start the test environment
- * npm run cli-tester:start
- *
- * # Inside container, test a CLI
- * ./scripts/test-cli.sh claude
- *
- * # Verify credentials
- * ./scripts/verify-auth.sh claude
- * ```
- */
-export { RelayPtyClient, createClient, getSocketPath, type InjectRequest, type InjectResponse, type StatusRequest, type StatusResponse, type RelayPtyResponse, } from './utils/socket-client.js';
-export { checkCredentials, clearCredentials, checkAllCredentials, clearAllCredentials, getCredentialPath, getConfigPaths, type CLIType, type CredentialCheck, } from './utils/credential-check.js';
-//# sourceMappingURL=index.d.ts.map

package/packages/cli-tester/dist/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EACL,cAAc,EACd,YAAY,EACZ,aAAa,EACb,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,gBAAgB,GACtB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,OAAO,EACZ,KAAK,eAAe,GACrB,MAAM,6BAA6B,CAAC"}

package/packages/cli-tester/dist/index.js

@@ -1,21 +0,0 @@
-/**
- * CLI Auth Tester - Manual interactive testing for CLI authentication flows
- *
- * This package provides utilities for testing CLI authentication in a Docker container.
- * Primary use case is debugging auth issues with various CLI tools (Claude, Codex, Gemini, etc.)
- *
- * @example
- * ```bash
- * # Start the test environment
- * npm run cli-tester:start
- *
- * # Inside container, test a CLI
- * ./scripts/test-cli.sh claude
- *
- * # Verify credentials
- * ./scripts/verify-auth.sh claude
- * ```
- */
-export { RelayPtyClient, createClient, getSocketPath, } from './utils/socket-client.js';
-export { checkCredentials, clearCredentials, checkAllCredentials, clearAllCredentials, getCredentialPath, getConfigPaths, } from './utils/credential-check.js';
-//# sourceMappingURL=index.js.map

package/packages/cli-tester/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EACL,cAAc,EACd,YAAY,EACZ,aAAa,GAMd,MAAM,0BAA0B,CAAC;AAElC,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,GAGf,MAAM,6BAA6B,CAAC"}

package/packages/cli-tester/dist/utils/credential-check.d.ts

@@ -1,56 +0,0 @@
-/**
- * Credential checking utilities for CLI authentication testing
- * Verifies and parses credential files for various CLI tools
- */
-export type CLIType = 'claude' | 'codex' | 'gemini' | 'cursor' | 'opencode' | 'droid';
-export interface CredentialCheck {
-    /** CLI type being checked */
-    cli: CLIType;
-    /** Whether the credential file exists */
-    exists: boolean;
-    /** Whether the credentials appear valid (have required fields) */
-    valid: boolean;
-    /** Whether an access token is present */
-    hasAccessToken: boolean;
-    /** Whether a refresh token is present */
-    hasRefreshToken: boolean;
-    /** Token expiration date if available */
-    expiresAt?: Date;
-    /** Path to the credential file */
-    filePath: string;
-    /** Raw credential data (tokens redacted) */
-    data?: Record<string, unknown>;
-    /** Error message if check failed */
-    error?: string;
-}
-/**
- * Get the credential file path for a CLI
- */
-export declare function getCredentialPath(cli: CLIType): string;
-/**
- * Get all config paths for a CLI (for clearing)
- */
-export declare function getConfigPaths(cli: CLIType): string[];
-/**
- * Check credentials for a specific CLI
- */
-export declare function checkCredentials(cli: CLIType): CredentialCheck;
-/**
- * Clear credentials for a specific CLI
- */
-export declare function clearCredentials(cli: CLIType): {
-    cleared: string[];
-    errors: string[];
-};
-/**
- * Clear all CLI credentials
- */
-export declare function clearAllCredentials(): Record<CLIType, {
-    cleared: string[];
-    errors: string[];
-}>;
-/**
- * Check all CLI credentials
- */
-export declare function checkAllCredentials(): Record<CLIType, CredentialCheck>;
-//# sourceMappingURL=credential-check.d.ts.map

package/packages/cli-tester/dist/utils/credential-check.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"credential-check.d.ts","sourceRoot":"","sources":["../../src/utils/credential-check.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,CAAC;AAEtF,MAAM,WAAW,eAAe;IAC9B,6BAA6B;IAC7B,GAAG,EAAE,OAAO,CAAC;IACb,yCAAyC;IACzC,MAAM,EAAE,OAAO,CAAC;IAChB,kEAAkE;IAClE,KAAK,EAAE,OAAO,CAAC;IACf,yCAAyC;IACzC,cAAc,EAAE,OAAO,CAAC;IACxB,yCAAyC;IACzC,eAAe,EAAE,OAAO,CAAC;IACzB,yCAAyC;IACzC,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAmBtD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,EAAE,CAkCrD;AAoGD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CAqC9D;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG;IAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAiBtF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAAC,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAS9F;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAStE"}