agent-relay 2.0.29 → 2.0.32

package/packages/api-types/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}

package/packages/api-types/vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['src/**/*.test.ts'],
+  },
+});

package/packages/benchmark/README.md

@@ -0,0 +1,200 @@
+# @agent-relay/benchmark
+
+Performance benchmarking for comparing agent configurations: single agents, sub-agents (hierarchical), and swarms (peer-to-peer).
+
+## Overview
+
+This package provides tools to measure and compare the performance of different agent configurations on the same tasks:
+
+| Configuration | Description | Communication |
+|---------------|-------------|---------------|
+| **Single** | One agent handles everything | None |
+| **Sub-agent** | Lead spawns and coordinates workers | Hierarchical (parent → child) |
+| **Swarm** | Peer agents coordinate as equals | Peer-to-peer via channels |
+
+## Installation
+
+```bash
+npm install @agent-relay/benchmark
+```
+
+For standalone mode (in-process daemon):
+```bash
+npm install @agent-relay/benchmark @agent-relay/daemon
+```
+
+## Quick Start
+
+### Programmatic Usage
+
+```typescript
+import { ComparisonBenchmark, type Task } from '@agent-relay/benchmark';
+
+const task: Task = {
+  id: 'refactor-auth',
+  description: 'Refactor authentication to use JWT',
+  files: ['src/auth/session.ts', 'src/auth/middleware.ts'],
+  expectedOutcome: 'All tests pass, JWT tokens used',
+  complexity: 'medium',
+};
+
+const benchmark = new ComparisonBenchmark();
+const comparison = await benchmark.runComparison(task);
+
+console.log(`Winner: ${comparison.winner}`);
+benchmark.printComparison(comparison);
+```
+
+### CLI Usage
+
+```bash
+# Run comparison on all configurations
+relay-benchmark run --dataset tasks.yaml --config all
+
+# Run specific configuration
+relay-benchmark run --dataset tasks.yaml --config swarm
+
+# List tasks in a dataset
+relay-benchmark list tasks.yaml
+
+# Output results to JSON
+relay-benchmark run --dataset tasks.yaml -o results.json
+```
+
+### Harbor Integration
+
+This package integrates with [Harbor](https://github.com/laude-institute/harbor) for large-scale agent evaluation:
+
+```bash
+# Install Harbor
+pip install harbor-bench
+
+# Run benchmark via Harbor
+harbor run \
+  --dataset tasks.yaml \
+  --agent @agent-relay/benchmark/harbor \
+  --parallel 10
+
+# Run at scale with cloud providers
+harbor run \
+  --dataset tasks.yaml \
+  --agent @agent-relay/benchmark/harbor \
+  --env daytona \
+  --parallel 100
+```
+
+## Task Dataset Format
+
+Tasks can be defined in YAML or JSON:
+
+```yaml
+name: My Tasks
+description: Tasks for benchmarking
+
+tasks:
+  - id: add-feature
+    description: Add user preferences feature
+    files:
+      - src/models/preferences.ts
+      - src/routes/preferences.ts
+      - tests/preferences.test.ts
+    expectedOutcome: Feature working, tests pass
+    complexity: medium  # low, medium, high
+    timeoutMs: 300000   # optional, default 5 minutes
+    tags:               # optional
+      - feature
+      - api
+```
+
+## Metrics Collected
+
+| Metric | Description |
+|--------|-------------|
+| `totalTimeMs` | Total execution time |
+| `timeToFirstActionMs` | Time until first agent action |
+| `messageCount` | Inter-agent messages sent |
+| `avgLatencyMs` | Average message latency |
+| `latencyP50Ms` | 50th percentile latency |
+| `latencyP99Ms` | 99th percentile latency |
+| `coordinationRounds` | Communication rounds |
+| `agentCount` | Agents used |
+| `totalTokensUsed` | LLM tokens consumed |
+| `peakMemoryMb` | Peak memory usage |
+| `success` | Task completed successfully |
+| `completionRate` | Partial completion (0-1) |
+
+## Scoring
+
+Results are scored on three components:
+
+- **Success (50 points)**: Task completion
+- **Time (30 points)**: Faster is better
+- **Efficiency (20 points)**: Fewer agents is better
+
+The configuration with the highest total score wins.
+
+## Configuration
+
+```typescript
+interface BenchmarkConfig {
+  configurations: ConfigurationType[];  // ['single', 'subagent', 'swarm']
+  cli: string;                          // CLI to use (default: 'claude')
+  cwd?: string;                         // Working directory
+  quiet: boolean;                       // Suppress output
+  cooldownMs: number;                   // Delay between runs
+  maxSwarmSize: number;                 // Max agents in swarm
+  socketPath?: string;                  // Custom relay socket
+}
+```
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    ComparisonBenchmark                       │
+│  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐         │
+│  │   Single    │  │  SubAgent   │  │    Swarm    │         │
+│  │   Runner    │  │   Runner    │  │   Runner    │         │
+│  └──────┬──────┘  └──────┬──────┘  └──────┬──────┘         │
+│         │                │                │                  │
+│         └────────────────┼────────────────┘                  │
+│                          │                                   │
+│                          ▼                                   │
+│              ┌───────────────────────┐                      │
+│              │   @agent-relay/sdk    │                      │
+│              │   (standalone mode)   │                      │
+│              └───────────────────────┘                      │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Example Output
+
+```
+============================================================
+COMPARISON RESULTS
+============================================================
+Task: refactor-auth
+Winner: SUBAGENT
+
++------------+--------+----------+-------+
+| Metric     | Single | Subagent | Swarm |
++------------+--------+----------+-------+
+| Success    | ✓      | ✓        | ✓     |
+| Time (s)   | 45.2   | 28.1     | 32.5  |
+| Agents     | 1      | 3        | 3     |
+| Messages   | 0      | 12       | 24    |
+| Completion | 100%   | 100%     | 100%  |
+| Score      | 65.3   | 78.2     | 71.8  |
++------------+--------+----------+-------+
+```
+
+## Included Datasets
+
+The package includes example datasets in `datasets/`:
+
+- `coding-tasks.yaml` - Standard software engineering tasks
+- `coordination-tasks.yaml` - Tasks requiring multi-agent coordination
+
+## License
+
+Apache-2.0

package/packages/benchmark/datasets/coding-tasks.yaml

@@ -0,0 +1,127 @@
+# Coding Tasks Dataset
+# Standard coding tasks for comparing agent configurations
+
+name: Coding Tasks
+description: Common software engineering tasks for benchmarking agent configurations
+version: "1.0.0"
+
+tasks:
+  # Low complexity tasks
+  - id: add-logging
+    description: Add structured logging to the API endpoints
+    files:
+      - src/routes/api.ts
+      - src/utils/logger.ts
+    expectedOutcome: All API endpoints log requests and responses with timestamps
+    complexity: low
+    tags:
+      - logging
+      - observability
+
+  - id: fix-typos
+    description: Fix typos and improve variable names in the codebase
+    files:
+      - src/utils/helpers.ts
+      - src/components/Form.tsx
+    expectedOutcome: All typos fixed, variable names follow conventions
+    complexity: low
+    tags:
+      - refactoring
+      - code-quality
+
+  # Medium complexity tasks
+  - id: add-rate-limiting
+    description: Add rate limiting middleware to protect API endpoints
+    files:
+      - src/middleware/rateLimit.ts
+      - src/routes/api.ts
+      - src/config/limits.ts
+    expectedOutcome: Rate limiting applied to all endpoints, configurable limits per route
+    complexity: medium
+    tags:
+      - security
+      - middleware
+
+  - id: refactor-auth-jwt
+    description: Refactor authentication from sessions to JWT tokens
+    files:
+      - src/auth/session.ts
+      - src/auth/jwt.ts
+      - src/middleware/auth.ts
+      - src/routes/login.ts
+      - tests/auth.test.ts
+    expectedOutcome: JWT-based auth working, all tests pass, sessions removed
+    complexity: medium
+    tags:
+      - auth
+      - security
+      - refactoring
+
+  - id: add-caching-layer
+    description: Add Redis caching to expensive database queries
+    files:
+      - src/services/cache.ts
+      - src/repositories/user.ts
+      - src/repositories/product.ts
+      - src/config/redis.ts
+    expectedOutcome: Caching implemented for user and product queries, cache invalidation working
+    complexity: medium
+    tags:
+      - performance
+      - caching
+
+  # High complexity tasks
+  - id: database-migration
+    description: Migrate from PostgreSQL to MongoDB while maintaining API compatibility
+    files:
+      - src/db/postgres/connection.ts
+      - src/db/mongo/connection.ts
+      - src/models/user.ts
+      - src/models/product.ts
+      - src/models/order.ts
+      - src/repositories/user.ts
+      - src/repositories/product.ts
+      - src/repositories/order.ts
+      - scripts/migrate-data.ts
+      - docker-compose.yml
+    expectedOutcome: All data migrated, API unchanged, tests pass on MongoDB
+    complexity: high
+    tags:
+      - database
+      - migration
+      - high-risk
+
+  - id: implement-rbac
+    description: Implement role-based access control across the application
+    files:
+      - src/auth/rbac.ts
+      - src/auth/permissions.ts
+      - src/middleware/authorize.ts
+      - src/routes/admin.ts
+      - src/routes/user.ts
+      - src/models/role.ts
+      - src/models/permission.ts
+      - tests/rbac.test.ts
+    expectedOutcome: RBAC fully implemented, admin routes protected, tests pass
+    complexity: high
+    tags:
+      - security
+      - auth
+      - permissions
+
+  - id: api-versioning
+    description: Implement API versioning with backward compatibility
+    files:
+      - src/routes/v1/index.ts
+      - src/routes/v2/index.ts
+      - src/middleware/version.ts
+      - src/transformers/v1-to-v2.ts
+      - src/docs/api-v1.yaml
+      - src/docs/api-v2.yaml
+      - tests/versioning.test.ts
+    expectedOutcome: V1 and V2 APIs working, automatic version negotiation, docs updated
+    complexity: high
+    tags:
+      - api
+      - versioning
+      - backward-compatibility

package/packages/benchmark/datasets/coordination-tasks.yaml

@@ -0,0 +1,122 @@
+# Coordination Tasks Dataset
+# Tasks specifically designed to test multi-agent coordination
+
+name: Coordination Tasks
+description: Tasks that require significant inter-agent coordination and communication
+version: "1.0.0"
+
+tasks:
+  # Tasks requiring parallel work
+  - id: parallel-refactor
+    description: Refactor 6 service files to use a new error handling pattern
+    files:
+      - src/services/userService.ts
+      - src/services/orderService.ts
+      - src/services/productService.ts
+      - src/services/paymentService.ts
+      - src/services/notificationService.ts
+      - src/services/analyticsService.ts
+    expectedOutcome: All services use new error handling, consistent pattern across all files
+    complexity: medium
+    tags:
+      - parallel-work
+      - refactoring
+
+  - id: parallel-testing
+    description: Write comprehensive unit tests for 4 independent modules
+    files:
+      - src/utils/validators.ts
+      - src/utils/formatters.ts
+      - src/utils/transformers.ts
+      - src/utils/parsers.ts
+      - tests/validators.test.ts
+      - tests/formatters.test.ts
+      - tests/transformers.test.ts
+      - tests/parsers.test.ts
+    expectedOutcome: 100% test coverage for all utility modules, all tests pass
+    complexity: medium
+    tags:
+      - testing
+      - parallel-work
+
+  # Tasks requiring coordination
+  - id: api-frontend-sync
+    description: Add a new feature requiring both API endpoints and frontend components
+    files:
+      - src/api/notifications.ts
+      - src/api/routes.ts
+      - src/components/NotificationBell.tsx
+      - src/components/NotificationList.tsx
+      - src/hooks/useNotifications.ts
+      - src/types/notification.ts
+    expectedOutcome: Notifications API and UI working together, types shared correctly
+    complexity: medium
+    tags:
+      - full-stack
+      - coordination
+
+  - id: schema-migration-chain
+    description: Database schema change requiring coordinated updates across layers
+    files:
+      - migrations/add_user_preferences.sql
+      - src/models/userPreferences.ts
+      - src/repositories/userPreferences.ts
+      - src/services/userService.ts
+      - src/routes/user.ts
+      - src/types/user.ts
+    expectedOutcome: Schema migrated, all layers updated, API returns new fields
+    complexity: high
+    tags:
+      - database
+      - coordination
+      - layered-architecture
+
+  # Tasks with dependencies
+  - id: dependency-chain
+    description: Build a feature where each component depends on the previous
+    files:
+      - src/core/eventBus.ts
+      - src/services/eventHandler.ts
+      - src/workers/eventProcessor.ts
+      - src/api/webhooks.ts
+      - tests/integration/events.test.ts
+    expectedOutcome: Event system working end-to-end, integration tests pass
+    complexity: high
+    tags:
+      - dependencies
+      - event-driven
+
+  # Tasks requiring consensus
+  - id: design-review
+    description: Review and improve API design across multiple endpoints
+    files:
+      - src/api/users.ts
+      - src/api/products.ts
+      - src/api/orders.ts
+      - src/api/payments.ts
+      - docs/api-design.md
+    expectedOutcome: Consistent API design patterns, documentation updated
+    complexity: medium
+    tags:
+      - review
+      - consensus
+      - documentation
+
+  # Large-scale coordination
+  - id: monorepo-sync
+    description: Update shared types across multiple packages in a monorepo
+    files:
+      - packages/shared/types/user.ts
+      - packages/shared/types/product.ts
+      - packages/api/src/routes/user.ts
+      - packages/api/src/routes/product.ts
+      - packages/web/src/api/user.ts
+      - packages/web/src/api/product.ts
+      - packages/mobile/src/api/user.ts
+      - packages/mobile/src/api/product.ts
+    expectedOutcome: Types consistent across all packages, no type errors
+    complexity: high
+    tags:
+      - monorepo
+      - types
+      - coordination

package/packages/benchmark/dist/benchmark.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Comparison Benchmark
+ *
+ * Main orchestrator for running comparison benchmarks across configurations.
+ */
+import type { ConfigurationType, Task, RunResult, ComparisonResult, BenchmarkConfig } from './types.js';
+/**
+ * Main benchmark orchestrator
+ */
+export declare class ComparisonBenchmark {
+    private config;
+    private runners;
+    constructor(config?: Partial<BenchmarkConfig>);
+    /**
+     * Create a runner for a configuration type
+     */
+    private createRunner;
+    /**
+     * Run a comparison across all configured configurations
+     */
+    runComparison(task: Task): Promise<ComparisonResult>;
+    /**
+     * Run a single configuration
+     */
+    runSingle(task: Task, configType: ConfigurationType): Promise<RunResult>;
+    /**
+     * Calculate score breakdown for a result
+     */
+    private calculateScore;
+    /**
+     * Determine the winning configuration
+     */
+    private determineWinner;
+    /**
+     * Print a single run result
+     */
+    private printRunResult;
+    /**
+     * Print comparison table
+     */
+    printComparison(comparison: ComparisonResult): void;
+}
+/**
+ * Quick helper to run a comparison benchmark
+ */
+export declare function runComparison(task: Task, config?: Partial<BenchmarkConfig>): Promise<ComparisonResult>;
+//# sourceMappingURL=benchmark.d.ts.map

package/packages/benchmark/dist/benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../src/benchmark.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,IAAI,EACJ,SAAS,EACT,gBAAgB,EAEhB,eAAe,EAChB,MAAM,YAAY,CAAC;AASpB;;GAEG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,OAAO,CAA8C;gBAEjD,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAUjD;;OAEG;IACH,OAAO,CAAC,YAAY;IAapB;;OAEG;IACG,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAgE1D;;OAEG;IACG,SAAS,CACb,IAAI,EAAE,IAAI,EACV,UAAU,EAAE,iBAAiB,GAC5B,OAAO,CAAC,SAAS,CAAC;IAarB;;OAEG;IACH,OAAO,CAAC,cAAc;IAwBtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAiBvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAWtB;;OAEG;IACH,eAAe,CAAC,UAAU,EAAE,gBAAgB,GAAG,IAAI;CAiFpD;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,IAAI,EACV,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAChC,OAAO,CAAC,gBAAgB,CAAC,CAG3B"}