agent-relay 1.0.22 → 1.2.0

package/scripts/test-cli-auth/README.md

@@ -0,0 +1,286 @@
+# CLI OAuth Flow Testing
+
+This directory contains tools for testing and validating the CLI-based OAuth authentication flow for AI providers.
+
+## Quick Start
+
+```bash
+# Make mock CLI executable
+chmod +x scripts/test-cli-auth/mock-cli.sh
+
+# Run all integration tests
+npx tsx scripts/test-cli-auth/test-oauth-flow.ts
+
+# Test a specific provider
+npx tsx scripts/test-cli-auth/test-oauth-flow.ts anthropic
+```
+
+## Architecture
+
+The CLI OAuth flow works as follows:
+
+```
+┌─────────────────┐      ┌──────────────┐      ┌─────────────────┐
+│  Dashboard UI   │─────▶│  Onboarding  │─────▶│  CLI via PTY    │
+│  (React)        │      │  API         │      │  (node-pty)     │
+└─────────────────┘      └──────────────┘      └─────────────────┘
+         ▲                      │                      │
+         │                      │                      ▼
+         │                      │               ┌─────────────────┐
+         │                      │               │  Interactive    │
+         │                      │◀──────────────│  Prompts        │
+         │                      │  auto-respond └─────────────────┘
+         │                      │                      │
+         │                      │                      ▼
+         │                      │               ┌─────────────────┐
+         │                      │               │  Auth URL       │
+         │◀─────────────────────┼───────────────│  Output         │
+         │   (opens popup)      │               └─────────────────┘
+         │                      │                      │
+         ▼                      │                      ▼
+┌─────────────────┐            │               ┌─────────────────┐
+│  OAuth Popup    │            │               │  Success        │
+│  (Browser)      │────────────┼──────────────▶│  Detection      │
+└─────────────────┘            │               └─────────────────┘
+```
+
+## Adding a New Provider
+
+### 1. Define the CLI Configuration
+
+Add a new entry to `CLI_AUTH_CONFIG` in `src/cloud/api/onboarding.ts`:
+
+```typescript
+export const CLI_AUTH_CONFIG: Record<string, CLIAuthConfig> = {
+  // ... existing providers
+
+  newprovider: {
+    // Required: CLI command to run
+    command: 'newcli',
+
+    // Required: Command arguments
+    args: ['auth', 'login'],
+
+    // Required: Pattern to extract auth URL (must have capture group)
+    urlPattern: /(https:\/\/[^\s]+)/,
+
+    // Optional: Path to credentials file after auth
+    credentialPath: '~/.newcli/credentials.json',
+
+    // Required: Display name for UI
+    displayName: 'NewProvider',
+
+    // Required: How long to wait for URL (ms)
+    waitTimeout: 3000,
+
+    // Required: Interactive prompts to auto-respond to
+    prompts: [
+      {
+        pattern: /do you trust this/i,
+        response: 'y\r',
+        delay: 100,
+        description: 'Trust prompt',
+      },
+    ],
+
+    // Required: Success indicators
+    successPatterns: [
+      /success/i,
+      /authenticated/i,
+    ],
+  },
+};
+```
+
+### 2. Add Mock CLI Behavior
+
+Update `scripts/test-cli-auth/mock-cli.sh` with the new provider's interactive flow:
+
+```bash
+newprovider)
+  echo -e "${BLUE}NewProvider CLI${NC}"
+  sleep "$DELAY"
+
+  echo -e "Do you trust this directory? [y/N] "
+  read -r -n 1 response 2>/dev/null || true
+  echo ""
+
+  echo -e "Auth URL:"
+  echo -e "${GREEN}https://newprovider.com/auth?session=test${NC}"
+
+  read -r -t 30 2>/dev/null || true
+  echo -e "${GREEN}Authenticated!${NC}"
+  ;;
+```
+
+### 3. Add Unit Tests
+
+Add tests to `src/cloud/api/onboarding.test.ts`:
+
+```typescript
+describe('newprovider', () => {
+  const config = CLI_AUTH_CONFIG.newprovider;
+
+  it('has correct command and args', () => {
+    expect(config.command).toBe('newcli');
+    expect(config.args).toEqual(['auth', 'login']);
+  });
+
+  it('extracts auth URL from output', () => {
+    const output = 'Visit https://newprovider.com/auth?id=xxx';
+    const match = output.match(config.urlPattern);
+    expect(match![1]).toContain('https://newprovider.com/auth');
+  });
+
+  // Test each prompt handler
+  describe('prompt handlers', () => {
+    it('detects trust prompt', () => {
+      const prompt = findMatchingPrompt(
+        'Do you trust this directory?',
+        config.prompts,
+        new Set()
+      );
+      expect(prompt!.description).toBe('Trust prompt');
+    });
+  });
+});
+```
+
+### 4. Run Tests
+
+```bash
+# Unit tests
+npx vitest run src/cloud/api/onboarding.test.ts
+
+# Integration tests
+npx tsx scripts/test-cli-auth/test-oauth-flow.ts newprovider
+```
+
+## Provider Checklist
+
+When adding or modifying a provider, ensure:
+
+- [ ] `command` is the correct CLI binary name
+- [ ] `args` includes any required subcommands (e.g., `['login']`)
+- [ ] `urlPattern` has a capture group `(...)` around the URL
+- [ ] `waitTimeout` is long enough for multi-step prompts
+- [ ] All interactive prompts are handled in `prompts` array
+- [ ] Each prompt has a unique `description` for deduplication
+- [ ] `successPatterns` cover all success messages the CLI outputs
+- [ ] Mock CLI simulates the real CLI's behavior accurately
+- [ ] Unit tests cover URL extraction and prompt detection
+- [ ] Integration test passes
+
+## Testing with Real CLIs
+
+For testing with actual CLIs (not mocks), you can:
+
+1. **Docker Container Test**: Spin up a container without credentials:
+   ```bash
+   docker run -it --rm node:20 bash
+   npm install -g @anthropic-ai/claude-code
+   # Run the onboarding flow
+   ```
+
+2. **Fresh VM**: Use a cloud VM with no cached credentials
+
+3. **Delete Credentials**: Remove local credential files:
+   ```bash
+   rm -rf ~/.claude
+   rm -rf ~/.codex
+   # etc.
+   ```
+
+## Troubleshooting
+
+### URL Not Extracted
+- Check if the CLI outputs the URL in expected format
+- Verify the `urlPattern` regex matches the output
+- Increase `waitTimeout` if prompts take longer
+
+### Prompts Not Detected
+- Run mock CLI manually to see exact prompt text
+- Check regex patterns are case-insensitive (`/i` flag)
+- Ensure ANSI codes are being stripped before matching
+
+### Success Not Detected
+- Verify CLI outputs one of the success patterns
+- Check for typos in pattern (e.g., `logged in` vs `loggedin`)
+- Add new patterns if CLI uses different success messages
+
+## CI Integration
+
+### GitHub Actions
+
+The workflow `.github/workflows/cli-oauth-test.yml` runs:
+
+1. **On every push/PR** that modifies:
+   - `src/cloud/api/onboarding.ts`
+   - `scripts/test-cli-auth/**`
+
+2. **Weekly schedule** (Sundays at midnight):
+   - Catches provider CLI changes early
+   - Auto-creates GitHub issues on failure
+
+### Running CI Tests Locally
+
+```bash
+# Build the test container with REAL CLIs (recommended)
+docker build -f scripts/test-cli-auth/Dockerfile.real \
+  -t cli-oauth-test-real scripts/test-cli-auth/
+
+# Run tests against real CLIs
+docker run --rm cli-oauth-test-real
+
+# Run with results output
+docker run --rm -v $(pwd)/test-results:/tmp cli-oauth-test-real
+cat test-results/cli-oauth-test-results.json
+
+# Interactive debugging
+docker run --rm -it cli-oauth-test-real bash
+claude  # Test Claude CLI manually
+```
+
+### Why Real CLIs?
+
+Using the actual CLIs instead of mocks:
+- **Catches real changes** in CLI behavior immediately
+- **No maintenance burden** of keeping mocks in sync
+- **Tests the actual code path** users will experience
+- **Detects new prompts** or changed output formats
+
+CLIs that aren't installed are skipped (not failed), so tests work even if some providers haven't published CLIs yet.
+
+### Test Output Format
+
+```json
+{
+  "timestamp": "2024-01-15T10:30:00.000Z",
+  "results": [
+    {
+      "provider": "anthropic",
+      "command": "claude",
+      "passed": true,
+      "urlExtracted": "https://console.anthropic.com/oauth/...",
+      "urlValid": true,
+      "promptsHandled": 3,
+      "exitCode": 0,
+      "duration": 1234
+    }
+  ],
+  "summary": {
+    "total": 5,
+    "passed": 5,
+    "failed": 0
+  }
+}
+```
+
+## Files
+
+- `mock-cli.sh` - Simulates CLI interactive flows for testing
+- `ci-test-runner.ts` - Docker-based CI test runner
+- `test-oauth-flow.ts` - Local integration test runner
+- `Dockerfile` - Test container definition
+- `package.json` - Test dependencies
+- `README.md` - This documentation

package/scripts/test-cli-auth/ci-test-real-clis.ts

@@ -0,0 +1,251 @@
+#!/usr/bin/env npx tsx
+/**
+ * CI Test Runner for Real CLI OAuth Flows
+ *
+ * Tests the actual CLI tools to verify:
+ * 1. URL extraction patterns work with real CLI output
+ * 2. Prompt detection works with real prompts
+ * 3. Auto-responses navigate through the flow correctly
+ *
+ * Exit codes:
+ *   0 - All available CLIs passed
+ *   1 - One or more tests failed
+ *
+ * Note: CLIs that aren't installed are skipped, not failed.
+ *
+ * IMPORTANT: This test uses the same runCLIAuthViaPTY function as production
+ * to ensure the PTY handling logic is consistent.
+ */
+
+import { execSync } from 'child_process';
+import { writeFileSync } from 'fs';
+
+// Import the actual config and PTY runner from cli-pty-runner.ts
+// This ensures tests use the EXACT SAME logic as production
+// In Docker: /app/src/cloud/api/cli-pty-runner.ts (maintains source tree structure)
+import {
+  CLI_AUTH_CONFIG,
+  runCLIAuthViaPTY,
+  type PTYAuthResult,
+} from './src/cloud/api/cli-pty-runner.js';
+
+interface TestResult {
+  provider: string;
+  command: string;
+  installed: boolean;
+  passed: boolean;
+  skipped: boolean;
+  urlExtracted: string | null;
+  promptsDetected: string[];
+  promptsResponded: string[];
+  successDetected: boolean;
+  exitCode: number | null;
+  duration: number;
+  rawOutput: string;
+  error?: string;
+}
+
+/**
+ * Check if a CLI is installed
+ */
+function isCliInstalled(command: string): boolean {
+  try {
+    execSync(`which ${command}`, { stdio: 'pipe' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Test a real CLI's OAuth flow using the shared PTY runner
+ *
+ * Uses the EXACT SAME runCLIAuthViaPTY function as production to ensure
+ * the PTY handling logic is consistent between tests and production.
+ */
+async function testRealCli(providerId: string): Promise<TestResult> {
+  const config = CLI_AUTH_CONFIG[providerId];
+  if (!config) {
+    return {
+      provider: providerId,
+      command: 'unknown',
+      installed: false,
+      passed: false,
+      skipped: true,
+      urlExtracted: null,
+      promptsDetected: [],
+      promptsResponded: [],
+      successDetected: false,
+      exitCode: null,
+      duration: 0,
+      rawOutput: '',
+      error: `Unknown provider: ${providerId}`,
+    };
+  }
+
+  const result: TestResult = {
+    provider: providerId,
+    command: `${config.command} ${config.args.join(' ')}`.trim(),
+    installed: isCliInstalled(config.command),
+    passed: false,
+    skipped: false,
+    urlExtracted: null,
+    promptsDetected: [],
+    promptsResponded: [],
+    successDetected: false,
+    exitCode: null,
+    duration: 0,
+    rawOutput: '',
+  };
+
+  // Skip if CLI not installed
+  if (!result.installed) {
+    result.skipped = true;
+    result.error = `CLI '${config.command}' not installed`;
+    return result;
+  }
+
+  const startTime = Date.now();
+
+  // Use the shared PTY runner - SAME code as production
+  const ptyResult: PTYAuthResult = await runCLIAuthViaPTY(config, {
+    onAuthUrl: (url) => {
+      result.urlExtracted = url;
+      console.log(`    [${providerId}] URL found: ${url.substring(0, 60)}...`);
+    },
+    onPromptHandled: (description) => {
+      result.promptsDetected.push(description);
+      result.promptsResponded.push(description);
+      console.log(`    [${providerId}] Responded to: ${description}`);
+    },
+    onOutput: (data) => {
+      result.rawOutput += data;
+    },
+  });
+
+  result.duration = Date.now() - startTime;
+  result.exitCode = ptyResult.exitCode;
+  result.successDetected = ptyResult.success;
+
+  // Pass if we got a URL (main goal of OAuth flow)
+  result.passed = !!result.urlExtracted;
+
+  if (!result.passed) {
+    result.error = ptyResult.error || 'Failed to extract auth URL from CLI output';
+  }
+
+  return result;
+}
+
+/**
+ * Run tests for all configured providers
+ */
+async function runAllTests() {
+  console.log('╔══════════════════════════════════════════════════════════════╗');
+  console.log('║        CLI OAuth Flow Tests - Real CLIs                       ║');
+  console.log('╚══════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  const results: TestResult[] = [];
+  const providerIds = Object.keys(CLI_AUTH_CONFIG);
+
+  for (const providerId of providerIds) {
+    const config = CLI_AUTH_CONFIG[providerId];
+    console.log(`Testing ${config.displayName} (${providerId})...`);
+
+    const result = await testRealCli(providerId);
+    results.push(result);
+
+    if (result.skipped) {
+      console.log(`  ⏭️  SKIPPED: ${result.error}`);
+    } else if (result.passed) {
+      console.log(`  ✅ PASSED`);
+    } else {
+      console.log(`  ❌ FAILED: ${result.error}`);
+    }
+
+    console.log(`     Installed: ${result.installed ? 'Yes' : 'No'}`);
+    if (!result.skipped) {
+      console.log(`     URL: ${result.urlExtracted ? 'Extracted' : 'NOT FOUND'}`);
+      console.log(`     Prompts: ${result.promptsResponded.length}/${config.prompts.length} handled`);
+      console.log(`     Duration: ${result.duration}ms`);
+    }
+    console.log('');
+  }
+
+  // Summary
+  const installed = results.filter(r => r.installed);
+  const skipped = results.filter(r => r.skipped);
+  const passed = results.filter(r => r.passed);
+  const failed = results.filter(r => !r.passed && !r.skipped);
+
+  console.log('═══════════════════════════════════════════════════════════════');
+  console.log(`Summary:`);
+  console.log(`  Installed: ${installed.length}/${results.length}`);
+  console.log(`  Passed: ${passed.length}`);
+  console.log(`  Failed: ${failed.length}`);
+  console.log(`  Skipped: ${skipped.length}`);
+  console.log('═══════════════════════════════════════════════════════════════');
+
+  // Write JSON results
+  const jsonResults = {
+    timestamp: new Date().toISOString(),
+    results: results.map(r => ({
+      provider: r.provider,
+      command: r.command,
+      installed: r.installed,
+      passed: r.passed,
+      skipped: r.skipped,
+      urlExtracted: r.urlExtracted ? true : false,
+      urlSample: r.urlExtracted?.substring(0, 80),
+      promptsDetected: r.promptsDetected,
+      promptsResponded: r.promptsResponded,
+      exitCode: r.exitCode,
+      duration: r.duration,
+      error: r.error,
+    })),
+    summary: {
+      total: results.length,
+      installed: installed.length,
+      passed: passed.length,
+      failed: failed.length,
+      skipped: skipped.length,
+    },
+  };
+
+  try {
+    writeFileSync('/tmp/cli-oauth-test-results.json', JSON.stringify(jsonResults, null, 2));
+    console.log('\nResults written to /tmp/cli-oauth-test-results.json');
+  } catch {
+    console.log('\n--- JSON Results ---');
+    console.log(JSON.stringify(jsonResults, null, 2));
+  }
+
+  // Exit with failure only if an installed CLI failed
+  // Skipped CLIs don't count as failures
+  if (failed.length > 0) {
+    console.log('\nFailed CLIs:');
+    for (const result of failed) {
+      console.log(`  - ${result.provider}: ${result.error}`);
+      if (result.rawOutput) {
+        console.log(`    Last 500 chars of output:`);
+        console.log(`    ${result.rawOutput.slice(-500).replace(/\n/g, '\n    ')}`);
+      }
+    }
+    process.exit(1);
+  }
+
+  // Warn if no CLIs were tested
+  if (installed.length === 0) {
+    console.log('\n⚠️  WARNING: No CLIs were installed - no actual testing performed!');
+    process.exit(0); // Don't fail, but warn
+  }
+
+  console.log('\n✅ All installed CLIs passed!');
+  process.exit(0);
+}
+
+runAllTests().catch((err) => {
+  console.error('Test runner failed:', err);
+  process.exit(1);
+});