agent-relay 1.1.0 → 1.2.0

package/scripts/test-cli-auth/ci-test-real-clis.ts

@@ -0,0 +1,251 @@
+#!/usr/bin/env npx tsx
+/**
+ * CI Test Runner for Real CLI OAuth Flows
+ *
+ * Tests the actual CLI tools to verify:
+ * 1. URL extraction patterns work with real CLI output
+ * 2. Prompt detection works with real prompts
+ * 3. Auto-responses navigate through the flow correctly
+ *
+ * Exit codes:
+ *   0 - All available CLIs passed
+ *   1 - One or more tests failed
+ *
+ * Note: CLIs that aren't installed are skipped, not failed.
+ *
+ * IMPORTANT: This test uses the same runCLIAuthViaPTY function as production
+ * to ensure the PTY handling logic is consistent.
+ */
+
+import { execSync } from 'child_process';
+import { writeFileSync } from 'fs';
+
+// Import the actual config and PTY runner from cli-pty-runner.ts
+// This ensures tests use the EXACT SAME logic as production
+// In Docker: /app/src/cloud/api/cli-pty-runner.ts (maintains source tree structure)
+import {
+  CLI_AUTH_CONFIG,
+  runCLIAuthViaPTY,
+  type PTYAuthResult,
+} from './src/cloud/api/cli-pty-runner.js';
+
+interface TestResult {
+  provider: string;
+  command: string;
+  installed: boolean;
+  passed: boolean;
+  skipped: boolean;
+  urlExtracted: string | null;
+  promptsDetected: string[];
+  promptsResponded: string[];
+  successDetected: boolean;
+  exitCode: number | null;
+  duration: number;
+  rawOutput: string;
+  error?: string;
+}
+
+/**
+ * Check if a CLI is installed
+ */
+function isCliInstalled(command: string): boolean {
+  try {
+    execSync(`which ${command}`, { stdio: 'pipe' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Test a real CLI's OAuth flow using the shared PTY runner
+ *
+ * Uses the EXACT SAME runCLIAuthViaPTY function as production to ensure
+ * the PTY handling logic is consistent between tests and production.
+ */
+async function testRealCli(providerId: string): Promise<TestResult> {
+  const config = CLI_AUTH_CONFIG[providerId];
+  if (!config) {
+    return {
+      provider: providerId,
+      command: 'unknown',
+      installed: false,
+      passed: false,
+      skipped: true,
+      urlExtracted: null,
+      promptsDetected: [],
+      promptsResponded: [],
+      successDetected: false,
+      exitCode: null,
+      duration: 0,
+      rawOutput: '',
+      error: `Unknown provider: ${providerId}`,
+    };
+  }
+
+  const result: TestResult = {
+    provider: providerId,
+    command: `${config.command} ${config.args.join(' ')}`.trim(),
+    installed: isCliInstalled(config.command),
+    passed: false,
+    skipped: false,
+    urlExtracted: null,
+    promptsDetected: [],
+    promptsResponded: [],
+    successDetected: false,
+    exitCode: null,
+    duration: 0,
+    rawOutput: '',
+  };
+
+  // Skip if CLI not installed
+  if (!result.installed) {
+    result.skipped = true;
+    result.error = `CLI '${config.command}' not installed`;
+    return result;
+  }
+
+  const startTime = Date.now();
+
+  // Use the shared PTY runner - SAME code as production
+  const ptyResult: PTYAuthResult = await runCLIAuthViaPTY(config, {
+    onAuthUrl: (url) => {
+      result.urlExtracted = url;
+      console.log(`    [${providerId}] URL found: ${url.substring(0, 60)}...`);
+    },
+    onPromptHandled: (description) => {
+      result.promptsDetected.push(description);
+      result.promptsResponded.push(description);
+      console.log(`    [${providerId}] Responded to: ${description}`);
+    },
+    onOutput: (data) => {
+      result.rawOutput += data;
+    },
+  });
+
+  result.duration = Date.now() - startTime;
+  result.exitCode = ptyResult.exitCode;
+  result.successDetected = ptyResult.success;
+
+  // Pass if we got a URL (main goal of OAuth flow)
+  result.passed = !!result.urlExtracted;
+
+  if (!result.passed) {
+    result.error = ptyResult.error || 'Failed to extract auth URL from CLI output';
+  }
+
+  return result;
+}
+
+/**
+ * Run tests for all configured providers
+ */
+async function runAllTests() {
+  console.log('╔══════════════════════════════════════════════════════════════╗');
+  console.log('║        CLI OAuth Flow Tests - Real CLIs                       ║');
+  console.log('╚══════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  const results: TestResult[] = [];
+  const providerIds = Object.keys(CLI_AUTH_CONFIG);
+
+  for (const providerId of providerIds) {
+    const config = CLI_AUTH_CONFIG[providerId];
+    console.log(`Testing ${config.displayName} (${providerId})...`);
+
+    const result = await testRealCli(providerId);
+    results.push(result);
+
+    if (result.skipped) {
+      console.log(`  ⏭️  SKIPPED: ${result.error}`);
+    } else if (result.passed) {
+      console.log(`  ✅ PASSED`);
+    } else {
+      console.log(`  ❌ FAILED: ${result.error}`);
+    }
+
+    console.log(`     Installed: ${result.installed ? 'Yes' : 'No'}`);
+    if (!result.skipped) {
+      console.log(`     URL: ${result.urlExtracted ? 'Extracted' : 'NOT FOUND'}`);
+      console.log(`     Prompts: ${result.promptsResponded.length}/${config.prompts.length} handled`);
+      console.log(`     Duration: ${result.duration}ms`);
+    }
+    console.log('');
+  }
+
+  // Summary
+  const installed = results.filter(r => r.installed);
+  const skipped = results.filter(r => r.skipped);
+  const passed = results.filter(r => r.passed);
+  const failed = results.filter(r => !r.passed && !r.skipped);
+
+  console.log('═══════════════════════════════════════════════════════════════');
+  console.log(`Summary:`);
+  console.log(`  Installed: ${installed.length}/${results.length}`);
+  console.log(`  Passed: ${passed.length}`);
+  console.log(`  Failed: ${failed.length}`);
+  console.log(`  Skipped: ${skipped.length}`);
+  console.log('═══════════════════════════════════════════════════════════════');
+
+  // Write JSON results
+  const jsonResults = {
+    timestamp: new Date().toISOString(),
+    results: results.map(r => ({
+      provider: r.provider,
+      command: r.command,
+      installed: r.installed,
+      passed: r.passed,
+      skipped: r.skipped,
+      urlExtracted: r.urlExtracted ? true : false,
+      urlSample: r.urlExtracted?.substring(0, 80),
+      promptsDetected: r.promptsDetected,
+      promptsResponded: r.promptsResponded,
+      exitCode: r.exitCode,
+      duration: r.duration,
+      error: r.error,
+    })),
+    summary: {
+      total: results.length,
+      installed: installed.length,
+      passed: passed.length,
+      failed: failed.length,
+      skipped: skipped.length,
+    },
+  };
+
+  try {
+    writeFileSync('/tmp/cli-oauth-test-results.json', JSON.stringify(jsonResults, null, 2));
+    console.log('\nResults written to /tmp/cli-oauth-test-results.json');
+  } catch {
+    console.log('\n--- JSON Results ---');
+    console.log(JSON.stringify(jsonResults, null, 2));
+  }
+
+  // Exit with failure only if an installed CLI failed
+  // Skipped CLIs don't count as failures
+  if (failed.length > 0) {
+    console.log('\nFailed CLIs:');
+    for (const result of failed) {
+      console.log(`  - ${result.provider}: ${result.error}`);
+      if (result.rawOutput) {
+        console.log(`    Last 500 chars of output:`);
+        console.log(`    ${result.rawOutput.slice(-500).replace(/\n/g, '\n    ')}`);
+      }
+    }
+    process.exit(1);
+  }
+
+  // Warn if no CLIs were tested
+  if (installed.length === 0) {
+    console.log('\n⚠️  WARNING: No CLIs were installed - no actual testing performed!');
+    process.exit(0); // Don't fail, but warn
+  }
+
+  console.log('\n✅ All installed CLIs passed!');
+  process.exit(0);
+}
+
+runAllTests().catch((err) => {
+  console.error('Test runner failed:', err);
+  process.exit(1);
+});

package/scripts/test-cli-auth/ci-test-runner.ts

@@ -0,0 +1,263 @@
+#!/usr/bin/env npx tsx
+/**
+ * CI Test Runner for CLI OAuth Flow
+ *
+ * This script runs in a Docker container and tests each provider's
+ * CLI OAuth flow to ensure URL extraction works correctly.
+ *
+ * Exit codes:
+ *   0 - All tests passed
+ *   1 - One or more tests failed
+ *
+ * Output format (JSON):
+ *   { "results": [...], "summary": { "passed": N, "failed": N } }
+ */
+
+import * as pty from 'node-pty';
+import { writeFileSync } from 'fs';
+
+// Provider configurations - must match CLI_AUTH_CONFIG in onboarding.ts
+const PROVIDERS = {
+  anthropic: {
+    command: 'claude',
+    args: [] as string[],
+    urlPattern: /(https:\/\/[^\s]+)/,
+    expectedUrlPrefix: 'https://console.anthropic.com',
+    prompts: [
+      { pattern: /dark\s*(mode|theme)/i, response: '\r' },
+      { pattern: /(subscription|api\s*key)/i, response: '\r' },
+      { pattern: /trust/i, response: 'y\r' },
+    ],
+  },
+  openai: {
+    command: 'codex',
+    args: ['login'],
+    urlPattern: /(https:\/\/[^\s]+)/,
+    expectedUrlPrefix: 'https://auth.openai.com',
+    prompts: [
+      { pattern: /trust/i, response: 'y\r' },
+    ],
+  },
+  google: {
+    command: 'gemini',
+    args: [] as string[],
+    urlPattern: /(https:\/\/[^\s]+)/,
+    expectedUrlPrefix: 'https://accounts.google.com',
+    prompts: [],
+  },
+  opencode: {
+    command: 'opencode',
+    args: [] as string[],
+    urlPattern: /(https:\/\/[^\s]+)/,
+    expectedUrlPrefix: 'https://opencode.ai',
+    prompts: [],
+  },
+  droid: {
+    command: 'droid',
+    args: [] as string[],
+    urlPattern: /(https:\/\/[^\s]+)/,
+    expectedUrlPrefix: 'https://factory.ai',
+    prompts: [],
+  },
+};
+
+interface TestResult {
+  provider: string;
+  command: string;
+  passed: boolean;
+  urlExtracted: string | null;
+  urlValid: boolean;
+  promptsHandled: number;
+  exitCode: number | null;
+  duration: number;
+  output: string;
+  error?: string;
+}
+
+function stripAnsi(text: string): string {
+  return text.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '');
+}
+
+async function testProvider(providerId: string): Promise<TestResult> {
+  const config = PROVIDERS[providerId as keyof typeof PROVIDERS];
+  if (!config) {
+    return {
+      provider: providerId,
+      command: 'unknown',
+      passed: false,
+      urlExtracted: null,
+      urlValid: false,
+      promptsHandled: 0,
+      exitCode: null,
+      duration: 0,
+      output: '',
+      error: `Unknown provider: ${providerId}`,
+    };
+  }
+
+  const startTime = Date.now();
+  const result: TestResult = {
+    provider: providerId,
+    command: `${config.command} ${config.args.join(' ')}`.trim(),
+    passed: false,
+    urlExtracted: null,
+    urlValid: false,
+    promptsHandled: 0,
+    exitCode: null,
+    duration: 0,
+    output: '',
+  };
+
+  return new Promise((resolve) => {
+    const respondedPrompts = new Set<number>();
+
+    try {
+      const proc = pty.spawn(config.command, config.args, {
+        name: 'xterm-256color',
+        cols: 120,
+        rows: 30,
+        env: { ...process.env, TERM: 'xterm-256color', NO_COLOR: '1' },
+      });
+
+      const timeout = setTimeout(() => {
+        proc.kill();
+        result.error = 'Timeout waiting for CLI';
+        result.duration = Date.now() - startTime;
+        resolve(result);
+      }, 15000);
+
+      proc.onData((data: string) => {
+        result.output += data;
+        const cleanText = stripAnsi(data);
+
+        // Check for prompts and respond
+        for (let i = 0; i < config.prompts.length; i++) {
+          if (respondedPrompts.has(i)) continue;
+          if (config.prompts[i].pattern.test(cleanText)) {
+            respondedPrompts.add(i);
+            result.promptsHandled++;
+            setTimeout(() => {
+              try {
+                proc.write(config.prompts[i].response);
+              } catch {
+                // Process may have exited
+              }
+            }, 100);
+          }
+        }
+
+        // Check for URL
+        const match = cleanText.match(config.urlPattern);
+        if (match && match[1] && !result.urlExtracted) {
+          result.urlExtracted = match[1];
+          result.urlValid = result.urlExtracted.startsWith(config.expectedUrlPrefix);
+        }
+      });
+
+      proc.onExit(({ exitCode }) => {
+        clearTimeout(timeout);
+        result.exitCode = exitCode;
+        result.duration = Date.now() - startTime;
+
+        // Determine pass/fail
+        result.passed = !!(
+          result.urlExtracted &&
+          result.urlValid &&
+          exitCode === 0
+        );
+
+        resolve(result);
+      });
+
+      // Send signal to continue after prompts are done
+      setTimeout(() => {
+        try {
+          proc.write('\n');
+        } catch {
+          // Ignore
+        }
+      }, 5000);
+    } catch (err) {
+      result.error = err instanceof Error ? err.message : 'Unknown error';
+      result.duration = Date.now() - startTime;
+      resolve(result);
+    }
+  });
+}
+
+async function runAllTests() {
+  console.log('╔══════════════════════════════════════════════════════════════╗');
+  console.log('║           CLI OAuth Flow CI Tests                            ║');
+  console.log('╚══════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  const results: TestResult[] = [];
+
+  for (const providerId of Object.keys(PROVIDERS)) {
+    process.stdout.write(`Testing ${providerId}... `);
+    const result = await testProvider(providerId);
+    results.push(result);
+
+    if (result.passed) {
+      console.log('✅ PASSED');
+    } else {
+      console.log(`❌ FAILED${result.error ? `: ${result.error}` : ''}`);
+    }
+
+    // Detailed output
+    console.log(`  Command: ${result.command}`);
+    console.log(`  URL: ${result.urlExtracted || 'NOT FOUND'}`);
+    console.log(`  Valid: ${result.urlValid ? 'Yes' : 'No'}`);
+    console.log(`  Prompts: ${result.promptsHandled}/${PROVIDERS[providerId as keyof typeof PROVIDERS].prompts.length}`);
+    console.log(`  Exit: ${result.exitCode}`);
+    console.log(`  Duration: ${result.duration}ms`);
+    console.log('');
+  }
+
+  // Summary
+  const passed = results.filter(r => r.passed).length;
+  const failed = results.filter(r => !r.passed).length;
+
+  console.log('═══════════════════════════════════════════════════════════════');
+  console.log(`Summary: ${passed} passed, ${failed} failed out of ${results.length} tests`);
+  console.log('═══════════════════════════════════════════════════════════════');
+
+  // Write JSON results for CI parsing
+  const jsonResults = {
+    timestamp: new Date().toISOString(),
+    results: results.map(r => ({
+      provider: r.provider,
+      command: r.command,
+      passed: r.passed,
+      urlExtracted: r.urlExtracted,
+      urlValid: r.urlValid,
+      promptsHandled: r.promptsHandled,
+      exitCode: r.exitCode,
+      duration: r.duration,
+      error: r.error,
+    })),
+    summary: {
+      total: results.length,
+      passed,
+      failed,
+    },
+  };
+
+  // Write to file for CI artifact
+  try {
+    writeFileSync('/tmp/cli-oauth-test-results.json', JSON.stringify(jsonResults, null, 2));
+    console.log('\nResults written to /tmp/cli-oauth-test-results.json');
+  } catch {
+    // Might not have write access, output to stdout instead
+    console.log('\n--- JSON Results ---');
+    console.log(JSON.stringify(jsonResults, null, 2));
+  }
+
+  // Exit with appropriate code
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+runAllTests().catch((err) => {
+  console.error('Test runner failed:', err);
+  process.exit(1);
+});

package/scripts/test-cli-auth/mock-cli.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# Mock CLI for testing OAuth flow prompt handling
+# Usage: ./mock-cli.sh <provider> [delay]
+#
+# This script simulates the interactive prompts of various AI CLI tools
+# for testing the onboarding OAuth flow without actual CLI binaries.
+#
+# When installed as symlinks (e.g., /usr/local/bin/claude -> mock-cli.sh),
+# it auto-detects the provider from the command name.
+
+# Detect provider from how the script was called
+SCRIPT_NAME=$(basename "$0")
+
+case "$SCRIPT_NAME" in
+  claude) PROVIDER="claude" ;;
+  codex) PROVIDER="codex" ;;
+  gemini) PROVIDER="gemini" ;;
+  opencode) PROVIDER="opencode" ;;
+  droid) PROVIDER="droid" ;;
+  mock-cli.sh|mock-cli|mock-cli-impl.sh)
+    PROVIDER="${1:-claude}"
+    shift 2>/dev/null || true
+    ;;
+  *) PROVIDER="${1:-claude}" ;;
+esac
+
+# Handle 'login' subcommand for codex
+if [ "$PROVIDER" = "codex" ] && [ "$1" = "login" ]; then
+  shift
+fi
+
+DELAY="${1:-0.3}"
+
+# Colors for output
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+case "$PROVIDER" in
+  claude|anthropic)
+    echo -e "${BLUE}Claude Code CLI${NC}"
+    echo ""
+    sleep "$DELAY"
+
+    # Dark mode prompt
+    echo -e "Would you like to use ${YELLOW}dark mode${NC}? (y/n) "
+    read -r -n 1 response 2>/dev/null || true
+    echo ""
+    sleep "$DELAY"
+
+    # Login method selection prompt (matches real Claude CLI)
+    echo -e "Select login method:"
+    echo ""
+    echo -e " ❯ 1. Claude account with ${YELLOW}subscription${NC} · Pro, Max, Team, or Enterprise"
+    echo ""
+    echo -e "   2. Anthropic Console account · API usage billing"
+    read -r -n 1 response 2>/dev/null || true
+    echo ""
+    sleep "$DELAY"
+
+    # Trust directory prompt
+    echo -e "Do you ${YELLOW}trust this directory${NC}? [y/N] "
+    read -r -n 1 response 2>/dev/null || true
+    echo ""
+    sleep "$DELAY"
+
+    # Auth URL
+    echo ""
+    echo -e "Please visit the following URL to authenticate:"
+    echo -e "${GREEN}https://console.anthropic.com/oauth/authorize?client_id=mock-test-123&state=abc${NC}"
+    echo ""
+    echo "Waiting for authentication..."
+
+    # Wait for completion signal (or timeout)
+    read -r -t 30 2>/dev/null || true
+    echo -e "${GREEN}Authentication successful!${NC}"
+    ;;
+
+  codex|openai)
+    echo -e "${BLUE}Codex CLI${NC}"
+    echo ""
+    sleep "$DELAY"
+
+    # Trust directory prompt
+    echo -e "Do you ${YELLOW}trust this workspace${NC}? [y/N] "
+    read -r -n 1 response 2>/dev/null || true
+    echo ""
+    sleep "$DELAY"
+
+    # Auth URL
+    echo ""
+    echo -e "Open this URL to log in:"
+    echo -e "${GREEN}https://auth.openai.com/authorize?client_id=mock-test-456&state=def${NC}"
+    echo ""
+    echo "Waiting..."
+
+    read -r -t 30 2>/dev/null || true
+    echo -e "${GREEN}Logged in successfully${NC}"
+    ;;
+
+  gemini|google)
+    echo -e "${BLUE}Gemini CLI${NC}"
+    echo ""
+    sleep "$DELAY"
+
+    # Auth URL
+    echo -e "Authenticate at:"
+    echo -e "${GREEN}https://accounts.google.com/o/oauth2/v2/auth?client_id=mock-test-789${NC}"
+    echo ""
+
+    read -r -t 30 2>/dev/null || true
+    echo -e "${GREEN}Authenticated!${NC}"
+    ;;
+
+  opencode)
+    echo -e "${BLUE}OpenCode CLI${NC}"
+    echo ""
+    sleep "$DELAY"
+
+    echo -e "Login URL:"
+    echo -e "${GREEN}https://opencode.ai/auth?session=mock-session${NC}"
+    echo ""
+
+    read -r -t 30 2>/dev/null || true
+    echo -e "${GREEN}Success${NC}"
+    ;;
+
+  droid)
+    echo -e "${BLUE}Droid CLI${NC}"
+    echo ""
+    sleep "$DELAY"
+
+    echo -e "Visit to authenticate:"
+    echo -e "${GREEN}https://factory.ai/droid/auth?id=mock-droid${NC}"
+    echo ""
+
+    read -r -t 30 2>/dev/null || true
+    echo -e "${GREEN}Authenticated${NC}"
+    ;;
+
+  *)
+    echo "Unknown provider: $PROVIDER"
+    echo "Supported: claude, codex, gemini, opencode, droid"
+    exit 1
+    ;;
+esac

package/scripts/test-cli-auth/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "cli-oauth-test",
+  "version": "1.0.0",
+  "description": "CLI OAuth flow testing for agent-relay",
+  "type": "module",
+  "scripts": {
+    "test": "tsx ci-test-runner.ts",
+    "test:local": "tsx test-oauth-flow.ts"
+  },
+  "dependencies": {
+    "node-pty": "^1.0.0",
+    "tsx": "^4.19.0"
+  }
+}