agent-browser 0.3.0 → 0.3.2

package/benchmark/benchmark.ts

@@ -1,521 +0,0 @@
-#!/usr/bin/env npx tsx
-/**
- * Benchmark: agent-browser vs playwright-mcp
- * 
- * Measures:
- * - Speed: cold start, navigation, click, snapshot operations
- * - Context usage: output/response size in bytes and estimated tokens
- */
-
-import { spawn, execSync, ChildProcess } from 'child_process';
-import * as readline from 'readline';
-
-// ============================================================================
-// Configuration
-// ============================================================================
-
-const TEST_URL = 'https://example.com';
-const ITERATIONS = 3;
-
-interface BenchmarkResult {
-  operation: string;
-  tool: string;
-  timeMs: number;
-  outputBytes: number;
-  estimatedTokens: number;
-}
-
-const results: BenchmarkResult[] = [];
-
-// Estimate tokens (~4 chars per token for English text)
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
-function formatBytes(bytes: number): string {
-  if (bytes < 1024) return `${bytes} B`;
-  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
-  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
-}
-
-// ============================================================================
-// Agent-Browser Benchmarks
-// ============================================================================
-
-async function runAgentBrowser(args: string[]): Promise<{ timeMs: number; output: string }> {
-  const start = performance.now();
-  
-  return new Promise((resolve, reject) => {
-    const proc = spawn('node', ['./dist/index.js', '--session', 'benchmark', '--json', ...args], {
-      cwd: process.cwd(),
-      env: { ...process.env },
-    });
-    
-    let output = '';
-    let stderr = '';
-    
-    proc.stdout.on('data', (data) => { output += data.toString(); });
-    proc.stderr.on('data', (data) => { stderr += data.toString(); });
-    
-    proc.on('close', (code) => {
-      const timeMs = performance.now() - start;
-      if (code !== 0 && !output.includes('"success"')) {
-        reject(new Error(`agent-browser failed: ${stderr || output}`));
-      } else {
-        resolve({ timeMs, output: output.trim() });
-      }
-    });
-    
-    proc.on('error', reject);
-  });
-}
-
-async function benchmarkAgentBrowser(): Promise<void> {
-  console.log('\n📦 Benchmarking agent-browser...\n');
-  
-  // Clean up any existing session
-  try {
-    await runAgentBrowser(['close']);
-  } catch {
-    // Ignore - session might not exist
-  }
-  
-  // Wait a bit for cleanup
-  await new Promise(r => setTimeout(r, 500));
-  
-  // Cold start (includes daemon startup + browser launch + navigation)
-  console.log('  ⏱️  Cold start (navigate)...');
-  const coldStart = await runAgentBrowser(['open', TEST_URL]);
-  results.push({
-    operation: 'cold_start_navigate',
-    tool: 'agent-browser',
-    timeMs: coldStart.timeMs,
-    outputBytes: coldStart.output.length,
-    estimatedTokens: estimateTokens(coldStart.output),
-  });
-  console.log(`     ${coldStart.timeMs.toFixed(0)}ms, ${formatBytes(coldStart.output.length)}`);
-  
-  // Warm operations
-  for (let i = 0; i < ITERATIONS; i++) {
-    // Navigate (warm)
-    console.log(`  ⏱️  Navigate (warm, iter ${i + 1})...`);
-    const nav = await runAgentBrowser(['open', TEST_URL]);
-    results.push({
-      operation: 'navigate_warm',
-      tool: 'agent-browser',
-      timeMs: nav.timeMs,
-      outputBytes: nav.output.length,
-      estimatedTokens: estimateTokens(nav.output),
-    });
-    console.log(`     ${nav.timeMs.toFixed(0)}ms, ${formatBytes(nav.output.length)}`);
-    
-    // Snapshot
-    console.log(`  ⏱️  Snapshot (iter ${i + 1})...`);
-    const snapshot = await runAgentBrowser(['snapshot']);
-    results.push({
-      operation: 'snapshot',
-      tool: 'agent-browser',
-      timeMs: snapshot.timeMs,
-      outputBytes: snapshot.output.length,
-      estimatedTokens: estimateTokens(snapshot.output),
-    });
-    console.log(`     ${snapshot.timeMs.toFixed(0)}ms, ${formatBytes(snapshot.output.length)}`);
-    
-    // Get title
-    console.log(`  ⏱️  Get title (iter ${i + 1})...`);
-    const title = await runAgentBrowser(['get', 'title']);
-    results.push({
-      operation: 'get_title',
-      tool: 'agent-browser',
-      timeMs: title.timeMs,
-      outputBytes: title.output.length,
-      estimatedTokens: estimateTokens(title.output),
-    });
-    console.log(`     ${title.timeMs.toFixed(0)}ms, ${formatBytes(title.output.length)}`);
-    
-    // Get URL
-    console.log(`  ⏱️  Get URL (iter ${i + 1})...`);
-    const url = await runAgentBrowser(['get', 'url']);
-    results.push({
-      operation: 'get_url',
-      tool: 'agent-browser',
-      timeMs: url.timeMs,
-      outputBytes: url.output.length,
-      estimatedTokens: estimateTokens(url.output),
-    });
-    console.log(`     ${url.timeMs.toFixed(0)}ms, ${formatBytes(url.output.length)}`);
-    
-    // Click (on a link that exists on example.com)
-    console.log(`  ⏱️  Click link (iter ${i + 1})...`);
-    const click = await runAgentBrowser(['click', 'a']);
-    results.push({
-      operation: 'click',
-      tool: 'agent-browser',
-      timeMs: click.timeMs,
-      outputBytes: click.output.length,
-      estimatedTokens: estimateTokens(click.output),
-    });
-    console.log(`     ${click.timeMs.toFixed(0)}ms, ${formatBytes(click.output.length)}`);
-    
-    // Navigate back for next iteration
-    await runAgentBrowser(['open', TEST_URL]);
-  }
-  
-  // Screenshot
-  console.log('  ⏱️  Screenshot...');
-  const screenshot = await runAgentBrowser(['screenshot']);
-  results.push({
-    operation: 'screenshot',
-    tool: 'agent-browser',
-    timeMs: screenshot.timeMs,
-    outputBytes: screenshot.output.length,
-    estimatedTokens: estimateTokens(screenshot.output),
-  });
-  console.log(`     ${screenshot.timeMs.toFixed(0)}ms, ${formatBytes(screenshot.output.length)}`);
-  
-  // Close
-  console.log('  ⏱️  Close...');
-  const close = await runAgentBrowser(['close']);
-  results.push({
-    operation: 'close',
-    tool: 'agent-browser',
-    timeMs: close.timeMs,
-    outputBytes: close.output.length,
-    estimatedTokens: estimateTokens(close.output),
-  });
-  console.log(`     ${close.timeMs.toFixed(0)}ms`);
-}
-
-// ============================================================================
-// Playwright-MCP Benchmarks
-// ============================================================================
-
-class MCPClient {
-  private proc: ChildProcess;
-  private rl: readline.Interface;
-  private responseBuffer: Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }> = new Map();
-  private requestId = 0;
-  private ready = false;
-
-  constructor() {
-    this.proc = spawn('node', ['./opensrc/repos/github.com/microsoft/playwright-mcp/cli.js', '--headless'], {
-      cwd: process.cwd(),
-      stdio: ['pipe', 'pipe', 'pipe'],
-    });
-
-    this.rl = readline.createInterface({ input: this.proc.stdout! });
-    
-    this.rl.on('line', (line) => {
-      try {
-        const msg = JSON.parse(line);
-        if (msg.id !== undefined && this.responseBuffer.has(msg.id)) {
-          const handler = this.responseBuffer.get(msg.id)!;
-          this.responseBuffer.delete(msg.id);
-          handler.resolve(msg);
-        }
-      } catch {
-        // Non-JSON output, ignore
-      }
-    });
-
-    this.proc.stderr?.on('data', (data) => {
-      // Debug output, ignore in benchmarks
-    });
-  }
-
-  async initialize(): Promise<{ timeMs: number; output: string }> {
-    const start = performance.now();
-    
-    // Send initialize request
-    const initResult = await this.sendRequest('initialize', {
-      protocolVersion: '2024-11-05',
-      capabilities: {},
-      clientInfo: { name: 'benchmark', version: '1.0.0' },
-    });
-    
-    // Send initialized notification
-    this.sendNotification('notifications/initialized', {});
-    
-    const timeMs = performance.now() - start;
-    const output = JSON.stringify(initResult);
-    
-    this.ready = true;
-    return { timeMs, output };
-  }
-
-  async callTool(name: string, args: Record<string, unknown>): Promise<{ timeMs: number; output: string }> {
-    const start = performance.now();
-    const result = await this.sendRequest('tools/call', { name, arguments: args });
-    const timeMs = performance.now() - start;
-    const output = JSON.stringify(result);
-    return { timeMs, output };
-  }
-
-  private sendRequest(method: string, params: Record<string, unknown>): Promise<any> {
-    const id = ++this.requestId;
-    const request = { jsonrpc: '2.0', id, method, params };
-    
-    return new Promise((resolve, reject) => {
-      this.responseBuffer.set(id, { resolve, reject });
-      this.proc.stdin!.write(JSON.stringify(request) + '\n');
-      
-      // Timeout after 30s
-      setTimeout(() => {
-        if (this.responseBuffer.has(id)) {
-          this.responseBuffer.delete(id);
-          reject(new Error(`Request timeout: ${method}`));
-        }
-      }, 30000);
-    });
-  }
-
-  private sendNotification(method: string, params: Record<string, unknown>): void {
-    const notification = { jsonrpc: '2.0', method, params };
-    this.proc.stdin!.write(JSON.stringify(notification) + '\n');
-  }
-
-  async close(): Promise<void> {
-    this.proc.kill();
-    this.rl.close();
-  }
-}
-
-async function benchmarkPlaywrightMCP(): Promise<void> {
-  console.log('\n📦 Benchmarking playwright-mcp...\n');
-  
-  let client: MCPClient | null = null;
-  
-  try {
-    // Cold start (includes server startup + initialization)
-    console.log('  ⏱️  Cold start (initialize + navigate)...');
-    const coldStartBegin = performance.now();
-    
-    client = new MCPClient();
-    const init = await client.initialize();
-    
-    // Navigate
-    const nav = await client.callTool('browser_navigate', { url: TEST_URL });
-    
-    const coldStartTime = performance.now() - coldStartBegin;
-    const coldStartOutput = init.output + nav.output;
-    
-    results.push({
-      operation: 'cold_start_navigate',
-      tool: 'playwright-mcp',
-      timeMs: coldStartTime,
-      outputBytes: coldStartOutput.length,
-      estimatedTokens: estimateTokens(coldStartOutput),
-    });
-    console.log(`     ${coldStartTime.toFixed(0)}ms, ${formatBytes(coldStartOutput.length)}`);
-    
-    // Warm operations
-    for (let i = 0; i < ITERATIONS; i++) {
-      // Navigate (warm)
-      console.log(`  ⏱️  Navigate (warm, iter ${i + 1})...`);
-      const navWarm = await client.callTool('browser_navigate', { url: TEST_URL });
-      results.push({
-        operation: 'navigate_warm',
-        tool: 'playwright-mcp',
-        timeMs: navWarm.timeMs,
-        outputBytes: navWarm.output.length,
-        estimatedTokens: estimateTokens(navWarm.output),
-      });
-      console.log(`     ${navWarm.timeMs.toFixed(0)}ms, ${formatBytes(navWarm.output.length)}`);
-      
-      // Snapshot
-      console.log(`  ⏱️  Snapshot (iter ${i + 1})...`);
-      const snapshot = await client.callTool('browser_snapshot', {});
-      results.push({
-        operation: 'snapshot',
-        tool: 'playwright-mcp',
-        timeMs: snapshot.timeMs,
-        outputBytes: snapshot.output.length,
-        estimatedTokens: estimateTokens(snapshot.output),
-      });
-      console.log(`     ${snapshot.timeMs.toFixed(0)}ms, ${formatBytes(snapshot.output.length)}`);
-      
-      // Note: playwright-mcp doesn't have separate get_title/get_url tools
-      // Title and URL are included in snapshot, so we'll skip those
-      
-      // Click
-      console.log(`  ⏱️  Click link (iter ${i + 1})...`);
-      // playwright-mcp uses ref from snapshot - we'll use a generic approach
-      const click = await client.callTool('browser_click', { 
-        element: 'More information link',
-        ref: 'a'  // This might not work exactly the same way
-      });
-      results.push({
-        operation: 'click',
-        tool: 'playwright-mcp',
-        timeMs: click.timeMs,
-        outputBytes: click.output.length,
-        estimatedTokens: estimateTokens(click.output),
-      });
-      console.log(`     ${click.timeMs.toFixed(0)}ms, ${formatBytes(click.output.length)}`);
-      
-      // Navigate back
-      await client.callTool('browser_navigate', { url: TEST_URL });
-    }
-    
-    // Screenshot
-    console.log('  ⏱️  Screenshot...');
-    const screenshot = await client.callTool('browser_take_screenshot', {});
-    results.push({
-      operation: 'screenshot',
-      tool: 'playwright-mcp',
-      timeMs: screenshot.timeMs,
-      outputBytes: screenshot.output.length,
-      estimatedTokens: estimateTokens(screenshot.output),
-    });
-    console.log(`     ${screenshot.timeMs.toFixed(0)}ms, ${formatBytes(screenshot.output.length)}`);
-    
-    // Close
-    console.log('  ⏱️  Close...');
-    const closeStart = performance.now();
-    const closeResult = await client.callTool('browser_close', {});
-    results.push({
-      operation: 'close',
-      tool: 'playwright-mcp',
-      timeMs: closeResult.timeMs,
-      outputBytes: closeResult.output.length,
-      estimatedTokens: estimateTokens(closeResult.output),
-    });
-    console.log(`     ${closeResult.timeMs.toFixed(0)}ms`);
-    
-  } catch (error) {
-    console.error('  ❌ playwright-mcp benchmark failed:', error);
-  } finally {
-    if (client) {
-      await client.close();
-    }
-  }
-}
-
-// ============================================================================
-// Results Summary
-// ============================================================================
-
-function printResults(): void {
-  console.log('\n' + '='.repeat(80));
-  console.log('📊 BENCHMARK RESULTS');
-  console.log('='.repeat(80));
-  
-  // Group by operation
-  const operations = [...new Set(results.map(r => r.operation))];
-  
-  console.log('\n📈 Speed Comparison (average across iterations):\n');
-  console.log('| Operation           | agent-browser | playwright-mcp | Difference |');
-  console.log('|---------------------|---------------|----------------|------------|');
-  
-  for (const op of operations) {
-    const agentResults = results.filter(r => r.operation === op && r.tool === 'agent-browser');
-    const mcpResults = results.filter(r => r.operation === op && r.tool === 'playwright-mcp');
-    
-    const agentAvg = agentResults.length > 0 
-      ? agentResults.reduce((sum, r) => sum + r.timeMs, 0) / agentResults.length 
-      : null;
-    const mcpAvg = mcpResults.length > 0 
-      ? mcpResults.reduce((sum, r) => sum + r.timeMs, 0) / mcpResults.length 
-      : null;
-    
-    const agentStr = agentAvg !== null ? `${agentAvg.toFixed(0)}ms`.padEnd(13) : 'N/A'.padEnd(13);
-    const mcpStr = mcpAvg !== null ? `${mcpAvg.toFixed(0)}ms`.padEnd(14) : 'N/A'.padEnd(14);
-    
-    let diff = '';
-    if (agentAvg !== null && mcpAvg !== null) {
-      const ratio = agentAvg / mcpAvg;
-      if (ratio < 1) {
-        diff = `${((1 - ratio) * 100).toFixed(0)}% faster`;
-      } else if (ratio > 1) {
-        diff = `${((ratio - 1) * 100).toFixed(0)}% slower`;
-      } else {
-        diff = 'same';
-      }
-    }
-    
-    console.log(`| ${op.padEnd(19)} | ${agentStr} | ${mcpStr} | ${diff.padEnd(10)} |`);
-  }
-  
-  console.log('\n📦 Context Usage (output size for AI consumption):\n');
-  console.log('| Operation           | agent-browser      | playwright-mcp     |');
-  console.log('|---------------------|--------------------|--------------------|');
-  
-  for (const op of operations) {
-    const agentResults = results.filter(r => r.operation === op && r.tool === 'agent-browser');
-    const mcpResults = results.filter(r => r.operation === op && r.tool === 'playwright-mcp');
-    
-    const agentAvgBytes = agentResults.length > 0 
-      ? agentResults.reduce((sum, r) => sum + r.outputBytes, 0) / agentResults.length 
-      : null;
-    const agentAvgTokens = agentResults.length > 0 
-      ? agentResults.reduce((sum, r) => sum + r.estimatedTokens, 0) / agentResults.length 
-      : null;
-    
-    const mcpAvgBytes = mcpResults.length > 0 
-      ? mcpResults.reduce((sum, r) => sum + r.outputBytes, 0) / mcpResults.length 
-      : null;
-    const mcpAvgTokens = mcpResults.length > 0 
-      ? mcpResults.reduce((sum, r) => sum + r.estimatedTokens, 0) / mcpResults.length 
-      : null;
-    
-    const agentStr = agentAvgBytes !== null 
-      ? `${formatBytes(agentAvgBytes)} (~${Math.round(agentAvgTokens!)} tok)`.padEnd(18)
-      : 'N/A'.padEnd(18);
-    const mcpStr = mcpAvgBytes !== null 
-      ? `${formatBytes(mcpAvgBytes)} (~${Math.round(mcpAvgTokens!)} tok)`.padEnd(18)
-      : 'N/A'.padEnd(18);
-    
-    console.log(`| ${op.padEnd(19)} | ${agentStr} | ${mcpStr} |`);
-  }
-  
-  // Total context usage
-  const agentTotal = results.filter(r => r.tool === 'agent-browser');
-  const mcpTotal = results.filter(r => r.tool === 'playwright-mcp');
-  
-  const agentTotalBytes = agentTotal.reduce((sum, r) => sum + r.outputBytes, 0);
-  const agentTotalTokens = agentTotal.reduce((sum, r) => sum + r.estimatedTokens, 0);
-  const mcpTotalBytes = mcpTotal.reduce((sum, r) => sum + r.outputBytes, 0);
-  const mcpTotalTokens = mcpTotal.reduce((sum, r) => sum + r.estimatedTokens, 0);
-  
-  console.log('\n📊 Total Context Usage (all operations combined):');
-  console.log(`   agent-browser:  ${formatBytes(agentTotalBytes)} (~${agentTotalTokens} tokens)`);
-  console.log(`   playwright-mcp: ${formatBytes(mcpTotalBytes)} (~${mcpTotalTokens} tokens)`);
-  
-  if (agentTotalBytes > 0 && mcpTotalBytes > 0) {
-    const ratio = agentTotalBytes / mcpTotalBytes;
-    if (ratio < 1) {
-      console.log(`   → agent-browser uses ${((1 - ratio) * 100).toFixed(0)}% less context`);
-    } else {
-      console.log(`   → playwright-mcp uses ${((1 - 1/ratio) * 100).toFixed(0)}% less context`);
-    }
-  }
-  
-  console.log('\n' + '='.repeat(80));
-}
-
-// ============================================================================
-// Main
-// ============================================================================
-
-async function main(): Promise<void> {
-  console.log('🚀 Browser Automation Benchmark');
-  console.log(`   Testing against: ${TEST_URL}`);
-  console.log(`   Iterations: ${ITERATIONS}`);
-  console.log('='.repeat(80));
-  
-  try {
-    // Build first
-    console.log('\n🔨 Building agent-browser...');
-    execSync('pnpm build', { cwd: process.cwd(), stdio: 'inherit' });
-    
-    await benchmarkAgentBrowser();
-    await benchmarkPlaywrightMCP();
-    
-    printResults();
-  } catch (error) {
-    console.error('\n❌ Benchmark failed:', error);
-    process.exit(1);
-  }
-}
-
-main();