agent-browser 0.1.3 → 0.3.0

package/benchmark/run.ts

@@ -0,0 +1,322 @@
+#!/usr/bin/env npx tsx
+/**
+ * Comprehensive Benchmark: agent-browser vs playwright-mcp
+ * 
+ * Tests realistic AI agent workflows on real websites.
+ */
+
+import { execSync, spawn } from 'child_process';
+import * as path from 'path';
+
+interface Result {
+  tool: string;
+  workflow: string;
+  operation: string;
+  timeMs: number;
+  outputBytes: number;
+}
+
+const results: Result[] = [];
+
+function formatTime(ms: number): string {
+  if (ms < 1000) return `${ms.toFixed(0)}ms`;
+  return `${(ms / 1000).toFixed(2)}s`;
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`;
+  return `${(bytes / 1024).toFixed(1)}KB`;
+}
+
+// ============================================================================
+// Agent-Browser Runner
+// ============================================================================
+
+function runAB(args: string[], session: string = 'bench'): { ms: number; output: string } {
+  const start = performance.now();
+  let output = '';
+  try {
+    output = execSync(`./bin/agent-browser ${args.join(' ')}`, {
+      stdio: 'pipe',
+      timeout: 30000,
+      env: { ...process.env, AGENT_BROWSER_SESSION: session },
+    }).toString();
+  } catch (e: any) {
+    output = e.stdout?.toString() || e.message || '';
+  }
+  return { ms: performance.now() - start, output };
+}
+
+// ============================================================================
+// Playwright-MCP Runner
+// ============================================================================
+
+interface MCPClient {
+  call: (tool: string, args: Record<string, unknown>) => Promise<{ ms: number; output: string }>;
+  close: () => void;
+}
+
+async function createMCPClient(): Promise<MCPClient> {
+  const mcpPath = path.join(process.cwd(), 'opensrc/repos/github.com/microsoft/playwright-mcp/cli.js');
+  const proc = spawn('node', [mcpPath, '--headless'], { stdio: ['pipe', 'pipe', 'pipe'] });
+  
+  let buffer = '';
+  let requestId = 0;
+  const pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>();
+
+  proc.stdout!.on('data', (data: Buffer) => {
+    buffer += data.toString();
+    const lines = buffer.split('\n');
+    buffer = lines.pop() || '';
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+        const msg = JSON.parse(line);
+        if (msg.id !== undefined && pending.has(msg.id)) {
+          pending.get(msg.id)!.resolve(msg);
+          pending.delete(msg.id);
+        }
+      } catch {}
+    }
+  });
+
+  const send = (method: string, params: Record<string, unknown>): Promise<any> => {
+    const id = ++requestId;
+    return new Promise((resolve, reject) => {
+      pending.set(id, { resolve, reject });
+      proc.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
+      setTimeout(() => {
+        if (pending.has(id)) {
+          pending.delete(id);
+          reject(new Error('Timeout'));
+        }
+      }, 30000);
+    });
+  };
+
+  // Initialize
+  await send('initialize', {
+    protocolVersion: '2024-11-05',
+    capabilities: {},
+    clientInfo: { name: 'benchmark', version: '1.0.0' },
+  });
+  proc.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized', params: {} }) + '\n');
+
+  return {
+    call: async (tool: string, args: Record<string, unknown>) => {
+      const start = performance.now();
+      const result = await send('tools/call', { name: tool, arguments: args });
+      return { ms: performance.now() - start, output: JSON.stringify(result) };
+    },
+    close: () => proc.kill(),
+  };
+}
+
+// ============================================================================
+// Workflows
+// ============================================================================
+
+interface Workflow {
+  name: string;
+  description: string;
+  steps: Array<{
+    name: string;
+    ab: string[];
+    mcp: { tool: string; args: Record<string, unknown> };
+  }>;
+}
+
+const workflows: Workflow[] = [
+  {
+    name: 'Wikipedia Research',
+    description: 'Navigate Wikipedia, read content, follow links',
+    steps: [
+      { name: 'Navigate', ab: ['open', 'https://en.wikipedia.org/wiki/Artificial_intelligence'], mcp: { tool: 'browser_navigate', args: { url: 'https://en.wikipedia.org/wiki/Artificial_intelligence' } } },
+      { name: 'Snapshot', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Get title', ab: ['get', 'title'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Snapshot 2', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Click link', ab: ['click', 'a[href="/wiki/Machine_learning"]'], mcp: { tool: 'browser_click', args: { element: 'Machine learning', ref: 'internal link' } } },
+      { name: 'Snapshot 3', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+    ],
+  },
+  {
+    name: 'GitHub Browse',
+    description: 'Browse a GitHub repository',
+    steps: [
+      { name: 'Navigate', ab: ['open', 'https://github.com/anthropics/anthropic-cookbook'], mcp: { tool: 'browser_navigate', args: { url: 'https://github.com/anthropics/anthropic-cookbook' } } },
+      { name: 'Snapshot', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Get URL', ab: ['get', 'url'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Eval (stars)', ab: ['eval', 'document.querySelector("#repo-stars-counter-star")?.textContent'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Snapshot 2', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+    ],
+  },
+  {
+    name: 'Hacker News',
+    description: 'Browse Hacker News front page',
+    steps: [
+      { name: 'Navigate', ab: ['open', 'https://news.ycombinator.com'], mcp: { tool: 'browser_navigate', args: { url: 'https://news.ycombinator.com' } } },
+      { name: 'Snapshot', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Eval (count)', ab: ['eval', 'document.querySelectorAll(".athing").length'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Snapshot 2', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Get title', ab: ['get', 'title'], mcp: { tool: 'browser_snapshot', args: {} } },
+    ],
+  },
+];
+
+// ============================================================================
+// Run Benchmarks
+// ============================================================================
+
+async function runAgentBrowser(workflow: Workflow, session: string): Promise<void> {
+  // Cleanup
+  try { runAB(['close'], session); } catch {}
+  await sleep(100);
+
+  for (const step of workflow.steps) {
+    const r = runAB(step.ab, session);
+    results.push({
+      tool: 'agent-browser',
+      workflow: workflow.name,
+      operation: step.name,
+      timeMs: r.ms,
+      outputBytes: r.output.length,
+    });
+  }
+  
+  try { runAB(['close'], session); } catch {}
+}
+
+async function runPlaywrightMCP(workflow: Workflow): Promise<void> {
+  let client: MCPClient | null = null;
+  try {
+    client = await createMCPClient();
+    
+    for (const step of workflow.steps) {
+      const r = await client.call(step.mcp.tool, step.mcp.args);
+      results.push({
+        tool: 'playwright-mcp',
+        workflow: workflow.name,
+        operation: step.name,
+        timeMs: r.ms,
+        outputBytes: r.output.length,
+      });
+    }
+    
+    await client.call('browser_close', {});
+  } catch (e) {
+    console.log(`  ⚠️  MCP error: ${e}`);
+  } finally {
+    client?.close();
+  }
+}
+
+// ============================================================================
+// Reporting
+// ============================================================================
+
+function printResults(): void {
+  console.log('\n' + '═'.repeat(80));
+  console.log('📊 DETAILED RESULTS');
+  console.log('═'.repeat(80));
+
+  for (const workflow of workflows) {
+    console.log(`\n📋 ${workflow.name}`);
+    console.log('─'.repeat(70));
+    console.log('│ Operation          │ agent-browser │ playwright-mcp │ Diff      │');
+    console.log('├────────────────────┼───────────────┼────────────────┼───────────┤');
+
+    let abTotal = 0, mcpTotal = 0;
+
+    for (const step of workflow.steps) {
+      const ab = results.find(r => r.tool === 'agent-browser' && r.workflow === workflow.name && r.operation === step.name);
+      const mcp = results.find(r => r.tool === 'playwright-mcp' && r.workflow === workflow.name && r.operation === step.name);
+
+      const abTime = ab?.timeMs || 0;
+      const mcpTime = mcp?.timeMs || 0;
+      abTotal += abTime;
+      mcpTotal += mcpTime;
+
+      const diff = mcpTime - abTime;
+      const diffStr = diff > 0 ? `+${formatTime(diff)}` : formatTime(diff);
+
+      console.log(`│ ${step.name.padEnd(18)} │ ${formatTime(abTime).padEnd(13)} │ ${formatTime(mcpTime).padEnd(14)} │ ${diffStr.padEnd(9)} │`);
+    }
+
+    console.log('├────────────────────┼───────────────┼────────────────┼───────────┤');
+    const totalDiff = mcpTotal - abTotal;
+    const totalDiffStr = totalDiff > 0 ? `+${formatTime(totalDiff)}` : formatTime(totalDiff);
+    console.log(`│ ${'TOTAL'.padEnd(18)} │ ${formatTime(abTotal).padEnd(13)} │ ${formatTime(mcpTotal).padEnd(14)} │ ${totalDiffStr.padEnd(9)} │`);
+    console.log('└────────────────────┴───────────────┴────────────────┴───────────┘');
+  }
+
+  // Summary
+  const abTotalAll = results.filter(r => r.tool === 'agent-browser').reduce((s, r) => s + r.timeMs, 0);
+  const mcpTotalAll = results.filter(r => r.tool === 'playwright-mcp').reduce((s, r) => s + r.timeMs, 0);
+  const abOps = results.filter(r => r.tool === 'agent-browser').length;
+  const mcpOps = results.filter(r => r.tool === 'playwright-mcp').length;
+
+  console.log('\n' + '═'.repeat(80));
+  console.log('📈 SUMMARY');
+  console.log('═'.repeat(80));
+  console.log(`\n  Workflows tested: ${workflows.length}`);
+  console.log(`  Total operations: ${abOps} (agent-browser), ${mcpOps} (playwright-mcp)`);
+  console.log(`\n  agent-browser total:  ${formatTime(abTotalAll)} (${(abTotalAll / abOps).toFixed(0)}ms avg/op)`);
+  console.log(`  playwright-mcp total: ${formatTime(mcpTotalAll)} (${(mcpTotalAll / mcpOps).toFixed(0)}ms avg/op)`);
+  
+  if (abTotalAll < mcpTotalAll) {
+    console.log(`\n  ✅ agent-browser is ${((mcpTotalAll - abTotalAll) / 1000).toFixed(2)}s faster overall`);
+  } else {
+    console.log(`\n  ⏱️  playwright-mcp is ${((abTotalAll - mcpTotalAll) / 1000).toFixed(2)}s faster overall`);
+  }
+
+  // Context usage
+  const abBytes = results.filter(r => r.tool === 'agent-browser').reduce((s, r) => s + r.outputBytes, 0);
+  const mcpBytes = results.filter(r => r.tool === 'playwright-mcp').reduce((s, r) => s + r.outputBytes, 0);
+  
+  console.log(`\n  Context usage:`);
+  console.log(`    agent-browser:  ${formatBytes(abBytes)} (~${Math.ceil(abBytes / 4)} tokens)`);
+  console.log(`    playwright-mcp: ${formatBytes(mcpBytes)} (~${Math.ceil(mcpBytes / 4)} tokens)`);
+
+  console.log('\n' + '═'.repeat(80));
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+async function main(): Promise<void> {
+  console.log('═'.repeat(80));
+  console.log('🚀 COMPREHENSIVE BENCHMARK: agent-browser vs playwright-mcp');
+  console.log('═'.repeat(80));
+  console.log('\nWorkflows:');
+  for (const w of workflows) {
+    console.log(`  • ${w.name}: ${w.description} (${w.steps.length} steps)`);
+  }
+
+  console.log('\n🔨 Building...');
+  execSync('pnpm build', { cwd: process.cwd(), stdio: 'inherit' });
+
+  for (const workflow of workflows) {
+    console.log(`\n${'─'.repeat(80)}`);
+    console.log(`📋 Running: ${workflow.name}`);
+    console.log('─'.repeat(80));
+
+    console.log('\n  agent-browser:');
+    await runAgentBrowser(workflow, `ab-${workflow.name.toLowerCase().replace(/\s+/g, '-')}`);
+    const abTime = results.filter(r => r.tool === 'agent-browser' && r.workflow === workflow.name).reduce((s, r) => s + r.timeMs, 0);
+    console.log(`    ✓ Completed in ${formatTime(abTime)}`);
+
+    console.log('\n  playwright-mcp:');
+    await runPlaywrightMCP(workflow);
+    const mcpTime = results.filter(r => r.tool === 'playwright-mcp' && r.workflow === workflow.name).reduce((s, r) => s + r.timeMs, 0);
+    console.log(`    ✓ Completed in ${formatTime(mcpTime)}`);
+  }
+
+  printResults();
+}
+
+main().catch(console.error);

package/cli/Cargo.lock

@@ -0,0 +1,114 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "agent-browser"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+
+[[package]]
+name = "libc"
+version = "0.2.180"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
+
+[[package]]
+name = "memchr"
+version = "2.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.105"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.149"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
+
+[[package]]
+name = "zmij"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8"

package/cli/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "agent-browser"
+version = "0.1.0"
+edition = "2021"
+description = "Fast browser automation CLI for AI agents"
+license = "Apache-2.0"
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+libc = "0.2"
+
+[profile.release]
+opt-level = 3
+lto = true
+codegen-units = 1
+strip = true