agent-browser 0.1.3 → 0.2.0

package/benchmark/run.ts

@@ -0,0 +1,322 @@
+#!/usr/bin/env npx tsx
+/**
+ * Comprehensive Benchmark: agent-browser vs playwright-mcp
+ * 
+ * Tests realistic AI agent workflows on real websites.
+ */
+
+import { execSync, spawn } from 'child_process';
+import * as path from 'path';
+
+interface Result {
+  tool: string;
+  workflow: string;
+  operation: string;
+  timeMs: number;
+  outputBytes: number;
+}
+
+const results: Result[] = [];
+
+function formatTime(ms: number): string {
+  if (ms < 1000) return `${ms.toFixed(0)}ms`;
+  return `${(ms / 1000).toFixed(2)}s`;
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`;
+  return `${(bytes / 1024).toFixed(1)}KB`;
+}
+
+// ============================================================================
+// Agent-Browser Runner
+// ============================================================================
+
+function runAB(args: string[], session: string = 'bench'): { ms: number; output: string } {
+  const start = performance.now();
+  let output = '';
+  try {
+    output = execSync(`./bin/agent-browser ${args.join(' ')}`, {
+      stdio: 'pipe',
+      timeout: 30000,
+      env: { ...process.env, AGENT_BROWSER_SESSION: session },
+    }).toString();
+  } catch (e: any) {
+    output = e.stdout?.toString() || e.message || '';
+  }
+  return { ms: performance.now() - start, output };
+}
+
+// ============================================================================
+// Playwright-MCP Runner
+// ============================================================================
+
+interface MCPClient {
+  call: (tool: string, args: Record<string, unknown>) => Promise<{ ms: number; output: string }>;
+  close: () => void;
+}
+
+async function createMCPClient(): Promise<MCPClient> {
+  const mcpPath = path.join(process.cwd(), 'opensrc/repos/github.com/microsoft/playwright-mcp/cli.js');
+  const proc = spawn('node', [mcpPath, '--headless'], { stdio: ['pipe', 'pipe', 'pipe'] });
+  
+  let buffer = '';
+  let requestId = 0;
+  const pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>();
+
+  proc.stdout!.on('data', (data: Buffer) => {
+    buffer += data.toString();
+    const lines = buffer.split('\n');
+    buffer = lines.pop() || '';
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+        const msg = JSON.parse(line);
+        if (msg.id !== undefined && pending.has(msg.id)) {
+          pending.get(msg.id)!.resolve(msg);
+          pending.delete(msg.id);
+        }
+      } catch {}
+    }
+  });
+
+  const send = (method: string, params: Record<string, unknown>): Promise<any> => {
+    const id = ++requestId;
+    return new Promise((resolve, reject) => {
+      pending.set(id, { resolve, reject });
+      proc.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
+      setTimeout(() => {
+        if (pending.has(id)) {
+          pending.delete(id);
+          reject(new Error('Timeout'));
+        }
+      }, 30000);
+    });
+  };
+
+  // Initialize
+  await send('initialize', {
+    protocolVersion: '2024-11-05',
+    capabilities: {},
+    clientInfo: { name: 'benchmark', version: '1.0.0' },
+  });
+  proc.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized', params: {} }) + '\n');
+
+  return {
+    call: async (tool: string, args: Record<string, unknown>) => {
+      const start = performance.now();
+      const result = await send('tools/call', { name: tool, arguments: args });
+      return { ms: performance.now() - start, output: JSON.stringify(result) };
+    },
+    close: () => proc.kill(),
+  };
+}
+
+// ============================================================================
+// Workflows
+// ============================================================================
+
+interface Workflow {
+  name: string;
+  description: string;
+  steps: Array<{
+    name: string;
+    ab: string[];
+    mcp: { tool: string; args: Record<string, unknown> };
+  }>;
+}
+
+const workflows: Workflow[] = [
+  {
+    name: 'Wikipedia Research',
+    description: 'Navigate Wikipedia, read content, follow links',
+    steps: [
+      { name: 'Navigate', ab: ['open', 'https://en.wikipedia.org/wiki/Artificial_intelligence'], mcp: { tool: 'browser_navigate', args: { url: 'https://en.wikipedia.org/wiki/Artificial_intelligence' } } },
+      { name: 'Snapshot', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Get title', ab: ['get', 'title'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Snapshot 2', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Click link', ab: ['click', 'a[href="/wiki/Machine_learning"]'], mcp: { tool: 'browser_click', args: { element: 'Machine learning', ref: 'internal link' } } },
+      { name: 'Snapshot 3', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+    ],
+  },
+  {
+    name: 'GitHub Browse',
+    description: 'Browse a GitHub repository',
+    steps: [
+      { name: 'Navigate', ab: ['open', 'https://github.com/anthropics/anthropic-cookbook'], mcp: { tool: 'browser_navigate', args: { url: 'https://github.com/anthropics/anthropic-cookbook' } } },
+      { name: 'Snapshot', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Get URL', ab: ['get', 'url'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Eval (stars)', ab: ['eval', 'document.querySelector("#repo-stars-counter-star")?.textContent'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Snapshot 2', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+    ],
+  },
+  {
+    name: 'Hacker News',
+    description: 'Browse Hacker News front page',
+    steps: [
+      { name: 'Navigate', ab: ['open', 'https://news.ycombinator.com'], mcp: { tool: 'browser_navigate', args: { url: 'https://news.ycombinator.com' } } },
+      { name: 'Snapshot', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Eval (count)', ab: ['eval', 'document.querySelectorAll(".athing").length'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Snapshot 2', ab: ['snapshot', '-i'], mcp: { tool: 'browser_snapshot', args: {} } },
+      { name: 'Get title', ab: ['get', 'title'], mcp: { tool: 'browser_snapshot', args: {} } },
+    ],
+  },
+];
+
+// ============================================================================
+// Run Benchmarks
+// ============================================================================
+
+async function runAgentBrowser(workflow: Workflow, session: string): Promise<void> {
+  // Cleanup
+  try { runAB(['close'], session); } catch {}
+  await sleep(100);
+
+  for (const step of workflow.steps) {
+    const r = runAB(step.ab, session);
+    results.push({
+      tool: 'agent-browser',
+      workflow: workflow.name,
+      operation: step.name,
+      timeMs: r.ms,
+      outputBytes: r.output.length,
+    });
+  }
+  
+  try { runAB(['close'], session); } catch {}
+}
+
+async function runPlaywrightMCP(workflow: Workflow): Promise<void> {
+  let client: MCPClient | null = null;
+  try {
+    client = await createMCPClient();
+    
+    for (const step of workflow.steps) {
+      const r = await client.call(step.mcp.tool, step.mcp.args);
+      results.push({
+        tool: 'playwright-mcp',
+        workflow: workflow.name,
+        operation: step.name,
+        timeMs: r.ms,
+        outputBytes: r.output.length,
+      });
+    }
+    
+    await client.call('browser_close', {});
+  } catch (e) {
+    console.log(`  ⚠️  MCP error: ${e}`);
+  } finally {
+    client?.close();
+  }
+}
+
+// ============================================================================
+// Reporting
+// ============================================================================
+
+function printResults(): void {
+  console.log('\n' + '═'.repeat(80));
+  console.log('📊 DETAILED RESULTS');
+  console.log('═'.repeat(80));
+
+  for (const workflow of workflows) {
+    console.log(`\n📋 ${workflow.name}`);
+    console.log('─'.repeat(70));
+    console.log('│ Operation          │ agent-browser │ playwright-mcp │ Diff      │');
+    console.log('├────────────────────┼───────────────┼────────────────┼───────────┤');
+
+    let abTotal = 0, mcpTotal = 0;
+
+    for (const step of workflow.steps) {
+      const ab = results.find(r => r.tool === 'agent-browser' && r.workflow === workflow.name && r.operation === step.name);
+      const mcp = results.find(r => r.tool === 'playwright-mcp' && r.workflow === workflow.name && r.operation === step.name);
+
+      const abTime = ab?.timeMs || 0;
+      const mcpTime = mcp?.timeMs || 0;
+      abTotal += abTime;
+      mcpTotal += mcpTime;
+
+      const diff = mcpTime - abTime;
+      const diffStr = diff > 0 ? `+${formatTime(diff)}` : formatTime(diff);
+
+      console.log(`│ ${step.name.padEnd(18)} │ ${formatTime(abTime).padEnd(13)} │ ${formatTime(mcpTime).padEnd(14)} │ ${diffStr.padEnd(9)} │`);
+    }
+
+    console.log('├────────────────────┼───────────────┼────────────────┼───────────┤');
+    const totalDiff = mcpTotal - abTotal;
+    const totalDiffStr = totalDiff > 0 ? `+${formatTime(totalDiff)}` : formatTime(totalDiff);
+    console.log(`│ ${'TOTAL'.padEnd(18)} │ ${formatTime(abTotal).padEnd(13)} │ ${formatTime(mcpTotal).padEnd(14)} │ ${totalDiffStr.padEnd(9)} │`);
+    console.log('└────────────────────┴───────────────┴────────────────┴───────────┘');
+  }
+
+  // Summary
+  const abTotalAll = results.filter(r => r.tool === 'agent-browser').reduce((s, r) => s + r.timeMs, 0);
+  const mcpTotalAll = results.filter(r => r.tool === 'playwright-mcp').reduce((s, r) => s + r.timeMs, 0);
+  const abOps = results.filter(r => r.tool === 'agent-browser').length;
+  const mcpOps = results.filter(r => r.tool === 'playwright-mcp').length;
+
+  console.log('\n' + '═'.repeat(80));
+  console.log('📈 SUMMARY');
+  console.log('═'.repeat(80));
+  console.log(`\n  Workflows tested: ${workflows.length}`);
+  console.log(`  Total operations: ${abOps} (agent-browser), ${mcpOps} (playwright-mcp)`);
+  console.log(`\n  agent-browser total:  ${formatTime(abTotalAll)} (${(abTotalAll / abOps).toFixed(0)}ms avg/op)`);
+  console.log(`  playwright-mcp total: ${formatTime(mcpTotalAll)} (${(mcpTotalAll / mcpOps).toFixed(0)}ms avg/op)`);
+  
+  if (abTotalAll < mcpTotalAll) {
+    console.log(`\n  ✅ agent-browser is ${((mcpTotalAll - abTotalAll) / 1000).toFixed(2)}s faster overall`);
+  } else {
+    console.log(`\n  ⏱️  playwright-mcp is ${((abTotalAll - mcpTotalAll) / 1000).toFixed(2)}s faster overall`);
+  }
+
+  // Context usage
+  const abBytes = results.filter(r => r.tool === 'agent-browser').reduce((s, r) => s + r.outputBytes, 0);
+  const mcpBytes = results.filter(r => r.tool === 'playwright-mcp').reduce((s, r) => s + r.outputBytes, 0);
+  
+  console.log(`\n  Context usage:`);
+  console.log(`    agent-browser:  ${formatBytes(abBytes)} (~${Math.ceil(abBytes / 4)} tokens)`);
+  console.log(`    playwright-mcp: ${formatBytes(mcpBytes)} (~${Math.ceil(mcpBytes / 4)} tokens)`);
+
+  console.log('\n' + '═'.repeat(80));
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+async function main(): Promise<void> {
+  console.log('═'.repeat(80));
+  console.log('🚀 COMPREHENSIVE BENCHMARK: agent-browser vs playwright-mcp');
+  console.log('═'.repeat(80));
+  console.log('\nWorkflows:');
+  for (const w of workflows) {
+    console.log(`  • ${w.name}: ${w.description} (${w.steps.length} steps)`);
+  }
+
+  console.log('\n🔨 Building...');
+  execSync('pnpm build', { cwd: process.cwd(), stdio: 'inherit' });
+
+  for (const workflow of workflows) {
+    console.log(`\n${'─'.repeat(80)}`);
+    console.log(`📋 Running: ${workflow.name}`);
+    console.log('─'.repeat(80));
+
+    console.log('\n  agent-browser:');
+    await runAgentBrowser(workflow, `ab-${workflow.name.toLowerCase().replace(/\s+/g, '-')}`);
+    const abTime = results.filter(r => r.tool === 'agent-browser' && r.workflow === workflow.name).reduce((s, r) => s + r.timeMs, 0);
+    console.log(`    ✓ Completed in ${formatTime(abTime)}`);
+
+    console.log('\n  playwright-mcp:');
+    await runPlaywrightMCP(workflow);
+    const mcpTime = results.filter(r => r.tool === 'playwright-mcp' && r.workflow === workflow.name).reduce((s, r) => s + r.timeMs, 0);
+    console.log(`    ✓ Completed in ${formatTime(mcpTime)}`);
+  }
+
+  printResults();
+}
+
+main().catch(console.error);

package/dist/actions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EAoGT,MAAM,YAAY,CAAC;AAQpB;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiPjG"}
+{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EAoGT,MAAM,YAAY,CAAC;AASpB;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiPjG"}

package/dist/actions.js

@@ -260,8 +260,9 @@ async function handleNavigate(command, browser) {
     });
 }
 async function handleClick(command, browser) {
-    const page = browser.getPage();
-    await page.click(command.selector, {
+    // Support both refs (@e1) and regular selectors
+    const locator = browser.getLocator(command.selector);
+    await locator.click({
         button: command.button,
         clickCount: command.clickCount,
         delay: command.delay,
@@ -269,11 +270,11 @@ async function handleClick(command, browser) {
     return successResponse(command.id, { clicked: true });
 }
 async function handleType(command, browser) {
-    const page = browser.getPage();
+    const locator = browser.getLocator(command.selector);
     if (command.clear) {
-        await page.fill(command.selector, '');
+        await locator.fill('');
     }
-    await page.type(command.selector, command.text, {
+    await locator.pressSequentially(command.text, {
         delay: command.delay,
     });
     return successResponse(command.id, { typed: true });
@@ -311,11 +312,21 @@ async function handleScreenshot(command, browser) {
     }
 }
 async function handleSnapshot(command, browser) {
-    const page = browser.getPage();
-    // Use ariaSnapshot which returns a string representation of the accessibility tree
-    const snapshot = await page.locator(':root').ariaSnapshot();
+    // Use enhanced snapshot with refs and optional filtering
+    const { tree, refs } = await browser.getSnapshot({
+        interactive: command.interactive,
+        maxDepth: command.maxDepth,
+        compact: command.compact,
+        selector: command.selector,
+    });
+    // Simplify refs for output (just role and name)
+    const simpleRefs = {};
+    for (const [ref, data] of Object.entries(refs)) {
+        simpleRefs[ref] = { role: data.role, name: data.name };
+    }
     return successResponse(command.id, {
-        snapshot: snapshot ?? 'Empty page',
+        snapshot: tree || 'Empty page',
+        refs: Object.keys(simpleRefs).length > 0 ? simpleRefs : undefined,
     });
 }
 async function handleEvaluate(command, browser) {
@@ -378,14 +389,14 @@ async function handleScroll(command, browser) {
     return successResponse(command.id, { scrolled: true });
 }
 async function handleSelect(command, browser) {
-    const page = browser.getPage();
+    const locator = browser.getLocator(command.selector);
     const values = Array.isArray(command.values) ? command.values : [command.values];
-    await page.selectOption(command.selector, values);
+    await locator.selectOption(values);
     return successResponse(command.id, { selected: values });
 }
 async function handleHover(command, browser) {
-    const page = browser.getPage();
-    await page.hover(command.selector);
+    const locator = browser.getLocator(command.selector);
+    await locator.hover();
     return successResponse(command.id, { hovered: true });
 }
 async function handleContent(command, browser) {
@@ -432,34 +443,34 @@ async function handleWindowNew(command, browser) {
 }
 // New handlers for enhanced Playwright parity
 async function handleFill(command, browser) {
-    const frame = browser.getFrame();
-    await frame.fill(command.selector, command.value);
+    const locator = browser.getLocator(command.selector);
+    await locator.fill(command.value);
     return successResponse(command.id, { filled: true });
 }
 async function handleCheck(command, browser) {
-    const frame = browser.getFrame();
-    await frame.check(command.selector);
+    const locator = browser.getLocator(command.selector);
+    await locator.check();
     return successResponse(command.id, { checked: true });
 }
 async function handleUncheck(command, browser) {
-    const frame = browser.getFrame();
-    await frame.uncheck(command.selector);
+    const locator = browser.getLocator(command.selector);
+    await locator.uncheck();
     return successResponse(command.id, { unchecked: true });
 }
 async function handleUpload(command, browser) {
-    const frame = browser.getFrame();
+    const locator = browser.getLocator(command.selector);
     const files = Array.isArray(command.files) ? command.files : [command.files];
-    await frame.setInputFiles(command.selector, files);
+    await locator.setInputFiles(files);
     return successResponse(command.id, { uploaded: files });
 }
 async function handleDoubleClick(command, browser) {
-    const frame = browser.getFrame();
-    await frame.dblclick(command.selector);
+    const locator = browser.getLocator(command.selector);
+    await locator.dblclick();
     return successResponse(command.id, { clicked: true });
 }
 async function handleFocus(command, browser) {
-    const frame = browser.getFrame();
-    await frame.focus(command.selector);
+    const locator = browser.getLocator(command.selector);
+    await locator.focus();
     return successResponse(command.id, { focused: true });
 }
 async function handleDrag(command, browser) {
@@ -702,28 +713,28 @@ async function handleTitle(command, browser) {
     return successResponse(command.id, { title });
 }
 async function handleGetAttribute(command, browser) {
-    const page = browser.getPage();
-    const value = await page.getAttribute(command.selector, command.attribute);
+    const locator = browser.getLocator(command.selector);
+    const value = await locator.getAttribute(command.attribute);
     return successResponse(command.id, { attribute: command.attribute, value });
 }
 async function handleGetText(command, browser) {
-    const page = browser.getPage();
-    const text = await page.textContent(command.selector);
+    const locator = browser.getLocator(command.selector);
+    const text = await locator.textContent();
     return successResponse(command.id, { text });
 }
 async function handleIsVisible(command, browser) {
-    const page = browser.getPage();
-    const visible = await page.isVisible(command.selector);
+    const locator = browser.getLocator(command.selector);
+    const visible = await locator.isVisible();
     return successResponse(command.id, { visible });
 }
 async function handleIsEnabled(command, browser) {
-    const page = browser.getPage();
-    const enabled = await page.isEnabled(command.selector);
+    const locator = browser.getLocator(command.selector);
+    const enabled = await locator.isEnabled();
     return successResponse(command.id, { enabled });
 }
 async function handleIsChecked(command, browser) {
-    const page = browser.getPage();
-    const checked = await page.isChecked(command.selector);
+    const locator = browser.getLocator(command.selector);
+    const checked = await locator.isChecked();
     return successResponse(command.id, { checked });
 }
 async function handleCount(command, browser) {