agent-browser 0.4.4 → 0.6.0

package/README.md

@@ -18,6 +18,8 @@ git clone https://github.com/vercel-labs/agent-browser
 cd agent-browser
 pnpm install
 pnpm build
+pnpm build:native   # Requires Rust (https://rustup.rs)
+pnpm link --global  # Makes agent-browser available globally
 agent-browser install
 ```
 
@@ -72,10 +74,11 @@ agent-browser scroll <dir> [px]       # Scroll (up/down/left/right)
 agent-browser scrollintoview <sel>    # Scroll element into view (alias: scrollinto)
 agent-browser drag <src> <tgt>        # Drag and drop
 agent-browser upload <sel> <files>    # Upload files
-agent-browser screenshot [path]       # Take screenshot (--full for full page)
+agent-browser screenshot [path]       # Take screenshot (--full for full page, base64 png to stdout if no path)
 agent-browser pdf <path>              # Save as PDF
 agent-browser snapshot                # Accessibility tree with refs (best for AI)
 agent-browser eval <js>               # Run JavaScript
+agent-browser connect <port>          # Connect to browser via CDP
 agent-browser close                   # Close browser (aliases: quit, exit)
 ```
 
@@ -300,6 +303,7 @@ agent-browser snapshot -i -c -d 5         # Combine options
 | `--name, -n` | Locator name filter |
 | `--exact` | Exact text match |
 | `--headed` | Show browser window (not headless) |
+| `--cdp <port>` | Connect via Chrome DevTools Protocol |
 | `--debug` | Debug output |
 
 ## Selectors
@@ -457,6 +461,136 @@ export async function handler() {
 }
 ```
 
+## CDP Mode
+
+Connect to an existing browser via Chrome DevTools Protocol:
+
+```bash
+# Start Chrome with: google-chrome --remote-debugging-port=9222
+
+# Connect once, then run commands without --cdp
+agent-browser connect 9222
+agent-browser snapshot
+agent-browser tab
+agent-browser close
+
+# Or pass --cdp on each command
+agent-browser --cdp 9222 snapshot
+```
+
+This enables control of:
+- Electron apps
+- Chrome/Chromium instances with remote debugging
+- WebView2 applications
+- Any browser exposing a CDP endpoint
+
+## Streaming (Browser Preview)
+
+Stream the browser viewport via WebSocket for live preview or "pair browsing" where a human can watch and interact alongside an AI agent.
+
+### Enable Streaming
+
+Set the `AGENT_BROWSER_STREAM_PORT` environment variable:
+
+```bash
+AGENT_BROWSER_STREAM_PORT=9223 agent-browser open example.com
+```
+
+This starts a WebSocket server on the specified port that streams the browser viewport and accepts input events.
+
+### WebSocket Protocol
+
+Connect to `ws://localhost:9223` to receive frames and send input:
+
+**Receive frames:**
+```json
+{
+  "type": "frame",
+  "data": "<base64-encoded-jpeg>",
+  "metadata": {
+    "deviceWidth": 1280,
+    "deviceHeight": 720,
+    "pageScaleFactor": 1,
+    "offsetTop": 0,
+    "scrollOffsetX": 0,
+    "scrollOffsetY": 0
+  }
+}
+```
+
+**Send mouse events:**
+```json
+{
+  "type": "input_mouse",
+  "eventType": "mousePressed",
+  "x": 100,
+  "y": 200,
+  "button": "left",
+  "clickCount": 1
+}
+```
+
+**Send keyboard events:**
+```json
+{
+  "type": "input_keyboard",
+  "eventType": "keyDown",
+  "key": "Enter",
+  "code": "Enter"
+}
+```
+
+**Send touch events:**
+```json
+{
+  "type": "input_touch",
+  "eventType": "touchStart",
+  "touchPoints": [{ "x": 100, "y": 200 }]
+}
+```
+
+### Programmatic API
+
+For advanced use, control streaming directly via the protocol:
+
+```typescript
+import { BrowserManager } from 'agent-browser';
+
+const browser = new BrowserManager();
+await browser.launch({ headless: true });
+await browser.navigate('https://example.com');
+
+// Start screencast
+await browser.startScreencast((frame) => {
+  // frame.data is base64-encoded image
+  // frame.metadata contains viewport info
+  console.log('Frame received:', frame.metadata.deviceWidth, 'x', frame.metadata.deviceHeight);
+}, {
+  format: 'jpeg',
+  quality: 80,
+  maxWidth: 1280,
+  maxHeight: 720,
+});
+
+// Inject mouse events
+await browser.injectMouseEvent({
+  type: 'mousePressed',
+  x: 100,
+  y: 200,
+  button: 'left',
+});
+
+// Inject keyboard events
+await browser.injectKeyboardEvent({
+  type: 'keyDown',
+  key: 'Enter',
+  code: 'Enter',
+});
+
+// Stop when done
+await browser.stopScreencast();
+```
+
 ## Architecture
 
 agent-browser uses a client-daemon architecture:

package/dist/actions.d.ts

@@ -1,5 +1,15 @@
-import type { BrowserManager } from './browser.js';
+import type { BrowserManager, ScreencastFrame } from './browser.js';
 import type { Command, Response } from './types.js';
+/**
+ * Set the callback for screencast frames
+ * This is called by the daemon to set up frame streaming
+ */
+export declare function setScreencastFrameCallback(callback: ((frame: ScreencastFrame) => void) | null): void;
+/**
+ * Convert Playwright errors to AI-friendly messages
+ * @internal Exported for testing
+ */
+export declare function toAIFriendlyError(error: unknown, selector: string): Error;
 /**
  * Execute a command and return a response
  */

package/dist/actions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EAoGT,MAAM,YAAY,CAAC;AAkDpB;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiPjG"}
+{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpE,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EAqHT,MAAM,YAAY,CAAC;AAMpB;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC,GAAG,IAAI,GAClD,IAAI,CAEN;AAQD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,KAAK,CA6CzE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAmQjG"}

package/dist/actions.js

@@ -1,8 +1,18 @@
 import { successResponse, errorResponse } from './protocol.js';
+// Callback for screencast frames - will be set by the daemon when streaming is active
+let screencastFrameCallback = null;
+/**
+ * Set the callback for screencast frames
+ * This is called by the daemon to set up frame streaming
+ */
+export function setScreencastFrameCallback(callback) {
+    screencastFrameCallback = callback;
+}
 /**
  * Convert Playwright errors to AI-friendly messages
+ * @internal Exported for testing
  */
-function toAIFriendlyError(error, selector) {
+export function toAIFriendlyError(error, selector) {
     const message = error instanceof Error ? error.message : String(error);
     // Handle strict mode violation (multiple elements match)
     if (message.includes('strict mode violation')) {
@@ -12,17 +22,23 @@ function toAIFriendlyError(error, selector) {
         return new Error(`Selector "${selector}" matched ${count} elements. ` +
             `Run 'snapshot' to get updated refs, or use a more specific CSS selector.`);
     }
-    // Handle element not found
+    // Handle element not interactable (must be checked BEFORE timeout case)
+    // This includes cases where an overlay/modal blocks the element
+    if (message.includes('intercepts pointer events')) {
+        return new Error(`Element "${selector}" is blocked by another element (likely a modal or overlay). ` +
+            `Try dismissing any modals/cookie banners first.`);
+    }
+    // Handle element not visible
+    if (message.includes('not visible') && !message.includes('Timeout')) {
+        return new Error(`Element "${selector}" is not visible. ` +
+            `Try scrolling it into view or check if it's hidden.`);
+    }
+    // Handle element not found (timeout waiting for element)
     if (message.includes('waiting for') &&
         (message.includes('to be visible') || message.includes('Timeout'))) {
         return new Error(`Element "${selector}" not found or not visible. ` +
             `Run 'snapshot' to see current page elements.`);
     }
-    // Handle element not interactable
-    if (message.includes('intercepts pointer events') || message.includes('not visible')) {
-        return new Error(`Element "${selector}" is not interactable (may be hidden or covered). ` +
-            `Try scrolling it into view or check if a modal/overlay is blocking it.`);
-    }
     // Return original error for unknown cases
     return error instanceof Error ? error : new Error(message);
 }
@@ -154,6 +170,8 @@ export async function executeCommand(command, browser) {
                 return await handleCount(command, browser);
             case 'boundingbox':
                 return await handleBoundingBox(command, browser);
+            case 'styles':
+                return await handleStyles(command, browser);
             case 'video_start':
                 return await handleVideoStart(command, browser);
             case 'video_stop':
@@ -260,6 +278,22 @@ export async function executeCommand(command, browser) {
                 return await handleWaitForDownload(command, browser);
             case 'responsebody':
                 return await handleResponseBody(command, browser);
+            case 'screencast_start':
+                return await handleScreencastStart(command, browser);
+            case 'screencast_stop':
+                return await handleScreencastStop(command, browser);
+            case 'input_mouse':
+                return await handleInputMouse(command, browser);
+            case 'input_keyboard':
+                return await handleInputKeyboard(command, browser);
+            case 'input_touch':
+                return await handleInputTouch(command, browser);
+            case 'recording_start':
+                return await handleRecordingStart(command, browser);
+            case 'recording_stop':
+                return await handleRecordingStop(command, browser);
+            case 'recording_restart':
+                return await handleRecordingRestart(command, browser);
             default: {
                 // TypeScript narrows to never here, but we handle it for safety
                 const unknownCommand = command;
@@ -467,6 +501,11 @@ async function handleClose(command, browser) {
 }
 async function handleTabNew(command, browser) {
     const result = await browser.newTab();
+    // Navigate to URL if provided (same pattern as handleNavigate)
+    if (command.url) {
+        const page = browser.getPage();
+        await page.goto(command.url, { waitUntil: 'domcontentloaded' });
+    }
     return successResponse(command.id, result);
 }
 async function handleTabList(command, browser) {
@@ -477,7 +516,7 @@ async function handleTabList(command, browser) {
     });
 }
 async function handleTabSwitch(command, browser) {
-    const result = browser.switchTo(command.index);
+    const result = await browser.switchTo(command.index);
     const page = browser.getPage();
     return successResponse(command.id, {
         ...result,
@@ -836,6 +875,50 @@ async function handleBoundingBox(command, browser) {
     const box = await page.locator(command.selector).boundingBox();
     return successResponse(command.id, { box });
 }
+async function handleStyles(command, browser) {
+    const page = browser.getPage();
+    // Shared extraction logic as a string to be eval'd in browser context
+    const extractStylesScript = `(function(el) {
+    const s = getComputedStyle(el);
+    const r = el.getBoundingClientRect();
+    return {
+      tag: el.tagName.toLowerCase(),
+      text: el.innerText?.trim().slice(0, 80) || null,
+      box: {
+        x: Math.round(r.x),
+        y: Math.round(r.y),
+        width: Math.round(r.width),
+        height: Math.round(r.height),
+      },
+      styles: {
+        fontSize: s.fontSize,
+        fontWeight: s.fontWeight,
+        fontFamily: s.fontFamily.split(',')[0].trim().replace(/"/g, ''),
+        color: s.color,
+        backgroundColor: s.backgroundColor,
+        borderRadius: s.borderRadius,
+        border: s.border !== 'none' && s.borderWidth !== '0px' ? s.border : null,
+        boxShadow: s.boxShadow !== 'none' ? s.boxShadow : null,
+        padding: s.padding,
+      },
+    };
+  })`;
+    // Check if it's a ref - single element
+    if (browser.isRef(command.selector)) {
+        const locator = browser.getLocator(command.selector);
+        const element = (await locator.evaluate((el, script) => {
+            const fn = eval(script);
+            return fn(el);
+        }, extractStylesScript));
+        return successResponse(command.id, { elements: [element] });
+    }
+    // CSS selector - can match multiple elements
+    const elements = (await page.$$eval(command.selector, (els, script) => {
+        const fn = eval(script);
+        return els.map((el) => fn(el));
+    }, extractStylesScript));
+    return successResponse(command.id, { elements });
+}
 // Advanced handlers
 async function handleVideoStart(command, browser) {
     // Video recording requires context-level setup at launch
@@ -967,8 +1050,8 @@ async function handleInnerHtml(command, browser) {
     return successResponse(command.id, { html });
 }
 async function handleInputValue(command, browser) {
-    const page = browser.getPage();
-    const value = await page.locator(command.selector).inputValue();
+    const locator = browser.getLocator(command.selector);
+    const value = await locator.inputValue();
     return successResponse(command.id, { value });
 }
 async function handleSetValue(command, browser) {
@@ -1233,4 +1316,78 @@ async function handleResponseBody(command, browser) {
         body: parsed,
     });
 }
+// Screencast and input injection handlers
+async function handleScreencastStart(command, browser) {
+    if (!screencastFrameCallback) {
+        throw new Error('Screencast frame callback not set. Start the streaming server first.');
+    }
+    await browser.startScreencast(screencastFrameCallback, {
+        format: command.format,
+        quality: command.quality,
+        maxWidth: command.maxWidth,
+        maxHeight: command.maxHeight,
+        everyNthFrame: command.everyNthFrame,
+    });
+    return successResponse(command.id, {
+        started: true,
+        format: command.format ?? 'jpeg',
+        quality: command.quality ?? 80,
+    });
+}
+async function handleScreencastStop(command, browser) {
+    await browser.stopScreencast();
+    return successResponse(command.id, { stopped: true });
+}
+async function handleInputMouse(command, browser) {
+    await browser.injectMouseEvent({
+        type: command.type,
+        x: command.x,
+        y: command.y,
+        button: command.button,
+        clickCount: command.clickCount,
+        deltaX: command.deltaX,
+        deltaY: command.deltaY,
+        modifiers: command.modifiers,
+    });
+    return successResponse(command.id, { injected: true });
+}
+async function handleInputKeyboard(command, browser) {
+    await browser.injectKeyboardEvent({
+        type: command.type,
+        key: command.key,
+        code: command.code,
+        text: command.text,
+        modifiers: command.modifiers,
+    });
+    return successResponse(command.id, { injected: true });
+}
+async function handleInputTouch(command, browser) {
+    await browser.injectTouchEvent({
+        type: command.type,
+        touchPoints: command.touchPoints,
+        modifiers: command.modifiers,
+    });
+    return successResponse(command.id, { injected: true });
+}
+// Recording handlers (Playwright native video recording)
+async function handleRecordingStart(command, browser) {
+    await browser.startRecording(command.path, command.url);
+    return successResponse(command.id, {
+        started: true,
+        path: command.path,
+    });
+}
+async function handleRecordingStop(command, browser) {
+    const result = await browser.stopRecording();
+    return successResponse(command.id, result);
+}
+async function handleRecordingRestart(command, browser) {
+    const result = await browser.restartRecording(command.path, command.url);
+    return successResponse(command.id, {
+        started: true,
+        path: command.path,
+        previousPath: result.previousPath,
+        stopped: result.stopped,
+    });
+}
 //# sourceMappingURL=actions.js.map