agent-browser 0.11.1 → 0.13.0

package/README.md

@@ -112,6 +112,7 @@ agent-browser scrollintoview <sel>    # Scroll element into view (alias: scrolli
 agent-browser drag <src> <tgt>        # Drag and drop
 agent-browser upload <sel> <files>    # Upload files
 agent-browser screenshot [path]       # Take screenshot (--full for full page, saves to a temporary directory if no path)
+agent-browser screenshot --annotate   # Annotated screenshot with numbered element labels
 agent-browser pdf <path>              # Save as PDF
 agent-browser snapshot                # Accessibility tree with refs (best for AI)
 agent-browser eval <js>               # Run JavaScript (-b for base64, --stdin for piped input)
@@ -253,6 +254,21 @@ agent-browser dialog accept [text]    # Accept (with optional prompt text)
 agent-browser dialog dismiss          # Dismiss
 ```
 
+### Diff
+
+```bash
+agent-browser diff snapshot                              # Compare current vs last snapshot
+agent-browser diff snapshot --baseline before.txt        # Compare current vs saved snapshot file
+agent-browser diff snapshot --selector "#main" --compact # Scoped snapshot diff
+agent-browser diff screenshot --baseline before.png      # Visual pixel diff against baseline
+agent-browser diff screenshot --baseline b.png -o d.png  # Save diff image to custom path
+agent-browser diff screenshot --baseline b.png -t 0.2    # Adjust color threshold (0-1)
+agent-browser diff url https://v1.com https://v2.com     # Compare two URLs (snapshot diff)
+agent-browser diff url https://v1.com https://v2.com --screenshot  # Also visual diff
+agent-browser diff url https://v1.com https://v2.com --wait-until networkidle  # Custom wait strategy
+agent-browser diff url https://v1.com https://v2.com --selector "#main"  # Scope to element
+```
+
 ### Debug
 
 ```bash
@@ -401,6 +417,27 @@ agent-browser snapshot -i -c -d 5         # Combine options
 
 The `-C` flag is useful for modern web apps that use custom clickable elements (divs, spans) instead of standard buttons/links.
 
+## Annotated Screenshots
+
+The `--annotate` flag overlays numbered labels on interactive elements in the screenshot. Each label `[N]` corresponds to ref `@eN`, so the same refs work for both visual and text-based workflows.
+
+```bash
+agent-browser screenshot --annotate
+# -> Screenshot saved to /tmp/screenshot-2026-02-17T12-00-00-abc123.png
+#    [1] @e1 button "Submit"
+#    [2] @e2 link "Home"
+#    [3] @e3 textbox "Email"
+```
+
+After an annotated screenshot, refs are cached so you can immediately interact with elements:
+
+```bash
+agent-browser screenshot --annotate ./page.png
+agent-browser click @e2     # Click the "Home" link labeled [2]
+```
+
+This is useful for multimodal AI models that can reason about visual layout, unlabeled icon buttons, canvas elements, or visual state that the text accessibility tree cannot capture.
+
 ## Options
 
 | Option | Description |
@@ -422,6 +459,7 @@ The `-C` flag is useful for modern web apps that use custom clickable elements (
 | `--device <name>` | iOS device name, e.g. "iPhone 15 Pro" (or `AGENT_BROWSER_IOS_DEVICE` env) |
 | `--json` | JSON output (for agents) |
 | `--full, -f` | Full page screenshot |
+| `--annotate` | Annotated screenshot with numbered element labels (or `AGENT_BROWSER_ANNOTATE` env) |
 | `--headed` | Show browser window (not headless) |
 | `--cdp <port\|url>` | Connect via Chrome DevTools Protocol (port or WebSocket URL) |
 | `--auto-connect` | Auto-discover and connect to running Chrome (or `AGENT_BROWSER_AUTO_CONNECT` env) |

package/dist/actions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAUpE,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EA4HT,MAAM,YAAY,CAAC;AAMpB;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC,GAAG,IAAI,GAClD,IAAI,CAEN;AAQD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,KAAK,CAqDzE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiRjG"}
+{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAUpE,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EAmIT,MAAM,YAAY,CAAC;AAQpB;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC,GAAG,IAAI,GAClD,IAAI,CAEN;AAQD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,KAAK,CAqDzE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAuRjG"}

package/dist/actions.js

@@ -4,6 +4,8 @@ import { mkdirSync } from 'node:fs';
 import { getAppDir } from './daemon.js';
 import { getSessionsDir, readStateFile, isValidSessionName, isEncryptedPayload, listStateFiles, cleanupExpiredStates, } from './state-utils.js';
 import { successResponse, errorResponse } from './protocol.js';
+import { diffSnapshots, diffScreenshots } from './diff.js';
+import { getEnhancedSnapshot } from './snapshot.js';
 // Callback for screencast frames - will be set by the daemon when streaming is active
 let screencastFrameCallback = null;
 /**
@@ -318,6 +320,12 @@ export async function executeCommand(command, browser) {
                 return await handleRecordingStop(command, browser);
             case 'recording_restart':
                 return await handleRecordingRestart(command, browser);
+            case 'diff_snapshot':
+                return await handleDiffSnapshot(command, browser);
+            case 'diff_screenshot':
+                return await handleDiffScreenshot(command, browser);
+            case 'diff_url':
+                return await handleDiffUrl(command, browser);
             default: {
                 // TypeScript narrows to never here, but we handle it for safety
                 const unknownCommand = command;
@@ -409,6 +417,12 @@ async function handlePress(command, browser) {
     }
     return successResponse(command.id, { pressed: true });
 }
+const ANNOTATION_OVERLAY_ID = '__agent_browser_annotations__';
+async function removeAnnotationOverlay(page) {
+    await page
+        .evaluate(`(() => { const el = document.getElementById(${JSON.stringify(ANNOTATION_OVERLAY_ID)}); if (el) el.remove(); })()`)
+        .catch(() => { });
+}
 async function handleScreenshot(command, browser) {
     const page = browser.getPage();
     const options = {
@@ -422,6 +436,7 @@ async function handleScreenshot(command, browser) {
     if (command.selector) {
         target = browser.getLocator(command.selector);
     }
+    let overlayInjected = false;
     try {
         let savePath = command.path;
         if (!savePath) {
@@ -433,10 +448,150 @@ async function handleScreenshot(command, browser) {
             mkdirSync(screenshotDir, { recursive: true });
             savePath = path.join(screenshotDir, filename);
         }
+        let annotations;
+        if (command.annotate) {
+            const { refs } = await browser.getSnapshot({ interactive: true });
+            const entries = Object.entries(refs);
+            const results = await Promise.all(entries.map(async ([ref, data]) => {
+                try {
+                    const locator = browser.getLocatorFromRef(ref);
+                    if (!locator)
+                        return null;
+                    const box = await locator.boundingBox();
+                    if (!box || box.width === 0 || box.height === 0)
+                        return null;
+                    const num = parseInt(ref.replace('e', ''), 10);
+                    return {
+                        ref,
+                        number: num,
+                        role: data.role,
+                        name: data.name || undefined,
+                        box: {
+                            x: Math.round(box.x),
+                            y: Math.round(box.y),
+                            width: Math.round(box.width),
+                            height: Math.round(box.height),
+                        },
+                    };
+                }
+                catch {
+                    return null;
+                }
+            }));
+            // When a selector is provided the screenshot is cropped to that element,
+            // so filter to annotations that overlap the target and shift coordinates.
+            let targetBox = null;
+            if (command.selector) {
+                const raw = await browser.getLocator(command.selector).boundingBox();
+                if (raw) {
+                    targetBox = {
+                        x: Math.round(raw.x),
+                        y: Math.round(raw.y),
+                        width: Math.round(raw.width),
+                        height: Math.round(raw.height),
+                    };
+                }
+            }
+            const filtered = results.filter((a) => a !== null);
+            // Filter by selector overlap if needed, but keep viewport-relative coords
+            // for overlay positioning. Coordinate shifting happens later for metadata only.
+            let overlayItems;
+            if (targetBox) {
+                const tb = targetBox;
+                overlayItems = filtered
+                    .filter((a) => {
+                    const ax2 = a.box.x + a.box.width;
+                    const ay2 = a.box.y + a.box.height;
+                    const bx2 = tb.x + tb.width;
+                    const by2 = tb.y + tb.height;
+                    return a.box.x < bx2 && ax2 > tb.x && a.box.y < by2 && ay2 > tb.y;
+                })
+                    .sort((a, b) => a.number - b.number);
+            }
+            else {
+                overlayItems = filtered.sort((a, b) => a.number - b.number);
+            }
+            if (overlayItems.length > 0) {
+                const overlayData = overlayItems.map((a) => ({
+                    number: a.number,
+                    x: a.box.x,
+                    y: a.box.y,
+                    width: a.box.width,
+                    height: a.box.height,
+                }));
+                // Uses position:absolute with document-relative coords so labels render
+                // correctly for both viewport and fullPage screenshots, and when the
+                // screenshot is scoped to a selector element.
+                await page.evaluate(`(() => {
+          var items = ${JSON.stringify(overlayData)};
+          var id = ${JSON.stringify(ANNOTATION_OVERLAY_ID)};
+          var sx = window.scrollX || 0;
+          var sy = window.scrollY || 0;
+          var c = document.createElement('div');
+          c.id = id;
+          c.style.cssText = 'position:absolute;top:0;left:0;width:0;height:0;pointer-events:none;z-index:2147483647;';
+          for (var i = 0; i < items.length; i++) {
+            var it = items[i];
+            var dx = it.x + sx;
+            var dy = it.y + sy;
+            var b = document.createElement('div');
+            b.style.cssText = 'position:absolute;left:' + dx + 'px;top:' + dy + 'px;width:' + it.width + 'px;height:' + it.height + 'px;border:2px solid rgba(255,0,0,0.8);box-sizing:border-box;pointer-events:none;';
+            var l = document.createElement('div');
+            l.textContent = String(it.number);
+            var labelTop = dy < 14 ? '2px' : '-14px';
+            l.style.cssText = 'position:absolute;top:' + labelTop + ';left:-2px;background:rgba(255,0,0,0.9);color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;white-space:nowrap;';
+            b.appendChild(l);
+            c.appendChild(b);
+          }
+          document.documentElement.appendChild(c);
+        })()`);
+                overlayInjected = true;
+            }
+            // Build returned annotation metadata with image-relative coordinates.
+            // Selector: shift to target-element-relative.
+            // fullPage: convert to document-relative (matching fullPage image origin).
+            // Default: viewport-relative (unchanged).
+            if (targetBox) {
+                const tb = targetBox;
+                annotations = overlayItems.map((a) => ({
+                    ...a,
+                    box: {
+                        x: a.box.x - tb.x,
+                        y: a.box.y - tb.y,
+                        width: a.box.width,
+                        height: a.box.height,
+                    },
+                }));
+            }
+            else if (command.fullPage) {
+                const scroll = (await page.evaluate(`({x: window.scrollX || 0, y: window.scrollY || 0})`));
+                annotations = overlayItems.map((a) => ({
+                    ...a,
+                    box: {
+                        x: a.box.x + scroll.x,
+                        y: a.box.y + scroll.y,
+                        width: a.box.width,
+                        height: a.box.height,
+                    },
+                }));
+            }
+            else {
+                annotations = overlayItems;
+            }
+        }
         await target.screenshot({ ...options, path: savePath });
-        return successResponse(command.id, { path: savePath });
+        if (overlayInjected) {
+            await removeAnnotationOverlay(page);
+        }
+        return successResponse(command.id, {
+            path: savePath,
+            ...(annotations && annotations.length > 0 ? { annotations } : {}),
+        });
     }
     catch (error) {
+        if (overlayInjected) {
+            await removeAnnotationOverlay(page);
+        }
         if (command.selector) {
             throw toAIFriendlyError(error, command.selector);
         }
@@ -1610,4 +1765,83 @@ async function handleRecordingRestart(command, browser) {
         stopped: result.stopped,
     });
 }
+// Diff handlers
+async function handleDiffSnapshot(command, browser) {
+    let before;
+    if (command.baseline) {
+        try {
+            before = fs.readFileSync(command.baseline, 'utf-8');
+        }
+        catch {
+            return errorResponse(command.id, `Cannot read baseline file: ${command.baseline}`);
+        }
+    }
+    else {
+        before = browser.getLastSnapshot();
+        if (!before) {
+            return errorResponse(command.id, 'No previous snapshot in this session. Take a snapshot first, or use --baseline <file>.');
+        }
+    }
+    const page = browser.getPage();
+    const { tree } = await getEnhancedSnapshot(page, {
+        selector: command.selector,
+        compact: command.compact,
+        maxDepth: command.maxDepth,
+    });
+    const after = tree || 'Empty page';
+    const result = diffSnapshots(before, after);
+    browser.setLastSnapshot(after);
+    return successResponse(command.id, result);
+}
+async function handleDiffScreenshot(command, browser) {
+    if (!fs.existsSync(command.baseline)) {
+        return errorResponse(command.id, `Baseline file not found: ${command.baseline}`);
+    }
+    const page = browser.getPage();
+    let screenshotBuffer;
+    if (command.selector) {
+        const locator = browser.getLocatorFromRef(command.selector) || page.locator(command.selector);
+        screenshotBuffer = await locator.screenshot({ type: 'png' });
+    }
+    else {
+        screenshotBuffer = await page.screenshot({ fullPage: command.fullPage, type: 'png' });
+    }
+    const baselineBuffer = fs.readFileSync(command.baseline);
+    const ext = path.extname(command.baseline).toLowerCase();
+    const baselineMime = ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : 'image/png';
+    const result = await diffScreenshots(page.context(), baselineBuffer, screenshotBuffer, {
+        threshold: command.threshold,
+        outputPath: command.output,
+        baselineMime,
+    });
+    return successResponse(command.id, result);
+}
+async function handleDiffUrl(command, browser) {
+    const page = browser.getPage();
+    const waitUntil = command.waitUntil ?? 'load';
+    const snapshotOpts = {
+        selector: command.selector,
+        compact: command.compact,
+        maxDepth: command.maxDepth,
+    };
+    // Capture state of url1
+    await page.goto(command.url1, { waitUntil });
+    const { tree: tree1 } = await getEnhancedSnapshot(page, snapshotOpts);
+    const snapshot1 = tree1 || 'Empty page';
+    let screenshot1;
+    if (command.screenshot) {
+        screenshot1 = await page.screenshot({ fullPage: command.fullPage, type: 'png' });
+    }
+    // Capture state of url2
+    await page.goto(command.url2, { waitUntil });
+    const { tree: tree2 } = await getEnhancedSnapshot(page, snapshotOpts);
+    const snapshot2 = tree2 || 'Empty page';
+    const snapshotDiff = diffSnapshots(snapshot1, snapshot2);
+    const result = { snapshot: snapshotDiff };
+    if (command.screenshot && screenshot1) {
+        const screenshot2 = await page.screenshot({ fullPage: command.fullPage, type: 'png' });
+        result.screenshot = await diffScreenshots(page.context(), screenshot1, screenshot2, {});
+    }
+    return successResponse(command.id, result);
+}
 //# sourceMappingURL=actions.js.map