assistme 0.3.0 → 0.3.2

package/src/browser/controller.ts

@@ -0,0 +1,1386 @@
+import { WebSocket } from "ws";
+import { platform } from "node:os";
+import type {
+  CDPTab,
+  CDPResponse,
+  CDPEvalResult,
+  CDPScreenshotResult,
+  BoundingBox,
+  RefEntry,
+  SnapshotResult,
+  ActionSpec,
+  ActionResult,
+} from "./types.js";
+
+export class BrowserController {
+  private ws: WebSocket | null = null;
+  private debugPort: number;
+  private messageId = 0;
+  private callbacks = new Map<number, (response: CDPResponse) => void>();
+  private connected = false;
+  private currentTabId: string | null = null;
+  private refCache: Map<number, RefEntry> = new Map();
+
+  constructor(port = 9222) {
+    this.debugPort = port;
+  }
+
+  // ── Connection ──────────────────────────────────────────────────
+
+  async isAvailable(): Promise<boolean> {
+    try {
+      const res = await fetch(`http://127.0.0.1:${this.debugPort}/json/version`, {
+        signal: AbortSignal.timeout(2000),
+      });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  async connect(tabIndex?: number): Promise<string> {
+    // Reuse existing connection if still open and targeting the same tab
+    if (this.connected && this.ws?.readyState === WebSocket.OPEN) {
+      if (tabIndex === undefined) {
+        return "Already connected to browser.";
+      }
+      // If a specific tab is requested, check if we're already on it
+      const tabs = await this.getTabs();
+      const pageTabs = tabs.filter((t) => t.type === "page");
+      const targetTab = pageTabs[tabIndex];
+      if (targetTab && targetTab.id === this.currentTabId) {
+        return `Already connected to tab: "${targetTab.title}"`;
+      }
+      // Need to switch — disconnect first
+      await this.disconnect();
+    }
+
+    const available = await this.isAvailable();
+    if (!available) {
+      throw new Error(
+        `Cannot connect to browser on port ${this.debugPort}. ` +
+          "Chrome remote debugging is not reachable. " +
+          "Please ensure Chrome is running with remote debugging enabled."
+      );
+    }
+
+    const tabs = await this.getTabs();
+    const pageTabs = tabs.filter((t) => t.type === "page");
+
+    if (pageTabs.length === 0) {
+      throw new Error("No browser tabs found. Please open at least one tab.");
+    }
+
+    const targetTab = pageTabs[tabIndex ?? 0];
+    if (!targetTab.webSocketDebuggerUrl) {
+      throw new Error("Tab does not expose a WebSocket debugger URL.");
+    }
+
+    this.currentTabId = targetTab.id;
+
+    return new Promise((resolve, reject) => {
+      let settled = false;
+      this.ws = new WebSocket(targetTab.webSocketDebuggerUrl!);
+
+      const connectTimeout = setTimeout(() => {
+        if (!settled) {
+          settled = true;
+          this.ws?.close();
+          reject(new Error("Connection timeout (5s)"));
+        }
+      }, 5000);
+
+      this.ws.on("open", () => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(connectTimeout);
+        this.connected = true;
+        // Enable required domains
+        this.send("Page.enable").catch(() => {});
+        this.send("Runtime.enable").catch(() => {});
+        this.send("DOM.enable").catch(() => {});
+        resolve(`Connected to tab: "${targetTab.title}" (${targetTab.url})`);
+      });
+
+      this.ws.on("message", (data) => {
+        try {
+          const msg = JSON.parse(data.toString()) as CDPResponse;
+          if (msg.id !== undefined && this.callbacks.has(msg.id)) {
+            this.callbacks.get(msg.id)!(msg);
+            this.callbacks.delete(msg.id);
+          }
+        } catch {
+          // Ignore non-JSON messages (events)
+        }
+      });
+
+      this.ws.on("error", (err) => {
+        this.connected = false;
+        if (!settled) {
+          settled = true;
+          clearTimeout(connectTimeout);
+          reject(new Error(`WebSocket error: ${err.message}`));
+        }
+      });
+
+      this.ws.on("close", () => {
+        this.connected = false;
+        this.ws = null;
+        // Reject pending CDP commands so they don't hang forever
+        for (const [id, cb] of this.callbacks) {
+          cb({ id, error: { code: -1, message: "WebSocket closed" } });
+        }
+        this.callbacks.clear();
+      });
+    });
+  }
+
+  async disconnect(): Promise<string> {
+    if (this.ws) {
+      this.ws.close();
+      this.ws = null;
+      this.connected = false;
+    }
+    return "Disconnected from browser.";
+  }
+
+  // ── CDP Protocol ────────────────────────────────────────────────
+
+  private async getTabs(): Promise<CDPTab[]> {
+    const res = await fetch(`http://127.0.0.1:${this.debugPort}/json`, {
+      signal: AbortSignal.timeout(3000),
+    });
+    return (await res.json()) as CDPTab[];
+  }
+
+  private send(method: string, params?: Record<string, unknown>): Promise<Record<string, unknown>> {
+    return new Promise((resolve, reject) => {
+      if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+        reject(new Error("Not connected to browser. Call browser_connect first."));
+        return;
+      }
+
+      const id = ++this.messageId;
+      const timeout = setTimeout(() => {
+        this.callbacks.delete(id);
+        reject(new Error(`CDP command timed out: ${method}`));
+      }, 15000);
+
+      this.callbacks.set(id, (response) => {
+        clearTimeout(timeout);
+        if (response.error) {
+          reject(new Error(`CDP error: ${response.error.message}`));
+        } else {
+          resolve(response.result || {});
+        }
+      });
+
+      this.ws.send(JSON.stringify({ id, method, params }));
+    });
+  }
+
+  private ensureConnected() {
+    if (!this.connected || !this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("Not connected to browser. Use browser_connect tool first.");
+    }
+  }
+
+  // ── Navigation ──────────────────────────────────────────────────
+
+  async navigate(url: string): Promise<string> {
+    this.ensureConnected();
+    await this.send("Page.navigate", { url });
+    // Wait for load
+    await this.waitForLoad();
+    const info = await this.getPageInfo();
+    return `Navigated to: ${info.title}\nURL: ${info.url}`;
+  }
+
+  async goBack(): Promise<string> {
+    this.ensureConnected();
+    await this.send("Page.navigateToHistoryEntry", {
+      entryId: -1,
+    }).catch(() => {});
+    // Fallback: use JS
+    await this.evaluate("window.history.back()");
+    await this.waitForLoad();
+    const info = await this.getPageInfo();
+    return `Went back to: ${info.title}`;
+  }
+
+  async reload(): Promise<string> {
+    this.ensureConnected();
+    await this.send("Page.reload");
+    await this.waitForLoad();
+    return "Page reloaded.";
+  }
+
+  // ── Page Reading ────────────────────────────────────────────────
+
+  async readPage(): Promise<string> {
+    this.ensureConnected();
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          // Get page title and URL
+          let output = "Title: " + document.title + "\\n";
+          output += "URL: " + window.location.href + "\\n\\n";
+
+          // Get main text content, cleaned up
+          const body = document.body.cloneNode(true);
+          // Remove scripts, styles, navs that add noise
+          body.querySelectorAll('script, style, noscript, svg, iframe').forEach(el => el.remove());
+
+          const text = body.innerText
+            .split('\\n')
+            .map(line => line.trim())
+            .filter(line => line.length > 0)
+            .join('\\n');
+
+          output += text;
+          return output.slice(0, 30000);
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    return ((result as CDPEvalResult).result?.value as string) || "Could not read page content.";
+  }
+
+  async readElement(selector: string): Promise<string> {
+    this.ensureConnected();
+    const selectorJS = JSON.stringify(selector);
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          const el = document.querySelector(${selectorJS});
+          if (!el) return 'Element not found: ' + ${selectorJS};
+          return el.innerText || el.textContent || el.value || '(empty)';
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    return ((result as CDPEvalResult).result?.value as string) || "Element not found.";
+  }
+
+  async getPageInfo(): Promise<{ title: string; url: string }> {
+    const result = await this.send("Runtime.evaluate", {
+      expression: `JSON.stringify({ title: document.title, url: window.location.href })`,
+      returnByValue: true,
+    });
+    try {
+      return JSON.parse(((result as CDPEvalResult).result?.value as string) || "{}");
+    } catch {
+      return { title: "Unknown", url: "unknown" };
+    }
+  }
+
+  // ── Screenshots (for Claude vision) ─────────────────────────────
+
+  async screenshot(): Promise<string> {
+    this.ensureConnected();
+    const result = await this.send("Page.captureScreenshot", {
+      format: "png",
+      quality: 80,
+      captureBeyondViewport: false,
+    });
+    // Returns base64-encoded PNG
+    return (result as CDPScreenshotResult).data || "";
+  }
+
+  // ── Interactions ────────────────────────────────────────────────
+
+  async click(selector: string): Promise<string> {
+    this.ensureConnected();
+    const selectorJS = JSON.stringify(selector);
+
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          var sel = ${selectorJS};
+
+          // Support :contains('text') pseudo-selector (not native CSS)
+          var containsMatch = sel.match(/^(.+?)?:contains\\(['"](.+?)['"]\\)$/);
+          if (containsMatch) {
+            var baseTag = (containsMatch[1] || '*').toLowerCase();
+            var searchText = containsMatch[2];
+            var candidates = document.querySelectorAll(baseTag === '*' ? '*' : baseTag);
+            var found = null;
+            for (var i = 0; i < candidates.length; i++) {
+              var c = candidates[i];
+              // Prefer exact text match on direct text content (not children)
+              var directText = Array.from(c.childNodes)
+                .filter(function(n) { return n.nodeType === 3; })
+                .map(function(n) { return n.textContent.trim(); })
+                .join(' ');
+              if (directText === searchText || c.textContent.trim() === searchText) {
+                // Prefer the deepest (most specific) matching element
+                if (!found || found.contains(c)) found = c;
+              }
+            }
+            if (!found) return 'Element not found: ' + sel;
+            found.scrollIntoView({ block: 'center', behavior: 'instant' });
+            found.click();
+            return 'Clicked: ' + (found.tagName || '') + ' ' + (found.textContent || '').slice(0, 50).trim();
+          }
+
+          var el = document.querySelector(sel);
+          if (!el) return 'Element not found: ' + sel;
+
+          // Scroll into view
+          el.scrollIntoView({ block: 'center', behavior: 'instant' });
+
+          // Click
+          el.click();
+          return 'Clicked: ' + (el.tagName || '') + ' ' + (el.textContent || '').slice(0, 50).trim();
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    // Small delay for any resulting navigation/animation
+    await new Promise((r) => setTimeout(r, 500));
+    return ((result as CDPEvalResult).result?.value as string) || "Click executed.";
+  }
+
+  async typeText(selector: string, text: string): Promise<string> {
+    this.ensureConnected();
+    // Use JSON.stringify for safe string interpolation into JS — handles all
+    // special characters (quotes, backslashes, newlines, unicode) correctly.
+    const selectorJS = JSON.stringify(selector);
+    const textJS = JSON.stringify(text);
+
+    // First clear and set value via JS, dispatching all relevant events
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          const el = document.querySelector(${selectorJS});
+          if (!el) return 'Element not found: ' + ${selectorJS};
+
+          el.focus();
+
+          // Clear existing value
+          const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
+            window.HTMLInputElement.prototype, 'value'
+          )?.set || Object.getOwnPropertyDescriptor(
+            window.HTMLTextAreaElement.prototype, 'value'
+          )?.set;
+          if (nativeInputValueSetter) {
+            nativeInputValueSetter.call(el, ${textJS});
+          } else {
+            el.value = ${textJS};
+          }
+
+          // Dispatch events that frameworks (React, Angular, Material) listen to
+          el.dispatchEvent(new Event('input', { bubbles: true, cancelable: true }));
+          el.dispatchEvent(new Event('change', { bubbles: true, cancelable: true }));
+          el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${textJS} }));
+          return 'Typed into: ' + (el.tagName || '') + ' [' + (el.name || el.id || '') + ']';
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    return ((result as CDPEvalResult).result?.value as string) || "Text entered.";
+  }
+
+  async pressKey(key: string): Promise<string> {
+    this.ensureConnected();
+
+    // Map common key names to CDP key codes
+    const keyMap: Record<string, { keyCode: number; code: string }> = {
+      Enter: { keyCode: 13, code: "Enter" },
+      Tab: { keyCode: 9, code: "Tab" },
+      Escape: { keyCode: 27, code: "Escape" },
+      Backspace: { keyCode: 8, code: "Backspace" },
+      ArrowDown: { keyCode: 40, code: "ArrowDown" },
+      ArrowUp: { keyCode: 38, code: "ArrowUp" },
+    };
+
+    const mapped = keyMap[key];
+    if (mapped) {
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyDown",
+        key,
+        code: mapped.code,
+        windowsVirtualKeyCode: mapped.keyCode,
+        nativeVirtualKeyCode: mapped.keyCode,
+      });
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key,
+        code: mapped.code,
+        windowsVirtualKeyCode: mapped.keyCode,
+        nativeVirtualKeyCode: mapped.keyCode,
+      });
+    } else {
+      // Single character key
+      await this.send("Input.dispatchKeyEvent", {
+        type: "char",
+        text: key,
+      });
+    }
+
+    return `Pressed key: ${key}`;
+  }
+
+  async scrollDown(): Promise<string> {
+    this.ensureConnected();
+    await this.send("Runtime.evaluate", {
+      expression: "window.scrollBy(0, window.innerHeight * 0.8)",
+    });
+    await new Promise((r) => setTimeout(r, 300));
+    return "Scrolled down.";
+  }
+
+  async scrollUp(): Promise<string> {
+    this.ensureConnected();
+    await this.send("Runtime.evaluate", {
+      expression: "window.scrollBy(0, -window.innerHeight * 0.8)",
+    });
+    await new Promise((r) => setTimeout(r, 300));
+    return "Scrolled up.";
+  }
+
+  // ── Annotated Snapshot (ref system) ─────────────────────────────
+
+  /**
+   * Take a snapshot of all interactive elements on the page.
+   *
+   * Strategy (informed by research — arxiv:2511.19477):
+   *  - **Text ref table is ALWAYS returned** — compact, low-token, works for
+   *    all page complexities including dense layouts (date pickers, tables).
+   *  - **Annotated screenshot is OPTIONAL** (annotate parameter):
+   *    - true:  overlay ref badges on screenshot (best for simple pages with
+   *             few interactive elements — gives visual context)
+   *    - false: plain screenshot without overlays (default — avoids label
+   *             clutter on dense pages; model still sees the page visually)
+   *    - Research shows text-based grounding outperforms visual annotations
+   *      on complex pages, and the hybrid approach (a11y text primary +
+   *      selective vision) achieves ~85% vs ~50% for pure vision.
+   */
+  async snapshot(annotate = false): Promise<SnapshotResult> {
+    this.ensureConnected();
+
+    // Wait for page to be ready (auto-wait like Playwright)
+    await this.waitForLoad(5000);
+
+    // 1. Find all interactive elements, assign ref IDs, get bounding boxes
+    const findResult = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          // Clean up previous refs
+          document.querySelectorAll('[data-assistme-ref]').forEach(function(el) {
+            el.removeAttribute('data-assistme-ref');
+          });
+
+          var selectors = [
+            'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea',
+            '[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]',
+            '[role="combobox"]', '[role="listbox"]', '[role="menuitem"]', '[role="tab"]',
+            '[role="switch"]', '[role="slider"]', '[role="option"]', '[role="searchbox"]',
+            '[onclick]', '[tabindex]:not([tabindex="-1"])',
+            '[contenteditable="true"]'
+          ].join(', ');
+
+          // Collect elements from main document AND same-origin iframes
+          var all = Array.from(document.querySelectorAll(selectors));
+          try {
+            var iframes = document.querySelectorAll('iframe');
+            for (var fi = 0; fi < iframes.length; fi++) {
+              try {
+                var iframeDoc = iframes[fi].contentDocument;
+                if (iframeDoc) {
+                  var iframeRect = iframes[fi].getBoundingClientRect();
+                  var iframeEls = iframeDoc.querySelectorAll(selectors);
+                  for (var fe = 0; fe < iframeEls.length; fe++) {
+                    // Tag iframe elements with offset for coordinate correction
+                    iframeEls[fe].__iframeOffset = { x: iframeRect.x, y: iframeRect.y };
+                    all.push(iframeEls[fe]);
+                  }
+                }
+              } catch(e) { /* cross-origin iframe, skip */ }
+            }
+          } catch(e) { /* iframe enumeration failed, continue */ }
+
+          var refs = [];
+          var vh = window.innerHeight;
+          var vw = window.innerWidth;
+
+          for (var i = 0; i < all.length && refs.length < 80; i++) {
+            var el = all[i];
+            var rect = el.getBoundingClientRect();
+
+            // Skip invisible / tiny elements
+            if (rect.width < 5 || rect.height < 5) continue;
+            var style = window.getComputedStyle(el);
+            if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') continue;
+
+            // Skip elements far outside viewport
+            if (rect.bottom < -50 || rect.top > vh + 50) continue;
+            if (rect.right < -50 || rect.left > vw + 50) continue;
+
+            // Determine role
+            var role = el.getAttribute('role') || '';
+            if (!role) {
+              var tag = el.tagName.toLowerCase();
+              if (tag === 'a') role = 'link';
+              else if (tag === 'button') role = 'button';
+              else if (tag === 'input') {
+                var t = (el.type || 'text').toLowerCase();
+                if (t === 'checkbox') role = 'checkbox';
+                else if (t === 'radio') role = 'radio';
+                else if (t === 'submit' || t === 'button') role = 'button';
+                else role = 'textbox';
+              }
+              else if (tag === 'select') role = 'combobox';
+              else if (tag === 'textarea') role = 'textbox';
+              else role = tag;
+            }
+
+            // Determine accessible name
+            var name = '';
+            var ariaLabel = el.getAttribute('aria-label');
+            var ariaLabelledBy = el.getAttribute('aria-labelledby');
+            if (ariaLabel) {
+              name = ariaLabel;
+            } else if (ariaLabelledBy) {
+              var labelEl = document.getElementById(ariaLabelledBy);
+              if (labelEl) name = labelEl.textContent.trim();
+            } else if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
+              if (el.id) {
+                var lbl = document.querySelector('label[for="' + CSS.escape(el.id) + '"]');
+                if (lbl) name = lbl.textContent.trim();
+              }
+              if (!name) name = el.getAttribute('placeholder') || el.getAttribute('name') || '';
+            } else {
+              name = (el.textContent || '').trim().slice(0, 60);
+            }
+
+            var refId = refs.length + 1;
+            el.setAttribute('data-assistme-ref', String(refId));
+
+            // Correct coordinates for elements inside iframes
+            var offsetX = el.__iframeOffset ? el.__iframeOffset.x : 0;
+            var offsetY = el.__iframeOffset ? el.__iframeOffset.y : 0;
+
+            refs.push({
+              id: refId,
+              role: role,
+              name: name,
+              tag: el.tagName.toLowerCase(),
+              type: el.getAttribute('type') || '',
+              box: {
+                x: Math.round(rect.x + offsetX),
+                y: Math.round(rect.y + offsetY),
+                width: Math.round(rect.width),
+                height: Math.round(rect.height)
+              }
+            });
+          }
+
+          return JSON.stringify(refs);
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    const refs: RefEntry[] = JSON.parse(
+      ((findResult as CDPEvalResult).result?.value as string) || "[]"
+    ).map((r: Record<string, unknown>) => ({
+      id: r.id as number,
+      role: r.role as string,
+      name: r.name as string,
+      tag: r.tag as string,
+      inputType: (r.type as string) || "",
+      box: r.box as BoundingBox,
+    }));
+
+    // 2. Optionally inject visual overlay with ref labels
+    //    (Skip for dense pages — labels would overlap and become unreadable)
+    if (annotate && refs.length <= 40) {
+      const refsJson = JSON.stringify(refs);
+      await this.send("Runtime.evaluate", {
+        expression: `
+          (function() {
+            var old = document.getElementById('__assistme_refs__');
+            if (old) old.remove();
+
+            var overlay = document.createElement('div');
+            overlay.id = '__assistme_refs__';
+            overlay.style.cssText = 'position:fixed;top:0;left:0;width:100%;height:100%;pointer-events:none;z-index:2147483647;';
+
+            var refs = ${refsJson};
+            var vh = window.innerHeight;
+            var vw = window.innerWidth;
+
+            for (var i = 0; i < refs.length; i++) {
+              var b = refs[i].box;
+              if (b.y + b.height < 0 || b.y > vh || b.x + b.width < 0 || b.x > vw) continue;
+
+              // Red badge with ref number
+              var badge = document.createElement('div');
+              var badgeTop = Math.max(0, b.y - 14);
+              var badgeLeft = Math.max(0, b.x);
+              badge.style.cssText = 'position:fixed;background:#e8384f;color:#fff;font:bold 10px/1.2 monospace;padding:1px 3px;border-radius:2px;white-space:nowrap;'
+                + 'left:' + badgeLeft + 'px;top:' + badgeTop + 'px;';
+              badge.textContent = String(refs[i].id);
+              overlay.appendChild(badge);
+
+              // Border around element
+              var border = document.createElement('div');
+              border.style.cssText = 'position:fixed;border:1.5px solid #e8384f;border-radius:2px;'
+                + 'left:' + b.x + 'px;top:' + b.y + 'px;width:' + b.width + 'px;height:' + b.height + 'px;';
+              overlay.appendChild(border);
+            }
+
+            document.documentElement.appendChild(overlay);
+          })()
+        `,
+      });
+    }
+
+    // 3. Take screenshot (with or without overlay)
+    const image = await this.screenshot();
+
+    // 4. Remove overlay if injected (keep data-assistme-ref attributes for later resolution)
+    if (annotate) {
+      await this.send("Runtime.evaluate", {
+        expression: `(function() { var el = document.getElementById('__assistme_refs__'); if (el) el.remove(); })()`,
+      });
+    }
+
+    // 5. Cache refs for subsequent act() calls
+    this.refCache.clear();
+    for (const ref of refs) {
+      this.refCache.set(ref.id, ref);
+    }
+
+    // 6. Get page info
+    const pageInfo = await this.getPageInfo();
+
+    return { image, refs, url: pageInfo.url, title: pageInfo.title };
+  }
+
+  /**
+   * Build a compact text table of refs for the model.
+   */
+  static formatRefTable(result: SnapshotResult): string {
+    let table = `Page: ${result.title}\nURL: ${result.url}\n\nRefs:\n`;
+    for (const ref of result.refs) {
+      const extra = ref.inputType ? ` (${ref.inputType})` : "";
+      const nameStr = ref.name ? ` "${ref.name}"` : "";
+      table += `[${ref.id}] ${ref.role}${nameStr}${extra}\n`;
+    }
+    if (result.refs.length === 0) {
+      table += "(no interactive elements found)\n";
+    }
+    return table;
+  }
+
+  // ── Ref Resolution ────────────────────────────────────────────────
+
+  /**
+   * Resolve a ref ID to its current center coordinates in the viewport.
+   * Uses two strategies:
+   *  1. Fast: find by data-assistme-ref attribute (set during snapshot)
+   *  2. Stable: search by role + accessible name (survives DOM changes)
+   *
+   * Includes actionability checks (like Playwright):
+   *  - Element must be visible (not display:none, not zero-size)
+   *  - Element must be in viewport (scrolls into view if needed)
+   *  - Element must not be covered by another element (checks elementFromPoint)
+   *
+   * Returns null if the element cannot be found or is not actionable.
+   * Returns { error: string } if found but not actionable (for diagnostics).
+   */
+  private async resolveRef(
+    refId: number
+  ): Promise<{ x: number; y: number; width: number; height: number; error?: string } | null> {
+    const cached = this.refCache.get(refId);
+    const role = cached?.role || "";
+    const name = cached?.name || "";
+    const roleJS = JSON.stringify(role);
+    const nameJS = JSON.stringify(name);
+
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          var refId = ${refId};
+          var role = ${roleJS};
+          var name = ${nameJS};
+
+          // Strategy 1: data attribute (fast, from last snapshot)
+          var el = document.querySelector('[data-assistme-ref="' + refId + '"]');
+
+          // Strategy 2: role + name search (stable, survives DOM changes)
+          if (!el && role && name) {
+            var selectorMap = {
+              textbox: 'input, textarea, [role="textbox"], [role="searchbox"]',
+              button: 'button, [role="button"], input[type="submit"], input[type="button"]',
+              link: 'a[href], [role="link"]',
+              combobox: 'select, [role="combobox"]',
+              checkbox: 'input[type="checkbox"], [role="checkbox"]',
+              radio: 'input[type="radio"], [role="radio"]',
+              tab: '[role="tab"]',
+              menuitem: '[role="menuitem"]',
+              option: '[role="option"], option',
+            };
+            var sel = selectorMap[role] || '*[role="' + role + '"]';
+            var candidates = document.querySelectorAll(sel);
+            for (var i = 0; i < candidates.length; i++) {
+              var c = candidates[i];
+              var cName = c.getAttribute('aria-label')
+                || c.getAttribute('placeholder')
+                || (c.textContent || '').trim().slice(0, 60);
+              if (cName === name) { el = c; break; }
+            }
+          }
+
+          if (!el) return 'null';
+
+          // ── Actionability checks (Playwright-style) ──────────────
+
+          // Check visibility
+          var style = window.getComputedStyle(el);
+          if (style.display === 'none')
+            return JSON.stringify({ error: 'Element is hidden (display:none)' });
+          if (style.visibility === 'hidden')
+            return JSON.stringify({ error: 'Element is hidden (visibility:hidden)' });
+          if (parseFloat(style.opacity) < 0.05)
+            return JSON.stringify({ error: 'Element is hidden (opacity:0)' });
+
+          // Check disabled
+          if (el.disabled || el.getAttribute('aria-disabled') === 'true')
+            return JSON.stringify({ error: 'Element is disabled' });
+
+          // Scroll into view
+          el.scrollIntoView({ block: 'center', behavior: 'instant' });
+          var r = el.getBoundingClientRect();
+
+          // Check non-zero size
+          if (r.width < 1 || r.height < 1)
+            return JSON.stringify({ error: 'Element has zero size (' + r.width + 'x' + r.height + ')' });
+
+          // Check element is in viewport
+          if (r.bottom < 0 || r.top > window.innerHeight || r.right < 0 || r.left > window.innerWidth)
+            return JSON.stringify({ error: 'Element is outside viewport after scroll' });
+
+          var cx = r.x + r.width / 2;
+          var cy = r.y + r.height / 2;
+
+          // Check not covered by another element (hit test)
+          var topEl = document.elementFromPoint(cx, cy);
+          if (topEl && topEl !== el && !el.contains(topEl) && !topEl.closest('[data-assistme-ref="' + refId + '"]')) {
+            // Check if the covering element is the overlay (ignore it)
+            if (!topEl.closest('#__assistme_refs__')) {
+              var coverTag = topEl.tagName.toLowerCase();
+              var coverText = (topEl.textContent || '').trim().slice(0, 30);
+              return JSON.stringify({
+                error: 'Element is covered by <' + coverTag + '>' + (coverText ? ' "' + coverText + '"' : ''),
+                x: cx, y: cy, width: r.width, height: r.height
+              });
+            }
+          }
+
+          return JSON.stringify({
+            x: cx,
+            y: cy,
+            width: r.width,
+            height: r.height
+          });
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    const value = (result as CDPEvalResult).result?.value as string;
+    if (!value || value === "null") return null;
+    try {
+      return JSON.parse(value);
+    } catch {
+      return null;
+    }
+  }
+
+  // ── Ref-based Interactions (CDP Input Events) ─────────────────────
+
+  /**
+   * Click an element by ref using CDP Input.dispatchMouseEvent.
+   * This simulates a real mouse click through the browser's input pipeline,
+   * triggering hover states, focus management, and all native browser events
+   * — more reliable than el.click() for framework components.
+   *
+   * Includes auto-wait: retries up to 3 times (with 500ms intervals) if the
+   * element is not yet actionable (e.g., covered by a loading overlay, still
+   * animating into view). This matches Playwright's auto-waiting behavior.
+   */
+  async clickRef(refId: number): Promise<string> {
+    this.ensureConnected();
+
+    // Auto-wait: retry up to 3 times if element is not actionable yet
+    const maxRetries = 3;
+    let lastError = "";
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      const resolved = await this.resolveRef(refId);
+
+      if (!resolved) {
+        return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+      }
+
+      if (resolved.error) {
+        lastError = resolved.error;
+        // If element is covered or hidden, wait and retry (it might be animating)
+        if (attempt < maxRetries - 1) {
+          await new Promise((r) => setTimeout(r, 500));
+          continue;
+        }
+        // Final attempt failed — report the actionability issue
+        const ref = this.refCache.get(refId);
+        return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+      }
+
+      // Element is actionable — small delay after scroll for rendering
+      if (attempt === 0) {
+        await new Promise((r) => setTimeout(r, 50));
+        // Re-read position after scroll settled
+        const settled = await this.resolveRef(refId);
+        if (settled && !settled.error) {
+          resolved.x = settled.x;
+          resolved.y = settled.y;
+        }
+      }
+
+      // Full mouse event sequence: move → press → release
+      await this.send("Input.dispatchMouseEvent", {
+        type: "mouseMoved",
+        x: resolved.x,
+        y: resolved.y,
+      });
+      await this.send("Input.dispatchMouseEvent", {
+        type: "mousePressed",
+        x: resolved.x,
+        y: resolved.y,
+        button: "left",
+        clickCount: 1,
+      });
+      await this.send("Input.dispatchMouseEvent", {
+        type: "mouseReleased",
+        x: resolved.x,
+        y: resolved.y,
+        button: "left",
+        clickCount: 1,
+      });
+
+      await new Promise((r) => setTimeout(r, 300));
+      const ref = this.refCache.get(refId);
+      return `Clicked [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+    }
+
+    // Should not reach here, but just in case
+    const ref = this.refCache.get(refId);
+    return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+  }
+
+  /**
+   * Type text into an element by ref using CDP Input events.
+   * Clicks to focus, selects all existing text (Ctrl/Cmd+A), then uses
+   * Input.insertText for reliable text insertion across all frameworks.
+   */
+  async typeRef(refId: number, text: string): Promise<string> {
+    this.ensureConnected();
+
+    // Click to focus the element
+    const clickResult = await this.clickRef(refId);
+    if (clickResult.includes("not found")) return clickResult;
+    await new Promise((r) => setTimeout(r, 100));
+
+    // Select all existing text (Cmd+A on macOS, Ctrl+A elsewhere)
+    const modifier = platform() === "darwin" ? 4 : 2;
+    await this.send("Input.dispatchKeyEvent", {
+      type: "keyDown",
+      modifiers: modifier,
+      key: "a",
+      code: "KeyA",
+      windowsVirtualKeyCode: 65,
+    });
+    await this.send("Input.dispatchKeyEvent", {
+      type: "keyUp",
+      key: "a",
+      code: "KeyA",
+    });
+
+    // Delete selected text
+    await this.send("Input.dispatchKeyEvent", {
+      type: "keyDown",
+      key: "Backspace",
+      code: "Backspace",
+      windowsVirtualKeyCode: 8,
+    });
+    await this.send("Input.dispatchKeyEvent", {
+      type: "keyUp",
+      key: "Backspace",
+      code: "Backspace",
+    });
+
+    // Insert text via CDP (goes through the browser's input pipeline)
+    await this.send("Input.insertText", { text });
+
+    await new Promise((r) => setTimeout(r, 100));
+    const ref = this.refCache.get(refId);
+    return `Typed "${text}" into [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+  }
+
+  /**
+   * Select a dropdown option by ref. Delegates to selectOption with the
+   * ref's data attribute as selector, handling both native <select> and
+   * custom dropdown components.
+   */
+  async selectRef(refId: number, option: string): Promise<string> {
+    this.ensureConnected();
+
+    // Check if ref exists
+    const cached = this.refCache.get(refId);
+    if (!cached) {
+      return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+    }
+
+    // Use the data attribute selector to find the element
+    const result = await this.selectOption(`[data-assistme-ref="${refId}"]`, option);
+    return result.replace(
+      /\[data-assistme-ref="\d+"\]/,
+      `[${refId}] ${cached.role} "${cached.name}"`
+    );
+  }
+
+  // ── Action Pipeline ───────────────────────────────────────────────
+
+  /**
+   * Execute a batch of actions sequentially using refs.
+   * Reduces round-trips: instead of one tool call per action, the model
+   * can specify a sequence of actions that execute atomically.
+   *
+   * Optionally takes a screenshot after all actions complete.
+   */
+  async act(
+    actions: ActionSpec[],
+    takeScreenshot = false
+  ): Promise<{ results: ActionResult[]; screenshot?: string }> {
+    this.ensureConnected();
+    const results: ActionResult[] = [];
+
+    for (const spec of actions) {
+      let result: string;
+      let success = true;
+
+      try {
+        switch (spec.action) {
+          case "click":
+            result = await this.clickRef(spec.ref);
+            success = !result.includes("not found");
+            break;
+          case "type":
+            result = await this.typeRef(spec.ref, spec.text);
+            success = !result.includes("not found");
+            break;
+          case "select":
+            result = await this.selectRef(spec.ref, spec.option);
+            success = !result.includes("not found");
+            break;
+          case "press":
+            result = await this.pressKey(spec.key);
+            break;
+          case "scroll":
+            result = spec.direction === "up" ? await this.scrollUp() : await this.scrollDown();
+            break;
+          case "wait":
+            await new Promise((r) => setTimeout(r, Math.min(spec.ms, 5000)));
+            result = `Waited ${spec.ms}ms`;
+            break;
+          default:
+            result = `Unknown action: ${(spec as { action: string }).action}`;
+            success = false;
+        }
+      } catch (err) {
+        result = `Error: ${err instanceof Error ? err.message : String(err)}`;
+        success = false;
+      }
+
+      results.push({
+        action: spec.action,
+        ref: "ref" in spec ? (spec as { ref: number }).ref : undefined,
+        result,
+        success,
+      });
+
+      // If an action failed, stop the batch (remaining refs may be stale)
+      if (!success) break;
+
+      // Brief pause between actions for DOM to settle
+      if (spec.action !== "wait") {
+        await new Promise((r) => setTimeout(r, 200));
+      }
+    }
+
+    let screenshot: string | undefined;
+    if (takeScreenshot) {
+      // Wait a bit for final DOM changes to settle
+      await new Promise((r) => setTimeout(r, 300));
+      screenshot = await this.screenshot();
+    }
+
+    return { results, screenshot };
+  }
+
+  // ── Dropdown/Select ─────────────────────────────────────────────
+
+  /**
+   * Select an option from a dropdown — handles both native <select> elements
+   * and custom Material Design / React / Angular dropdown components.
+   *
+   * Strategy:
+   * 1. Try native <select> first (by selector or label text)
+   * 2. Fall back to custom dropdown: click to open, then click the option by text
+   */
+  async selectOption(selector: string, optionText: string): Promise<string> {
+    this.ensureConnected();
+    const selectorJS = JSON.stringify(selector);
+    const optionJS = JSON.stringify(optionText);
+
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          var sel = ${selectorJS};
+          var optText = ${optionJS};
+
+          // Strategy 1: Native <select> element
+          var selectEl = document.querySelector(sel);
+          if (selectEl && selectEl.tagName === 'SELECT') {
+            var options = selectEl.querySelectorAll('option');
+            for (var i = 0; i < options.length; i++) {
+              if (options[i].textContent.trim() === optText) {
+                selectEl.value = options[i].value;
+                selectEl.dispatchEvent(new Event('change', { bubbles: true }));
+                selectEl.dispatchEvent(new Event('input', { bubbles: true }));
+                return 'Selected "' + optText + '" in native select';
+              }
+            }
+            return 'Option "' + optText + '" not found in select. Available: ' +
+              Array.from(options).map(function(o) { return o.textContent.trim(); }).join(', ');
+          }
+
+          // Strategy 2: Custom dropdown — find the trigger element
+          var trigger = selectEl;
+          if (!trigger) {
+            // Try finding by label/placeholder text
+            var allEls = document.querySelectorAll('*');
+            for (var j = 0; j < allEls.length; j++) {
+              var el = allEls[j];
+              var ownText = Array.from(el.childNodes)
+                .filter(function(n) { return n.nodeType === 3; })
+                .map(function(n) { return n.textContent.trim(); })
+                .join('');
+              if (ownText === sel || el.getAttribute('aria-label') === sel) {
+                trigger = el;
+                break;
+              }
+            }
+          }
+
+          if (!trigger) return 'Dropdown not found: ' + sel;
+
+          // Click to open the dropdown
+          trigger.scrollIntoView({ block: 'center', behavior: 'instant' });
+          trigger.click();
+
+          // Wait a frame for the dropdown menu to render, then select the option
+          return new Promise(function(resolve) {
+            setTimeout(function() {
+              // Look for the option in listbox/menu/dropdown overlays
+              var optionContainers = document.querySelectorAll(
+                '[role="listbox"], [role="menu"], [role="presentation"], .MuiMenu-list, .MuiList-root, ul.mdc-list, .VfPpkd-xl07Ob'
+              );
+
+              // Also check all visible elements as fallback
+              var searchIn = optionContainers.length > 0
+                ? Array.from(optionContainers).flatMap(function(c) { return Array.from(c.querySelectorAll('*')); })
+                : Array.from(document.querySelectorAll('li, [role="option"], [role="menuitem"], div[data-value]'));
+
+              for (var k = 0; k < searchIn.length; k++) {
+                var opt = searchIn[k];
+                var txt = opt.textContent ? opt.textContent.trim() : '';
+                if (txt === optText) {
+                  opt.scrollIntoView({ block: 'center', behavior: 'instant' });
+                  opt.click();
+                  resolve('Selected "' + optText + '" from custom dropdown');
+                  return;
+                }
+              }
+
+              // Broader search: any visible element with exact text match
+              var everything = document.querySelectorAll('*');
+              for (var m = 0; m < everything.length; m++) {
+                var candidate = everything[m];
+                if (candidate.textContent && candidate.textContent.trim() === optText &&
+                    candidate.offsetParent !== null && candidate.children.length === 0) {
+                  candidate.click();
+                  resolve('Selected "' + optText + '" (broad match)');
+                  return;
+                }
+              }
+
+              resolve('Option "' + optText + '" not found in dropdown');
+            }, 300);
+          });
+        })()
+      `,
+      returnByValue: true,
+      awaitPromise: true,
+    });
+
+    await new Promise((r) => setTimeout(r, 500));
+    return ((result as CDPEvalResult).result?.value as string) || "Selection attempted.";
+  }
+
+  // ── JavaScript Evaluation ───────────────────────────────────────
+
+  async evaluate(expression: string): Promise<string> {
+    this.ensureConnected();
+    const result = await this.send("Runtime.evaluate", {
+      expression,
+      returnByValue: true,
+      awaitPromise: true,
+    });
+
+    const evalResult = (result as CDPEvalResult).result;
+    const value = evalResult?.value;
+    if (value === undefined) {
+      const desc = evalResult?.description;
+      return desc || "(undefined)";
+    }
+    return typeof value === "string" ? value : JSON.stringify(value, null, 2);
+  }
+
+  // ── Tab Management ──────────────────────────────────────────────
+
+  async listTabs(): Promise<string> {
+    const tabs = await this.getTabs();
+    const pageTabs = tabs.filter((t) => t.type === "page");
+
+    if (pageTabs.length === 0) return "No tabs open.";
+
+    return pageTabs
+      .map(
+        (t, i) =>
+          `[${i}] ${t.title.slice(0, 60)}${this.currentTabId === t.id ? " (active)" : ""}\n    ${t.url}`
+      )
+      .join("\n\n");
+  }
+
+  async switchTab(index: number): Promise<string> {
+    const tabs = await this.getTabs();
+    const pageTabs = tabs.filter((t) => t.type === "page");
+
+    if (index < 0 || index >= pageTabs.length) {
+      return `Invalid tab index. Available: 0-${pageTabs.length - 1}`;
+    }
+
+    // Disconnect from current tab
+    await this.disconnect();
+
+    // Connect to new tab
+    return this.connect(index);
+  }
+
+  async openNewTab(url?: string): Promise<string> {
+    const targetUrl = url || "about:blank";
+    const res = await fetch(
+      `http://127.0.0.1:${this.debugPort}/json/new?${encodeURIComponent(targetUrl)}`,
+      { signal: AbortSignal.timeout(5000) }
+    );
+    const tab = (await res.json()) as CDPTab;
+
+    // Connect to the new tab
+    await this.disconnect();
+    const tabs = await this.getTabs();
+    const idx = tabs.filter((t) => t.type === "page").findIndex((t) => t.id === tab.id);
+    if (idx >= 0) {
+      await this.connect(idx);
+    }
+
+    return `Opened new tab: ${targetUrl}`;
+  }
+
+  // ── Helpers ─────────────────────────────────────────────────────
+
+  private async waitForLoad(timeoutMs = 8000): Promise<void> {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+      try {
+        const result = await this.send("Runtime.evaluate", {
+          expression: "document.readyState",
+          returnByValue: true,
+        });
+        const state = (result as CDPEvalResult).result?.value;
+        if (state === "complete" || state === "interactive") {
+          // Extra small wait for dynamic content
+          await new Promise((r) => setTimeout(r, 500));
+          return;
+        }
+      } catch {
+        // Tab might be navigating
+      }
+      await new Promise((r) => setTimeout(r, 300));
+    }
+  }
+
+  /**
+   * Find interactive elements on the page for the AI to understand what's clickable
+   */
+  async getInteractiveElements(): Promise<string> {
+    this.ensureConnected();
+    const result = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          const elements = [];
+          const selectors = 'a, button, input, select, textarea, [role="button"], [onclick]';
+          const all = document.querySelectorAll(selectors);
+          for (let i = 0; i < all.length && elements.length < 50; i++) {
+            const el = all[i];
+            const rect = el.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) continue; // Skip hidden
+
+            // Build a reliable CSS selector
+            let selector;
+            if (el.id) {
+              selector = '#' + CSS.escape(el.id);
+            } else if (el.getAttribute('data-testid')) {
+              selector = '[data-testid="' + el.getAttribute('data-testid') + '"]';
+            } else {
+              // Build a path-based selector: find nth-of-type among siblings
+              const tag = el.tagName.toLowerCase();
+              const parent = el.parentElement;
+              if (parent) {
+                const siblings = parent.querySelectorAll(':scope > ' + tag);
+                const idx = Array.from(siblings).indexOf(el) + 1;
+                selector = tag + ':nth-of-type(' + idx + ')';
+              } else {
+                selector = tag;
+              }
+            }
+
+            elements.push({
+              tag: el.tagName.toLowerCase(),
+              text: (el.textContent || '').trim().slice(0, 80),
+              type: el.getAttribute('type') || '',
+              name: el.getAttribute('name') || '',
+              id: el.id || '',
+              href: el.getAttribute('href') || '',
+              placeholder: el.getAttribute('placeholder') || '',
+              selector: selector,
+            });
+          }
+          return JSON.stringify(elements, null, 2);
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    return ((result as CDPEvalResult).result?.value as string) || "[]";
+  }
+
+  isConnected(): boolean {
+    return this.connected && this.ws?.readyState === WebSocket.OPEN;
+  }
+
+  // ── Login Detection ────────────────────────────────────────────
+
+  /**
+   * Detect if the current page appears to be a login/authentication page.
+   * Checks URL patterns, password input fields, and login form actions.
+   */
+  async detectLoginPage(): Promise<{ isLoginPage: boolean; reason: string }> {
+    try {
+      const result = await this.send("Runtime.evaluate", {
+        expression: `
+          (function() {
+            var url = window.location.href.toLowerCase();
+
+            // Exclude signup/registration pages — these are NOT login pages
+            var signupPatterns = [
+              '/signup', '/sign-up', '/sign_up', '/register',
+              '/registration', '/create-account', '/create_account',
+              '/join', '/enroll',
+              'accounts.google.com/lifecycle/steps/signup',
+              'signup.live.com',
+            ];
+            for (var s = 0; s < signupPatterns.length; s++) {
+              if (url.indexOf(signupPatterns[s]) !== -1) {
+                return JSON.stringify({ isLoginPage: false, reason: '' });
+              }
+            }
+
+            // URL-based detection
+            var loginPatterns = [
+              '/login', '/signin', '/sign-in', '/sign_in',
+              '/auth/', '/sso/', '/oauth/', '/session/new',
+              '/accounts/login', '/users/sign_in',
+              'accounts.google.com/v3/signin',
+              'accounts.google.com/servicelogin',
+              'login.microsoftonline.com',
+              'github.com/login', 'github.com/session',
+              'login.live.com', 'appleid.apple.com'
+            ];
+            for (var i = 0; i < loginPatterns.length; i++) {
+              if (url.indexOf(loginPatterns[i]) !== -1) {
+                return JSON.stringify({
+                  isLoginPage: true,
+                  reason: 'URL contains login pattern: ' + loginPatterns[i]
+                });
+              }
+            }
+
+            // Password input detection (visible only)
+            var passwordInputs = document.querySelectorAll('input[type="password"]');
+            for (var j = 0; j < passwordInputs.length; j++) {
+              var input = passwordInputs[j];
+              var rect = input.getBoundingClientRect();
+              var style = window.getComputedStyle(input);
+              if (rect.width > 0 && rect.height > 0 &&
+                  style.display !== 'none' && style.visibility !== 'hidden') {
+                return JSON.stringify({
+                  isLoginPage: true,
+                  reason: 'Page contains visible password input field'
+                });
+              }
+            }
+
+            // Login form action detection
+            var formSelectors = [
+              'form[action*="login"]', 'form[action*="signin"]',
+              'form[action*="session"]', 'form[action*="auth"]',
+              'form[action*="authenticate"]'
+            ];
+            var loginForms = document.querySelectorAll(formSelectors.join(','));
+            if (loginForms.length > 0) {
+              return JSON.stringify({
+                isLoginPage: true,
+                reason: 'Page contains login form'
+              });
+            }
+
+            return JSON.stringify({ isLoginPage: false, reason: '' });
+          })()
+        `,
+        returnByValue: true,
+      });
+
+      const value = (result as CDPEvalResult).result?.value as string;
+      return JSON.parse(value || '{"isLoginPage":false,"reason":""}');
+    } catch {
+      return { isLoginPage: false, reason: "" };
+    }
+  }
+}