assistme 0.3.4 → 0.3.6

package/src/browser/controller.ts

@@ -12,6 +12,12 @@ import type {
   ActionResult,
   RefActionResult,
 } from "./types.js";
+import {
+  REF_CACHE_MAX_SIZE,
+  FRAME_CONTEXTS_MAX_SIZE,
+  CDP_COMMAND_TIMEOUT_MS,
+  WS_CONNECT_TIMEOUT_MS,
+} from "../utils/constants.js";
 
 export class BrowserController {
   private ws: WebSocket | null = null;
@@ -21,6 +27,7 @@ export class BrowserController {
   private connected = false;
   private currentTabId: string | null = null;
   private refCache: Map<number, RefEntry> = new Map();
+  private frameContexts: Map<number, number> = new Map(); // refId → contextId
 
   constructor(port = 9222) {
     this.debugPort = port;
@@ -87,9 +94,9 @@ export class BrowserController {
         if (!settled) {
           settled = true;
           this.ws?.close();
-          reject(new Error("Connection timeout (5s)"));
+          reject(new Error(`Connection timeout (${WS_CONNECT_TIMEOUT_MS}ms)`));
         }
-      }, 5000);
+      }, WS_CONNECT_TIMEOUT_MS);
 
       this.ws.on("open", () => {
         if (settled) return;
@@ -142,6 +149,9 @@ export class BrowserController {
       this.ws = null;
       this.connected = false;
     }
+    // Clear caches to prevent memory leaks across task boundaries
+    this.refCache.clear();
+    this.frameContexts.clear();
     return "Disconnected from browser.";
   }
 
@@ -165,7 +175,7 @@ export class BrowserController {
       const timeout = setTimeout(() => {
         this.callbacks.delete(id);
         reject(new Error(`CDP command timed out: ${method}`));
-      }, 15000);
+      }, CDP_COMMAND_TIMEOUT_MS);
 
       this.callbacks.set(id, (response) => {
         clearTimeout(timeout);
@@ -367,17 +377,50 @@ export class BrowserController {
     const selectorJS = JSON.stringify(selector);
     const textJS = JSON.stringify(text);
 
-    // First clear and set value via JS, dispatching all relevant events
+    // First try to find the element in main document, then in same-origin iframes
     const result = await this.send("Runtime.evaluate", {
       expression: `
         (function() {
-          const el = document.querySelector(${selectorJS});
+          var el = document.querySelector(${selectorJS});
+
+          // If not found in main document, search same-origin iframes
+          if (!el) {
+            var iframes = document.querySelectorAll('iframe');
+            for (var i = 0; i < iframes.length; i++) {
+              try {
+                var iframeDoc = iframes[i].contentDocument;
+                if (iframeDoc) {
+                  el = iframeDoc.querySelector(${selectorJS});
+                  if (el) break;
+                }
+              } catch(e) { /* cross-origin, skip */ }
+            }
+          }
+
           if (!el) return 'Element not found: ' + ${selectorJS};
 
           el.focus();
 
-          // Clear existing value
-          const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
+          // Check if this is a contenteditable element (rich text editor)
+          var isContentEditable = el.isContentEditable ||
+            el.getAttribute('contenteditable') === 'true' ||
+            el.getAttribute('contenteditable') === '';
+
+          if (isContentEditable) {
+            // For contenteditable: select all content, then replace
+            var ownerDoc = el.ownerDocument;
+            var sel = ownerDoc.defaultView.getSelection();
+            var range = ownerDoc.createRange();
+            range.selectNodeContents(el);
+            sel.removeAllRanges();
+            sel.addRange(range);
+            // Use insertText command which respects undo stack and triggers input events
+            ownerDoc.execCommand('insertText', false, ${textJS});
+            return 'Typed into: ' + (el.tagName || '') + ' [contenteditable]';
+          }
+
+          // For input/textarea: clear and set value
+          var nativeInputValueSetter = Object.getOwnPropertyDescriptor(
             window.HTMLInputElement.prototype, 'value'
           )?.set || Object.getOwnPropertyDescriptor(
             window.HTMLTextAreaElement.prototype, 'value'
@@ -398,7 +441,36 @@ export class BrowserController {
       returnByValue: true,
     });
 
-    return ((result as CDPEvalResult).result?.value as string) || "Text entered.";
+    const textResult = ((result as CDPEvalResult).result?.value as string) || "";
+
+    // If element still not found, try typing into the currently focused element via CDP
+    if (textResult.startsWith("Element not found")) {
+      return this.typeAtFocus(text);
+    }
+
+    return textResult || "Text entered.";
+  }
+
+  /**
+   * Type text into the currently focused element using CDP Input.insertText.
+   * This bypasses DOM queries entirely and works with any focused element,
+   * including those inside cross-origin iframes or shadow DOM.
+   */
+  async typeAtFocus(text: string): Promise<string> {
+    this.ensureConnected();
+
+    // Optionally clear existing content: select all then delete
+    const modKey = platform() === "darwin" ? "Meta" : "Control";
+    await this.pressKey(`${modKey}+a`);
+    await new Promise((r) => setTimeout(r, 50));
+    await this.pressKey("Backspace");
+    await new Promise((r) => setTimeout(r, 50));
+
+    // Insert text via CDP — goes through the browser's native input pipeline
+    await this.send("Input.insertText", { text });
+    await new Promise((r) => setTimeout(r, 100));
+
+    return "Text entered (into focused element).";
   }
 
   async pressKey(key: string): Promise<string> {
@@ -670,6 +742,9 @@ export class BrowserController {
       box: r.box as BoundingBox,
     }));
 
+    // 1b. Discover elements in cross-origin iframes via CDP frame targeting
+    await this.discoverCrossOriginFrameRefs(refs);
+
     // 2. Optionally inject visual overlay with ref labels
     //    (Skip for dense pages — labels would overlap and become unreadable)
     if (annotate && refs.length <= 40) {
@@ -725,6 +800,8 @@ export class BrowserController {
     }
 
     // 5. Cache refs for subsequent act() calls
+    // Note: frameContexts is populated by discoverCrossOriginFrameRefs above,
+    // so we only clear refCache here (frameContexts was cleared at start of discover)
     this.refCache.clear();
     for (const ref of refs) {
       this.refCache.set(ref.id, ref);
@@ -752,6 +829,237 @@ export class BrowserController {
     return table;
   }
 
+  // ── Cross-Origin Iframe Discovery ────────────────────────────────
+
+  /**
+   * Use CDP's Page.getFrameTree + Runtime.evaluate with contextId to discover
+   * interactive elements inside cross-origin iframes (e.g., ProtonMail editor,
+   * Google Docs, embedded rich text editors).
+   *
+   * Same-origin iframes are already handled inline by the main snapshot JS.
+   * This method handles the ones that threw cross-origin errors.
+   */
+  private async discoverCrossOriginFrameRefs(refs: RefEntry[]): Promise<void> {
+    this.frameContexts.clear();
+    try {
+      // Get the frame tree to find all child frames
+      const frameTree = (await this.send("Page.getFrameTree")) as {
+        frameTree?: {
+          frame: { id: string };
+          childFrames?: Array<{ frame: { id: string; url: string; name?: string } }>;
+        };
+      };
+
+      const mainFrameId = frameTree.frameTree?.frame?.id;
+      const childFrames = frameTree.frameTree?.childFrames || [];
+      if (childFrames.length === 0) return;
+
+      // Get all execution contexts to map frameId → contextId
+      // We need to enable Runtime events and collect contexts
+      const contexts = await this.getFrameContexts(mainFrameId || "");
+
+      for (const child of childFrames) {
+        const frameId = child.frame.id;
+        const contextId = contexts.get(frameId);
+        if (!contextId) continue;
+
+        // Get the iframe's bounding rect from the parent frame for coordinate offset
+        const iframeOffsetResult = await this.send("Runtime.evaluate", {
+          expression: `
+            (function() {
+              var iframes = document.querySelectorAll('iframe');
+              for (var i = 0; i < iframes.length; i++) {
+                try {
+                  // Match by frame src or name
+                  var f = iframes[i];
+                  if (f.contentWindow) {
+                    var r = f.getBoundingClientRect();
+                    if (r.width > 10 && r.height > 10) {
+                      return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, index: i });
+                    }
+                  }
+                } catch(e) {}
+              }
+              return 'null';
+            })()
+          `,
+          returnByValue: true,
+        });
+
+        let iframeOffset = { x: 0, y: 0 };
+        try {
+          const parsed = JSON.parse(
+            ((iframeOffsetResult as CDPEvalResult).result?.value as string) || "null"
+          );
+          if (parsed) iframeOffset = { x: parsed.x, y: parsed.y };
+        } catch {
+          /* ignore */
+        }
+
+        // Evaluate inside the child frame's execution context
+        const startRefId = refs.length + 1;
+        try {
+          const frameResult = await this.send("Runtime.evaluate", {
+            expression: `
+              (function() {
+                var selectors = [
+                  'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea',
+                  '[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]',
+                  '[role="combobox"]', '[role="listbox"]', '[role="menuitem"]', '[role="tab"]',
+                  '[role="switch"]', '[role="slider"]', '[role="option"]', '[role="searchbox"]',
+                  '[onclick]', '[tabindex]:not([tabindex="-1"])',
+                  '[contenteditable="true"]', '[contenteditable=""]'
+                ].join(', ');
+
+                var all = document.querySelectorAll(selectors);
+                // Also check if the body itself is contenteditable
+                if (document.body && (document.body.isContentEditable || document.body.getAttribute('contenteditable') === 'true')) {
+                  all = [document.body].concat(Array.from(all));
+                }
+
+                var refs = [];
+                var startId = ${startRefId};
+                var vh = window.innerHeight;
+                var vw = window.innerWidth;
+
+                for (var i = 0; i < all.length && refs.length < 20; i++) {
+                  var el = all[i];
+                  var rect = el.getBoundingClientRect();
+                  if (rect.width < 5 || rect.height < 5) continue;
+                  var style = window.getComputedStyle(el);
+                  if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') continue;
+
+                  var role = el.getAttribute('role') || '';
+                  if (!role) {
+                    var tag = el.tagName.toLowerCase();
+                    if (tag === 'a') role = 'link';
+                    else if (tag === 'button') role = 'button';
+                    else if (tag === 'input') {
+                      var t = (el.type || 'text').toLowerCase();
+                      if (t === 'checkbox') role = 'checkbox';
+                      else if (t === 'radio') role = 'radio';
+                      else if (t === 'submit' || t === 'button') role = 'button';
+                      else role = 'textbox';
+                    }
+                    else if (tag === 'select') role = 'combobox';
+                    else if (tag === 'textarea') role = 'textbox';
+                    else if (el.isContentEditable) role = 'textbox';
+                    else role = tag;
+                  }
+
+                  var name = '';
+                  var ariaLabel = el.getAttribute('aria-label');
+                  if (ariaLabel) {
+                    name = ariaLabel;
+                  } else if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
+                    name = el.getAttribute('placeholder') || el.getAttribute('name') || '';
+                  } else if (el.isContentEditable) {
+                    name = 'compose body';
+                  } else {
+                    name = (el.textContent || '').trim().slice(0, 60);
+                  }
+
+                  var refId = startId + refs.length;
+                  el.setAttribute('data-assistme-ref', String(refId));
+
+                  refs.push({
+                    id: refId,
+                    role: role,
+                    name: name,
+                    tag: el.tagName.toLowerCase(),
+                    type: el.getAttribute('type') || '',
+                    box: {
+                      x: Math.round(rect.x),
+                      y: Math.round(rect.y),
+                      width: Math.round(rect.width),
+                      height: Math.round(rect.height)
+                    },
+                    inFrame: true
+                  });
+                }
+
+                return JSON.stringify(refs);
+              })()
+            `,
+            contextId,
+            returnByValue: true,
+          });
+
+          const frameRefs = JSON.parse(
+            ((frameResult as CDPEvalResult).result?.value as string) || "[]"
+          );
+
+          for (const r of frameRefs) {
+            refs.push({
+              id: r.id as number,
+              role: r.role as string,
+              name: r.name as string,
+              tag: r.tag as string,
+              inputType: (r.type as string) || "",
+              box: {
+                x: Math.round((r.box.x as number) + iframeOffset.x),
+                y: Math.round((r.box.y as number) + iframeOffset.y),
+                width: r.box.width as number,
+                height: r.box.height as number,
+              },
+            });
+            // Store frame context for later resolution (with size guard)
+            if (this.frameContexts.size < FRAME_CONTEXTS_MAX_SIZE) {
+              this.frameContexts.set(r.id as number, contextId);
+            }
+          }
+        } catch {
+          // Frame evaluation failed (e.g., about:blank, pdf viewer) — skip
+        }
+      }
+    } catch {
+      // Frame tree unavailable — not critical, skip silently
+    }
+  }
+
+  /**
+   * Get execution context IDs for each frame in the page.
+   * Uses Runtime.executionContextCreated events collected during the session,
+   * or falls back to evaluating in known frames.
+   */
+  private async getFrameContexts(_mainFrameId: string): Promise<Map<string, number>> {
+    const contexts = new Map<string, number>();
+    try {
+      // Enable Runtime domain to get context descriptions (may already be enabled)
+      await this.send("Runtime.enable").catch(() => {});
+
+      // Use Page.getFrameTree to get frame IDs, then try to create isolated worlds
+      // for each frame to get their execution context IDs
+      const frameTree = (await this.send("Page.getFrameTree")) as {
+        frameTree?: {
+          frame: { id: string };
+          childFrames?: Array<{ frame: { id: string } }>;
+        };
+      };
+
+      const childFrames = frameTree.frameTree?.childFrames || [];
+      for (const child of childFrames) {
+        try {
+          // Create an isolated world in the frame to get a context ID
+          const world = (await this.send("Page.createIsolatedWorld", {
+            frameId: child.frame.id,
+            worldName: "assistme-snapshot",
+            grantUniveralAccess: true,
+          })) as { executionContextId?: number };
+
+          if (world.executionContextId) {
+            contexts.set(child.frame.id, world.executionContextId);
+          }
+        } catch {
+          // Frame might not support isolated worlds — skip
+        }
+      }
+    } catch {
+      // Fallback: no contexts available
+    }
+    return contexts;
+  }
+
   // ── Ref Resolution ────────────────────────────────────────────────
 
   /**
@@ -869,9 +1177,103 @@ export class BrowserController {
     });
 
     const value = (result as CDPEvalResult).result?.value as string;
-    if (!value || value === "null") return null;
+    if (value && value !== "null") {
+      try {
+        return JSON.parse(value);
+      } catch {
+        /* fall through to frame search */
+      }
+    }
+
+    // Strategy 3: search in cross-origin iframe contexts
+    const frameContextId = this.frameContexts.get(refId);
+    if (frameContextId) {
+      return this.resolveRefInFrame(refId, frameContextId, role, name);
+    }
+
+    return null;
+  }
+
+  /**
+   * Resolve a ref inside a cross-origin iframe using its execution context.
+   * Returns coordinates adjusted by the iframe's viewport offset.
+   */
+  private async resolveRefInFrame(
+    refId: number,
+    contextId: number,
+    role: string,
+    name: string
+  ): Promise<{ x: number; y: number; width: number; height: number; error?: string } | null> {
+    const roleJS = JSON.stringify(role);
+    const nameJS = JSON.stringify(name);
+
     try {
-      return JSON.parse(value);
+      // Get iframe offset from main document
+      const offsetResult = await this.send("Runtime.evaluate", {
+        expression: `
+          (function() {
+            var iframes = document.querySelectorAll('iframe');
+            for (var i = 0; i < iframes.length; i++) {
+              var r = iframes[i].getBoundingClientRect();
+              if (r.width > 10 && r.height > 10) {
+                return JSON.stringify({ x: r.x, y: r.y });
+              }
+            }
+            return JSON.stringify({ x: 0, y: 0 });
+          })()
+        `,
+        returnByValue: true,
+      });
+      const offset = JSON.parse(
+        ((offsetResult as CDPEvalResult).result?.value as string) || '{"x":0,"y":0}'
+      );
+
+      // Resolve element inside the frame
+      const frameResult = await this.send("Runtime.evaluate", {
+        expression: `
+          (function() {
+            var el = document.querySelector('[data-assistme-ref="${refId}"]');
+            if (!el && ${roleJS} && ${nameJS}) {
+              // Fallback: search by role
+              var candidates = document.querySelectorAll('*');
+              for (var i = 0; i < candidates.length; i++) {
+                var c = candidates[i];
+                if (c.isContentEditable || c.getAttribute('contenteditable') === 'true') {
+                  el = c; break;
+                }
+              }
+            }
+            if (!el) return 'null';
+
+            el.scrollIntoView({ block: 'center', behavior: 'instant' });
+            var r = el.getBoundingClientRect();
+            if (r.width < 1 || r.height < 1) return JSON.stringify({ error: 'Zero size' });
+
+            return JSON.stringify({
+              x: r.x + r.width / 2,
+              y: r.y + r.height / 2,
+              width: r.width,
+              height: r.height
+            });
+          })()
+        `,
+        contextId,
+        returnByValue: true,
+      });
+
+      const value = (frameResult as CDPEvalResult).result?.value as string;
+      if (!value || value === "null") return null;
+
+      const parsed = JSON.parse(value);
+      if (parsed.error) return parsed;
+
+      // Adjust coordinates by iframe offset
+      return {
+        x: parsed.x + offset.x,
+        y: parsed.y + offset.y,
+        width: parsed.width,
+        height: parsed.height,
+      };
     } catch {
       return null;
     }
@@ -981,11 +1383,24 @@ export class BrowserController {
     await new Promise((r) => setTimeout(r, 50));
 
     // 2. Verify the field is empty; if not, fall back to JS-based clearing
-    const cleared = await this.send("Runtime.evaluate", {
+    // Determine which context to evaluate in (main doc or iframe)
+    const frameContextId = this.frameContexts.get(refId);
+    const clearEvalOpts: Record<string, unknown> = {
       expression: `
         (function() {
           var el = document.querySelector('[data-assistme-ref="${refId}"]');
           if (!el) return 'no_element';
+
+          // For contenteditable elements, check textContent instead of value
+          if (el.isContentEditable || el.getAttribute('contenteditable') === 'true') {
+            if (el.textContent && el.textContent.trim() !== '') {
+              el.textContent = '';
+              el.dispatchEvent(new Event('input', { bubbles: true }));
+              return 'js_cleared';
+            }
+            return 'ok';
+          }
+
           if (el.value !== undefined && el.value !== '') {
             // Ctrl+A didn't work (some frameworks intercept it) — clear via JS
             var setter = Object.getOwnPropertyDescriptor(
@@ -1003,9 +1418,15 @@ export class BrowserController {
         })()
       `,
       returnByValue: true,
-    });
+    };
+    // If element is in a cross-origin iframe, evaluate in its context
+    if (frameContextId) {
+      clearEvalOpts.contextId = frameContextId;
+    }
+    const cleared = await this.send("Runtime.evaluate", clearEvalOpts);
     const clearStatus = ((cleared as CDPEvalResult).result?.value as string) || "ok";
-    if (clearStatus === "no_element") {
+    if (clearStatus === "no_element" && !frameContextId) {
+      // Element not found in main doc and no frame context — truly missing
       return {
         success: false,
         message: `Ref ${refLabel} not found after click. Take a new snapshot.`,

package/src/mcp/browser-server.ts

@@ -92,7 +92,7 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
       ),
       tool(
         "browser_type",
-        "Type text into an input field in the user's browser.",
+        "Type text into an input field in the user's browser. If the CSS selector fails, automatically falls back to typing into the currently focused element. Works with contenteditable elements (rich text editors) and cross-origin iframes.",
         {
           selector: z.string().describe("CSS selector of the input element"),
           text: z.string().describe("Text to type"),