assistme 0.3.2 → 0.3.3

package/dist/index.js

@@ -493,11 +493,20 @@ URL: ${info.url}`;
   }
   async goBack() {
     this.ensureConnected();
-    await this.send("Page.navigateToHistoryEntry", {
-      entryId: -1
-    }).catch(() => {
-    });
-    await this.evaluate("window.history.back()");
+    try {
+      const history = await this.send("Page.getNavigationHistory");
+      const idx = history.currentIndex ?? 0;
+      const entries = history.entries ?? [];
+      if (idx > 0 && entries[idx - 1]) {
+        await this.send("Page.navigateToHistoryEntry", {
+          entryId: entries[idx - 1].id
+        });
+      } else {
+        await this.evaluate("window.history.back()");
+      }
+    } catch {
+      await this.evaluate("window.history.back()");
+    }
     await this.waitForLoad();
     const info = await this.getPageInfo();
     return `Went back to: ${info.title}`;
@@ -665,29 +674,80 @@ URL: ${info.url}`;
       Tab: { keyCode: 9, code: "Tab" },
       Escape: { keyCode: 27, code: "Escape" },
       Backspace: { keyCode: 8, code: "Backspace" },
+      Delete: { keyCode: 46, code: "Delete" },
       ArrowDown: { keyCode: 40, code: "ArrowDown" },
-      ArrowUp: { keyCode: 38, code: "ArrowUp" }
+      ArrowUp: { keyCode: 38, code: "ArrowUp" },
+      ArrowLeft: { keyCode: 37, code: "ArrowLeft" },
+      ArrowRight: { keyCode: 39, code: "ArrowRight" },
+      Home: { keyCode: 36, code: "Home" },
+      End: { keyCode: 35, code: "End" },
+      Space: { keyCode: 32, code: "Space" }
+    };
+    const modifierMap = {
+      Alt: 1,
+      Control: 2,
+      Meta: 4,
+      Shift: 8
     };
-    const mapped = keyMap[key];
+    const parts = key.split("+");
+    let modifiers = 0;
+    let actualKey = parts[parts.length - 1];
+    for (let i = 0; i < parts.length - 1; i++) {
+      const mod = modifierMap[parts[i]];
+      if (mod) modifiers |= mod;
+    }
+    const mapped = keyMap[actualKey];
     if (mapped) {
       await this.send("Input.dispatchKeyEvent", {
         type: "keyDown",
-        key,
+        key: actualKey,
         code: mapped.code,
         windowsVirtualKeyCode: mapped.keyCode,
-        nativeVirtualKeyCode: mapped.keyCode
+        nativeVirtualKeyCode: mapped.keyCode,
+        modifiers
       });
       await this.send("Input.dispatchKeyEvent", {
         type: "keyUp",
-        key,
+        key: actualKey,
         code: mapped.code,
         windowsVirtualKeyCode: mapped.keyCode,
-        nativeVirtualKeyCode: mapped.keyCode
+        nativeVirtualKeyCode: mapped.keyCode,
+        modifiers
+      });
+    } else if (actualKey.length === 1) {
+      const code = `Key${actualKey.toUpperCase()}`;
+      const keyCode = actualKey.toUpperCase().charCodeAt(0);
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyDown",
+        key: actualKey,
+        code,
+        windowsVirtualKeyCode: keyCode,
+        nativeVirtualKeyCode: keyCode,
+        modifiers
+      });
+      if (!modifiers) {
+        await this.send("Input.dispatchKeyEvent", {
+          type: "char",
+          text: actualKey,
+          modifiers
+        });
+      }
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key: actualKey,
+        code,
+        modifiers
       });
     } else {
       await this.send("Input.dispatchKeyEvent", {
-        type: "char",
-        text: key
+        type: "keyDown",
+        key: actualKey,
+        modifiers
+      });
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key: actualKey,
+        modifiers
       });
     }
     return `Pressed key: ${key}`;
@@ -1061,12 +1121,17 @@ Refs:
    */
   async clickRef(refId) {
     this.ensureConnected();
+    const ref = this.refCache.get(refId);
+    const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
     const maxRetries = 3;
     let lastError = "";
     for (let attempt = 0; attempt < maxRetries; attempt++) {
       const resolved = await this.resolveRef(refId);
       if (!resolved) {
-        return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+        return {
+          success: false,
+          message: `Ref ${refLabel} not found. Take a new snapshot with browser_snapshot.`
+        };
       }
       if (resolved.error) {
         lastError = resolved.error;
@@ -1074,8 +1139,7 @@ Refs:
           await new Promise((r) => setTimeout(r, 500));
           continue;
         }
-        const ref3 = this.refCache.get(refId);
-        return `Cannot click [${refId}] ${ref3?.role || ""} "${ref3?.name || ""}": ${lastError}`;
+        return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
       }
       if (attempt === 0) {
         await new Promise((r) => setTimeout(r, 50));
@@ -1105,11 +1169,9 @@ Refs:
         clickCount: 1
       });
       await new Promise((r) => setTimeout(r, 300));
-      const ref2 = this.refCache.get(refId);
-      return `Clicked [${refId}] ${ref2?.role || ""} "${ref2?.name || ""}"`;
+      return { success: true, message: `Clicked ${refLabel}` };
     }
-    const ref = this.refCache.get(refId);
-    return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+    return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
   }
   /**
    * Type text into an element by ref using CDP Input events.
@@ -1118,37 +1180,49 @@ Refs:
    */
   async typeRef(refId, text) {
     this.ensureConnected();
+    const ref = this.refCache.get(refId);
+    const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
     const clickResult = await this.clickRef(refId);
-    if (clickResult.includes("not found")) return clickResult;
+    if (!clickResult.success) return clickResult;
     await new Promise((r) => setTimeout(r, 100));
-    const modifier = platform() === "darwin" ? 4 : 2;
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyDown",
-      modifiers: modifier,
-      key: "a",
-      code: "KeyA",
-      windowsVirtualKeyCode: 65
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyUp",
-      key: "a",
-      code: "KeyA"
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyDown",
-      key: "Backspace",
-      code: "Backspace",
-      windowsVirtualKeyCode: 8
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyUp",
-      key: "Backspace",
-      code: "Backspace"
+    const selectAllKey = platform() === "darwin" ? "Meta+a" : "Control+a";
+    await this.pressKey(selectAllKey);
+    await new Promise((r) => setTimeout(r, 50));
+    await this.pressKey("Backspace");
+    await new Promise((r) => setTimeout(r, 50));
+    const cleared = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          var el = document.querySelector('[data-assistme-ref="${refId}"]');
+          if (!el) return 'no_element';
+          if (el.value !== undefined && el.value !== '') {
+            // Ctrl+A didn't work (some frameworks intercept it) \u2014 clear via JS
+            var setter = Object.getOwnPropertyDescriptor(
+              window.HTMLInputElement.prototype, 'value'
+            )?.set || Object.getOwnPropertyDescriptor(
+              window.HTMLTextAreaElement.prototype, 'value'
+            )?.set;
+            if (setter) setter.call(el, '');
+            else el.value = '';
+            el.dispatchEvent(new Event('input', { bubbles: true }));
+            el.dispatchEvent(new Event('change', { bubbles: true }));
+            return 'js_cleared';
+          }
+          return 'ok';
+        })()
+      `,
+      returnByValue: true
     });
+    const clearStatus = cleared.result?.value || "ok";
+    if (clearStatus === "no_element") {
+      return {
+        success: false,
+        message: `Ref ${refLabel} not found after click. Take a new snapshot.`
+      };
+    }
     await this.send("Input.insertText", { text });
     await new Promise((r) => setTimeout(r, 100));
-    const ref = this.refCache.get(refId);
-    return `Typed "${text}" into [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+    return { success: true, message: `Typed "${text}" into ${refLabel}` };
   }
   /**
    * Select a dropdown option by ref. Delegates to selectOption with the
@@ -1159,13 +1233,16 @@ Refs:
     this.ensureConnected();
     const cached = this.refCache.get(refId);
     if (!cached) {
-      return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+      return {
+        success: false,
+        message: `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`
+      };
     }
+    const refLabel = `[${refId}] ${cached.role} "${cached.name}"`;
     const result = await this.selectOption(`[data-assistme-ref="${refId}"]`, option);
-    return result.replace(
-      /\[data-assistme-ref="\d+"\]/,
-      `[${refId}] ${cached.role} "${cached.name}"`
-    );
+    const message = result.replace(/\[data-assistme-ref="\d+"\]/, refLabel);
+    const success = !result.includes("not found");
+    return { success, message };
   }
   // ── Action Pipeline ───────────────────────────────────────────────
   /**
@@ -1183,18 +1260,24 @@ Refs:
       let success = true;
       try {
         switch (spec.action) {
-          case "click":
-            result = await this.clickRef(spec.ref);
-            success = !result.includes("not found");
+          case "click": {
+            const r = await this.clickRef(spec.ref);
+            result = r.message;
+            success = r.success;
             break;
-          case "type":
-            result = await this.typeRef(spec.ref, spec.text);
-            success = !result.includes("not found");
+          }
+          case "type": {
+            const r = await this.typeRef(spec.ref, spec.text);
+            result = r.message;
+            success = r.success;
             break;
-          case "select":
-            result = await this.selectRef(spec.ref, spec.option);
-            success = !result.includes("not found");
+          }
+          case "select": {
+            const r = await this.selectRef(spec.ref, spec.option);
+            result = r.message;
+            success = r.success;
             break;
+          }
           case "press":
             result = await this.pressKey(spec.key);
             break;
@@ -1269,15 +1352,24 @@ Refs:
           // Strategy 2: Custom dropdown \u2014 find the trigger element
           var trigger = selectEl;
           if (!trigger) {
-            // Try finding by label/placeholder text
-            var allEls = document.querySelectorAll('*');
-            for (var j = 0; j < allEls.length; j++) {
-              var el = allEls[j];
+            // Try finding by aria-label first (fast, indexed)
+            trigger = document.querySelector('[aria-label="' + sel.replace(/"/g, '\\"') + '"]');
+          }
+          if (!trigger) {
+            // Try finding by label/placeholder text in likely dropdown elements
+            var dropdownCandidates = document.querySelectorAll(
+              'button, [role="combobox"], [role="listbox"], [role="button"], ' +
+              'select, input, .MuiSelect-root, .MuiInput-root, ' +
+              '[class*="select"], [class*="dropdown"], [class*="picker"]'
+            );
+            for (var j = 0; j < dropdownCandidates.length; j++) {
+              var el = dropdownCandidates[j];
               var ownText = Array.from(el.childNodes)
                 .filter(function(n) { return n.nodeType === 3; })
                 .map(function(n) { return n.textContent.trim(); })
                 .join('');
-              if (ownText === sel || el.getAttribute('aria-label') === sel) {
+              if (ownText === sel || el.getAttribute('aria-label') === sel ||
+                  el.getAttribute('placeholder') === sel) {
                 trigger = el;
                 break;
               }
@@ -1314,10 +1406,13 @@ Refs:
                 }
               }
 
-              // Broader search: any visible element with exact text match
-              var everything = document.querySelectorAll('*');
-              for (var m = 0; m < everything.length; m++) {
-                var candidate = everything[m];
+              // Broader search: visible leaf elements in interactive containers
+              var broadCandidates = document.querySelectorAll(
+                'li, span, div, a, button, label, [role="option"], [role="menuitem"], ' +
+                '[role="menuitemradio"], [role="menuitemcheckbox"], [data-value]'
+              );
+              for (var m = 0; m < broadCandidates.length; m++) {
+                var candidate = broadCandidates[m];
                 if (candidate.textContent && candidate.textContent.trim() === optText &&
                     candidate.offsetParent !== null && candidate.children.length === 0) {
                   candidate.click();
@@ -1390,6 +1485,7 @@ Refs:
   // ── Helpers ─────────────────────────────────────────────────────
   async waitForLoad(timeoutMs = 8e3) {
     const start = Date.now();
+    let sawInteractive = false;
     while (Date.now() - start < timeoutMs) {
       try {
         const result = await this.send("Runtime.evaluate", {
@@ -1397,67 +1493,22 @@ Refs:
           returnByValue: true
         });
         const state = result.result?.value;
-        if (state === "complete" || state === "interactive") {
-          await new Promise((r) => setTimeout(r, 500));
+        if (state === "complete") {
+          await new Promise((r) => setTimeout(r, 300));
           return;
         }
+        if (state === "interactive") {
+          if (!sawInteractive) {
+            sawInteractive = true;
+          }
+        }
       } catch {
       }
       await new Promise((r) => setTimeout(r, 300));
     }
-  }
-  /**
-   * Find interactive elements on the page for the AI to understand what's clickable
-   */
-  async getInteractiveElements() {
-    this.ensureConnected();
-    const result = await this.send("Runtime.evaluate", {
-      expression: `
-        (function() {
-          const elements = [];
-          const selectors = 'a, button, input, select, textarea, [role="button"], [onclick]';
-          const all = document.querySelectorAll(selectors);
-          for (let i = 0; i < all.length && elements.length < 50; i++) {
-            const el = all[i];
-            const rect = el.getBoundingClientRect();
-            if (rect.width === 0 || rect.height === 0) continue; // Skip hidden
-
-            // Build a reliable CSS selector
-            let selector;
-            if (el.id) {
-              selector = '#' + CSS.escape(el.id);
-            } else if (el.getAttribute('data-testid')) {
-              selector = '[data-testid="' + el.getAttribute('data-testid') + '"]';
-            } else {
-              // Build a path-based selector: find nth-of-type among siblings
-              const tag = el.tagName.toLowerCase();
-              const parent = el.parentElement;
-              if (parent) {
-                const siblings = parent.querySelectorAll(':scope > ' + tag);
-                const idx = Array.from(siblings).indexOf(el) + 1;
-                selector = tag + ':nth-of-type(' + idx + ')';
-              } else {
-                selector = tag;
-              }
-            }
-
-            elements.push({
-              tag: el.tagName.toLowerCase(),
-              text: (el.textContent || '').trim().slice(0, 80),
-              type: el.getAttribute('type') || '',
-              name: el.getAttribute('name') || '',
-              id: el.id || '',
-              href: el.getAttribute('href') || '',
-              placeholder: el.getAttribute('placeholder') || '',
-              selector: selector,
-            });
-          }
-          return JSON.stringify(elements, null, 2);
-        })()
-      `,
-      returnByValue: true
-    });
-    return result.result?.value || "[]";
+    if (sawInteractive) {
+      await new Promise((r) => setTimeout(r, 300));
+    }
   }
   isConnected() {
     return this.connected && this.ws?.readyState === WebSocket.OPEN;
@@ -1796,12 +1847,14 @@ async function ensureBrowserAvailable(port = 9222) {
     detail: "Could not start browser with remote debugging. Possible causes:\n  1) Another assistme debug browser is already using port " + port + "\n  2) The browser crashed on startup\nTry: rm -rf ~/.assistme/browser-profile && assistme"
   };
 }
-var browserInstance = null;
+var browserInstances = /* @__PURE__ */ new Map();
 function getBrowser(port = 9222) {
-  if (!browserInstance) {
-    browserInstance = new BrowserController(port);
+  let instance = browserInstances.get(port);
+  if (!instance) {
+    instance = new BrowserController(port);
+    browserInstances.set(port, instance);
   }
-  return browserInstance;
+  return instance;
 }
 
 // src/commands/browser.ts
@@ -3508,9 +3561,6 @@ async function executeTool(name, input) {
     case "browser_scroll":
       await ensureConnected(browser);
       return input.direction === "up" ? browser.scrollUp() : browser.scrollDown();
-    case "browser_get_elements":
-      await ensureConnected(browser);
-      return browser.getInteractiveElements();
     case "browser_select":
       await ensureConnected(browser);
       return browser.selectOption(input.selector, input.option);
@@ -3691,7 +3741,6 @@ var BROWSER_TOOL_NAMES = [
   "browser_type",
   "browser_press_key",
   "browser_scroll",
-  "browser_get_elements",
   "browser_select",
   "browser_snapshot",
   "browser_act",
@@ -3734,13 +3783,7 @@ function createBrowserMcpServer() {
           const base64 = await executeTool("browser_screenshot", {});
           if (base64.length > 100) {
             return {
-              content: [
-                {
-                  type: "image",
-                  data: base64,
-                  mimeType: "image/png"
-                }
-              ]
+              content: [{ type: "image", data: base64, mimeType: "image/png" }]
             };
           }
           return { content: [{ type: "text", text: base64 }] };
@@ -3773,12 +3816,6 @@ function createBrowserMcpServer() {
         { direction: z.string().describe("'down' or 'up'") },
         async (args) => callTool("browser_scroll", args)
       ),
-      tool(
-        "browser_get_elements",
-        "Find all interactive elements (links, buttons, inputs) on the current page.",
-        {},
-        async () => callTool("browser_get_elements", {})
-      ),
       tool(
         "browser_select",
         "Select an option from a dropdown menu. Handles both native <select> elements and custom dropdowns (Material Design, React, Angular). Use this instead of manually clicking dropdown items.",
@@ -3807,11 +3844,7 @@ function createBrowserMcpServer() {
           const imageData = parts[1] || "";
           const content = [];
           if (imageData.length > 100) {
-            content.push({
-              type: "image",
-              data: imageData,
-              mimeType: "image/png"
-            });
+            content.push({ type: "image", data: imageData, mimeType: "image/png" });
           }
           content.push({ type: "text", text: refTable });
           return { content };
@@ -3847,11 +3880,7 @@ function createBrowserMcpServer() {
           const content = [];
           content.push({ type: "text", text: actionText });
           if (screenshotData.length > 100) {
-            content.push({
-              type: "image",
-              data: screenshotData,
-              mimeType: "image/png"
-            });
+            content.push({ type: "image", data: screenshotData, mimeType: "image/png" });
           }
           return { content };
         }
@@ -5125,7 +5154,7 @@ Available capabilities:
    - Refs persist across actions unless the page navigates. Re-snapshot after navigation or major DOM changes.
 
    **Legacy tools (still available, use when refs don't work):**
-   - browser_click, browser_type, browser_select, browser_get_elements, browser_screenshot, browser_evaluate
+   - browser_click, browser_type, browser_select, browser_screenshot, browser_evaluate
    - browser_click supports :contains('text') pseudo-selectors
    - browser_select handles native and custom dropdowns
 
@@ -5441,7 +5470,9 @@ var TaskProcessor = class {
       } finally {
         clearTimeout(timeoutId);
       }
-      await withRetry(() => completeTask(task.id, finalResponse, tokenUsage), {
+      const MAX_CONTENT_LENGTH = 5e4;
+      const truncatedResponse = finalResponse.length > MAX_CONTENT_LENGTH ? finalResponse.slice(0, MAX_CONTENT_LENGTH) + "\n\n[Response truncated]" : finalResponse;
+      await withRetry(() => completeTask(task.id, truncatedResponse, tokenUsage), {
         maxRetries: 2,
         baseDelayMs: 300,
         label: "completeTask"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "assistme",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "AssistMe CLI Agent - AI-powered assistant that controls your real browser",
   "type": "module",
   "main": "dist/index.js",

package/src/agent/processor.ts

@@ -305,8 +305,15 @@ export class TaskProcessor {
         clearTimeout(timeoutId);
       }
 
+      // Truncate finalResponse to avoid edge function payload limits
+      const MAX_CONTENT_LENGTH = 50_000;
+      const truncatedResponse =
+        finalResponse.length > MAX_CONTENT_LENGTH
+          ? finalResponse.slice(0, MAX_CONTENT_LENGTH) + "\n\n[Response truncated]"
+          : finalResponse;
+
       // Complete the task (with retry for transient DB failures)
-      await withRetry(() => completeTask(task.id, finalResponse, tokenUsage), {
+      await withRetry(() => completeTask(task.id, truncatedResponse, tokenUsage), {
         maxRetries: 2,
         baseDelayMs: 300,
         label: "completeTask",

package/src/agent/system-prompt.ts

@@ -28,7 +28,7 @@ Available capabilities:
    - Refs persist across actions unless the page navigates. Re-snapshot after navigation or major DOM changes.
 
    **Legacy tools (still available, use when refs don't work):**
-   - browser_click, browser_type, browser_select, browser_get_elements, browser_screenshot, browser_evaluate
+   - browser_click, browser_type, browser_select, browser_screenshot, browser_evaluate
    - browser_click supports :contains('text') pseudo-selectors
    - browser_select handles native and custom dropdowns
 

package/src/browser/chrome-launcher.ts

@@ -545,11 +545,13 @@ export async function ensureBrowserAvailable(port = 9222): Promise<AutoLaunchRes
 
 // ── Singleton ───────────────────────────────────────────────────────
 
-let browserInstance: BrowserController | null = null;
+const browserInstances = new Map<number, BrowserController>();
 
 export function getBrowser(port = 9222): BrowserController {
-  if (!browserInstance) {
-    browserInstance = new BrowserController(port);
+  let instance = browserInstances.get(port);
+  if (!instance) {
+    instance = new BrowserController(port);
+    browserInstances.set(port, instance);
   }
-  return browserInstance;
+  return instance;
 }

package/src/browser/controller.ts

@@ -10,6 +10,7 @@ import type {
   SnapshotResult,
   ActionSpec,
   ActionResult,
+  RefActionResult,
 } from "./types.js";
 
 export class BrowserController {
@@ -198,11 +199,26 @@ export class BrowserController {
 
   async goBack(): Promise<string> {
     this.ensureConnected();
-    await this.send("Page.navigateToHistoryEntry", {
-      entryId: -1,
-    }).catch(() => {});
-    // Fallback: use JS
-    await this.evaluate("window.history.back()");
+    try {
+      // Get navigation history and go to the previous entry
+      const history = (await this.send("Page.getNavigationHistory")) as {
+        currentIndex?: number;
+        entries?: Array<{ id: number }>;
+      };
+      const idx = history.currentIndex ?? 0;
+      const entries = history.entries ?? [];
+      if (idx > 0 && entries[idx - 1]) {
+        await this.send("Page.navigateToHistoryEntry", {
+          entryId: entries[idx - 1].id,
+        });
+      } else {
+        // No previous entry in CDP history — use JS fallback
+        await this.evaluate("window.history.back()");
+      }
+    } catch {
+      // CDP history API failed — use JS fallback
+      await this.evaluate("window.history.back()");
+    }
     await this.waitForLoad();
     const info = await this.getPageInfo();
     return `Went back to: ${info.title}`;
@@ -394,31 +410,88 @@ export class BrowserController {
       Tab: { keyCode: 9, code: "Tab" },
       Escape: { keyCode: 27, code: "Escape" },
       Backspace: { keyCode: 8, code: "Backspace" },
+      Delete: { keyCode: 46, code: "Delete" },
       ArrowDown: { keyCode: 40, code: "ArrowDown" },
       ArrowUp: { keyCode: 38, code: "ArrowUp" },
+      ArrowLeft: { keyCode: 37, code: "ArrowLeft" },
+      ArrowRight: { keyCode: 39, code: "ArrowRight" },
+      Home: { keyCode: 36, code: "Home" },
+      End: { keyCode: 35, code: "End" },
+      Space: { keyCode: 32, code: "Space" },
     };
 
-    const mapped = keyMap[key];
+    // CDP modifier bitmask values
+    const modifierMap: Record<string, number> = {
+      Alt: 1,
+      Control: 2,
+      Meta: 4,
+      Shift: 8,
+    };
+
+    // Parse modifier combos like "Control+a", "Meta+Shift+z"
+    const parts = key.split("+");
+    let modifiers = 0;
+    let actualKey = parts[parts.length - 1];
+    for (let i = 0; i < parts.length - 1; i++) {
+      const mod = modifierMap[parts[i]];
+      if (mod) modifiers |= mod;
+    }
+
+    const mapped = keyMap[actualKey];
     if (mapped) {
       await this.send("Input.dispatchKeyEvent", {
         type: "keyDown",
-        key,
+        key: actualKey,
         code: mapped.code,
         windowsVirtualKeyCode: mapped.keyCode,
         nativeVirtualKeyCode: mapped.keyCode,
+        modifiers,
       });
       await this.send("Input.dispatchKeyEvent", {
         type: "keyUp",
-        key,
+        key: actualKey,
         code: mapped.code,
         windowsVirtualKeyCode: mapped.keyCode,
         nativeVirtualKeyCode: mapped.keyCode,
+        modifiers,
+      });
+    } else if (actualKey.length === 1) {
+      // Single character key (e.g., "a", "z")
+      const code = `Key${actualKey.toUpperCase()}`;
+      const keyCode = actualKey.toUpperCase().charCodeAt(0);
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyDown",
+        key: actualKey,
+        code,
+        windowsVirtualKeyCode: keyCode,
+        nativeVirtualKeyCode: keyCode,
+        modifiers,
+      });
+      if (!modifiers) {
+        // Only insert text for unmodified single characters
+        await this.send("Input.dispatchKeyEvent", {
+          type: "char",
+          text: actualKey,
+          modifiers,
+        });
+      }
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key: actualKey,
+        code,
+        modifiers,
       });
     } else {
-      // Single character key
+      // Unknown key name — try as-is
       await this.send("Input.dispatchKeyEvent", {
-        type: "char",
-        text: key,
+        type: "keyDown",
+        key: actualKey,
+        modifiers,
+      });
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key: actualKey,
+        modifiers,
       });
     }
 
@@ -816,8 +889,10 @@ export class BrowserController {
    * element is not yet actionable (e.g., covered by a loading overlay, still
    * animating into view). This matches Playwright's auto-waiting behavior.
    */
-  async clickRef(refId: number): Promise<string> {
+  async clickRef(refId: number): Promise<RefActionResult> {
     this.ensureConnected();
+    const ref = this.refCache.get(refId);
+    const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
 
     // Auto-wait: retry up to 3 times if element is not actionable yet
     const maxRetries = 3;
@@ -827,7 +902,10 @@ export class BrowserController {
       const resolved = await this.resolveRef(refId);
 
       if (!resolved) {
-        return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+        return {
+          success: false,
+          message: `Ref ${refLabel} not found. Take a new snapshot with browser_snapshot.`,
+        };
       }
 
       if (resolved.error) {
@@ -837,9 +915,7 @@ export class BrowserController {
           await new Promise((r) => setTimeout(r, 500));
           continue;
         }
-        // Final attempt failed — report the actionability issue
-        const ref = this.refCache.get(refId);
-        return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+        return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
       }
 
       // Element is actionable — small delay after scroll for rendering
@@ -875,13 +951,10 @@ export class BrowserController {
       });
 
       await new Promise((r) => setTimeout(r, 300));
-      const ref = this.refCache.get(refId);
-      return `Clicked [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+      return { success: true, message: `Clicked ${refLabel}` };
     }
 
-    // Should not reach here, but just in case
-    const ref = this.refCache.get(refId);
-    return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+    return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
   }
 
   /**
@@ -889,48 +962,61 @@ export class BrowserController {
    * Clicks to focus, selects all existing text (Ctrl/Cmd+A), then uses
    * Input.insertText for reliable text insertion across all frameworks.
    */
-  async typeRef(refId: number, text: string): Promise<string> {
+  async typeRef(refId: number, text: string): Promise<RefActionResult> {
     this.ensureConnected();
+    const ref = this.refCache.get(refId);
+    const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
 
     // Click to focus the element
     const clickResult = await this.clickRef(refId);
-    if (clickResult.includes("not found")) return clickResult;
+    if (!clickResult.success) return clickResult;
     await new Promise((r) => setTimeout(r, 100));
 
-    // Select all existing text (Cmd+A on macOS, Ctrl+A elsewhere)
-    const modifier = platform() === "darwin" ? 4 : 2;
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyDown",
-      modifiers: modifier,
-      key: "a",
-      code: "KeyA",
-      windowsVirtualKeyCode: 65,
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyUp",
-      key: "a",
-      code: "KeyA",
-    });
+    // Clear existing text using multiple strategies for reliability:
+    // 1. Try Ctrl/Cmd+A to select all, then Backspace to delete
+    const selectAllKey = platform() === "darwin" ? "Meta+a" : "Control+a";
+    await this.pressKey(selectAllKey);
+    await new Promise((r) => setTimeout(r, 50));
+    await this.pressKey("Backspace");
+    await new Promise((r) => setTimeout(r, 50));
 
-    // Delete selected text
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyDown",
-      key: "Backspace",
-      code: "Backspace",
-      windowsVirtualKeyCode: 8,
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyUp",
-      key: "Backspace",
-      code: "Backspace",
+    // 2. Verify the field is empty; if not, fall back to JS-based clearing
+    const cleared = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          var el = document.querySelector('[data-assistme-ref="${refId}"]');
+          if (!el) return 'no_element';
+          if (el.value !== undefined && el.value !== '') {
+            // Ctrl+A didn't work (some frameworks intercept it) — clear via JS
+            var setter = Object.getOwnPropertyDescriptor(
+              window.HTMLInputElement.prototype, 'value'
+            )?.set || Object.getOwnPropertyDescriptor(
+              window.HTMLTextAreaElement.prototype, 'value'
+            )?.set;
+            if (setter) setter.call(el, '');
+            else el.value = '';
+            el.dispatchEvent(new Event('input', { bubbles: true }));
+            el.dispatchEvent(new Event('change', { bubbles: true }));
+            return 'js_cleared';
+          }
+          return 'ok';
+        })()
+      `,
+      returnByValue: true,
     });
+    const clearStatus = ((cleared as CDPEvalResult).result?.value as string) || "ok";
+    if (clearStatus === "no_element") {
+      return {
+        success: false,
+        message: `Ref ${refLabel} not found after click. Take a new snapshot.`,
+      };
+    }
 
     // Insert text via CDP (goes through the browser's input pipeline)
     await this.send("Input.insertText", { text });
 
     await new Promise((r) => setTimeout(r, 100));
-    const ref = this.refCache.get(refId);
-    return `Typed "${text}" into [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+    return { success: true, message: `Typed "${text}" into ${refLabel}` };
   }
 
   /**
@@ -938,21 +1024,22 @@ export class BrowserController {
    * ref's data attribute as selector, handling both native <select> and
    * custom dropdown components.
    */
-  async selectRef(refId: number, option: string): Promise<string> {
+  async selectRef(refId: number, option: string): Promise<RefActionResult> {
     this.ensureConnected();
 
-    // Check if ref exists
     const cached = this.refCache.get(refId);
     if (!cached) {
-      return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+      return {
+        success: false,
+        message: `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`,
+      };
     }
 
-    // Use the data attribute selector to find the element
+    const refLabel = `[${refId}] ${cached.role} "${cached.name}"`;
     const result = await this.selectOption(`[data-assistme-ref="${refId}"]`, option);
-    return result.replace(
-      /\[data-assistme-ref="\d+"\]/,
-      `[${refId}] ${cached.role} "${cached.name}"`
-    );
+    const message = result.replace(/\[data-assistme-ref="\d+"\]/, refLabel);
+    const success = !result.includes("not found");
+    return { success, message };
   }
 
   // ── Action Pipeline ───────────────────────────────────────────────
@@ -977,18 +1064,24 @@ export class BrowserController {
 
       try {
         switch (spec.action) {
-          case "click":
-            result = await this.clickRef(spec.ref);
-            success = !result.includes("not found");
+          case "click": {
+            const r = await this.clickRef(spec.ref);
+            result = r.message;
+            success = r.success;
             break;
-          case "type":
-            result = await this.typeRef(spec.ref, spec.text);
-            success = !result.includes("not found");
+          }
+          case "type": {
+            const r = await this.typeRef(spec.ref, spec.text);
+            result = r.message;
+            success = r.success;
             break;
-          case "select":
-            result = await this.selectRef(spec.ref, spec.option);
-            success = !result.includes("not found");
+          }
+          case "select": {
+            const r = await this.selectRef(spec.ref, spec.option);
+            result = r.message;
+            success = r.success;
             break;
+          }
           case "press":
             result = await this.pressKey(spec.key);
             break;
@@ -1074,15 +1167,24 @@ export class BrowserController {
           // Strategy 2: Custom dropdown — find the trigger element
           var trigger = selectEl;
           if (!trigger) {
-            // Try finding by label/placeholder text
-            var allEls = document.querySelectorAll('*');
-            for (var j = 0; j < allEls.length; j++) {
-              var el = allEls[j];
+            // Try finding by aria-label first (fast, indexed)
+            trigger = document.querySelector('[aria-label="' + sel.replace(/"/g, '\\"') + '"]');
+          }
+          if (!trigger) {
+            // Try finding by label/placeholder text in likely dropdown elements
+            var dropdownCandidates = document.querySelectorAll(
+              'button, [role="combobox"], [role="listbox"], [role="button"], ' +
+              'select, input, .MuiSelect-root, .MuiInput-root, ' +
+              '[class*="select"], [class*="dropdown"], [class*="picker"]'
+            );
+            for (var j = 0; j < dropdownCandidates.length; j++) {
+              var el = dropdownCandidates[j];
               var ownText = Array.from(el.childNodes)
                 .filter(function(n) { return n.nodeType === 3; })
                 .map(function(n) { return n.textContent.trim(); })
                 .join('');
-              if (ownText === sel || el.getAttribute('aria-label') === sel) {
+              if (ownText === sel || el.getAttribute('aria-label') === sel ||
+                  el.getAttribute('placeholder') === sel) {
                 trigger = el;
                 break;
               }
@@ -1119,10 +1221,13 @@ export class BrowserController {
                 }
               }
 
-              // Broader search: any visible element with exact text match
-              var everything = document.querySelectorAll('*');
-              for (var m = 0; m < everything.length; m++) {
-                var candidate = everything[m];
+              // Broader search: visible leaf elements in interactive containers
+              var broadCandidates = document.querySelectorAll(
+                'li, span, div, a, button, label, [role="option"], [role="menuitem"], ' +
+                '[role="menuitemradio"], [role="menuitemcheckbox"], [data-value]'
+              );
+              for (var m = 0; m < broadCandidates.length; m++) {
+                var candidate = broadCandidates[m];
                 if (candidate.textContent && candidate.textContent.trim() === optText &&
                     candidate.offsetParent !== null && candidate.children.length === 0) {
                   candidate.click();
@@ -1217,6 +1322,7 @@ export class BrowserController {
 
   private async waitForLoad(timeoutMs = 8000): Promise<void> {
     const start = Date.now();
+    let sawInteractive = false;
     while (Date.now() - start < timeoutMs) {
       try {
         const result = await this.send("Runtime.evaluate", {
@@ -1224,71 +1330,27 @@ export class BrowserController {
           returnByValue: true,
         });
         const state = (result as CDPEvalResult).result?.value;
-        if (state === "complete" || state === "interactive") {
-          // Extra small wait for dynamic content
-          await new Promise((r) => setTimeout(r, 500));
+        if (state === "complete") {
+          // Fully loaded — brief wait for dynamic content
+          await new Promise((r) => setTimeout(r, 300));
           return;
         }
+        if (state === "interactive") {
+          if (!sawInteractive) {
+            sawInteractive = true;
+            // DOM is ready but sub-resources still loading — give it more
+            // time to reach "complete" before settling for "interactive"
+          }
+        }
       } catch {
         // Tab might be navigating
       }
       await new Promise((r) => setTimeout(r, 300));
     }
-  }
-
-  /**
-   * Find interactive elements on the page for the AI to understand what's clickable
-   */
-  async getInteractiveElements(): Promise<string> {
-    this.ensureConnected();
-    const result = await this.send("Runtime.evaluate", {
-      expression: `
-        (function() {
-          const elements = [];
-          const selectors = 'a, button, input, select, textarea, [role="button"], [onclick]';
-          const all = document.querySelectorAll(selectors);
-          for (let i = 0; i < all.length && elements.length < 50; i++) {
-            const el = all[i];
-            const rect = el.getBoundingClientRect();
-            if (rect.width === 0 || rect.height === 0) continue; // Skip hidden
-
-            // Build a reliable CSS selector
-            let selector;
-            if (el.id) {
-              selector = '#' + CSS.escape(el.id);
-            } else if (el.getAttribute('data-testid')) {
-              selector = '[data-testid="' + el.getAttribute('data-testid') + '"]';
-            } else {
-              // Build a path-based selector: find nth-of-type among siblings
-              const tag = el.tagName.toLowerCase();
-              const parent = el.parentElement;
-              if (parent) {
-                const siblings = parent.querySelectorAll(':scope > ' + tag);
-                const idx = Array.from(siblings).indexOf(el) + 1;
-                selector = tag + ':nth-of-type(' + idx + ')';
-              } else {
-                selector = tag;
-              }
-            }
-
-            elements.push({
-              tag: el.tagName.toLowerCase(),
-              text: (el.textContent || '').trim().slice(0, 80),
-              type: el.getAttribute('type') || '',
-              name: el.getAttribute('name') || '',
-              id: el.id || '',
-              href: el.getAttribute('href') || '',
-              placeholder: el.getAttribute('placeholder') || '',
-              selector: selector,
-            });
-          }
-          return JSON.stringify(elements, null, 2);
-        })()
-      `,
-      returnByValue: true,
-    });
-
-    return ((result as CDPEvalResult).result?.value as string) || "[]";
+    // Timed out — if we at least saw "interactive", that's usually good enough
+    if (sawInteractive) {
+      await new Promise((r) => setTimeout(r, 300));
+    }
   }
 
   isConnected(): boolean {

package/src/browser/types.ts

@@ -62,6 +62,12 @@ export interface ActionResult {
   success: boolean;
 }
 
+/** Structured result from ref-based interactions (click, type, select). */
+export interface RefActionResult {
+  success: boolean;
+  message: string;
+}
+
 export interface AutoLaunchResult {
   success: boolean;
   action: "already_available" | "launched" | "chrome_not_found" | "launch_failed" | "port_conflict";

package/src/mcp/browser-server.ts

@@ -9,11 +9,15 @@ import { getLimiterForTool } from "../utils/rate-limiter.js";
 
 // ── Helper ──────────────────────────────────────────────────────────
 
+/** MCP content block — text or image. */
+type ContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image"; data: string; mimeType: string };
+
+type ToolResult = { content: ContentBlock[] };
+
 /** Wrap executeTool with rate limiting and text result. */
-async function callTool(
-  name: string,
-  input: Record<string, unknown>
-): Promise<{ content: Array<{ type: "text"; text: string }> }> {
+async function callTool(name: string, input: Record<string, unknown>): Promise<ToolResult> {
   const limiter = getLimiterForTool(name);
   if (limiter) await limiter.acquire();
   const result = await executeTool(name, input);
@@ -31,7 +35,6 @@ export const BROWSER_TOOL_NAMES = [
   "browser_type",
   "browser_press_key",
   "browser_scroll",
-  "browser_get_elements",
   "browser_select",
   "browser_snapshot",
   "browser_act",
@@ -69,19 +72,13 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
         "browser_screenshot",
         "Take a screenshot of the current browser page. Returns a base64-encoded PNG image.",
         {},
-        async () => {
+        async (): Promise<ToolResult> => {
           const limiter = getLimiterForTool("browser_screenshot");
           if (limiter) await limiter.acquire();
           const base64 = await executeTool("browser_screenshot", {});
           if (base64.length > 100) {
             return {
-              content: [
-                {
-                  type: "image" as const,
-                  data: base64,
-                  mimeType: "image/png",
-                } as unknown as { type: "text"; text: string },
-              ],
+              content: [{ type: "image", data: base64, mimeType: "image/png" }],
             };
           }
           return { content: [{ type: "text", text: base64 }] };
@@ -114,12 +111,6 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
         { direction: z.string().describe("'down' or 'up'") },
         async (args) => callTool("browser_scroll", args)
       ),
-      tool(
-        "browser_get_elements",
-        "Find all interactive elements (links, buttons, inputs) on the current page.",
-        {},
-        async () => callTool("browser_get_elements", {})
-      ),
       tool(
         "browser_select",
         "Select an option from a dropdown menu. Handles both native <select> elements and custom dropdowns (Material Design, React, Angular). Use this instead of manually clicking dropdown items.",
@@ -149,7 +140,7 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
               "Overlay ref badges on the screenshot. Default false. Use true for simple pages where visual context helps."
             ),
         },
-        async (args) => {
+        async (args): Promise<ToolResult> => {
           const limiter = getLimiterForTool("browser_snapshot");
           if (limiter) await limiter.acquire();
           const result = await executeTool("browser_snapshot", args);
@@ -159,13 +150,9 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
           const refTable = parts[0];
           const imageData = parts[1] || "";
 
-          const content: Array<{ type: "text"; text: string }> = [];
+          const content: ContentBlock[] = [];
           if (imageData.length > 100) {
-            content.push({
-              type: "image" as const,
-              data: imageData,
-              mimeType: "image/png",
-            } as unknown as { type: "text"; text: string });
+            content.push({ type: "image", data: imageData, mimeType: "image/png" });
           }
           content.push({ type: "text", text: refTable });
 
@@ -197,7 +184,7 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
             .optional()
             .describe("Take screenshot after actions (default: false)"),
         },
-        async (args) => {
+        async (args): Promise<ToolResult> => {
           const limiter = getLimiterForTool("browser_act");
           if (limiter) await limiter.acquire();
           const result = await executeTool("browser_act", {
@@ -210,14 +197,10 @@ export function createBrowserMcpServer(): McpSdkServerConfigWithInstance {
           const actionText = parts[0];
           const screenshotData = parts[1] || "";
 
-          const content: Array<{ type: "text"; text: string }> = [];
+          const content: ContentBlock[] = [];
           content.push({ type: "text", text: actionText });
           if (screenshotData.length > 100) {
-            content.push({
-              type: "image" as const,
-              data: screenshotData,
-              mimeType: "image/png",
-            } as unknown as { type: "text"; text: string });
+            content.push({ type: "image", data: screenshotData, mimeType: "image/png" });
           }
 
           return { content };

package/src/tools/browser.ts

@@ -14,6 +14,7 @@ export type {
   SnapshotResult,
   ActionSpec,
   ActionResult,
+  RefActionResult,
   AutoLaunchResult,
 } from "../browser/types.js";
 

package/src/tools/index.ts

@@ -169,9 +169,6 @@ export async function executeTool(name: string, input: Record<string, unknown>):
     case "browser_scroll":
       await ensureConnected(browser);
       return (input.direction as string) === "up" ? browser.scrollUp() : browser.scrollDown();
-    case "browser_get_elements":
-      await ensureConnected(browser);
-      return browser.getInteractiveElements();
     case "browser_select":
       await ensureConnected(browser);
       return browser.selectOption(input.selector as string, input.option as string);