assistme 0.3.3 → 0.3.5

package/src/agent/processor.ts

@@ -31,6 +31,55 @@ import {
 import { createEventHooks } from "./event-hooks.js";
 import { BASE_SYSTEM_PROMPT } from "./system-prompt.js";
 
+/**
+ * Manages the task wall-clock timeout.
+ * Supports pausing while the agent is waiting for user input (ask_user)
+ * so that idle wait time doesn't count toward the timeout.
+ */
+class TaskTimeout {
+  private timeoutId: ReturnType<typeof setTimeout> | null = null;
+  private remainingMs: number;
+  private resumedAt: number;
+
+  constructor(
+    private abortController: AbortController,
+    timeoutMs: number
+  ) {
+    this.remainingMs = timeoutMs;
+    this.resumedAt = Date.now();
+    this.schedule();
+  }
+
+  private schedule(): void {
+    this.timeoutId = setTimeout(() => {
+      this.abortController.abort();
+    }, this.remainingMs);
+  }
+
+  /** Pause the timeout (e.g. while waiting for user). */
+  pause(): void {
+    if (this.timeoutId) {
+      clearTimeout(this.timeoutId);
+      this.timeoutId = null;
+      const elapsed = Date.now() - this.resumedAt;
+      this.remainingMs = Math.max(0, this.remainingMs - elapsed);
+    }
+  }
+
+  /** Resume the timeout after user interaction completes. */
+  resume(): void {
+    this.resumedAt = Date.now();
+    this.schedule();
+  }
+
+  clear(): void {
+    if (this.timeoutId) {
+      clearTimeout(this.timeoutId);
+      this.timeoutId = null;
+    }
+  }
+}
+
 const MAX_HISTORY_ENTRIES = 10;
 const MAX_RESPONSE_LENGTH = 1500;
 
@@ -143,6 +192,9 @@ export class TaskProcessor {
         systemPrompt += historyPrompt;
       }
 
+      const abortController = new AbortController();
+      const taskTimeout = new TaskTimeout(abortController, taskTimeoutMs);
+
       // Create MCP servers for custom tools
       const browserServer = createBrowserMcpServer();
       const agentToolsServer = createAgentToolsServer({
@@ -150,6 +202,8 @@ export class TaskProcessor {
         skillManager: this.skillManager,
         taskId: task.id,
         sessionId: this.sessionId || undefined,
+        onUserWaitStart: () => taskTimeout.pause(),
+        onUserWaitEnd: () => taskTimeout.resume(),
       });
 
       // Create event hooks for Supabase event emission
@@ -203,7 +257,6 @@ export class TaskProcessor {
         };
       }
 
-      const abortController = new AbortController();
       const options: Options = {
         model: config.model,
         systemPrompt,
@@ -221,11 +274,7 @@ export class TaskProcessor {
         abortController,
       };
 
-      // Wall-clock timeout via abort
       const taskStartTime = Date.now();
-      const timeoutId = setTimeout(() => {
-        abortController.abort();
-      }, taskTimeoutMs);
 
       try {
         for await (const message of query({
@@ -302,7 +351,7 @@ export class TaskProcessor {
           }
         }
       } finally {
-        clearTimeout(timeoutId);
+        taskTimeout.clear();
       }
 
       // Truncate finalResponse to avoid edge function payload limits

package/src/agent/system-prompt.ts

@@ -41,12 +41,16 @@ Available capabilities:
    - Bash tool for shell commands
    - Glob and Grep for file search
 
-3. MEMORY:
+3. MEMORY & CREDENTIALS:
    - You can remember things about the user using memory_store
    - Use this when you learn preferences, important facts, or standing instructions
    - Your stored memories persist across conversations
    - PROACTIVELY use memory_store during tasks when you discover user preferences, habits, or important context
    - Before completing a task, consider if anything learned should be remembered for future conversations
+   - CRITICAL — Credential Storage: When you create, register, or receive any account credentials (username, password, API keys, tokens), you MUST use credential_set to save them locally. NEVER use memory_store for credentials — memory_store is for preferences and facts, credential_set is for secrets. Examples:
+     * After registering a new email/account → credential_set with type "login" and data { "username": "...", "password": "...", "email": "..." }
+     * After generating an API key → credential_set with type "api_key" and data { "api_key": "..." }
+     * Credentials saved via credential_set are encrypted on disk and viewable in the desktop app's Credentials panel
 
 4. SKILL-AWARE EXECUTION (CRITICAL — follow this for EVERY task):
    Step A — Search: Before executing ANY task, check if an existing skill matches (use skill_invoke or skill_search).

package/src/browser/controller.ts

@@ -21,6 +21,7 @@ export class BrowserController {
   private connected = false;
   private currentTabId: string | null = null;
   private refCache: Map<number, RefEntry> = new Map();
+  private frameContexts: Map<number, number> = new Map(); // refId → contextId
 
   constructor(port = 9222) {
     this.debugPort = port;
@@ -367,17 +368,50 @@ export class BrowserController {
     const selectorJS = JSON.stringify(selector);
     const textJS = JSON.stringify(text);
 
-    // First clear and set value via JS, dispatching all relevant events
+    // First try to find the element in main document, then in same-origin iframes
     const result = await this.send("Runtime.evaluate", {
       expression: `
         (function() {
-          const el = document.querySelector(${selectorJS});
+          var el = document.querySelector(${selectorJS});
+
+          // If not found in main document, search same-origin iframes
+          if (!el) {
+            var iframes = document.querySelectorAll('iframe');
+            for (var i = 0; i < iframes.length; i++) {
+              try {
+                var iframeDoc = iframes[i].contentDocument;
+                if (iframeDoc) {
+                  el = iframeDoc.querySelector(${selectorJS});
+                  if (el) break;
+                }
+              } catch(e) { /* cross-origin, skip */ }
+            }
+          }
+
           if (!el) return 'Element not found: ' + ${selectorJS};
 
           el.focus();
 
-          // Clear existing value
-          const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
+          // Check if this is a contenteditable element (rich text editor)
+          var isContentEditable = el.isContentEditable ||
+            el.getAttribute('contenteditable') === 'true' ||
+            el.getAttribute('contenteditable') === '';
+
+          if (isContentEditable) {
+            // For contenteditable: select all content, then replace
+            var ownerDoc = el.ownerDocument;
+            var sel = ownerDoc.defaultView.getSelection();
+            var range = ownerDoc.createRange();
+            range.selectNodeContents(el);
+            sel.removeAllRanges();
+            sel.addRange(range);
+            // Use insertText command which respects undo stack and triggers input events
+            ownerDoc.execCommand('insertText', false, ${textJS});
+            return 'Typed into: ' + (el.tagName || '') + ' [contenteditable]';
+          }
+
+          // For input/textarea: clear and set value
+          var nativeInputValueSetter = Object.getOwnPropertyDescriptor(
             window.HTMLInputElement.prototype, 'value'
           )?.set || Object.getOwnPropertyDescriptor(
             window.HTMLTextAreaElement.prototype, 'value'
@@ -398,7 +432,36 @@ export class BrowserController {
       returnByValue: true,
     });
 
-    return ((result as CDPEvalResult).result?.value as string) || "Text entered.";
+    const textResult = ((result as CDPEvalResult).result?.value as string) || "";
+
+    // If element still not found, try typing into the currently focused element via CDP
+    if (textResult.startsWith("Element not found")) {
+      return this.typeAtFocus(text);
+    }
+
+    return textResult || "Text entered.";
+  }
+
+  /**
+   * Type text into the currently focused element using CDP Input.insertText.
+   * This bypasses DOM queries entirely and works with any focused element,
+   * including those inside cross-origin iframes or shadow DOM.
+   */
+  async typeAtFocus(text: string): Promise<string> {
+    this.ensureConnected();
+
+    // Optionally clear existing content: select all then delete
+    const modKey = platform() === "darwin" ? "Meta" : "Control";
+    await this.pressKey(`${modKey}+a`);
+    await new Promise((r) => setTimeout(r, 50));
+    await this.pressKey("Backspace");
+    await new Promise((r) => setTimeout(r, 50));
+
+    // Insert text via CDP — goes through the browser's native input pipeline
+    await this.send("Input.insertText", { text });
+    await new Promise((r) => setTimeout(r, 100));
+
+    return "Text entered (into focused element).";
   }
 
   async pressKey(key: string): Promise<string> {
@@ -670,6 +733,9 @@ export class BrowserController {
       box: r.box as BoundingBox,
     }));
 
+    // 1b. Discover elements in cross-origin iframes via CDP frame targeting
+    await this.discoverCrossOriginFrameRefs(refs);
+
     // 2. Optionally inject visual overlay with ref labels
     //    (Skip for dense pages — labels would overlap and become unreadable)
     if (annotate && refs.length <= 40) {
@@ -725,6 +791,8 @@ export class BrowserController {
     }
 
     // 5. Cache refs for subsequent act() calls
+    // Note: frameContexts is populated by discoverCrossOriginFrameRefs above,
+    // so we only clear refCache here (frameContexts was cleared at start of discover)
     this.refCache.clear();
     for (const ref of refs) {
       this.refCache.set(ref.id, ref);
@@ -752,6 +820,235 @@ export class BrowserController {
     return table;
   }
 
+  // ── Cross-Origin Iframe Discovery ────────────────────────────────
+
+  /**
+   * Use CDP's Page.getFrameTree + Runtime.evaluate with contextId to discover
+   * interactive elements inside cross-origin iframes (e.g., ProtonMail editor,
+   * Google Docs, embedded rich text editors).
+   *
+   * Same-origin iframes are already handled inline by the main snapshot JS.
+   * This method handles the ones that threw cross-origin errors.
+   */
+  private async discoverCrossOriginFrameRefs(refs: RefEntry[]): Promise<void> {
+    this.frameContexts.clear();
+    try {
+      // Get the frame tree to find all child frames
+      const frameTree = (await this.send("Page.getFrameTree")) as {
+        frameTree?: {
+          frame: { id: string };
+          childFrames?: Array<{ frame: { id: string; url: string; name?: string } }>;
+        };
+      };
+
+      const mainFrameId = frameTree.frameTree?.frame?.id;
+      const childFrames = frameTree.frameTree?.childFrames || [];
+      if (childFrames.length === 0) return;
+
+      // Get all execution contexts to map frameId → contextId
+      // We need to enable Runtime events and collect contexts
+      const contexts = await this.getFrameContexts(mainFrameId || "");
+
+      for (const child of childFrames) {
+        const frameId = child.frame.id;
+        const contextId = contexts.get(frameId);
+        if (!contextId) continue;
+
+        // Get the iframe's bounding rect from the parent frame for coordinate offset
+        const iframeOffsetResult = await this.send("Runtime.evaluate", {
+          expression: `
+            (function() {
+              var iframes = document.querySelectorAll('iframe');
+              for (var i = 0; i < iframes.length; i++) {
+                try {
+                  // Match by frame src or name
+                  var f = iframes[i];
+                  if (f.contentWindow) {
+                    var r = f.getBoundingClientRect();
+                    if (r.width > 10 && r.height > 10) {
+                      return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, index: i });
+                    }
+                  }
+                } catch(e) {}
+              }
+              return 'null';
+            })()
+          `,
+          returnByValue: true,
+        });
+
+        let iframeOffset = { x: 0, y: 0 };
+        try {
+          const parsed = JSON.parse(
+            ((iframeOffsetResult as CDPEvalResult).result?.value as string) || "null"
+          );
+          if (parsed) iframeOffset = { x: parsed.x, y: parsed.y };
+        } catch {
+          /* ignore */
+        }
+
+        // Evaluate inside the child frame's execution context
+        const startRefId = refs.length + 1;
+        try {
+          const frameResult = await this.send("Runtime.evaluate", {
+            expression: `
+              (function() {
+                var selectors = [
+                  'a[href]', 'button', 'input:not([type="hidden"])', 'select', 'textarea',
+                  '[role="button"]', '[role="link"]', '[role="checkbox"]', '[role="radio"]',
+                  '[role="combobox"]', '[role="listbox"]', '[role="menuitem"]', '[role="tab"]',
+                  '[role="switch"]', '[role="slider"]', '[role="option"]', '[role="searchbox"]',
+                  '[onclick]', '[tabindex]:not([tabindex="-1"])',
+                  '[contenteditable="true"]', '[contenteditable=""]'
+                ].join(', ');
+
+                var all = document.querySelectorAll(selectors);
+                // Also check if the body itself is contenteditable
+                if (document.body && (document.body.isContentEditable || document.body.getAttribute('contenteditable') === 'true')) {
+                  all = [document.body].concat(Array.from(all));
+                }
+
+                var refs = [];
+                var startId = ${startRefId};
+                var vh = window.innerHeight;
+                var vw = window.innerWidth;
+
+                for (var i = 0; i < all.length && refs.length < 20; i++) {
+                  var el = all[i];
+                  var rect = el.getBoundingClientRect();
+                  if (rect.width < 5 || rect.height < 5) continue;
+                  var style = window.getComputedStyle(el);
+                  if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') continue;
+
+                  var role = el.getAttribute('role') || '';
+                  if (!role) {
+                    var tag = el.tagName.toLowerCase();
+                    if (tag === 'a') role = 'link';
+                    else if (tag === 'button') role = 'button';
+                    else if (tag === 'input') {
+                      var t = (el.type || 'text').toLowerCase();
+                      if (t === 'checkbox') role = 'checkbox';
+                      else if (t === 'radio') role = 'radio';
+                      else if (t === 'submit' || t === 'button') role = 'button';
+                      else role = 'textbox';
+                    }
+                    else if (tag === 'select') role = 'combobox';
+                    else if (tag === 'textarea') role = 'textbox';
+                    else if (el.isContentEditable) role = 'textbox';
+                    else role = tag;
+                  }
+
+                  var name = '';
+                  var ariaLabel = el.getAttribute('aria-label');
+                  if (ariaLabel) {
+                    name = ariaLabel;
+                  } else if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
+                    name = el.getAttribute('placeholder') || el.getAttribute('name') || '';
+                  } else if (el.isContentEditable) {
+                    name = 'compose body';
+                  } else {
+                    name = (el.textContent || '').trim().slice(0, 60);
+                  }
+
+                  var refId = startId + refs.length;
+                  el.setAttribute('data-assistme-ref', String(refId));
+
+                  refs.push({
+                    id: refId,
+                    role: role,
+                    name: name,
+                    tag: el.tagName.toLowerCase(),
+                    type: el.getAttribute('type') || '',
+                    box: {
+                      x: Math.round(rect.x),
+                      y: Math.round(rect.y),
+                      width: Math.round(rect.width),
+                      height: Math.round(rect.height)
+                    },
+                    inFrame: true
+                  });
+                }
+
+                return JSON.stringify(refs);
+              })()
+            `,
+            contextId,
+            returnByValue: true,
+          });
+
+          const frameRefs = JSON.parse(
+            ((frameResult as CDPEvalResult).result?.value as string) || "[]"
+          );
+
+          for (const r of frameRefs) {
+            refs.push({
+              id: r.id as number,
+              role: r.role as string,
+              name: r.name as string,
+              tag: r.tag as string,
+              inputType: (r.type as string) || "",
+              box: {
+                x: Math.round((r.box.x as number) + iframeOffset.x),
+                y: Math.round((r.box.y as number) + iframeOffset.y),
+                width: r.box.width as number,
+                height: r.box.height as number,
+              },
+            });
+            // Store frame context for later resolution
+            this.frameContexts.set(r.id as number, contextId);
+          }
+        } catch {
+          // Frame evaluation failed (e.g., about:blank, pdf viewer) — skip
+        }
+      }
+    } catch {
+      // Frame tree unavailable — not critical, skip silently
+    }
+  }
+
+  /**
+   * Get execution context IDs for each frame in the page.
+   * Uses Runtime.executionContextCreated events collected during the session,
+   * or falls back to evaluating in known frames.
+   */
+  private async getFrameContexts(_mainFrameId: string): Promise<Map<string, number>> {
+    const contexts = new Map<string, number>();
+    try {
+      // Enable Runtime domain to get context descriptions (may already be enabled)
+      await this.send("Runtime.enable").catch(() => {});
+
+      // Use Page.getFrameTree to get frame IDs, then try to create isolated worlds
+      // for each frame to get their execution context IDs
+      const frameTree = (await this.send("Page.getFrameTree")) as {
+        frameTree?: {
+          frame: { id: string };
+          childFrames?: Array<{ frame: { id: string } }>;
+        };
+      };
+
+      const childFrames = frameTree.frameTree?.childFrames || [];
+      for (const child of childFrames) {
+        try {
+          // Create an isolated world in the frame to get a context ID
+          const world = (await this.send("Page.createIsolatedWorld", {
+            frameId: child.frame.id,
+            worldName: "assistme-snapshot",
+            grantUniveralAccess: true,
+          })) as { executionContextId?: number };
+
+          if (world.executionContextId) {
+            contexts.set(child.frame.id, world.executionContextId);
+          }
+        } catch {
+          // Frame might not support isolated worlds — skip
+        }
+      }
+    } catch {
+      // Fallback: no contexts available
+    }
+    return contexts;
+  }
+
   // ── Ref Resolution ────────────────────────────────────────────────
 
   /**
@@ -869,9 +1166,103 @@ export class BrowserController {
     });
 
     const value = (result as CDPEvalResult).result?.value as string;
-    if (!value || value === "null") return null;
+    if (value && value !== "null") {
+      try {
+        return JSON.parse(value);
+      } catch {
+        /* fall through to frame search */
+      }
+    }
+
+    // Strategy 3: search in cross-origin iframe contexts
+    const frameContextId = this.frameContexts.get(refId);
+    if (frameContextId) {
+      return this.resolveRefInFrame(refId, frameContextId, role, name);
+    }
+
+    return null;
+  }
+
+  /**
+   * Resolve a ref inside a cross-origin iframe using its execution context.
+   * Returns coordinates adjusted by the iframe's viewport offset.
+   */
+  private async resolveRefInFrame(
+    refId: number,
+    contextId: number,
+    role: string,
+    name: string
+  ): Promise<{ x: number; y: number; width: number; height: number; error?: string } | null> {
+    const roleJS = JSON.stringify(role);
+    const nameJS = JSON.stringify(name);
+
     try {
-      return JSON.parse(value);
+      // Get iframe offset from main document
+      const offsetResult = await this.send("Runtime.evaluate", {
+        expression: `
+          (function() {
+            var iframes = document.querySelectorAll('iframe');
+            for (var i = 0; i < iframes.length; i++) {
+              var r = iframes[i].getBoundingClientRect();
+              if (r.width > 10 && r.height > 10) {
+                return JSON.stringify({ x: r.x, y: r.y });
+              }
+            }
+            return JSON.stringify({ x: 0, y: 0 });
+          })()
+        `,
+        returnByValue: true,
+      });
+      const offset = JSON.parse(
+        ((offsetResult as CDPEvalResult).result?.value as string) || '{"x":0,"y":0}'
+      );
+
+      // Resolve element inside the frame
+      const frameResult = await this.send("Runtime.evaluate", {
+        expression: `
+          (function() {
+            var el = document.querySelector('[data-assistme-ref="${refId}"]');
+            if (!el && ${roleJS} && ${nameJS}) {
+              // Fallback: search by role
+              var candidates = document.querySelectorAll('*');
+              for (var i = 0; i < candidates.length; i++) {
+                var c = candidates[i];
+                if (c.isContentEditable || c.getAttribute('contenteditable') === 'true') {
+                  el = c; break;
+                }
+              }
+            }
+            if (!el) return 'null';
+
+            el.scrollIntoView({ block: 'center', behavior: 'instant' });
+            var r = el.getBoundingClientRect();
+            if (r.width < 1 || r.height < 1) return JSON.stringify({ error: 'Zero size' });
+
+            return JSON.stringify({
+              x: r.x + r.width / 2,
+              y: r.y + r.height / 2,
+              width: r.width,
+              height: r.height
+            });
+          })()
+        `,
+        contextId,
+        returnByValue: true,
+      });
+
+      const value = (frameResult as CDPEvalResult).result?.value as string;
+      if (!value || value === "null") return null;
+
+      const parsed = JSON.parse(value);
+      if (parsed.error) return parsed;
+
+      // Adjust coordinates by iframe offset
+      return {
+        x: parsed.x + offset.x,
+        y: parsed.y + offset.y,
+        width: parsed.width,
+        height: parsed.height,
+      };
     } catch {
       return null;
     }
@@ -981,11 +1372,24 @@ export class BrowserController {
     await new Promise((r) => setTimeout(r, 50));
 
     // 2. Verify the field is empty; if not, fall back to JS-based clearing
-    const cleared = await this.send("Runtime.evaluate", {
+    // Determine which context to evaluate in (main doc or iframe)
+    const frameContextId = this.frameContexts.get(refId);
+    const clearEvalOpts: Record<string, unknown> = {
       expression: `
         (function() {
           var el = document.querySelector('[data-assistme-ref="${refId}"]');
           if (!el) return 'no_element';
+
+          // For contenteditable elements, check textContent instead of value
+          if (el.isContentEditable || el.getAttribute('contenteditable') === 'true') {
+            if (el.textContent && el.textContent.trim() !== '') {
+              el.textContent = '';
+              el.dispatchEvent(new Event('input', { bubbles: true }));
+              return 'js_cleared';
+            }
+            return 'ok';
+          }
+
           if (el.value !== undefined && el.value !== '') {
             // Ctrl+A didn't work (some frameworks intercept it) — clear via JS
             var setter = Object.getOwnPropertyDescriptor(
@@ -1003,9 +1407,15 @@ export class BrowserController {
         })()
       `,
       returnByValue: true,
-    });
+    };
+    // If element is in a cross-origin iframe, evaluate in its context
+    if (frameContextId) {
+      clearEvalOpts.contextId = frameContextId;
+    }
+    const cleared = await this.send("Runtime.evaluate", clearEvalOpts);
     const clearStatus = ((cleared as CDPEvalResult).result?.value as string) || "ok";
-    if (clearStatus === "no_element") {
+    if (clearStatus === "no_element" && !frameContextId) {
+      // Element not found in main doc and no frame context — truly missing
       return {
         success: false,
         message: `Ref ${refLabel} not found after click. Take a new snapshot.`,

package/src/db/event.ts

@@ -2,6 +2,36 @@ import { callMcpHandler } from "./api-client.js";
 import { log } from "../utils/logger.js";
 import type { EventType } from "./types.js";
 
+const MAX_EMIT_RETRIES = 2;
+const EMIT_RETRY_DELAY_MS = 500;
+
+async function emitWithRetry(
+  messageId: string,
+  eventType: EventType,
+  eventData: Record<string, unknown>,
+  seq: number
+): Promise<void> {
+  for (let attempt = 0; attempt <= MAX_EMIT_RETRIES; attempt++) {
+    try {
+      await callMcpHandler("event.emit", {
+        message_id: messageId,
+        event_type: eventType,
+        event_data: eventData,
+        seq,
+      });
+      return;
+    } catch (err) {
+      if (attempt < MAX_EMIT_RETRIES) {
+        await new Promise((r) => setTimeout(r, EMIT_RETRY_DELAY_MS * (attempt + 1)));
+      } else {
+        log.warn(
+          `Failed to emit event after ${MAX_EMIT_RETRIES + 1} attempts: ${err instanceof Error ? err.message : err}`
+        );
+      }
+    }
+  }
+}
+
 /**
  * Per-task event emitter. Each task gets its own sequence counter
  * to avoid cross-task sequence number collisions.
@@ -13,16 +43,7 @@ export class TaskEventEmitter {
 
   async emit(eventType: EventType, eventData: Record<string, unknown>): Promise<void> {
     this.sequence++;
-    try {
-      await callMcpHandler("event.emit", {
-        message_id: this.messageId,
-        event_type: eventType,
-        event_data: eventData,
-        seq: this.sequence,
-      });
-    } catch (err) {
-      log.warn(`Failed to emit event: ${err instanceof Error ? err.message : err}`);
-    }
+    await emitWithRetry(this.messageId, eventType, eventData, this.sequence);
   }
 }
 
@@ -39,14 +60,5 @@ export async function emitEvent(
   eventData: Record<string, unknown>
 ): Promise<void> {
   eventSequence++;
-  try {
-    await callMcpHandler("event.emit", {
-      message_id: messageId,
-      event_type: eventType,
-      event_data: eventData,
-      seq: eventSequence,
-    });
-  } catch (err) {
-    log.warn(`Failed to emit event: ${err instanceof Error ? err.message : err}`);
-  }
+  await emitWithRetry(messageId, eventType, eventData, eventSequence);
 }