assistme 0.3.2 → 0.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "assistme",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "AssistMe CLI Agent - AI-powered assistant that controls your real browser",
   "type": "module",
   "main": "dist/index.js",

package/src/agent/processor.ts

@@ -31,6 +31,55 @@ import {
 import { createEventHooks } from "./event-hooks.js";
 import { BASE_SYSTEM_PROMPT } from "./system-prompt.js";
 
+/**
+ * Manages the task wall-clock timeout.
+ * Supports pausing while the agent is waiting for user input (ask_user)
+ * so that idle wait time doesn't count toward the timeout.
+ */
+class TaskTimeout {
+  private timeoutId: ReturnType<typeof setTimeout> | null = null;
+  private remainingMs: number;
+  private resumedAt: number;
+
+  constructor(
+    private abortController: AbortController,
+    timeoutMs: number
+  ) {
+    this.remainingMs = timeoutMs;
+    this.resumedAt = Date.now();
+    this.schedule();
+  }
+
+  private schedule(): void {
+    this.timeoutId = setTimeout(() => {
+      this.abortController.abort();
+    }, this.remainingMs);
+  }
+
+  /** Pause the timeout (e.g. while waiting for user). */
+  pause(): void {
+    if (this.timeoutId) {
+      clearTimeout(this.timeoutId);
+      this.timeoutId = null;
+      const elapsed = Date.now() - this.resumedAt;
+      this.remainingMs = Math.max(0, this.remainingMs - elapsed);
+    }
+  }
+
+  /** Resume the timeout after user interaction completes. */
+  resume(): void {
+    this.resumedAt = Date.now();
+    this.schedule();
+  }
+
+  clear(): void {
+    if (this.timeoutId) {
+      clearTimeout(this.timeoutId);
+      this.timeoutId = null;
+    }
+  }
+}
+
 const MAX_HISTORY_ENTRIES = 10;
 const MAX_RESPONSE_LENGTH = 1500;
 
@@ -143,6 +192,9 @@ export class TaskProcessor {
         systemPrompt += historyPrompt;
       }
 
+      const abortController = new AbortController();
+      const taskTimeout = new TaskTimeout(abortController, taskTimeoutMs);
+
       // Create MCP servers for custom tools
       const browserServer = createBrowserMcpServer();
       const agentToolsServer = createAgentToolsServer({
@@ -150,6 +202,8 @@ export class TaskProcessor {
         skillManager: this.skillManager,
         taskId: task.id,
         sessionId: this.sessionId || undefined,
+        onUserWaitStart: () => taskTimeout.pause(),
+        onUserWaitEnd: () => taskTimeout.resume(),
       });
 
       // Create event hooks for Supabase event emission
@@ -203,7 +257,6 @@ export class TaskProcessor {
         };
       }
 
-      const abortController = new AbortController();
       const options: Options = {
         model: config.model,
         systemPrompt,
@@ -221,11 +274,7 @@ export class TaskProcessor {
         abortController,
       };
 
-      // Wall-clock timeout via abort
       const taskStartTime = Date.now();
-      const timeoutId = setTimeout(() => {
-        abortController.abort();
-      }, taskTimeoutMs);
 
       try {
         for await (const message of query({
@@ -302,11 +351,18 @@ export class TaskProcessor {
           }
         }
       } finally {
-        clearTimeout(timeoutId);
+        taskTimeout.clear();
       }
 
+      // Truncate finalResponse to avoid edge function payload limits
+      const MAX_CONTENT_LENGTH = 50_000;
+      const truncatedResponse =
+        finalResponse.length > MAX_CONTENT_LENGTH
+          ? finalResponse.slice(0, MAX_CONTENT_LENGTH) + "\n\n[Response truncated]"
+          : finalResponse;
+
       // Complete the task (with retry for transient DB failures)
-      await withRetry(() => completeTask(task.id, finalResponse, tokenUsage), {
+      await withRetry(() => completeTask(task.id, truncatedResponse, tokenUsage), {
         maxRetries: 2,
         baseDelayMs: 300,
         label: "completeTask",

package/src/agent/system-prompt.ts

@@ -28,7 +28,7 @@ Available capabilities:
    - Refs persist across actions unless the page navigates. Re-snapshot after navigation or major DOM changes.
 
    **Legacy tools (still available, use when refs don't work):**
-   - browser_click, browser_type, browser_select, browser_get_elements, browser_screenshot, browser_evaluate
+   - browser_click, browser_type, browser_select, browser_screenshot, browser_evaluate
    - browser_click supports :contains('text') pseudo-selectors
    - browser_select handles native and custom dropdowns
 
@@ -41,12 +41,16 @@ Available capabilities:
    - Bash tool for shell commands
    - Glob and Grep for file search
 
-3. MEMORY:
+3. MEMORY & CREDENTIALS:
    - You can remember things about the user using memory_store
    - Use this when you learn preferences, important facts, or standing instructions
    - Your stored memories persist across conversations
    - PROACTIVELY use memory_store during tasks when you discover user preferences, habits, or important context
    - Before completing a task, consider if anything learned should be remembered for future conversations
+   - CRITICAL — Credential Storage: When you create, register, or receive any account credentials (username, password, API keys, tokens), you MUST use credential_set to save them locally. NEVER use memory_store for credentials — memory_store is for preferences and facts, credential_set is for secrets. Examples:
+     * After registering a new email/account → credential_set with type "login" and data { "username": "...", "password": "...", "email": "..." }
+     * After generating an API key → credential_set with type "api_key" and data { "api_key": "..." }
+     * Credentials saved via credential_set are encrypted on disk and viewable in the desktop app's Credentials panel
 
 4. SKILL-AWARE EXECUTION (CRITICAL — follow this for EVERY task):
    Step A — Search: Before executing ANY task, check if an existing skill matches (use skill_invoke or skill_search).

package/src/browser/chrome-launcher.ts

@@ -545,11 +545,13 @@ export async function ensureBrowserAvailable(port = 9222): Promise<AutoLaunchRes
 
 // ── Singleton ───────────────────────────────────────────────────────
 
-let browserInstance: BrowserController | null = null;
+const browserInstances = new Map<number, BrowserController>();
 
 export function getBrowser(port = 9222): BrowserController {
-  if (!browserInstance) {
-    browserInstance = new BrowserController(port);
+  let instance = browserInstances.get(port);
+  if (!instance) {
+    instance = new BrowserController(port);
+    browserInstances.set(port, instance);
   }
-  return browserInstance;
+  return instance;
 }

package/src/browser/controller.ts

@@ -10,6 +10,7 @@ import type {
   SnapshotResult,
   ActionSpec,
   ActionResult,
+  RefActionResult,
 } from "./types.js";
 
 export class BrowserController {
@@ -198,11 +199,26 @@ export class BrowserController {
 
   async goBack(): Promise<string> {
     this.ensureConnected();
-    await this.send("Page.navigateToHistoryEntry", {
-      entryId: -1,
-    }).catch(() => {});
-    // Fallback: use JS
-    await this.evaluate("window.history.back()");
+    try {
+      // Get navigation history and go to the previous entry
+      const history = (await this.send("Page.getNavigationHistory")) as {
+        currentIndex?: number;
+        entries?: Array<{ id: number }>;
+      };
+      const idx = history.currentIndex ?? 0;
+      const entries = history.entries ?? [];
+      if (idx > 0 && entries[idx - 1]) {
+        await this.send("Page.navigateToHistoryEntry", {
+          entryId: entries[idx - 1].id,
+        });
+      } else {
+        // No previous entry in CDP history — use JS fallback
+        await this.evaluate("window.history.back()");
+      }
+    } catch {
+      // CDP history API failed — use JS fallback
+      await this.evaluate("window.history.back()");
+    }
     await this.waitForLoad();
     const info = await this.getPageInfo();
     return `Went back to: ${info.title}`;
@@ -394,31 +410,88 @@ export class BrowserController {
       Tab: { keyCode: 9, code: "Tab" },
       Escape: { keyCode: 27, code: "Escape" },
       Backspace: { keyCode: 8, code: "Backspace" },
+      Delete: { keyCode: 46, code: "Delete" },
       ArrowDown: { keyCode: 40, code: "ArrowDown" },
       ArrowUp: { keyCode: 38, code: "ArrowUp" },
+      ArrowLeft: { keyCode: 37, code: "ArrowLeft" },
+      ArrowRight: { keyCode: 39, code: "ArrowRight" },
+      Home: { keyCode: 36, code: "Home" },
+      End: { keyCode: 35, code: "End" },
+      Space: { keyCode: 32, code: "Space" },
     };
 
-    const mapped = keyMap[key];
+    // CDP modifier bitmask values
+    const modifierMap: Record<string, number> = {
+      Alt: 1,
+      Control: 2,
+      Meta: 4,
+      Shift: 8,
+    };
+
+    // Parse modifier combos like "Control+a", "Meta+Shift+z"
+    const parts = key.split("+");
+    let modifiers = 0;
+    let actualKey = parts[parts.length - 1];
+    for (let i = 0; i < parts.length - 1; i++) {
+      const mod = modifierMap[parts[i]];
+      if (mod) modifiers |= mod;
+    }
+
+    const mapped = keyMap[actualKey];
     if (mapped) {
       await this.send("Input.dispatchKeyEvent", {
         type: "keyDown",
-        key,
+        key: actualKey,
         code: mapped.code,
         windowsVirtualKeyCode: mapped.keyCode,
         nativeVirtualKeyCode: mapped.keyCode,
+        modifiers,
       });
       await this.send("Input.dispatchKeyEvent", {
         type: "keyUp",
-        key,
+        key: actualKey,
         code: mapped.code,
         windowsVirtualKeyCode: mapped.keyCode,
         nativeVirtualKeyCode: mapped.keyCode,
+        modifiers,
+      });
+    } else if (actualKey.length === 1) {
+      // Single character key (e.g., "a", "z")
+      const code = `Key${actualKey.toUpperCase()}`;
+      const keyCode = actualKey.toUpperCase().charCodeAt(0);
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyDown",
+        key: actualKey,
+        code,
+        windowsVirtualKeyCode: keyCode,
+        nativeVirtualKeyCode: keyCode,
+        modifiers,
+      });
+      if (!modifiers) {
+        // Only insert text for unmodified single characters
+        await this.send("Input.dispatchKeyEvent", {
+          type: "char",
+          text: actualKey,
+          modifiers,
+        });
+      }
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key: actualKey,
+        code,
+        modifiers,
       });
     } else {
-      // Single character key
+      // Unknown key name — try as-is
       await this.send("Input.dispatchKeyEvent", {
-        type: "char",
-        text: key,
+        type: "keyDown",
+        key: actualKey,
+        modifiers,
+      });
+      await this.send("Input.dispatchKeyEvent", {
+        type: "keyUp",
+        key: actualKey,
+        modifiers,
       });
     }
 
@@ -816,8 +889,10 @@ export class BrowserController {
    * element is not yet actionable (e.g., covered by a loading overlay, still
    * animating into view). This matches Playwright's auto-waiting behavior.
    */
-  async clickRef(refId: number): Promise<string> {
+  async clickRef(refId: number): Promise<RefActionResult> {
     this.ensureConnected();
+    const ref = this.refCache.get(refId);
+    const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
 
     // Auto-wait: retry up to 3 times if element is not actionable yet
     const maxRetries = 3;
@@ -827,7 +902,10 @@ export class BrowserController {
       const resolved = await this.resolveRef(refId);
 
       if (!resolved) {
-        return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+        return {
+          success: false,
+          message: `Ref ${refLabel} not found. Take a new snapshot with browser_snapshot.`,
+        };
       }
 
       if (resolved.error) {
@@ -837,9 +915,7 @@ export class BrowserController {
           await new Promise((r) => setTimeout(r, 500));
           continue;
         }
-        // Final attempt failed — report the actionability issue
-        const ref = this.refCache.get(refId);
-        return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+        return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
       }
 
       // Element is actionable — small delay after scroll for rendering
@@ -875,13 +951,10 @@ export class BrowserController {
       });
 
       await new Promise((r) => setTimeout(r, 300));
-      const ref = this.refCache.get(refId);
-      return `Clicked [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+      return { success: true, message: `Clicked ${refLabel}` };
     }
 
-    // Should not reach here, but just in case
-    const ref = this.refCache.get(refId);
-    return `Cannot click [${refId}] ${ref?.role || ""} "${ref?.name || ""}": ${lastError}`;
+    return { success: false, message: `Cannot click ${refLabel}: ${lastError}` };
   }
 
   /**
@@ -889,48 +962,61 @@ export class BrowserController {
    * Clicks to focus, selects all existing text (Ctrl/Cmd+A), then uses
    * Input.insertText for reliable text insertion across all frameworks.
    */
-  async typeRef(refId: number, text: string): Promise<string> {
+  async typeRef(refId: number, text: string): Promise<RefActionResult> {
     this.ensureConnected();
+    const ref = this.refCache.get(refId);
+    const refLabel = `[${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
 
     // Click to focus the element
     const clickResult = await this.clickRef(refId);
-    if (clickResult.includes("not found")) return clickResult;
+    if (!clickResult.success) return clickResult;
     await new Promise((r) => setTimeout(r, 100));
 
-    // Select all existing text (Cmd+A on macOS, Ctrl+A elsewhere)
-    const modifier = platform() === "darwin" ? 4 : 2;
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyDown",
-      modifiers: modifier,
-      key: "a",
-      code: "KeyA",
-      windowsVirtualKeyCode: 65,
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyUp",
-      key: "a",
-      code: "KeyA",
-    });
+    // Clear existing text using multiple strategies for reliability:
+    // 1. Try Ctrl/Cmd+A to select all, then Backspace to delete
+    const selectAllKey = platform() === "darwin" ? "Meta+a" : "Control+a";
+    await this.pressKey(selectAllKey);
+    await new Promise((r) => setTimeout(r, 50));
+    await this.pressKey("Backspace");
+    await new Promise((r) => setTimeout(r, 50));
 
-    // Delete selected text
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyDown",
-      key: "Backspace",
-      code: "Backspace",
-      windowsVirtualKeyCode: 8,
-    });
-    await this.send("Input.dispatchKeyEvent", {
-      type: "keyUp",
-      key: "Backspace",
-      code: "Backspace",
+    // 2. Verify the field is empty; if not, fall back to JS-based clearing
+    const cleared = await this.send("Runtime.evaluate", {
+      expression: `
+        (function() {
+          var el = document.querySelector('[data-assistme-ref="${refId}"]');
+          if (!el) return 'no_element';
+          if (el.value !== undefined && el.value !== '') {
+            // Ctrl+A didn't work (some frameworks intercept it) — clear via JS
+            var setter = Object.getOwnPropertyDescriptor(
+              window.HTMLInputElement.prototype, 'value'
+            )?.set || Object.getOwnPropertyDescriptor(
+              window.HTMLTextAreaElement.prototype, 'value'
+            )?.set;
+            if (setter) setter.call(el, '');
+            else el.value = '';
+            el.dispatchEvent(new Event('input', { bubbles: true }));
+            el.dispatchEvent(new Event('change', { bubbles: true }));
+            return 'js_cleared';
+          }
+          return 'ok';
+        })()
+      `,
+      returnByValue: true,
     });
+    const clearStatus = ((cleared as CDPEvalResult).result?.value as string) || "ok";
+    if (clearStatus === "no_element") {
+      return {
+        success: false,
+        message: `Ref ${refLabel} not found after click. Take a new snapshot.`,
+      };
+    }
 
     // Insert text via CDP (goes through the browser's input pipeline)
     await this.send("Input.insertText", { text });
 
     await new Promise((r) => setTimeout(r, 100));
-    const ref = this.refCache.get(refId);
-    return `Typed "${text}" into [${refId}] ${ref?.role || ""} "${ref?.name || ""}"`;
+    return { success: true, message: `Typed "${text}" into ${refLabel}` };
   }
 
   /**
@@ -938,21 +1024,22 @@ export class BrowserController {
    * ref's data attribute as selector, handling both native <select> and
    * custom dropdown components.
    */
-  async selectRef(refId: number, option: string): Promise<string> {
+  async selectRef(refId: number, option: string): Promise<RefActionResult> {
     this.ensureConnected();
 
-    // Check if ref exists
     const cached = this.refCache.get(refId);
     if (!cached) {
-      return `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`;
+      return {
+        success: false,
+        message: `Ref [${refId}] not found. Take a new snapshot with browser_snapshot.`,
+      };
     }
 
-    // Use the data attribute selector to find the element
+    const refLabel = `[${refId}] ${cached.role} "${cached.name}"`;
     const result = await this.selectOption(`[data-assistme-ref="${refId}"]`, option);
-    return result.replace(
-      /\[data-assistme-ref="\d+"\]/,
-      `[${refId}] ${cached.role} "${cached.name}"`
-    );
+    const message = result.replace(/\[data-assistme-ref="\d+"\]/, refLabel);
+    const success = !result.includes("not found");
+    return { success, message };
   }
 
   // ── Action Pipeline ───────────────────────────────────────────────
@@ -977,18 +1064,24 @@ export class BrowserController {
 
       try {
         switch (spec.action) {
-          case "click":
-            result = await this.clickRef(spec.ref);
-            success = !result.includes("not found");
+          case "click": {
+            const r = await this.clickRef(spec.ref);
+            result = r.message;
+            success = r.success;
             break;
-          case "type":
-            result = await this.typeRef(spec.ref, spec.text);
-            success = !result.includes("not found");
+          }
+          case "type": {
+            const r = await this.typeRef(spec.ref, spec.text);
+            result = r.message;
+            success = r.success;
             break;
-          case "select":
-            result = await this.selectRef(spec.ref, spec.option);
-            success = !result.includes("not found");
+          }
+          case "select": {
+            const r = await this.selectRef(spec.ref, spec.option);
+            result = r.message;
+            success = r.success;
             break;
+          }
           case "press":
             result = await this.pressKey(spec.key);
             break;
@@ -1074,15 +1167,24 @@ export class BrowserController {
           // Strategy 2: Custom dropdown — find the trigger element
           var trigger = selectEl;
           if (!trigger) {
-            // Try finding by label/placeholder text
-            var allEls = document.querySelectorAll('*');
-            for (var j = 0; j < allEls.length; j++) {
-              var el = allEls[j];
+            // Try finding by aria-label first (fast, indexed)
+            trigger = document.querySelector('[aria-label="' + sel.replace(/"/g, '\\"') + '"]');
+          }
+          if (!trigger) {
+            // Try finding by label/placeholder text in likely dropdown elements
+            var dropdownCandidates = document.querySelectorAll(
+              'button, [role="combobox"], [role="listbox"], [role="button"], ' +
+              'select, input, .MuiSelect-root, .MuiInput-root, ' +
+              '[class*="select"], [class*="dropdown"], [class*="picker"]'
+            );
+            for (var j = 0; j < dropdownCandidates.length; j++) {
+              var el = dropdownCandidates[j];
               var ownText = Array.from(el.childNodes)
                 .filter(function(n) { return n.nodeType === 3; })
                 .map(function(n) { return n.textContent.trim(); })
                 .join('');
-              if (ownText === sel || el.getAttribute('aria-label') === sel) {
+              if (ownText === sel || el.getAttribute('aria-label') === sel ||
+                  el.getAttribute('placeholder') === sel) {
                 trigger = el;
                 break;
               }
@@ -1119,10 +1221,13 @@ export class BrowserController {
                 }
               }
 
-              // Broader search: any visible element with exact text match
-              var everything = document.querySelectorAll('*');
-              for (var m = 0; m < everything.length; m++) {
-                var candidate = everything[m];
+              // Broader search: visible leaf elements in interactive containers
+              var broadCandidates = document.querySelectorAll(
+                'li, span, div, a, button, label, [role="option"], [role="menuitem"], ' +
+                '[role="menuitemradio"], [role="menuitemcheckbox"], [data-value]'
+              );
+              for (var m = 0; m < broadCandidates.length; m++) {
+                var candidate = broadCandidates[m];
                 if (candidate.textContent && candidate.textContent.trim() === optText &&
                     candidate.offsetParent !== null && candidate.children.length === 0) {
                   candidate.click();
@@ -1217,6 +1322,7 @@ export class BrowserController {
 
   private async waitForLoad(timeoutMs = 8000): Promise<void> {
     const start = Date.now();
+    let sawInteractive = false;
     while (Date.now() - start < timeoutMs) {
       try {
         const result = await this.send("Runtime.evaluate", {
@@ -1224,71 +1330,27 @@ export class BrowserController {
           returnByValue: true,
         });
         const state = (result as CDPEvalResult).result?.value;
-        if (state === "complete" || state === "interactive") {
-          // Extra small wait for dynamic content
-          await new Promise((r) => setTimeout(r, 500));
+        if (state === "complete") {
+          // Fully loaded — brief wait for dynamic content
+          await new Promise((r) => setTimeout(r, 300));
           return;
         }
+        if (state === "interactive") {
+          if (!sawInteractive) {
+            sawInteractive = true;
+            // DOM is ready but sub-resources still loading — give it more
+            // time to reach "complete" before settling for "interactive"
+          }
+        }
       } catch {
         // Tab might be navigating
       }
       await new Promise((r) => setTimeout(r, 300));
     }
-  }
-
-  /**
-   * Find interactive elements on the page for the AI to understand what's clickable
-   */
-  async getInteractiveElements(): Promise<string> {
-    this.ensureConnected();
-    const result = await this.send("Runtime.evaluate", {
-      expression: `
-        (function() {
-          const elements = [];
-          const selectors = 'a, button, input, select, textarea, [role="button"], [onclick]';
-          const all = document.querySelectorAll(selectors);
-          for (let i = 0; i < all.length && elements.length < 50; i++) {
-            const el = all[i];
-            const rect = el.getBoundingClientRect();
-            if (rect.width === 0 || rect.height === 0) continue; // Skip hidden
-
-            // Build a reliable CSS selector
-            let selector;
-            if (el.id) {
-              selector = '#' + CSS.escape(el.id);
-            } else if (el.getAttribute('data-testid')) {
-              selector = '[data-testid="' + el.getAttribute('data-testid') + '"]';
-            } else {
-              // Build a path-based selector: find nth-of-type among siblings
-              const tag = el.tagName.toLowerCase();
-              const parent = el.parentElement;
-              if (parent) {
-                const siblings = parent.querySelectorAll(':scope > ' + tag);
-                const idx = Array.from(siblings).indexOf(el) + 1;
-                selector = tag + ':nth-of-type(' + idx + ')';
-              } else {
-                selector = tag;
-              }
-            }
-
-            elements.push({
-              tag: el.tagName.toLowerCase(),
-              text: (el.textContent || '').trim().slice(0, 80),
-              type: el.getAttribute('type') || '',
-              name: el.getAttribute('name') || '',
-              id: el.id || '',
-              href: el.getAttribute('href') || '',
-              placeholder: el.getAttribute('placeholder') || '',
-              selector: selector,
-            });
-          }
-          return JSON.stringify(elements, null, 2);
-        })()
-      `,
-      returnByValue: true,
-    });
-
-    return ((result as CDPEvalResult).result?.value as string) || "[]";
+    // Timed out — if we at least saw "interactive", that's usually good enough
+    if (sawInteractive) {
+      await new Promise((r) => setTimeout(r, 300));
+    }
   }
 
   isConnected(): boolean {

package/src/browser/types.ts

@@ -62,6 +62,12 @@ export interface ActionResult {
   success: boolean;
 }
 
+/** Structured result from ref-based interactions (click, type, select). */
+export interface RefActionResult {
+  success: boolean;
+  message: string;
+}
+
 export interface AutoLaunchResult {
   success: boolean;
   action: "already_available" | "launched" | "chrome_not_found" | "launch_failed" | "port_conflict";