btcp-browser-agent 0.1.11 → 0.1.14

package/package.json

@@ -1,69 +1,69 @@
-{
-  "name": "btcp-browser-agent",
-  "version": "0.1.11",
-  "description": "Give AI agents the power to control browsers. A foundation for building agentic systems with smart DOM snapshots and stable element references.",
-  "type": "module",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "default": "./dist/index.js"
-    },
-    "./core": {
-      "types": "./packages/core/dist/index.d.ts",
-      "import": "./packages/core/dist/index.js",
-      "default": "./packages/core/dist/index.js"
-    },
-    "./extension": {
-      "types": "./packages/extension/dist/index.d.ts",
-      "import": "./packages/extension/dist/index.js",
-      "default": "./packages/extension/dist/index.js"
-    },
-    "./extension/content": {
-      "types": "./packages/extension/dist/content.d.ts",
-      "import": "./packages/extension/dist/content.js",
-      "default": "./packages/extension/dist/content.js"
-    },
-    "./extension/background": {
-      "types": "./packages/extension/dist/background.d.ts",
-      "import": "./packages/extension/dist/background.js",
-      "default": "./packages/extension/dist/background.js"
-    }
-  },
-  "scripts": {
-    "build": "npm run build:packages && tsc -p tsconfig.build.json",
-    "build:packages": "tsc -p packages/core/tsconfig.json && tsc -p packages/extension/tsconfig.json && tsc -p packages/cli/tsconfig.json",
-    "clean": "rm -rf dist packages/*/dist",
-    "prepare": "npm run build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
-  },
-  "workspaces": [
-    "packages/core",
-    "packages/extension",
-    "packages/cli"
-  ],
-  "files": [
-    "dist",
-    "packages/core/dist",
-    "packages/extension/dist",
-    "!**/__tests__",
-    "!**/*.map"
-  ],
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/browser-tool-calling-protocol/btcp-browser-agent.git"
-  },
-  "dependencies": {},
-  "devDependencies": {
-    "@types/chrome": "^0.0.268",
-    "@types/node": "^20.10.0",
-    "jsdom": "^24.0.0",
-    "typescript": "^5.3.0",
-    "vitest": "^2.0.0"
-  }
-}
+{
+  "name": "btcp-browser-agent",
+  "version": "0.1.14",
+  "description": "Give AI agents the power to control browsers. A foundation for building agentic systems with smart DOM snapshots and stable element references.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./core": {
+      "types": "./packages/core/dist/index.d.ts",
+      "import": "./packages/core/dist/index.js",
+      "default": "./packages/core/dist/index.js"
+    },
+    "./extension": {
+      "types": "./packages/extension/dist/index.d.ts",
+      "import": "./packages/extension/dist/index.js",
+      "default": "./packages/extension/dist/index.js"
+    },
+    "./extension/content": {
+      "types": "./packages/extension/dist/content.d.ts",
+      "import": "./packages/extension/dist/content.js",
+      "default": "./packages/extension/dist/content.js"
+    },
+    "./extension/background": {
+      "types": "./packages/extension/dist/background.d.ts",
+      "import": "./packages/extension/dist/background.js",
+      "default": "./packages/extension/dist/background.js"
+    }
+  },
+  "scripts": {
+    "build": "npm run build:packages && tsc -p tsconfig.build.json",
+    "build:packages": "tsc -p packages/core/tsconfig.json && tsc -p packages/extension/tsconfig.json && tsc -p packages/cli/tsconfig.json",
+    "clean": "rm -rf dist packages/*/dist",
+    "prepare": "npm run build",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  },
+  "workspaces": [
+    "packages/core",
+    "packages/extension",
+    "packages/cli"
+  ],
+  "files": [
+    "dist",
+    "packages/core/dist",
+    "packages/extension/dist",
+    "!**/__tests__",
+    "!**/*.map"
+  ],
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/browser-tool-calling-protocol/btcp-browser-agent.git"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@types/chrome": "^0.0.268",
+    "@types/node": "^20.10.0",
+    "jsdom": "^24.0.0",
+    "typescript": "^5.3.0",
+    "vitest": "^2.0.0"
+  }
+}

package/packages/core/dist/actions.js

@@ -349,31 +349,99 @@ export class DOMActions {
     }
     async type(selector, text, options = {}) {
         const element = this.getElement(selector);
-        if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement)) {
+        // Check if element is contenteditable
+        const isContentEditable = element.getAttribute('contenteditable') === 'true' ||
+            element.getAttribute('contenteditable') === '';
+        if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || isContentEditable)) {
             const actualType = element.tagName.toLowerCase();
             const availableActions = this.getAvailableActionsForElement(element);
-            throw createElementNotCompatibleError(selector, 'type', actualType, ['input', 'textarea'], availableActions);
+            throw createElementNotCompatibleError(selector, 'type', actualType, ['input', 'textarea', 'contenteditable'], availableActions);
         }
-        element.focus();
-        if (options.clear) {
-            element.value = '';
-            element.dispatchEvent(new Event('input', { bubbles: true }));
+        // Focus the element (cast to HTMLElement for contenteditable)
+        if (element instanceof HTMLElement) {
+            element.focus();
         }
-        for (const char of text) {
-            element.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
-            element.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
-            element.value += char;
-            element.dispatchEvent(new Event('input', { bubbles: true }));
-            element.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
-            if (options.delay) {
-                await this.sleep(options.delay);
+        // Handle contenteditable elements differently
+        if (isContentEditable) {
+            const htmlElement = element;
+            if (options.clear) {
+                htmlElement.textContent = '';
+                htmlElement.dispatchEvent(new Event('input', { bubbles: true }));
+            }
+            for (const char of text) {
+                htmlElement.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
+                htmlElement.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
+                // Insert text at cursor position or append
+                const selection = this.window.getSelection();
+                if (selection && selection.rangeCount > 0) {
+                    const range = selection.getRangeAt(0);
+                    range.deleteContents();
+                    const textNode = this.document.createTextNode(char);
+                    range.insertNode(textNode);
+                    range.setStartAfter(textNode);
+                    range.setEndAfter(textNode);
+                    selection.removeAllRanges();
+                    selection.addRange(range);
+                }
+                else {
+                    htmlElement.textContent += char;
+                }
+                htmlElement.dispatchEvent(new Event('input', { bubbles: true }));
+                htmlElement.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
+                if (options.delay) {
+                    await this.sleep(options.delay);
+                }
+            }
+            htmlElement.dispatchEvent(new Event('change', { bubbles: true }));
+            // Wait for verification that textContent contains typed text
+            const result = await waitForAssertion(() => {
+                const content = htmlElement.textContent || '';
+                const expected = text;
+                const actual = content;
+                if (!content.includes(text)) {
+                    return {
+                        success: false,
+                        error: `Expected textContent to contain "${text}"`,
+                        description: 'textContent check',
+                        expected,
+                        actual
+                    };
+                }
+                return {
+                    success: true,
+                    error: null,
+                    description: 'textContent check',
+                    expected,
+                    actual
+                };
+            }, { timeout: 1000, interval: 50 });
+            if (!result.success) {
+                throw createVerificationError('type', result, selector);
             }
         }
-        element.dispatchEvent(new Event('change', { bubbles: true }));
-        // Wait for verification that value contains typed text
-        const result = await waitForAssertion(() => assertValueContains(element, text), { timeout: 1000, interval: 50 });
-        if (!result.success) {
-            throw createVerificationError('type', result, selector);
+        else {
+            // Handle regular input/textarea elements
+            const inputElement = element;
+            if (options.clear) {
+                inputElement.value = '';
+                inputElement.dispatchEvent(new Event('input', { bubbles: true }));
+            }
+            for (const char of text) {
+                inputElement.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
+                inputElement.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
+                inputElement.value += char;
+                inputElement.dispatchEvent(new Event('input', { bubbles: true }));
+                inputElement.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
+                if (options.delay) {
+                    await this.sleep(options.delay);
+                }
+            }
+            inputElement.dispatchEvent(new Event('change', { bubbles: true }));
+            // Wait for verification that value contains typed text
+            const result = await waitForAssertion(() => assertValueContains(inputElement, text), { timeout: 1000, interval: 50 });
+            if (!result.success) {
+                throw createVerificationError('type', result, selector);
+            }
         }
         return { success: true, error: null };
     }
@@ -796,15 +864,15 @@ export class DOMActions {
         // Create overlay container with absolute positioning covering entire document
         this.overlayContainer = this.document.createElement('div');
         this.overlayContainer.id = 'btcp-highlight-overlay';
-        this.overlayContainer.style.cssText = `
-      position: absolute;
-      top: 0;
-      left: 0;
-      width: ${this.document.documentElement.scrollWidth}px;
-      height: ${this.document.documentElement.scrollHeight}px;
-      pointer-events: none;
-      z-index: 999999;
-      contain: layout style paint;
+        this.overlayContainer.style.cssText = `
+      position: absolute;
+      top: 0;
+      left: 0;
+      width: ${this.document.documentElement.scrollWidth}px;
+      height: ${this.document.documentElement.scrollHeight}px;
+      pointer-events: none;
+      z-index: 999999;
+      contain: layout style paint;
     `;
         let highlightedCount = 0;
         // Create border overlays and labels for each ref
@@ -825,17 +893,17 @@ export class DOMActions {
                 const border = this.document.createElement('div');
                 border.className = 'btcp-ref-border';
                 border.dataset.ref = ref;
-                border.style.cssText = `
-          position: absolute;
-          width: ${bbox.width}px;
-          height: ${bbox.height}px;
-          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
-          border: 2px solid rgba(59, 130, 246, 0.8);
-          border-radius: 2px;
-          box-shadow: 0 0 0 1px rgba(59, 130, 246, 0.2);
-          pointer-events: none;
-          will-change: transform;
-          contain: layout style paint;
+                border.style.cssText = `
+          position: absolute;
+          width: ${bbox.width}px;
+          height: ${bbox.height}px;
+          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
+          border: 2px solid rgba(59, 130, 246, 0.8);
+          border-radius: 2px;
+          box-shadow: 0 0 0 1px rgba(59, 130, 246, 0.2);
+          pointer-events: none;
+          will-change: transform;
+          contain: layout style paint;
         `;
                 // Create label
                 const label = this.document.createElement('div');
@@ -843,21 +911,21 @@ export class DOMActions {
                 label.dataset.ref = ref;
                 // Extract number from ref (e.g., "@ref:5" -> "5")
                 label.textContent = ref.replace('@ref:', '');
-                label.style.cssText = `
-          position: absolute;
-          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
-          background: rgba(59, 130, 246, 0.9);
-          color: white;
-          padding: 2px 6px;
-          border-radius: 3px;
-          font-family: monospace;
-          font-size: 11px;
-          font-weight: bold;
-          box-shadow: 0 2px 4px rgba(0,0,0,0.3);
-          pointer-events: none;
-          white-space: nowrap;
-          will-change: transform;
-          contain: layout style paint;
+                label.style.cssText = `
+          position: absolute;
+          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
+          background: rgba(59, 130, 246, 0.9);
+          color: white;
+          padding: 2px 6px;
+          border-radius: 3px;
+          font-family: monospace;
+          font-size: 11px;
+          font-weight: bold;
+          box-shadow: 0 2px 4px rgba(0,0,0,0.3);
+          pointer-events: none;
+          white-space: nowrap;
+          will-change: transform;
+          contain: layout style paint;
         `;
                 this.overlayContainer.appendChild(border);
                 this.overlayContainer.appendChild(label);

package/packages/core/dist/snapshot.js

@@ -110,6 +110,11 @@ function getRole(element) {
         const type = element.type || 'text';
         return INPUT_ROLES[type] || 'textbox';
     }
+    // Detect contenteditable elements (ProseMirror, Quill, TinyMCE, etc.)
+    const contentEditable = element.getAttribute('contenteditable');
+    if (contentEditable === 'true' || contentEditable === '') {
+        return 'textbox';
+    }
     return IMPLICIT_ROLES[tagName] || null;
 }
 /**
@@ -320,6 +325,21 @@ function getAccessibleName(element) {
     if (isImage) {
         return getImageLabel(element);
     }
+    // Handle contenteditable elements (ProseMirror, Quill, TinyMCE, etc.)
+    const contentEditable = element.getAttribute('contenteditable');
+    if (contentEditable === 'true' || contentEditable === '') {
+        // Try data-placeholder attribute (common in rich text editors)
+        const placeholder = element.getAttribute('data-placeholder');
+        if (placeholder)
+            return placeholder.trim();
+        // Try finding placeholder in child paragraph element
+        const placeholderEl = element.querySelector('[data-placeholder]');
+        if (placeholderEl) {
+            const placeholderText = placeholderEl.getAttribute('data-placeholder');
+            if (placeholderText)
+                return placeholderText.trim();
+        }
+    }
     const ariaLabel = element.getAttribute('aria-label');
     if (ariaLabel)
         return ariaLabel.trim();

package/packages/extension/dist/background.d.ts

@@ -91,22 +91,23 @@ export declare class BackgroundAgent {
      */
     tab(tabId: number): TabHandle;
     /**
-     * Get the currently active tab (only if in session)
+     * Get the currently active tab (ensures session exists, creates if needed)
+     * This is the core "get or create" method that enables automatic session management.
      */
-    getActiveTab(): Promise<ChromeTab | null>;
+    getActiveTab(): Promise<ChromeTab>;
     /**
-     * List all tabs (only in session group)
+     * List all tabs in session (ensures session exists, creates if needed)
      */
     listTabs(): Promise<TabInfo[]>;
     /**
-     * Create a new tab (only in session group)
+     * Create a new tab in session (ensures session exists, creates if needed)
      */
     newTab(options?: {
         url?: string;
         active?: boolean;
     }): Promise<TabInfo>;
     /**
-     * Check if a tab is in the active session
+     * Check if a tab is in the active session (ensures session exists, creates if needed)
      */
     private isTabInSession;
     /**
@@ -118,7 +119,7 @@ export declare class BackgroundAgent {
      */
     switchTab(tabId: number): Promise<void>;
     /**
-     * Navigate to a URL (only in session tabs)
+     * Navigate to a URL (session auto-created if needed)
      * Always waits for page to be idle before returning.
      * Verifies navigation completed to the expected origin.
      */
@@ -126,25 +127,25 @@ export declare class BackgroundAgent {
         waitUntil?: 'load' | 'domcontentloaded';
     }): Promise<void>;
     /**
-     * Go back in history
+     * Go back in history (session auto-created if needed)
      */
     back(): Promise<void>;
     /**
-     * Go forward in history
+     * Go forward in history (session auto-created if needed)
      */
     forward(): Promise<void>;
     /**
-     * Reload the current page
+     * Reload the current page (session auto-created if needed)
      */
     reload(options?: {
         bypassCache?: boolean;
     }): Promise<void>;
     /**
-     * Get the current URL
+     * Get the current URL (session auto-created if needed)
      */
     getUrl(): Promise<string>;
     /**
-     * Get the page title
+     * Get the page title (session auto-created if needed)
      */
     getTitle(): Promise<string>;
     /**
@@ -180,6 +181,7 @@ export declare class BackgroundAgent {
     }): Promise<Response>;
     /**
      * Send a command to the ContentAgent in a specific tab
+     * Session is automatically created if it doesn't exist.
      */
     sendToContentAgent(command: Command, tabId?: number): Promise<Response>;
     /**

package/packages/extension/dist/background.js

@@ -151,38 +151,19 @@ export class BackgroundAgent {
     // TAB MANAGEMENT
     // ============================================================================
     /**
-     * Get the currently active tab (only if in session)
+     * Get the currently active tab (ensures session exists, creates if needed)
+     * This is the core "get or create" method that enables automatic session management.
      */
     async getActiveTab() {
-        const sessionGroupId = this.sessionManager.getActiveSessionGroupId();
-        // If no session, return null
-        if (sessionGroupId === null) {
-            return null;
-        }
-        // Get active tab and verify it's in the session
-        return new Promise((resolve) => {
-            chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
-                const activeTab = tabs[0];
-                // Only return if it's in the session group
-                if (activeTab && activeTab.groupId === sessionGroupId) {
-                    resolve(activeTab);
-                }
-                else {
-                    resolve(null);
-                }
-            });
-        });
+        const tabId = await this.sessionManager.getSessionTab();
+        return chrome.tabs.get(tabId);
     }
     /**
-     * List all tabs (only in session group)
+     * List all tabs in session (ensures session exists, creates if needed)
      */
     async listTabs() {
-        // Get active session group ID
-        const sessionGroupId = this.sessionManager.getActiveSessionGroupId();
-        // Session is required
-        if (sessionGroupId === null) {
-            throw new Error('No active session. Create a session first to manage tabs.');
-        }
+        // Ensure session exists (creates if needed)
+        const sessionGroupId = await this.sessionManager.ensureSession();
         // Only return tabs in the session group
         const tabs = await new Promise((resolve) => {
             chrome.tabs.query({ groupId: sessionGroupId }, (t) => resolve(t));
@@ -196,14 +177,11 @@ export class BackgroundAgent {
         }));
     }
     /**
-     * Create a new tab (only in session group)
+     * Create a new tab in session (ensures session exists, creates if needed)
      */
     async newTab(options) {
-        // Require active session
-        const sessionGroupId = this.sessionManager.getActiveSessionGroupId();
-        if (sessionGroupId === null) {
-            throw new Error('No active session. Create a session first to manage tabs.');
-        }
+        // Ensure session exists (creates if needed)
+        await this.sessionManager.ensureSession();
         const tab = await new Promise((resolve) => {
             chrome.tabs.create({ url: options?.url, active: options?.active ?? true }, (t) => resolve(t));
         });
@@ -231,14 +209,11 @@ export class BackgroundAgent {
         };
     }
     /**
-     * Check if a tab is in the active session
+     * Check if a tab is in the active session (ensures session exists, creates if needed)
      */
     async isTabInSession(tabId) {
-        const sessionGroupId = this.sessionManager.getActiveSessionGroupId();
-        // Session is required - no operations allowed without it
-        if (sessionGroupId === null) {
-            throw new Error('No active session. Create a session first to manage tabs.');
-        }
+        // Ensure session exists (creates if needed)
+        const sessionGroupId = await this.sessionManager.ensureSession();
         // Check if tab is in the session group
         const tab = await chrome.tabs.get(tabId);
         return tab.groupId === sessionGroupId;
@@ -282,19 +257,13 @@ export class BackgroundAgent {
     // NAVIGATION
     // ============================================================================
     /**
-     * Navigate to a URL (only in session tabs)
+     * Navigate to a URL (session auto-created if needed)
      * Always waits for page to be idle before returning.
      * Verifies navigation completed to the expected origin.
      */
     async navigate(url, _options) {
-        const tabId = this.activeTabId ?? (await this.getActiveTab())?.id;
-        if (!tabId)
-            throw new Error('No active tab');
-        // Validate tab is in session
-        const inSession = await this.isTabInSession(tabId);
-        if (!inSession) {
-            throw new Error('Cannot navigate: tab is not in the active session');
-        }
+        // getActiveTab() ensures session exists and always returns a valid tab
+        const tabId = this.activeTabId ?? (await this.getActiveTab()).id;
         await new Promise((resolve) => {
             chrome.tabs.update(tabId, { url }, () => resolve());
         });
@@ -334,52 +303,46 @@ export class BackgroundAgent {
         }
     }
     /**
-     * Go back in history
+     * Go back in history (session auto-created if needed)
      */
     async back() {
-        const tabId = this.activeTabId ?? (await this.getActiveTab())?.id;
-        if (!tabId)
-            throw new Error('No active tab');
+        const tabId = this.activeTabId ?? (await this.getActiveTab()).id;
         await new Promise((resolve) => {
             chrome.tabs.goBack(tabId, () => resolve());
         });
     }
     /**
-     * Go forward in history
+     * Go forward in history (session auto-created if needed)
      */
     async forward() {
-        const tabId = this.activeTabId ?? (await this.getActiveTab())?.id;
-        if (!tabId)
-            throw new Error('No active tab');
+        const tabId = this.activeTabId ?? (await this.getActiveTab()).id;
         await new Promise((resolve) => {
             chrome.tabs.goForward(tabId, () => resolve());
         });
     }
     /**
-     * Reload the current page
+     * Reload the current page (session auto-created if needed)
      */
     async reload(options) {
-        const tabId = this.activeTabId ?? (await this.getActiveTab())?.id;
-        if (!tabId)
-            throw new Error('No active tab');
+        const tabId = this.activeTabId ?? (await this.getActiveTab()).id;
         await new Promise((resolve) => {
             chrome.tabs.reload(tabId, { bypassCache: options?.bypassCache }, () => resolve());
         });
         await this.waitForTabLoad(tabId);
     }
     /**
-     * Get the current URL
+     * Get the current URL (session auto-created if needed)
      */
     async getUrl() {
         const tab = await this.getActiveTab();
-        return tab?.url || '';
+        return tab.url || '';
     }
     /**
-     * Get the page title
+     * Get the page title (session auto-created if needed)
      */
     async getTitle() {
         const tab = await this.getActiveTab();
-        return tab?.title || '';
+        return tab.title || '';
     }
     // ============================================================================
     // SCREENSHOTS
@@ -449,19 +412,14 @@ export class BackgroundAgent {
     }
     /**
      * Send a command to the ContentAgent in a specific tab
+     * Session is automatically created if it doesn't exist.
      */
     async sendToContentAgent(command, tabId) {
         // Ensure command has an ID for internal use
         const id = command.id || generateBgCommandId();
         const internalCmd = { ...command, id };
-        const targetTabId = tabId ?? this.activeTabId ?? (await this.getActiveTab())?.id;
-        if (!targetTabId) {
-            return {
-                id,
-                success: false,
-                error: 'No active tab for DOM command',
-            };
-        }
+        // getActiveTab() ensures session exists and always returns a valid tab
+        const targetTabId = tabId ?? this.activeTabId ?? (await this.getActiveTab()).id;
         // Try sending with automatic retry and recovery
         return this.sendMessageWithRetry(targetTabId, internalCmd);
     }
@@ -693,7 +651,7 @@ export class BackgroundAgent {
             case 'popupInitialize': {
                 console.log('[BackgroundAgent] Popup initializing, checking for session reconnection...');
                 // Check if we have a stored session but no active connection
-                const sessionGroupId = this.sessionManager.getActiveSessionGroupId();
+                const sessionGroupId = await this.sessionManager.getActiveSessionGroupIdAsync();
                 if (sessionGroupId === null) {
                     // Try to reconnect from storage
                     const result = await chrome.storage.session.get('btcp_active_session');