btcp-browser-agent 0.1.10 → 0.1.12

package/package.json

@@ -1,69 +1,69 @@
-{
-  "name": "btcp-browser-agent",
-  "version": "0.1.10",
-  "description": "Give AI agents the power to control browsers. A foundation for building agentic systems with smart DOM snapshots and stable element references.",
-  "type": "module",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "default": "./dist/index.js"
-    },
-    "./core": {
-      "types": "./packages/core/dist/index.d.ts",
-      "import": "./packages/core/dist/index.js",
-      "default": "./packages/core/dist/index.js"
-    },
-    "./extension": {
-      "types": "./packages/extension/dist/index.d.ts",
-      "import": "./packages/extension/dist/index.js",
-      "default": "./packages/extension/dist/index.js"
-    },
-    "./extension/content": {
-      "types": "./packages/extension/dist/content.d.ts",
-      "import": "./packages/extension/dist/content.js",
-      "default": "./packages/extension/dist/content.js"
-    },
-    "./extension/background": {
-      "types": "./packages/extension/dist/background.d.ts",
-      "import": "./packages/extension/dist/background.js",
-      "default": "./packages/extension/dist/background.js"
-    }
-  },
-  "scripts": {
-    "build": "npm run build:packages && tsc -p tsconfig.build.json",
-    "build:packages": "tsc -p packages/core/tsconfig.json && tsc -p packages/extension/tsconfig.json && tsc -p packages/cli/tsconfig.json",
-    "clean": "rm -rf dist packages/*/dist",
-    "prepare": "npm run build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
-  },
-  "workspaces": [
-    "packages/core",
-    "packages/extension",
-    "packages/cli"
-  ],
-  "files": [
-    "dist",
-    "packages/core/dist",
-    "packages/extension/dist",
-    "!**/__tests__",
-    "!**/*.map"
-  ],
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/browser-tool-calling-protocol/btcp-browser-agent.git"
-  },
-  "dependencies": {},
-  "devDependencies": {
-    "@types/chrome": "^0.0.268",
-    "@types/node": "^20.10.0",
-    "jsdom": "^24.0.0",
-    "typescript": "^5.3.0",
-    "vitest": "^2.0.0"
-  }
-}
+{
+  "name": "btcp-browser-agent",
+  "version": "0.1.12",
+  "description": "Give AI agents the power to control browsers. A foundation for building agentic systems with smart DOM snapshots and stable element references.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./core": {
+      "types": "./packages/core/dist/index.d.ts",
+      "import": "./packages/core/dist/index.js",
+      "default": "./packages/core/dist/index.js"
+    },
+    "./extension": {
+      "types": "./packages/extension/dist/index.d.ts",
+      "import": "./packages/extension/dist/index.js",
+      "default": "./packages/extension/dist/index.js"
+    },
+    "./extension/content": {
+      "types": "./packages/extension/dist/content.d.ts",
+      "import": "./packages/extension/dist/content.js",
+      "default": "./packages/extension/dist/content.js"
+    },
+    "./extension/background": {
+      "types": "./packages/extension/dist/background.d.ts",
+      "import": "./packages/extension/dist/background.js",
+      "default": "./packages/extension/dist/background.js"
+    }
+  },
+  "scripts": {
+    "build": "npm run build:packages && tsc -p tsconfig.build.json",
+    "build:packages": "tsc -p packages/core/tsconfig.json && tsc -p packages/extension/tsconfig.json && tsc -p packages/cli/tsconfig.json",
+    "clean": "rm -rf dist packages/*/dist",
+    "prepare": "npm run build",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  },
+  "workspaces": [
+    "packages/core",
+    "packages/extension",
+    "packages/cli"
+  ],
+  "files": [
+    "dist",
+    "packages/core/dist",
+    "packages/extension/dist",
+    "!**/__tests__",
+    "!**/*.map"
+  ],
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/browser-tool-calling-protocol/btcp-browser-agent.git"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@types/chrome": "^0.0.268",
+    "@types/node": "^20.10.0",
+    "jsdom": "^24.0.0",
+    "typescript": "^5.3.0",
+    "vitest": "^2.0.0"
+  }
+}

package/packages/core/dist/actions.js

@@ -349,31 +349,99 @@ export class DOMActions {
     }
     async type(selector, text, options = {}) {
         const element = this.getElement(selector);
-        if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement)) {
+        // Check if element is contenteditable
+        const isContentEditable = element.getAttribute('contenteditable') === 'true' ||
+            element.getAttribute('contenteditable') === '';
+        if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || isContentEditable)) {
             const actualType = element.tagName.toLowerCase();
             const availableActions = this.getAvailableActionsForElement(element);
-            throw createElementNotCompatibleError(selector, 'type', actualType, ['input', 'textarea'], availableActions);
+            throw createElementNotCompatibleError(selector, 'type', actualType, ['input', 'textarea', 'contenteditable'], availableActions);
         }
-        element.focus();
-        if (options.clear) {
-            element.value = '';
-            element.dispatchEvent(new Event('input', { bubbles: true }));
+        // Focus the element (cast to HTMLElement for contenteditable)
+        if (element instanceof HTMLElement) {
+            element.focus();
         }
-        for (const char of text) {
-            element.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
-            element.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
-            element.value += char;
-            element.dispatchEvent(new Event('input', { bubbles: true }));
-            element.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
-            if (options.delay) {
-                await this.sleep(options.delay);
+        // Handle contenteditable elements differently
+        if (isContentEditable) {
+            const htmlElement = element;
+            if (options.clear) {
+                htmlElement.textContent = '';
+                htmlElement.dispatchEvent(new Event('input', { bubbles: true }));
+            }
+            for (const char of text) {
+                htmlElement.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
+                htmlElement.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
+                // Insert text at cursor position or append
+                const selection = this.window.getSelection();
+                if (selection && selection.rangeCount > 0) {
+                    const range = selection.getRangeAt(0);
+                    range.deleteContents();
+                    const textNode = this.document.createTextNode(char);
+                    range.insertNode(textNode);
+                    range.setStartAfter(textNode);
+                    range.setEndAfter(textNode);
+                    selection.removeAllRanges();
+                    selection.addRange(range);
+                }
+                else {
+                    htmlElement.textContent += char;
+                }
+                htmlElement.dispatchEvent(new Event('input', { bubbles: true }));
+                htmlElement.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
+                if (options.delay) {
+                    await this.sleep(options.delay);
+                }
+            }
+            htmlElement.dispatchEvent(new Event('change', { bubbles: true }));
+            // Wait for verification that textContent contains typed text
+            const result = await waitForAssertion(() => {
+                const content = htmlElement.textContent || '';
+                const expected = text;
+                const actual = content;
+                if (!content.includes(text)) {
+                    return {
+                        success: false,
+                        error: `Expected textContent to contain "${text}"`,
+                        description: 'textContent check',
+                        expected,
+                        actual
+                    };
+                }
+                return {
+                    success: true,
+                    error: null,
+                    description: 'textContent check',
+                    expected,
+                    actual
+                };
+            }, { timeout: 1000, interval: 50 });
+            if (!result.success) {
+                throw createVerificationError('type', result, selector);
             }
         }
-        element.dispatchEvent(new Event('change', { bubbles: true }));
-        // Wait for verification that value contains typed text
-        const result = await waitForAssertion(() => assertValueContains(element, text), { timeout: 1000, interval: 50 });
-        if (!result.success) {
-            throw createVerificationError('type', result, selector);
+        else {
+            // Handle regular input/textarea elements
+            const inputElement = element;
+            if (options.clear) {
+                inputElement.value = '';
+                inputElement.dispatchEvent(new Event('input', { bubbles: true }));
+            }
+            for (const char of text) {
+                inputElement.dispatchEvent(new KeyboardEvent('keydown', { key: char, bubbles: true }));
+                inputElement.dispatchEvent(new KeyboardEvent('keypress', { key: char, bubbles: true }));
+                inputElement.value += char;
+                inputElement.dispatchEvent(new Event('input', { bubbles: true }));
+                inputElement.dispatchEvent(new KeyboardEvent('keyup', { key: char, bubbles: true }));
+                if (options.delay) {
+                    await this.sleep(options.delay);
+                }
+            }
+            inputElement.dispatchEvent(new Event('change', { bubbles: true }));
+            // Wait for verification that value contains typed text
+            const result = await waitForAssertion(() => assertValueContains(inputElement, text), { timeout: 1000, interval: 50 });
+            if (!result.success) {
+                throw createVerificationError('type', result, selector);
+            }
         }
         return { success: true, error: null };
     }
@@ -796,15 +864,15 @@ export class DOMActions {
         // Create overlay container with absolute positioning covering entire document
         this.overlayContainer = this.document.createElement('div');
         this.overlayContainer.id = 'btcp-highlight-overlay';
-        this.overlayContainer.style.cssText = `
-      position: absolute;
-      top: 0;
-      left: 0;
-      width: ${this.document.documentElement.scrollWidth}px;
-      height: ${this.document.documentElement.scrollHeight}px;
-      pointer-events: none;
-      z-index: 999999;
-      contain: layout style paint;
+        this.overlayContainer.style.cssText = `
+      position: absolute;
+      top: 0;
+      left: 0;
+      width: ${this.document.documentElement.scrollWidth}px;
+      height: ${this.document.documentElement.scrollHeight}px;
+      pointer-events: none;
+      z-index: 999999;
+      contain: layout style paint;
     `;
         let highlightedCount = 0;
         // Create border overlays and labels for each ref
@@ -825,17 +893,17 @@ export class DOMActions {
                 const border = this.document.createElement('div');
                 border.className = 'btcp-ref-border';
                 border.dataset.ref = ref;
-                border.style.cssText = `
-          position: absolute;
-          width: ${bbox.width}px;
-          height: ${bbox.height}px;
-          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
-          border: 2px solid rgba(59, 130, 246, 0.8);
-          border-radius: 2px;
-          box-shadow: 0 0 0 1px rgba(59, 130, 246, 0.2);
-          pointer-events: none;
-          will-change: transform;
-          contain: layout style paint;
+                border.style.cssText = `
+          position: absolute;
+          width: ${bbox.width}px;
+          height: ${bbox.height}px;
+          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
+          border: 2px solid rgba(59, 130, 246, 0.8);
+          border-radius: 2px;
+          box-shadow: 0 0 0 1px rgba(59, 130, 246, 0.2);
+          pointer-events: none;
+          will-change: transform;
+          contain: layout style paint;
         `;
                 // Create label
                 const label = this.document.createElement('div');
@@ -843,21 +911,21 @@ export class DOMActions {
                 label.dataset.ref = ref;
                 // Extract number from ref (e.g., "@ref:5" -> "5")
                 label.textContent = ref.replace('@ref:', '');
-                label.style.cssText = `
-          position: absolute;
-          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
-          background: rgba(59, 130, 246, 0.9);
-          color: white;
-          padding: 2px 6px;
-          border-radius: 3px;
-          font-family: monospace;
-          font-size: 11px;
-          font-weight: bold;
-          box-shadow: 0 2px 4px rgba(0,0,0,0.3);
-          pointer-events: none;
-          white-space: nowrap;
-          will-change: transform;
-          contain: layout style paint;
+                label.style.cssText = `
+          position: absolute;
+          transform: translate3d(${bbox.left + this.window.scrollX}px, ${bbox.top + this.window.scrollY}px, 0);
+          background: rgba(59, 130, 246, 0.9);
+          color: white;
+          padding: 2px 6px;
+          border-radius: 3px;
+          font-family: monospace;
+          font-size: 11px;
+          font-weight: bold;
+          box-shadow: 0 2px 4px rgba(0,0,0,0.3);
+          pointer-events: none;
+          white-space: nowrap;
+          will-change: transform;
+          contain: layout style paint;
         `;
                 this.overlayContainer.appendChild(border);
                 this.overlayContainer.appendChild(label);

package/packages/core/dist/snapshot.js

@@ -110,6 +110,11 @@ function getRole(element) {
         const type = element.type || 'text';
         return INPUT_ROLES[type] || 'textbox';
     }
+    // Detect contenteditable elements (ProseMirror, Quill, TinyMCE, etc.)
+    const contentEditable = element.getAttribute('contenteditable');
+    if (contentEditable === 'true' || contentEditable === '') {
+        return 'textbox';
+    }
     return IMPLICIT_ROLES[tagName] || null;
 }
 /**
@@ -320,6 +325,21 @@ function getAccessibleName(element) {
     if (isImage) {
         return getImageLabel(element);
     }
+    // Handle contenteditable elements (ProseMirror, Quill, TinyMCE, etc.)
+    const contentEditable = element.getAttribute('contenteditable');
+    if (contentEditable === 'true' || contentEditable === '') {
+        // Try data-placeholder attribute (common in rich text editors)
+        const placeholder = element.getAttribute('data-placeholder');
+        if (placeholder)
+            return placeholder.trim();
+        // Try finding placeholder in child paragraph element
+        const placeholderEl = element.querySelector('[data-placeholder]');
+        if (placeholderEl) {
+            const placeholderText = placeholderEl.getAttribute('data-placeholder');
+            if (placeholderText)
+                return placeholderText.trim();
+        }
+    }
     const ariaLabel = element.getAttribute('aria-label');
     if (ariaLabel)
         return ariaLabel.trim();

package/packages/extension/dist/background.js

@@ -594,7 +594,7 @@ export class BackgroundAgent {
             'tabNew', 'tabClose', 'tabSwitch', 'tabList',
             'groupCreate', 'groupUpdate', 'groupDelete', 'groupList',
             'groupAddTabs', 'groupRemoveTabs', 'groupGet',
-            'sessionGetCurrent', 'popupInitialize',
+            'sessionGetCurrent', 'sessionUseGroup', 'popupInitialize',
         ];
         return extensionActions.includes(command.action);
     }
@@ -679,6 +679,17 @@ export class BackgroundAgent {
                 const session = await this.sessionManager.getCurrentSession();
                 return { id: command.id, success: true, data: { session } };
             }
+            case 'sessionUseGroup': {
+                const used = await this.sessionManager.useExistingGroupAsSession(command.groupId);
+                if (!used) {
+                    return {
+                        id: command.id,
+                        success: false,
+                        error: `Failed to use group ${command.groupId} as session. Group may not exist.`,
+                    };
+                }
+                return { id: command.id, success: true, data: { groupId: command.groupId, used: true } };
+            }
             case 'popupInitialize': {
                 console.log('[BackgroundAgent] Popup initializing, checking for session reconnection...');
                 // Check if we have a stored session but no active connection

package/packages/extension/dist/index.d.ts

@@ -39,7 +39,7 @@
  * await client.click('@ref:5');
  * ```
  */
-import type { Command, Response, TabInfo } from './types.js';
+import type { Command, Response } from './types.js';
 import { BackgroundAgent as _BackgroundAgent, getBackgroundAgent as _getBackgroundAgent, setupMessageListener as _setupMessageListener, BrowserAgent as _BrowserAgent, getBrowserAgent as _getBrowserAgent } from './background.js';
 export * from './types.js';
 export { createScriptMessenger, createMethodMessenger, type MessageDefinitions, type ScriptMessenger, type MethodMessenger, type ScriptMessengerOptions, type PayloadOf, type ResultOf, } from './script-messenger.js';
@@ -130,124 +130,22 @@ export interface Client {
         quality?: number;
     }): Promise<string>;
     /**
-     * Open a new tab
+     * Wait for a selector to appear
      */
-    tabNew(options?: {
-        url?: string;
-        active?: boolean;
-    }): Promise<{
-        tabId: number;
-        url?: string;
-    }>;
-    /**
-     * Close a tab
-     */
-    tabClose(tabId?: number): Promise<Response>;
-    /**
-     * Switch to a tab
-     */
-    tabSwitch(tabId: number): Promise<Response>;
-    /**
-     * List all tabs
-     */
-    tabList(): Promise<TabInfo[]>;
-    /**
-     * Create a new tab group
-     */
-    groupCreate(options?: {
-        tabIds?: number[];
-        title?: string;
-        color?: string;
-        collapsed?: boolean;
-    }): Promise<{
-        group: import('./session-types.js').GroupInfo;
-    }>;
-    /**
-     * Update a tab group
-     */
-    groupUpdate(groupId: number, options: {
-        title?: string;
-        color?: string;
-        collapsed?: boolean;
-    }): Promise<{
-        group: import('./session-types.js').GroupInfo;
-    }>;
-    /**
-     * Delete a tab group (closes all tabs)
-     */
-    groupDelete(groupId: number): Promise<Response>;
-    /**
-     * List all tab groups
-     */
-    groupList(): Promise<import('./session-types.js').GroupInfo[]>;
-    /**
-     * Add tabs to a group
-     */
-    groupAddTabs(groupId: number, tabIds: number[]): Promise<Response>;
-    /**
-     * Remove tabs from their group
-     */
-    groupRemoveTabs(tabIds: number[]): Promise<Response>;
-    /**
-     * Get a specific tab group
-     */
-    groupGet(groupId: number): Promise<{
-        group: import('./session-types.js').GroupInfo;
-    }>;
-    /**
-     * Get current active session
-     */
-    sessionGetCurrent(): Promise<{
-        session: import('./session-types.js').SessionInfo | null;
-    }>;
-    /**
-     * Initialize popup (triggers session reconnection check)
-     */
-    popupInitialize(): Promise<{
-        initialized: boolean;
-        reconnected: boolean;
-    }>;
+    wait(options?: {
+        selector?: string;
+        timeout?: number;
+    }): Promise<Response>;
     /**
-     * Inject a script into the page's main world
-     *
-     * The script runs in the page context (not the content script isolated world),
-     * allowing access to page-level APIs like window, fetch interceptors, etc.
-     *
-     * @example
-     * ```typescript
-     * await client.scriptInject(`
-     *   window.addEventListener('message', (event) => {
-     *     if (event.data?.type !== 'btcp:script-command') return;
-     *     if (event.data.scriptId !== 'helper') return;
-     *     const { commandId, payload } = event.data;
-     *     // Handle and ack
-     *     window.postMessage({ type: 'btcp:script-ack', commandId, result: { ok: true } }, '*');
-     *   });
-     * `, { scriptId: 'helper' });
-     * ```
+     * Press a key
      */
-    scriptInject(code: string, options?: {
-        scriptId?: string;
-    }): Promise<{
-        scriptId: string;
-        injected: boolean;
-    }>;
+    press(key: string, options?: {
+        selector?: string;
+    }): Promise<Response>;
     /**
-     * Send a command to an injected script and wait for acknowledgment
-     *
-     * @example
-     * ```typescript
-     * const result = await client.scriptSend(
-     *   { action: 'getData', selector: '.items' },
-     *   { scriptId: 'helper', timeout: 5000 }
-     * );
-     * console.log(result); // { items: [...] }
-     * ```
+     * Evaluate JavaScript in the page context
      */
-    scriptSend(payload: unknown, options?: {
-        scriptId?: string;
-        timeout?: number;
-    }): Promise<unknown>;
+    evaluate(expression: string): Promise<unknown>;
 }
 /**
  * Generate a unique command ID for BTCP commands