browser-use 0.0.1 → 0.1.0

package/dist/controller/service.js

@@ -0,0 +1,1176 @@
+import fs from 'node:fs';
+import { ActionResult } from '../agent/views.js';
+import { BrowserError } from '../browser/views.js';
+import { FileSystem } from '../filesystem/file-system.js';
+import { ClickElementActionSchema, CloseTabActionSchema, DoneActionSchema, ExtractStructuredDataActionSchema, DropdownOptionsActionSchema, SelectDropdownActionSchema, GoToUrlActionSchema, InputTextActionSchema, NoParamsActionSchema, ReadFileActionSchema, ReplaceFileStrActionSchema, ScrollActionSchema, ScrollToTextActionSchema, SearchGoogleActionSchema, StructuredOutputActionSchema, SwitchTabActionSchema, UploadFileActionSchema, WaitActionSchema, WriteFileActionSchema, SendKeysActionSchema, SheetsRangeActionSchema, SheetsUpdateActionSchema, SheetsInputActionSchema, } from './views.js';
+import { Registry } from './registry/service.js';
+import TurndownService from 'turndown';
+import { UserMessage } from '../llm/messages.js';
+import { createLogger } from '../logging-config.js';
+const DEFAULT_WAIT_OFFSET = 3;
+const MAX_WAIT_SECONDS = 10;
+const toActionEntries = (action) => {
+    if (!action) {
+        return [];
+    }
+    return Object.entries(action).filter(([, params]) => params != null);
+};
+const createAbortError = (reason) => {
+    if (reason instanceof Error) {
+        return reason;
+    }
+    const error = new Error('Operation aborted');
+    error.name = 'AbortError';
+    return error;
+};
+const isAbortError = (error) => {
+    return error instanceof Error && error.name === 'AbortError';
+};
+const throwIfAborted = (signal) => {
+    if (signal?.aborted) {
+        throw createAbortError(signal.reason);
+    }
+};
+const waitWithSignal = async (timeoutMs, signal) => {
+    if (timeoutMs <= 0) {
+        throwIfAborted(signal);
+        return;
+    }
+    await new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+            cleanup();
+            resolve();
+        }, timeoutMs);
+        const onAbort = () => {
+            clearTimeout(timeout);
+            cleanup();
+            reject(createAbortError(signal?.reason));
+        };
+        const cleanup = () => {
+            signal?.removeEventListener('abort', onAbort);
+        };
+        if (signal) {
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
+            signal.addEventListener('abort', onAbort, { once: true });
+        }
+    });
+};
+const runWithTimeoutAndSignal = async (operation, timeoutMs, signal, timeoutMessage = 'Operation timed out') => {
+    throwIfAborted(signal);
+    if (timeoutMs <= 0) {
+        return operation();
+    }
+    return await new Promise((resolve, reject) => {
+        let settled = false;
+        const onAbort = () => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            cleanup();
+            reject(createAbortError(signal?.reason));
+        };
+        const onTimeout = () => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            cleanup();
+            reject(new Error(timeoutMessage));
+        };
+        const cleanup = () => {
+            if (timeout) {
+                clearTimeout(timeout);
+            }
+            signal?.removeEventListener('abort', onAbort);
+        };
+        const timeout = setTimeout(onTimeout, timeoutMs);
+        if (signal) {
+            if (signal.aborted) {
+                onAbort();
+                return;
+            }
+            signal.addEventListener('abort', onAbort, { once: true });
+        }
+        void operation()
+            .then((value) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            cleanup();
+            resolve(value);
+        })
+            .catch((error) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            cleanup();
+            reject(error);
+        });
+    });
+};
+export class Controller {
+    registry;
+    displayFilesInDoneText;
+    logger;
+    constructor(options = {}) {
+        const { exclude_actions = [], output_model = null, display_files_in_done_text = true, } = options;
+        this.registry = new Registry(exclude_actions);
+        this.displayFilesInDoneText = display_files_in_done_text;
+        this.logger = createLogger('browser_use.controller');
+        this.registerDefaultActions(output_model);
+    }
+    registerDefaultActions(outputModel) {
+        this.registerDoneAction(outputModel);
+        this.registerNavigationActions();
+        this.registerElementActions();
+        this.registerTabActions();
+        this.registerContentActions();
+        this.registerScrollActions();
+        this.registerFileSystemActions();
+        this.registerKeyboardActions();
+        this.registerDropdownActions();
+        this.registerSheetsActions();
+    }
+    registerNavigationActions() {
+        this.registry.action('Search the query in Google...', {
+            param_model: SearchGoogleActionSchema,
+        })(async function search_google(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(params.query)}&udm=14`;
+            const page = await browser_session.get_current_page();
+            const currentUrl = page?.url().replace(/\/+$/, '');
+            if (currentUrl === 'https://www.google.com') {
+                await browser_session.navigate_to(searchUrl, { signal });
+            }
+            else {
+                await browser_session.create_new_tab(searchUrl, { signal });
+            }
+            const msg = `🔍  Searched for "${params.query}" in Google`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: `Searched Google for '${params.query}'`,
+            });
+        });
+        this.registry.action('Navigate to URL...', {
+            param_model: GoToUrlActionSchema,
+        })(async function go_to_url(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            try {
+                if (params.new_tab) {
+                    await browser_session.create_new_tab(params.url, { signal });
+                    const tabIdx = browser_session.active_tab_index;
+                    const msg = `🔗  Opened new tab #${tabIdx} with url ${params.url}`;
+                    return new ActionResult({
+                        extracted_content: msg,
+                        include_in_memory: true,
+                        long_term_memory: `Opened new tab with URL ${params.url}`,
+                    });
+                }
+                await browser_session.navigate_to(params.url, { signal });
+                const msg = `🔗 Navigated to ${params.url}`;
+                return new ActionResult({
+                    extracted_content: msg,
+                    include_in_memory: true,
+                    long_term_memory: `Navigated to ${params.url}`,
+                });
+            }
+            catch (error) {
+                const errorMsg = String(error?.message ?? error ?? '');
+                const networkFailures = [
+                    'ERR_NAME_NOT_RESOLVED',
+                    'ERR_INTERNET_DISCONNECTED',
+                    'ERR_CONNECTION_REFUSED',
+                    'ERR_TIMED_OUT',
+                    'net::',
+                ];
+                if (networkFailures.some((needle) => errorMsg.includes(needle))) {
+                    const message = `Site unavailable: ${params.url} - ${errorMsg}`;
+                    throw new BrowserError(message);
+                }
+                throw error;
+            }
+        });
+        this.registry.action('Go back', { param_model: NoParamsActionSchema })(async function go_back(_params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            await browser_session.go_back({ signal });
+            const msg = '🔙  Navigated back';
+            return new ActionResult({ extracted_content: msg });
+        });
+        this.registry.action('Wait for x seconds default 3 (max 10 seconds). This can be used to wait until the page is fully loaded.', { param_model: WaitActionSchema })(async function wait(params, { signal }) {
+            const seconds = params.seconds ?? 3;
+            const actualSeconds = Math.min(Math.max(seconds - DEFAULT_WAIT_OFFSET, 0), MAX_WAIT_SECONDS);
+            const msg = `🕒  Waiting for ${actualSeconds + DEFAULT_WAIT_OFFSET} seconds`;
+            if (actualSeconds > 0) {
+                await waitWithSignal(actualSeconds * 1000, signal);
+            }
+            return new ActionResult({ extracted_content: msg });
+        });
+    }
+    registerElementActions() {
+        this.registry.action('Click element by index', {
+            param_model: ClickElementActionSchema,
+        })(async function click_element_by_index(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const element = await browser_session.get_dom_element_by_index(params.index, { signal });
+            if (!element) {
+                throw new BrowserError(`Element index ${params.index} does not exist - retry or use alternative actions`);
+            }
+            const initialTabs = Array.isArray(browser_session.tabs)
+                ? browser_session.tabs.length
+                : 0;
+            if (browser_session.is_file_input?.(element)) {
+                const msg = `Index ${params.index} - has an element which opens file upload dialog.`;
+                return new ActionResult({
+                    extracted_content: msg,
+                    include_in_memory: true,
+                    success: false,
+                    long_term_memory: msg,
+                });
+            }
+            const downloadPath = await browser_session._click_element_node(element, {
+                signal,
+            });
+            let msg = '';
+            if (downloadPath) {
+                msg = `💾 Downloaded file to ${downloadPath}`;
+            }
+            else {
+                const snippet = element.get_all_text_till_next_clickable_element?.(2) ?? '';
+                msg = `🖱️  Clicked button with index ${params.index}: ${snippet}`;
+            }
+            if (Array.isArray(browser_session.tabs) &&
+                browser_session.tabs.length > initialTabs) {
+                msg += ' - New tab opened - switching to it';
+                await browser_session.switch_to_tab(-1, { signal });
+            }
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: msg,
+            });
+        });
+        this.registry.action('Click and input text into an input interactive element', { param_model: InputTextActionSchema })(async function input_text(params, { browser_session, has_sensitive_data, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const element = await browser_session.get_dom_element_by_index(params.index, { signal });
+            if (!element) {
+                throw new BrowserError(`Element index ${params.index} does not exist - retry or use alternative actions`);
+            }
+            await browser_session._input_text_element_node(element, params.text, {
+                signal,
+            });
+            const msg = has_sensitive_data
+                ? `⌨️  Input sensitive data into index ${params.index}`
+                : `⌨️  Input ${params.text} into index ${params.index}`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: `Input '${params.text}' into element ${params.index}.`,
+            });
+        });
+        this.registry.action('Upload file to interactive element with file path', {
+            param_model: UploadFileActionSchema,
+        })(async function upload_file(params, { browser_session, available_file_paths, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            if (!available_file_paths?.includes(params.path)) {
+                throw new BrowserError(`File path ${params.path} is not available`);
+            }
+            if (!fs.existsSync(params.path)) {
+                throw new BrowserError(`File ${params.path} does not exist`);
+            }
+            const node = await browser_session.find_file_upload_element_by_index(params.index, 3, 3, { signal });
+            if (!node) {
+                throw new BrowserError(`No file upload element found at index ${params.index}`);
+            }
+            const locator = await browser_session.get_locate_element(node);
+            if (!locator) {
+                throw new BrowserError(`No file upload element found at index ${params.index}`);
+            }
+            await locator.setInputFiles(params.path);
+            const msg = `📁 Successfully uploaded file to index ${params.index}`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: `Uploaded file ${params.path} to element ${params.index}`,
+            });
+        });
+    }
+    registerTabActions() {
+        this.registry.action('Switch tab', { param_model: SwitchTabActionSchema })(async function switch_tab(params, ctx) {
+            const { browser_session, signal } = ctx;
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            await browser_session.switch_to_tab(params.page_id, { signal });
+            const page = await browser_session.get_current_page();
+            try {
+                await page?.wait_for_load_state?.('domcontentloaded', {
+                    timeout: 5000,
+                });
+            }
+            catch {
+                /* ignore */
+            }
+            const msg = `🔄  Switched to tab #${params.page_id} with url ${page?.url ?? ''}`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: `Switched to tab ${params.page_id}`,
+            });
+        });
+        this.registry.action('Close an existing tab', {
+            param_model: CloseTabActionSchema,
+        })(async function close_tab(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            await browser_session.switch_to_tab(params.page_id, { signal });
+            const page = await browser_session.get_current_page();
+            const url = page?.url ?? '';
+            await page?.close?.();
+            const newPage = await browser_session.get_current_page();
+            const newIndex = browser_session.active_tab_index;
+            const msg = `❌  Closed tab #${params.page_id} with ${url}, now focused on tab #${newIndex} with url ${newPage?.url ?? ''}`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: `Closed tab ${params.page_id} with url ${url}, now focused on tab ${newIndex} with url ${newPage?.url ?? ''}.`,
+            });
+        });
+    }
+    registerContentActions() {
+        this.registry.action('Extract structured, semantic data from the current webpage based on a textual query.', {
+            param_model: ExtractStructuredDataActionSchema,
+        })(async function extract_structured_data(params, { page, page_extraction_llm, file_system, signal }) {
+            throwIfAborted(signal);
+            if (!page) {
+                throw new BrowserError('No active page available for extraction.');
+            }
+            if (!page_extraction_llm) {
+                throw new BrowserError('page_extraction_llm is not configured.');
+            }
+            const fsInstance = file_system ?? new FileSystem(process.cwd(), false);
+            const html = await page.content?.();
+            if (!html) {
+                throw new BrowserError('Unable to extract page content.');
+            }
+            const turndown = new TurndownService({
+                headingStyle: 'atx',
+                codeBlockStyle: 'fenced',
+            });
+            let rawHtml = html;
+            if (!params.extract_links) {
+                rawHtml = rawHtml.replace(/<a\b[^>]*>/gi, '').replace(/<\/a>/gi, '');
+            }
+            let content = turndown.turndown(rawHtml);
+            content = content.replace(/\n+/g, '\n');
+            // Manually append iframe text into the content so it's readable by the LLM (includes cross-origin iframes)
+            const frames = page.frames?.() || [];
+            for (const iframe of frames) {
+                throwIfAborted(signal);
+                try {
+                    // Wait for iframe to load with aggressive timeout
+                    await runWithTimeoutAndSignal(async () => {
+                        await iframe.waitForLoadState?.('load');
+                    }, 2000, signal, 'Iframe load timeout');
+                }
+                catch (error) {
+                    if (isAbortError(error)) {
+                        throw error;
+                    }
+                    // Ignore iframe load errors
+                }
+                const iframeUrl = iframe.url?.();
+                const pageUrl = page.url?.();
+                if (iframeUrl &&
+                    pageUrl &&
+                    iframeUrl !== pageUrl &&
+                    !iframeUrl.startsWith('data:') &&
+                    !iframeUrl.startsWith('about:')) {
+                    content += `\n\nIFRAME ${iframeUrl}:\n`;
+                    try {
+                        const iframeHtml = await runWithTimeoutAndSignal(async () => (await iframe.content?.()) ?? '', 2000, signal, 'Iframe content extraction timeout');
+                        const iframeMarkdown = turndown.turndown(iframeHtml || '');
+                        content += iframeMarkdown;
+                    }
+                    catch (error) {
+                        if (isAbortError(error)) {
+                            throw error;
+                        }
+                        // Skip failed iframes
+                    }
+                }
+            }
+            // Replace multiple sequential \n with a single \n
+            content = content.replace(/\n+/g, '\n');
+            const maxChars = 30000;
+            if (content.length > maxChars) {
+                const head = content.slice(0, maxChars / 2);
+                const tail = content.slice(-maxChars / 2);
+                content = `${head}\n... left out the middle because it was too long ...\n${tail}`;
+            }
+            const prompt = `You convert websites into structured information. Extract information from this webpage based on the query. Focus only on content relevant to the query. If 
+1. The query is vague
+2. Does not make sense for the page
+3. Some/all of the information is not available
+
+Explain the content of the page and that the requested information is not available in the page. Respond in JSON format.
+Query: ${params.query}
+Website:
+${content}`;
+            const extraction = await page_extraction_llm.ainvoke([new UserMessage(prompt)], undefined, { signal: signal ?? undefined });
+            throwIfAborted(signal);
+            const completion = extraction?.completion ?? '';
+            const extracted_content = `Page Link: ${page.url}\nQuery: ${params.query}\nExtracted Content:\n${completion}`;
+            let includeOnce = false;
+            let memory = extracted_content;
+            const MAX_MEMORY_SIZE = 600;
+            if (extracted_content.length > MAX_MEMORY_SIZE) {
+                const lines = extracted_content.split('\n');
+                let display = '';
+                let count = 0;
+                for (const line of lines) {
+                    if (display.length + line.length > MAX_MEMORY_SIZE)
+                        break;
+                    display += `${line}\n`;
+                    count += 1;
+                }
+                const saveResult = await fsInstance.save_extracted_content(extracted_content);
+                // NOTE: Do NOT mention file_system tag here as it misleads LLM to use read_file action
+                // The extracted content preview above is sufficient for most tasks
+                memory = `Extracted content from ${page.url}\n<query>${params.query}</query>\n<extracted_content>\n${display}${lines.length - count} more lines (auto-saved, no need to read)...\n</extracted_content>`;
+                includeOnce = true;
+            }
+            return new ActionResult({
+                extracted_content,
+                include_extracted_content_only_once: includeOnce,
+                long_term_memory: memory,
+            });
+        });
+    }
+    registerScrollActions() {
+        const scrollLogger = this.logger; // Capture logger reference for use in named function
+        // Define the scroll handler implementation (shared by multiple action names for LLM compatibility)
+        const scrollImpl = async (params, { browser_session, signal }) => {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            if (!page || !page.evaluate) {
+                throw new BrowserError('Unable to access current page for scrolling.');
+            }
+            // Helper function to get window height with retries
+            const getWindowHeight = async (retries = 3) => {
+                for (let i = 0; i < retries; i++) {
+                    throwIfAborted(signal);
+                    try {
+                        const height = await page.evaluate(() => window.innerHeight);
+                        return height || 0;
+                    }
+                    catch (error) {
+                        if (i === retries - 1) {
+                            throw new Error(`Scroll failed due to an error: ${error}`);
+                        }
+                        await waitWithSignal(1000, signal);
+                    }
+                }
+                return 0;
+            };
+            const windowHeight = await getWindowHeight();
+            const scrollAmount = Math.floor(windowHeight * params.num_pages);
+            const pagesScrolled = params.num_pages;
+            const dy = params.down ? scrollAmount : -scrollAmount;
+            const direction = params.down ? 'down' : 'up';
+            let scrollTarget = 'the page';
+            // Element-specific scrolling if index is provided
+            if (params.index !== undefined && params.index !== null) {
+                try {
+                    const elementNode = await browser_session.get_dom_element_by_index(params.index, { signal });
+                    if (!elementNode) {
+                        throw new Error(`Element index ${params.index} does not exist - retry or use alternative actions`);
+                    }
+                    // Try direct container scrolling (no events that might close dropdowns)
+                    const containerScrollJs = `
+						(params) => {
+							const { dy, elementXPath } = params;
+
+							// Get the target element by XPath
+							const targetElement = document.evaluate(elementXPath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+							if (!targetElement) {
+								return { success: false, reason: 'Element not found by XPath' };
+							}
+
+							console.log('[SCROLL DEBUG] Starting direct container scroll for element:', targetElement.tagName);
+
+							// Try to find scrollable containers in the hierarchy (starting from element itself)
+							let currentElement = targetElement;
+							let scrollSuccess = false;
+							let scrolledElement = null;
+							let scrollDelta = 0;
+							let attempts = 0;
+
+							// Check up to 10 elements in hierarchy (including the target element itself)
+							while (currentElement && attempts < 10) {
+								const computedStyle = window.getComputedStyle(currentElement);
+								const hasScrollableY = /(auto|scroll|overlay)/.test(computedStyle.overflowY);
+								const canScrollVertically = currentElement.scrollHeight > currentElement.clientHeight;
+
+								console.log('[SCROLL DEBUG] Checking element:', currentElement.tagName,
+									'hasScrollableY:', hasScrollableY,
+									'canScrollVertically:', canScrollVertically,
+									'scrollHeight:', currentElement.scrollHeight,
+									'clientHeight:', currentElement.clientHeight);
+
+								if (hasScrollableY && canScrollVertically) {
+									const beforeScroll = currentElement.scrollTop;
+									const maxScroll = currentElement.scrollHeight - currentElement.clientHeight;
+
+									// Calculate scroll amount (1/3 of provided dy for gentler scrolling)
+									let scrollAmount = dy / 3;
+
+									// Ensure we don't scroll beyond bounds
+									if (scrollAmount > 0) {
+										scrollAmount = Math.min(scrollAmount, maxScroll - beforeScroll);
+									} else {
+										scrollAmount = Math.max(scrollAmount, -beforeScroll);
+									}
+
+									// Try direct scrollTop manipulation (most reliable)
+									currentElement.scrollTop = beforeScroll + scrollAmount;
+
+									const afterScroll = currentElement.scrollTop;
+									const actualScrollDelta = afterScroll - beforeScroll;
+
+									console.log('[SCROLL DEBUG] Scroll attempt:', currentElement.tagName,
+										'before:', beforeScroll, 'after:', afterScroll, 'delta:', actualScrollDelta);
+
+									if (Math.abs(actualScrollDelta) > 0.5) {
+										scrollSuccess = true;
+										scrolledElement = currentElement;
+										scrollDelta = actualScrollDelta;
+										console.log('[SCROLL DEBUG] Successfully scrolled container:', currentElement.tagName, 'delta:', actualScrollDelta);
+										break;
+									}
+								}
+
+								// Move to parent (but don't go beyond body for dropdown case)
+								if (currentElement === document.body || currentElement === document.documentElement) {
+									break;
+								}
+								currentElement = currentElement.parentElement;
+								attempts++;
+							}
+
+							if (scrollSuccess) {
+								// Successfully scrolled a container
+								return {
+									success: true,
+									method: 'direct_container_scroll',
+									containerType: 'element',
+									containerTag: scrolledElement.tagName.toLowerCase(),
+									containerClass: scrolledElement.className || '',
+									containerId: scrolledElement.id || '',
+									scrollDelta: scrollDelta
+								};
+							} else {
+								// No container found or could scroll
+								console.log('[SCROLL DEBUG] No scrollable container found for element');
+								return {
+									success: false,
+									reason: 'No scrollable container found',
+									needsPageScroll: true
+								};
+							}
+						}
+						`;
+                    const scrollParams = { dy, elementXPath: elementNode.xpath };
+                    const result = (await page.evaluate(containerScrollJs, scrollParams));
+                    if (result.success) {
+                        if (result.containerType === 'element') {
+                            let containerInfo = result.containerTag;
+                            if (result.containerId) {
+                                containerInfo += `#${result.containerId}`;
+                            }
+                            else if (result.containerClass) {
+                                containerInfo += `.${result.containerClass.split(' ')[0]}`;
+                            }
+                            scrollTarget = `element ${params.index}'s scroll container (${containerInfo})`;
+                            // Don't do additional page scrolling since we successfully scrolled the container
+                        }
+                        else {
+                            scrollTarget = `the page (fallback from element ${params.index})`;
+                        }
+                    }
+                    else {
+                        // Container scroll failed, need page-level scrolling
+                        scrollLogger.debug(`Container scroll failed for element ${params.index}: ${result.reason || 'Unknown'}`);
+                        scrollTarget = `the page (no container found for element ${params.index})`;
+                        // This will trigger page-level scrolling below
+                    }
+                }
+                catch (error) {
+                    scrollLogger.debug(`Element-specific scrolling failed for index ${params.index}: ${error}`);
+                    scrollTarget = `the page (fallback from element ${params.index})`;
+                    // Fall through to page-level scrolling
+                }
+            }
+            // Page-level scrolling (default or fallback)
+            if (scrollTarget === 'the page' ||
+                scrollTarget.includes('fallback') ||
+                scrollTarget.includes('no container found') ||
+                scrollTarget.includes('mouse wheel failed')) {
+                scrollLogger.debug(`🔄 Performing page-level scrolling. Reason: ${scrollTarget}`);
+                try {
+                    await browser_session._scrollContainer(dy);
+                }
+                catch (error) {
+                    // Hard fallback: always works on root scroller
+                    await page.evaluate((y) => window.scrollBy(0, y), dy);
+                    scrollLogger.debug('Smart scroll failed; used window.scrollBy fallback', error);
+                }
+            }
+            // Create descriptive message
+            let longTermMemory;
+            if (pagesScrolled === 1.0) {
+                longTermMemory = `Scrolled ${direction} ${scrollTarget} by one page`;
+            }
+            else {
+                longTermMemory = `Scrolled ${direction} ${scrollTarget} by ${pagesScrolled} pages`;
+            }
+            const msg = `🔍 ${longTermMemory}`;
+            scrollLogger.info(msg);
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: longTermMemory,
+            });
+        };
+        // Register scroll action with multiple names for LLM compatibility
+        // Different LLMs may use different names: scroll, scroll_page, scroll_down
+        const scrollDescription = 'Scroll the page by specified number of pages (set down=True to scroll down, down=False to scroll up, num_pages=number of pages to scroll like 0.5 for half page, 1.0 for one page, etc.). Optional index parameter to scroll within a specific element or its scroll container (works well for dropdowns and custom UI components).';
+        // Create named functions that wrap the implementation
+        // Different LLMs may use different names: scroll, scroll_page, scroll_down, scroll_by, scroll_page_by, scroll_up
+        const scrollAction = async function scroll(p, ctx) { return scrollImpl(p, ctx); };
+        const scrollPageAction = async function scroll_page(p, ctx) { return scrollImpl(p, ctx); };
+        const scrollDownAction = async function scroll_down(p, ctx) { return scrollImpl(p, ctx); };
+        const scrollByAction = async function scroll_by(p, ctx) { return scrollImpl(p, ctx); };
+        const scrollPageByAction = async function scroll_page_by(p, ctx) { return scrollImpl(p, ctx); };
+        const scrollUpAction = async function scroll_up(p, ctx) { return scrollImpl(p, ctx); };
+        this.registry.action(scrollDescription, { param_model: ScrollActionSchema })(scrollAction);
+        this.registry.action(scrollDescription, { param_model: ScrollActionSchema })(scrollPageAction);
+        this.registry.action(scrollDescription, { param_model: ScrollActionSchema })(scrollDownAction);
+        this.registry.action(scrollDescription, { param_model: ScrollActionSchema })(scrollByAction);
+        this.registry.action(scrollDescription, { param_model: ScrollActionSchema })(scrollPageByAction);
+        this.registry.action(scrollDescription, { param_model: ScrollActionSchema })(scrollUpAction);
+        this.registry.action('Scroll to a text in the current page', {
+            param_model: ScrollToTextActionSchema,
+        })(async function scroll_to_text(params, { browser_session }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            const page = await browser_session.get_current_page();
+            if (!page?.evaluate) {
+                throw new BrowserError('Unable to access page for scrolling.');
+            }
+            const success = await page.evaluate(({ text }) => {
+                const iterator = document.createNodeIterator(document.body, NodeFilter.SHOW_ELEMENT);
+                let node;
+                while ((node = iterator.nextNode())) {
+                    const el = node;
+                    if (!el || !el.textContent)
+                        continue;
+                    if (el.textContent.toLowerCase().includes(text.toLowerCase())) {
+                        el.scrollIntoView({ behavior: 'smooth', block: 'center' });
+                        return true;
+                    }
+                }
+                return false;
+            }, { text: params.text });
+            if (!success) {
+                throw new BrowserError(`Text '${params.text}' not found on page`);
+            }
+            const msg = `🔍  Scrolled to text: ${params.text}`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: msg,
+            });
+        });
+    }
+    registerFileSystemActions() {
+        this.registry.action('Read file_name from file system', {
+            param_model: ReadFileActionSchema,
+        })(async function read_file(params, { file_system, available_file_paths }) {
+            const fsInstance = file_system ?? new FileSystem(process.cwd(), false);
+            const allowed = Array.isArray(available_file_paths) &&
+                available_file_paths.includes(params.file_name);
+            const result = await fsInstance.read_file(params.file_name, allowed);
+            const MAX_MEMORY_SIZE = 1000;
+            let memory = result;
+            if (result.length > MAX_MEMORY_SIZE) {
+                const lines = result.split('\n');
+                let preview = '';
+                let used = 0;
+                for (const line of lines) {
+                    if (preview.length + line.length > MAX_MEMORY_SIZE)
+                        break;
+                    preview += `${line}\n`;
+                    used += 1;
+                }
+                const remaining = lines.length - used;
+                memory =
+                    remaining > 0 ? `${preview}${remaining} more lines...` : preview;
+            }
+            return new ActionResult({
+                extracted_content: result,
+                include_in_memory: true,
+                long_term_memory: memory,
+                include_extracted_content_only_once: true,
+            });
+        });
+        this.registry.action('Write content to file', {
+            param_model: WriteFileActionSchema,
+        })(async function write_file(params, { file_system }) {
+            const fsInstance = file_system ?? new FileSystem(process.cwd(), false);
+            let content = params.content;
+            const trailing = params.trailing_newline ?? true;
+            const leading = params.leading_newline ?? false;
+            if (trailing) {
+                content = `${content}\n`;
+            }
+            if (leading) {
+                content = `\n${content}`;
+            }
+            const append = params.append ?? false;
+            const result = append
+                ? await fsInstance.append_file(params.file_name, content)
+                : await fsInstance.write_file(params.file_name, content);
+            const msg = `📝  ${result}`;
+            return new ActionResult({
+                extracted_content: result,
+                include_in_memory: true,
+                long_term_memory: result,
+            });
+        });
+        this.registry.action('Replace text within an existing file', {
+            param_model: ReplaceFileStrActionSchema,
+        })(async function replace_file_str(params, { file_system }) {
+            const fsInstance = file_system ?? new FileSystem(process.cwd(), false);
+            const result = await fsInstance.replace_file_str(params.file_name, params.old_str, params.new_str);
+            return new ActionResult({
+                extracted_content: result,
+                include_in_memory: true,
+                long_term_memory: result,
+            });
+        });
+    }
+    registerKeyboardActions() {
+        this.registry.action('Send keys to the active page', {
+            param_model: SendKeysActionSchema,
+        })(async function send_keys(params, { browser_session }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            const page = await browser_session.get_current_page();
+            const keyboard = page?.keyboard;
+            if (!keyboard) {
+                throw new BrowserError('Keyboard input is not available on the current page.');
+            }
+            try {
+                await keyboard.press(params.keys);
+            }
+            catch (error) {
+                if (error instanceof Error && error.message.includes('Unknown key')) {
+                    for (const char of params.keys) {
+                        await keyboard.press(char);
+                    }
+                }
+                else {
+                    throw error;
+                }
+            }
+            const msg = `⌨️  Sent keys: ${params.keys}`;
+            return new ActionResult({
+                extracted_content: msg,
+                include_in_memory: true,
+                long_term_memory: msg,
+            });
+        });
+    }
+    registerDropdownActions() {
+        this.registry.action('Get all options from a native dropdown or ARIA menu', { param_model: DropdownOptionsActionSchema })(async function get_dropdown_options(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            const domElement = await browser_session.get_dom_element_by_index(params.index, { signal });
+            if (!domElement) {
+                throw new BrowserError(`Element index ${params.index} does not exist.`);
+            }
+            if (!page?.evaluate) {
+                throw new BrowserError('Unable to evaluate dropdown options on current page.');
+            }
+            if (!domElement.xpath) {
+                throw new BrowserError('DOM element does not include an XPath selector.');
+            }
+            const payload = await page.evaluate(({ xpath }) => {
+                const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                if (!element)
+                    return null;
+                if (element.tagName?.toLowerCase() === 'select') {
+                    const options = Array.from(element.options).map((opt, index) => ({
+                        text: opt.text,
+                        value: opt.value,
+                        index,
+                    }));
+                    return { type: 'select', options };
+                }
+                const ariaRoles = new Set(['menu', 'listbox', 'combobox']);
+                const role = element.getAttribute('role');
+                if (role && ariaRoles.has(role)) {
+                    const nodes = element.querySelectorAll('[role="menuitem"],[role="option"]');
+                    const options = Array.from(nodes).map((node, index) => ({
+                        text: node.textContent?.trim() ?? '',
+                        value: node.textContent?.trim() ?? '',
+                        index,
+                    }));
+                    return { type: 'aria', options };
+                }
+                return null;
+            }, { xpath: domElement.xpath });
+            if (!payload || !payload.options?.length) {
+                throw new BrowserError('No options found for the specified dropdown.');
+            }
+            const formatted = payload.options.map((opt) => `${opt.index}: text=${JSON.stringify(opt.text ?? '')}`);
+            formatted.push('Use the exact text string in select_dropdown_option');
+            const message = formatted.join('\n');
+            return new ActionResult({
+                extracted_content: message,
+                include_in_memory: true,
+                include_extracted_content_only_once: true,
+                long_term_memory: `Found dropdown options for index ${params.index}.`,
+            });
+        });
+        this.registry.action('Select dropdown option or ARIA menu item by text', {
+            param_model: SelectDropdownActionSchema,
+        })(async function select_dropdown_option(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            const domElement = await browser_session.get_dom_element_by_index(params.index, { signal });
+            if (!domElement?.xpath) {
+                throw new BrowserError('DOM element does not include an XPath selector.');
+            }
+            if (!page) {
+                throw new BrowserError('No active page for selection.');
+            }
+            for (const frame of page.frames ?? []) {
+                try {
+                    const typeInfo = await frame.evaluate((xpath) => {
+                        const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                        if (!element)
+                            return { found: false };
+                        const tagName = element.tagName?.toLowerCase();
+                        const role = element.getAttribute?.('role');
+                        if (tagName === 'select')
+                            return { found: true, type: 'select' };
+                        if (role && ['menu', 'listbox', 'combobox'].includes(role))
+                            return { found: true, type: 'aria' };
+                        return { found: false };
+                    }, domElement.xpath);
+                    if (!typeInfo?.found)
+                        continue;
+                    if (typeInfo.type === 'select') {
+                        await frame
+                            .locator(domElement.xpath)
+                            .first()
+                            .select_option({ label: params.text });
+                        const msg = `Selected option ${params.text}`;
+                        return new ActionResult({
+                            extracted_content: msg,
+                            include_in_memory: true,
+                            long_term_memory: msg,
+                        });
+                    }
+                    const clicked = await frame.evaluate(({ xpath, text }) => {
+                        const root = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                        if (!root)
+                            return false;
+                        const nodes = root.querySelectorAll('[role="menuitem"],[role="option"]');
+                        for (const node of Array.from(nodes)) {
+                            if (node.textContent?.trim() === text) {
+                                node.click();
+                                return true;
+                            }
+                        }
+                        return false;
+                    }, { xpath: domElement.xpath, text: params.text });
+                    if (clicked) {
+                        const msg = `Selected menu item ${params.text}`;
+                        return new ActionResult({
+                            extracted_content: msg,
+                            include_in_memory: true,
+                            long_term_memory: msg,
+                        });
+                    }
+                }
+                catch (error) {
+                    continue;
+                }
+            }
+            throw new BrowserError(`Could not select option '${params.text}' for index ${params.index}`);
+        });
+    }
+    registerSheetsActions() {
+        // Capture 'this' for use in callbacks
+        const self = this;
+        this.registry.action('Google Sheets: Get the contents of the entire sheet', {
+            domains: ['https://docs.google.com'],
+        })(async function sheets_get_contents(_params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            await page?.keyboard?.press('Enter');
+            await page?.keyboard?.press('Escape');
+            await page?.keyboard?.press('ControlOrMeta+A');
+            await page?.keyboard?.press('ControlOrMeta+C');
+            const content = await page?.evaluate?.(() => navigator.clipboard.readText());
+            return new ActionResult({
+                extracted_content: content ?? '',
+                include_in_memory: true,
+                long_term_memory: 'Retrieved sheet contents',
+                include_extracted_content_only_once: true,
+            });
+        });
+        this.registry.action('Google Sheets: Get the contents of a cell or range of cells', {
+            domains: ['https://docs.google.com'],
+            param_model: SheetsRangeActionSchema,
+        })(async function sheets_get_range(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            await self.gotoSheetsRange(page, params.cell_or_range, signal);
+            await page?.keyboard?.press('ControlOrMeta+C');
+            await waitWithSignal(100, signal);
+            const content = await page?.evaluate?.(() => navigator.clipboard.readText());
+            return new ActionResult({
+                extracted_content: content ?? '',
+                include_in_memory: true,
+                long_term_memory: `Retrieved contents from ${params.cell_or_range}`,
+                include_extracted_content_only_once: true,
+            });
+        });
+        this.registry.action('Google Sheets: Update the content of a cell or range of cells', {
+            domains: ['https://docs.google.com'],
+            param_model: SheetsUpdateActionSchema,
+        })(async function sheets_update(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            await self.gotoSheetsRange(page, params.cell_or_range, signal);
+            await page?.evaluate?.((value) => {
+                const clipboardData = new DataTransfer();
+                clipboardData.setData('text/plain', value);
+                document.activeElement?.dispatchEvent(new ClipboardEvent('paste', { clipboardData }));
+            }, params.value);
+            return new ActionResult({
+                extracted_content: `Updated cells: ${params.cell_or_range} = ${params.value}`,
+                long_term_memory: `Updated cells ${params.cell_or_range} with ${params.value}`,
+            });
+        });
+        this.registry.action('Google Sheets: Clear whatever cells are currently selected', {
+            domains: ['https://docs.google.com'],
+            param_model: SheetsRangeActionSchema,
+        })(async function sheets_clear(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            await self.gotoSheetsRange(page, params.cell_or_range, signal);
+            await page?.keyboard?.press('Backspace');
+            return new ActionResult({
+                extracted_content: `Cleared cells: ${params.cell_or_range}`,
+                long_term_memory: `Cleared cells ${params.cell_or_range}`,
+            });
+        });
+        this.registry.action('Google Sheets: Select a specific cell or range of cells', {
+            domains: ['https://docs.google.com'],
+            param_model: SheetsRangeActionSchema,
+        })(async function sheets_select(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            await self.gotoSheetsRange(page, params.cell_or_range, signal);
+            return new ActionResult({
+                extracted_content: `Selected cells: ${params.cell_or_range}`,
+                long_term_memory: `Selected cells ${params.cell_or_range}`,
+            });
+        });
+        this.registry.action('Google Sheets: Fallback method to type text into the currently selected cell', {
+            domains: ['https://docs.google.com'],
+            param_model: SheetsInputActionSchema,
+        })(async function sheets_input(params, { browser_session, signal }) {
+            if (!browser_session)
+                throw new Error('Browser session missing');
+            throwIfAborted(signal);
+            const page = await browser_session.get_current_page();
+            await page?.keyboard?.type(params.text, { delay: 100 });
+            await page?.keyboard?.press('Enter');
+            await page?.keyboard?.press('ArrowUp');
+            return new ActionResult({
+                extracted_content: `Inputted text ${params.text}`,
+                long_term_memory: `Inputted text '${params.text}' into cell`,
+            });
+        });
+    }
+    async gotoSheetsRange(page, cell_or_range, signal = null) {
+        if (!page?.keyboard) {
+            throw new BrowserError('No keyboard available for Google Sheets actions.');
+        }
+        throwIfAborted(signal);
+        await page.keyboard.press('Enter');
+        await page.keyboard.press('Escape');
+        await waitWithSignal(100, signal);
+        await page.keyboard.press('Home');
+        await page.keyboard.press('ArrowUp');
+        await waitWithSignal(100, signal);
+        await page.keyboard.press('Control+G');
+        await waitWithSignal(200, signal);
+        await page.keyboard.type(cell_or_range, { delay: 50 });
+        await page.keyboard.press('Enter');
+        await waitWithSignal(200, signal);
+        await page.keyboard.press('Escape');
+    }
+    registerDoneAction(outputModel) {
+        // Capture 'this' for use in callbacks
+        const self = this;
+        if (outputModel) {
+            const structuredSchema = StructuredOutputActionSchema(outputModel);
+            this.registry.action('Complete task - with return text and success flag.', { param_model: structuredSchema })(async function done(params) {
+                const payload = { ...params.data };
+                for (const key of Object.keys(payload)) {
+                    const value = payload[key];
+                    if (value && typeof value === 'object' && 'value' in value) {
+                        payload[key] = value.value;
+                    }
+                }
+                return new ActionResult({
+                    is_done: true,
+                    success: params.success,
+                    extracted_content: JSON.stringify(payload),
+                    long_term_memory: `Task completed. Success Status: ${params.success}`,
+                });
+            });
+            return;
+        }
+        this.registry.action('Complete task - provide a summary to the user.', {
+            param_model: DoneActionSchema,
+        })(async function done(params, { file_system }) {
+            const fsInstance = file_system ?? new FileSystem(process.cwd(), false);
+            let userMessage = params.text;
+            const lenMaxMemory = 100;
+            let memory = `Task completed: ${params.success} - ${params.text.slice(0, lenMaxMemory)}`;
+            if (params.text.length > lenMaxMemory) {
+                memory += ` - ${params.text.length - lenMaxMemory} more characters`;
+            }
+            const attachments = [];
+            if (params.files_to_display) {
+                if (self.displayFilesInDoneText) {
+                    let attachmentText = '';
+                    for (const fileName of params.files_to_display) {
+                        if (fileName === 'todo.md') {
+                            continue;
+                        }
+                        const content = fsInstance.display_file(fileName);
+                        if (content) {
+                            attachmentText += `\n\n${fileName}:\n${content}`;
+                            attachments.push(fileName);
+                        }
+                    }
+                    if (attachmentText) {
+                        userMessage += '\n\nAttachments:';
+                        userMessage += attachmentText;
+                    }
+                }
+                else {
+                    for (const fileName of params.files_to_display) {
+                        if (fileName === 'todo.md') {
+                            continue;
+                        }
+                        const content = fsInstance.display_file(fileName);
+                        if (content) {
+                            attachments.push(fileName);
+                        }
+                    }
+                }
+            }
+            const attachmentPaths = attachments.map((name) => `${fsInstance.get_dir()}/${name}`);
+            return new ActionResult({
+                is_done: true,
+                success: params.success,
+                extracted_content: userMessage,
+                long_term_memory: memory,
+                attachments: attachmentPaths,
+            });
+        });
+    }
+    use_structured_output_action(outputModel) {
+        this.registerDoneAction(outputModel);
+    }
+    action(description, options = {}) {
+        return this.registry.action(description, options);
+    }
+    async act(action, { browser_session, page_extraction_llm = null, sensitive_data = null, available_file_paths = null, file_system = null, context = null, signal = null, }) {
+        const entries = toActionEntries(action);
+        for (const [actionName, params] of entries) {
+            try {
+                const result = await this.registry.execute_action(actionName, params, {
+                    browser_session,
+                    page_extraction_llm,
+                    sensitive_data,
+                    available_file_paths,
+                    file_system,
+                    context,
+                    signal,
+                });
+                if (typeof result === 'string') {
+                    return new ActionResult({ extracted_content: result });
+                }
+                if (result instanceof ActionResult) {
+                    return result;
+                }
+                if (result == null) {
+                    return new ActionResult();
+                }
+                return new ActionResult({ extracted_content: JSON.stringify(result) });
+            }
+            catch (error) {
+                return new ActionResult({
+                    error: String(error?.message ?? error ?? ''),
+                });
+            }
+        }
+        return new ActionResult();
+    }
+}