browser-use 0.0.1 → 0.1.0

package/dist/dom/history-tree-processor/service.d.ts

@@ -0,0 +1,14 @@
+import { DOMHistoryElement, HashedDomElement } from './view.js';
+import { DOMElementNode } from '../views.js';
+export declare class HistoryTreeProcessor {
+    static convert_dom_element_to_history_element(dom_element: DOMElementNode, css_selector?: string | null): DOMHistoryElement;
+    static find_history_element_in_tree(dom_history_element: DOMHistoryElement, tree: DOMElementNode): DOMElementNode | null;
+    static compare_history_element_and_dom_element(dom_history_element: DOMHistoryElement, dom_element: DOMElementNode): boolean;
+    static _hash_dom_history_element(dom_history_element: DOMHistoryElement): HashedDomElement;
+    static _hash_dom_element(dom_element: DOMElementNode): HashedDomElement;
+    static _get_parent_branch_path(dom_element: DOMElementNode): string[];
+    static _parent_branch_path_hash(parent_branch_path: string[]): string;
+    static _attributes_hash(attributes: Record<string, string>): string;
+    static _xpath_hash(xpath: string): string;
+    static _text_hash(dom_element: DOMElementNode): string;
+}

package/dist/dom/history-tree-processor/service.js

@@ -0,0 +1,75 @@
+import crypto from 'node:crypto';
+import { DOMHistoryElement, HashedDomElement } from './view.js';
+import { DOMElementNode } from '../views.js';
+const sha256 = (value) => crypto.createHash('sha256').update(value).digest('hex');
+export class HistoryTreeProcessor {
+    static convert_dom_element_to_history_element(dom_element, css_selector = null) {
+        const parent_branch_path = this._get_parent_branch_path(dom_element);
+        return new DOMHistoryElement(dom_element.tag_name, dom_element.xpath, dom_element.highlight_index ?? null, parent_branch_path, dom_element.attributes, dom_element.shadow_root, css_selector, dom_element.page_coordinates, dom_element.viewport_coordinates, dom_element.viewport_info);
+    }
+    static find_history_element_in_tree(dom_history_element, tree) {
+        const hashed = this._hash_dom_history_element(dom_history_element);
+        const process_node = (node) => {
+            if (node.highlight_index !== null && node.highlight_index !== undefined) {
+                const hashed_node = this._hash_dom_element(node);
+                if (hashed_node.equals(hashed)) {
+                    return node;
+                }
+            }
+            for (const child of node.children) {
+                if (child instanceof DOMElementNode) {
+                    const result = process_node(child);
+                    if (result) {
+                        return result;
+                    }
+                }
+            }
+            return null;
+        };
+        return process_node(tree);
+    }
+    static compare_history_element_and_dom_element(dom_history_element, dom_element) {
+        const hashed_history = this._hash_dom_history_element(dom_history_element);
+        const hashed_dom = this._hash_dom_element(dom_element);
+        return hashed_history.equals(hashed_dom);
+    }
+    static _hash_dom_history_element(dom_history_element) {
+        const branch_path_hash = this._parent_branch_path_hash(dom_history_element.entire_parent_branch_path);
+        const attributes_hash = this._attributes_hash(dom_history_element.attributes);
+        const xpath_hash = this._xpath_hash(dom_history_element.xpath);
+        return new HashedDomElement(branch_path_hash, attributes_hash, xpath_hash);
+    }
+    static _hash_dom_element(dom_element) {
+        const parent_branch_path = this._get_parent_branch_path(dom_element);
+        const branch_path_hash = this._parent_branch_path_hash(parent_branch_path);
+        const attributes_hash = this._attributes_hash(dom_element.attributes);
+        const xpath_hash = this._xpath_hash(dom_element.xpath);
+        return new HashedDomElement(branch_path_hash, attributes_hash, xpath_hash);
+    }
+    static _get_parent_branch_path(dom_element) {
+        const parents = [];
+        let current = dom_element;
+        while (current && current.parent) {
+            parents.push(current);
+            current = current.parent;
+        }
+        parents.reverse();
+        return parents.map((parent) => parent.tag_name);
+    }
+    static _parent_branch_path_hash(parent_branch_path) {
+        return sha256(parent_branch_path.join('/'));
+    }
+    static _attributes_hash(attributes) {
+        const attributes_string = Object.entries(attributes)
+            .map(([key, value]) => `${key}=${value}`)
+            .join('');
+        return sha256(attributes_string);
+    }
+    static _xpath_hash(xpath) {
+        return sha256(xpath);
+    }
+    static _text_hash(dom_element) {
+        const text = dom_element.get_all_text_till_next_clickable_element();
+        return sha256(text);
+    }
+}

package/dist/dom/history-tree-processor/view.d.ts

@@ -0,0 +1,54 @@
+export declare class HashedDomElement {
+    branch_path_hash: string;
+    attributes_hash: string;
+    xpath_hash: string;
+    constructor(branch_path_hash: string, attributes_hash: string, xpath_hash: string);
+    /**
+     * Check equality with another HashedDomElement
+     */
+    equals(other: HashedDomElement): boolean;
+}
+export interface Coordinates {
+    x: number;
+    y: number;
+}
+export interface CoordinateSet {
+    top_left: Coordinates;
+    top_right: Coordinates;
+    bottom_left: Coordinates;
+    bottom_right: Coordinates;
+    center: Coordinates;
+    width: number;
+    height: number;
+}
+export interface ViewportInfo {
+    scroll_x?: number | null;
+    scroll_y?: number | null;
+    width: number;
+    height: number;
+}
+export declare class DOMHistoryElement {
+    tag_name: string;
+    xpath: string;
+    highlight_index: number | null;
+    entire_parent_branch_path: string[];
+    attributes: Record<string, string>;
+    shadow_root: boolean;
+    css_selector: string | null;
+    page_coordinates: CoordinateSet | null;
+    viewport_coordinates: CoordinateSet | null;
+    viewport_info: ViewportInfo | null;
+    constructor(tag_name: string, xpath: string, highlight_index: number | null, entire_parent_branch_path: string[], attributes: Record<string, string>, shadow_root?: boolean, css_selector?: string | null, page_coordinates?: CoordinateSet | null, viewport_coordinates?: CoordinateSet | null, viewport_info?: ViewportInfo | null);
+    to_dict(): {
+        tag_name: string;
+        xpath: string;
+        highlight_index: number | null;
+        entire_parent_branch_path: string[];
+        attributes: Record<string, string>;
+        shadow_root: boolean;
+        css_selector: string | null;
+        page_coordinates: CoordinateSet | null;
+        viewport_coordinates: CoordinateSet | null;
+        viewport_info: ViewportInfo | null;
+    };
+}

package/dist/dom/history-tree-processor/view.js

@@ -0,0 +1,56 @@
+export class HashedDomElement {
+    branch_path_hash;
+    attributes_hash;
+    xpath_hash;
+    constructor(branch_path_hash, attributes_hash, xpath_hash) {
+        this.branch_path_hash = branch_path_hash;
+        this.attributes_hash = attributes_hash;
+        this.xpath_hash = xpath_hash;
+    }
+    /**
+     * Check equality with another HashedDomElement
+     */
+    equals(other) {
+        return (this.branch_path_hash === other.branch_path_hash &&
+            this.attributes_hash === other.attributes_hash &&
+            this.xpath_hash === other.xpath_hash);
+    }
+}
+export class DOMHistoryElement {
+    tag_name;
+    xpath;
+    highlight_index;
+    entire_parent_branch_path;
+    attributes;
+    shadow_root;
+    css_selector;
+    page_coordinates;
+    viewport_coordinates;
+    viewport_info;
+    constructor(tag_name, xpath, highlight_index, entire_parent_branch_path, attributes, shadow_root = false, css_selector = null, page_coordinates = null, viewport_coordinates = null, viewport_info = null) {
+        this.tag_name = tag_name;
+        this.xpath = xpath;
+        this.highlight_index = highlight_index;
+        this.entire_parent_branch_path = entire_parent_branch_path;
+        this.attributes = attributes;
+        this.shadow_root = shadow_root;
+        this.css_selector = css_selector;
+        this.page_coordinates = page_coordinates;
+        this.viewport_coordinates = viewport_coordinates;
+        this.viewport_info = viewport_info;
+    }
+    to_dict() {
+        return {
+            tag_name: this.tag_name,
+            xpath: this.xpath,
+            highlight_index: this.highlight_index,
+            entire_parent_branch_path: this.entire_parent_branch_path,
+            attributes: this.attributes,
+            shadow_root: this.shadow_root,
+            css_selector: this.css_selector,
+            page_coordinates: this.page_coordinates,
+            viewport_coordinates: this.viewport_coordinates,
+            viewport_info: this.viewport_info,
+        };
+    }
+}

package/dist/dom/playground/extraction.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Interactive DOM element testing tool.
+ *
+ * This playground allows you to:
+ * - Navigate to websites
+ * - Extract DOM state and clickable elements
+ * - Interactively click elements by index
+ * - Input text into elements
+ * - Copy element JSON to clipboard
+ * - Analyze token counts for LLM prompts
+ *
+ * Usage:
+ * - Enter an element index to click it
+ * - Enter 'index,text' to input text into an element
+ * - Enter 'c,index' to copy element JSON to clipboard
+ * - Enter 'q' to quit
+ */
+declare function testFocusVsAllElements(): Promise<void>;
+export { testFocusVsAllElements };

package/dist/dom/playground/extraction.js

@@ -0,0 +1,187 @@
+import * as fs from 'fs/promises';
+import * as readline from 'readline';
+import { promisify } from 'util';
+import { BrowserProfile, BrowserSession } from '../../browser/index.js';
+import { DomService } from '../service.js';
+import { DEFAULT_INCLUDE_ATTRIBUTES } from '../views.js';
+import { AgentMessagePrompt } from '../../agent/prompts.js';
+import { FileSystem } from '../../filesystem/file-system.js';
+const TIMEOUT = 60;
+/**
+ * Interactive DOM element testing tool.
+ *
+ * This playground allows you to:
+ * - Navigate to websites
+ * - Extract DOM state and clickable elements
+ * - Interactively click elements by index
+ * - Input text into elements
+ * - Copy element JSON to clipboard
+ * - Analyze token counts for LLM prompts
+ *
+ * Usage:
+ * - Enter an element index to click it
+ * - Enter 'index,text' to input text into an element
+ * - Enter 'c,index' to copy element JSON to clipboard
+ * - Enter 'q' to quit
+ */
+async function testFocusVsAllElements() {
+    const browserSession = new BrowserSession({
+        browser_profile: new BrowserProfile({
+            window_size: { width: 1100, height: 1000 },
+            disable_security: true,
+            wait_for_network_idle_page_load_time: 1,
+            headless: false,
+        }),
+    });
+    const websites = [
+        'https://google.com',
+        'https://www.ycombinator.com/companies',
+        'https://kayak.com/flights',
+        'https://docs.google.com/spreadsheets/d/1INaIcfpYXlMRWO__de61SHFCaqt1lfHlcvtXZPItlpI/edit',
+        'https://www.zeiss.com/career/en/job-search.html?page=1',
+        'https://www.mlb.com/yankees/stats/',
+        'https://www.amazon.com/s?k=laptop&s=review-rank',
+        'https://reddit.com',
+        'https://codepen.io/geheimschriftstift/pen/mPLvQz',
+        'https://www.google.com/search?q=google+hi',
+        'https://amazon.com',
+        'https://github.com',
+    ];
+    await browserSession.start();
+    const page = await browserSession.getCurrentPage();
+    if (!page) {
+        throw new Error('No page available');
+    }
+    const domService = new DomService(page);
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+    });
+    const question = promisify(rl.question).bind(rl);
+    for (const website of websites) {
+        await page.goto(website);
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+        let lastClickedIndex = null;
+        while (true) {
+            try {
+                console.log(`\n${'='.repeat(50)}\nTesting ${website}\n${'='.repeat(50)}`);
+                console.log('\nGetting page state...');
+                const startTime = Date.now();
+                const allElementsState = await browserSession.get_state_summary(true);
+                const endTime = Date.now();
+                console.log(`get_state_summary took ${((endTime - startTime) / 1000).toFixed(2)} seconds`);
+                const selectorMap = allElementsState.selector_map;
+                const totalElements = Object.keys(selectorMap).length;
+                console.log(`Total number of elements: ${totalElements}`);
+                const prompt = new AgentMessagePrompt({
+                    browser_state_summary: allElementsState,
+                    file_system: new FileSystem('./tmp'),
+                    include_attributes: DEFAULT_INCLUDE_ATTRIBUTES,
+                    step_info: null,
+                });
+                const userMessage = prompt.get_user_message(false).text || '';
+                const textToSave = userMessage;
+                await fs.mkdir('./tmp', { recursive: true });
+                await fs.writeFile('./tmp/user_message.txt', textToSave, 'utf-8');
+                await fs.writeFile('./tmp/element_tree.json', JSON.stringify(allElementsState.element_tree.toJSON(), null, 0), 'utf-8');
+                try {
+                    // Optional: tiktoken is not installed by default
+                    // @ts-ignore - tiktoken is an optional dependency
+                    const { encoding_for_model } = await import('tiktoken');
+                    const encoding = encoding_for_model('gpt-4o');
+                    const tokenCount = encoding.encode(textToSave).length;
+                    console.log(`Token count: ${tokenCount}`);
+                }
+                catch (error) {
+                    console.log('Could not calculate token count (tiktoken not installed):', error.message);
+                }
+                console.log('User message written to ./tmp/user_message.txt');
+                console.log('Element tree written to ./tmp/element_tree.json');
+                const answer = String(await question("Enter element index to click, 'index,text' to input, 'c,index' to copy element JSON, or 'q' to quit: "));
+                if (answer.toLowerCase().trim() === 'q') {
+                    break;
+                }
+                try {
+                    if (answer.toLowerCase().startsWith('c,')) {
+                        const parts = answer.split(',', 2);
+                        if (parts.length === 2) {
+                            try {
+                                const targetIndex = parseInt(parts[1].trim(), 10);
+                                if (targetIndex in selectorMap) {
+                                    const elementNode = selectorMap[targetIndex];
+                                    const elementJson = JSON.stringify(elementNode.toJSON(), null, 2);
+                                    console.log(`Element ${targetIndex} JSON:`);
+                                    console.log(elementJson);
+                                    console.log(`\nElement: ${elementNode.tag_name}`);
+                                }
+                                else {
+                                    console.log(`Invalid index: ${targetIndex}`);
+                                }
+                            }
+                            catch {
+                                console.log(`Invalid index format: ${parts[1]}`);
+                            }
+                        }
+                        else {
+                            console.log("Invalid input format. Use 'c,index'.");
+                        }
+                    }
+                    else if (answer.includes(',')) {
+                        const parts = answer.split(',', 2);
+                        if (parts.length === 2) {
+                            try {
+                                const targetIndex = parseInt(parts[0].trim(), 10);
+                                const textToInput = parts[1];
+                                if (targetIndex in selectorMap) {
+                                    const elementNode = selectorMap[targetIndex];
+                                    console.log(`Inputting text '${textToInput}' into element ${targetIndex}: ${elementNode.tag_name}`);
+                                    await browserSession._inputTextElementNode(elementNode, textToInput);
+                                    console.log('Input successful.');
+                                }
+                                else {
+                                    console.log(`Invalid index: ${targetIndex}`);
+                                }
+                            }
+                            catch {
+                                console.log(`Invalid index format: ${parts[0]}`);
+                            }
+                        }
+                        else {
+                            console.log("Invalid input format. Use 'index,text'.");
+                        }
+                    }
+                    else {
+                        try {
+                            const clickedIndex = parseInt(answer, 10);
+                            if (clickedIndex in selectorMap) {
+                                const elementNode = selectorMap[clickedIndex];
+                                console.log(`Clicking element ${clickedIndex}: ${elementNode.tag_name}`);
+                                await browserSession._clickElementNode(elementNode);
+                                console.log('Click successful.');
+                            }
+                            else {
+                                console.log(`Invalid index: ${clickedIndex}`);
+                            }
+                        }
+                        catch {
+                            console.log(`Invalid input: '${answer}'. Enter an index, 'index,text', 'c,index', or 'q'.`);
+                        }
+                    }
+                }
+                catch (actionError) {
+                    console.log(`Action failed: ${actionError.message}`);
+                }
+            }
+            catch (error) {
+                console.log(`Error in loop: ${error.message}`);
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+            }
+        }
+    }
+    rl.close();
+    await browserSession.close();
+}
+if (require.main === module) {
+    testFocusVsAllElements().catch(console.error);
+}
+export { testFocusVsAllElements };

package/dist/dom/playground/process-dom.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/dom/playground/process-dom.js

@@ -0,0 +1,5 @@
+async function main() {
+    console.log('Process DOM playground');
+}
+main();
+export {};

package/dist/dom/playground/test-accessibility.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Accessibility Tree Playground for browser-use
+ *
+ * - Launches a browser and navigates to a target URL (default: amazon.com)
+ * - Extracts both the full and interesting-only accessibility trees using Playwright
+ * - Prints and saves both trees to JSON files
+ * - Recursively prints relevant info for each node (role, name, value, description, focusable, focused, checked, selected, disabled, children count)
+ * - Explains the difference between the accessibility tree and the DOM tree
+ * - Notes on React/Vue/SPA apps
+ * - Easy to modify for your own experiments
+ *
+ * Run with: npx tsx src/dom/playground/test-accessibility.ts
+ */
+type AXNode = {
+    role?: string;
+    name?: string;
+    value?: string | number;
+    description?: string;
+    focusable?: boolean;
+    focused?: boolean;
+    checked?: boolean | 'mixed';
+    selected?: boolean;
+    disabled?: boolean;
+    children?: AXNode[];
+    [key: string]: any;
+};
+/**
+ * Helper to recursively print relevant info from the accessibility tree
+ */
+declare function printAxTree(node: AXNode | null, depth?: number): void;
+/**
+ * Helper to print all available accessibility node attributes
+ * Prints all key-value pairs for each node (except 'children'), then recurses into children
+ */
+declare function printAllFields(node: AXNode | null, depth?: number): void;
+/**
+ * Flatten the accessibility tree into a list of "role name" strings
+ */
+declare function flattenAxTree(node: AXNode | null, lines: string[]): void;
+/**
+ * Get and analyze the accessibility tree for a given URL
+ */
+declare function getAxTree(targetUrl: string): Promise<void>;
+export { printAxTree, printAllFields, flattenAxTree, getAxTree };

package/dist/dom/playground/test-accessibility.js

@@ -0,0 +1,111 @@
+/**
+ * Accessibility Tree Playground for browser-use
+ *
+ * - Launches a browser and navigates to a target URL (default: amazon.com)
+ * - Extracts both the full and interesting-only accessibility trees using Playwright
+ * - Prints and saves both trees to JSON files
+ * - Recursively prints relevant info for each node (role, name, value, description, focusable, focused, checked, selected, disabled, children count)
+ * - Explains the difference between the accessibility tree and the DOM tree
+ * - Notes on React/Vue/SPA apps
+ * - Easy to modify for your own experiments
+ *
+ * Run with: npx tsx src/dom/playground/test-accessibility.ts
+ */
+import { chromium } from 'playwright';
+/**
+ * Helper to recursively print relevant info from the accessibility tree
+ */
+function printAxTree(node, depth = 0) {
+    if (!node) {
+        return;
+    }
+    const indent = '  '.repeat(depth);
+    const info = [
+        `role=${JSON.stringify(node.role)}`,
+        node.name ? `name=${JSON.stringify(node.name)}` : null,
+        node.value !== undefined ? `value=${JSON.stringify(node.value)}` : null,
+        node.description ? `desc=${JSON.stringify(node.description)}` : null,
+        'focusable' in node ? `focusable=${JSON.stringify(node.focusable)}` : null,
+        'focused' in node ? `focused=${JSON.stringify(node.focused)}` : null,
+        'checked' in node ? `checked=${JSON.stringify(node.checked)}` : null,
+        'selected' in node ? `selected=${JSON.stringify(node.selected)}` : null,
+        'disabled' in node ? `disabled=${JSON.stringify(node.disabled)}` : null,
+        node.children ? `children=${node.children.length}` : null,
+    ];
+    console.log('--------------------------------');
+    console.log(indent + info.filter((x) => x !== null).join(', '));
+    for (const child of node.children || []) {
+        printAxTree(child, depth + 1);
+    }
+}
+/**
+ * Helper to print all available accessibility node attributes
+ * Prints all key-value pairs for each node (except 'children'), then recurses into children
+ */
+function printAllFields(node, depth = 0) {
+    if (!node) {
+        return;
+    }
+    const indent = '  '.repeat(depth);
+    for (const [k, v] of Object.entries(node)) {
+        if (k !== 'children') {
+            console.log(`${indent}${k}: ${JSON.stringify(v)}`);
+        }
+    }
+    if ('children' in node && node.children) {
+        console.log(`${indent}children: ${node.children.length}`);
+        for (const child of node.children) {
+            printAllFields(child, depth + 1);
+        }
+    }
+}
+/**
+ * Flatten the accessibility tree into a list of "role name" strings
+ */
+function flattenAxTree(node, lines) {
+    if (!node) {
+        return;
+    }
+    const role = node.role || '';
+    const name = node.name || '';
+    lines.push(`${role} ${name}`);
+    for (const child of node.children || []) {
+        flattenAxTree(child, lines);
+    }
+}
+/**
+ * Get and analyze the accessibility tree for a given URL
+ */
+async function getAxTree(targetUrl) {
+    const browser = await chromium.launch({ headless: true });
+    const page = await browser.newPage();
+    console.log(`Navigating to ${targetUrl}`);
+    await page.goto(targetUrl, { waitUntil: 'load' });
+    const axTreeInteresting = await page.accessibility.snapshot({
+        interestingOnly: true,
+    });
+    const lines = [];
+    flattenAxTree(axTreeInteresting, lines);
+    console.log(lines);
+    console.log(`length of ax_tree_interesting: ${lines.length}`);
+    await browser.close();
+}
+/**
+ * Main execution
+ */
+async function main() {
+    const TARGET_URLS = [
+        // 'https://amazon.com/',
+        // 'https://www.google.com/',
+        // 'https://www.facebook.com/',
+        // 'https://platform.openai.com/tokenizer',
+        'https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/input/checkbox',
+    ];
+    for (const url of TARGET_URLS) {
+        await getAxTree(url);
+    }
+}
+if (require.main === module) {
+    main().catch(console.error);
+}
+export { printAxTree, printAllFields, flattenAxTree, getAxTree };

package/dist/dom/service.d.ts

@@ -0,0 +1,19 @@
+import type { Page } from '../browser/types.js';
+import { DOMState } from './views.js';
+export declare class DomService {
+    private readonly page;
+    private readonly logger;
+    private readonly jsCode;
+    constructor(page: Page, logger?: import("../logging-config.js").Logger);
+    get_clickable_elements(highlight_elements?: boolean, focus_element?: number, viewport_expansion?: number): Promise<DOMState>;
+    get_cross_origin_iframes(): Promise<any>;
+    private _build_dom_tree;
+    private _construct_dom_tree;
+    private _parse_node;
+    private safeHostname;
+    private getFrames;
+    private getFrameUrl;
+    private isAdUrl;
+    private getPageUrl;
+    private isDebugEnabled;
+}