browser-use 0.0.1 → 0.1.0

package/dist/dom/service.js

@@ -0,0 +1,227 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { createLogger } from '../logging-config.js';
+import { observe_debug } from '../observability.js';
+import { time_execution_async } from '../utils.js';
+import { is_new_tab_page } from '../utils.js';
+import { DOMElementNode, DOMState, DOMTextNode, } from './views.js';
+const DOM_TREE_SCRIPT = fs.readFileSync(fileURLToPath(new URL('./dom_tree/index.js', import.meta.url)), 'utf-8');
+export class DomService {
+    page;
+    logger;
+    jsCode;
+    constructor(page, logger = createLogger('browser_use.dom.service')) {
+        this.page = page;
+        this.logger = logger;
+        this.jsCode = DOM_TREE_SCRIPT;
+    }
+    // @ts-ignore - Decorator type mismatch with TypeScript strict mode
+    async get_clickable_elements(highlight_elements = true, focus_element = -1, viewport_expansion = 0) {
+        const [element_tree, selector_map] = await this._build_dom_tree(highlight_elements, focus_element, viewport_expansion);
+        return new DOMState(element_tree, selector_map);
+    }
+    // @ts-ignore - Decorator type mismatch with TypeScript strict mode
+    async get_cross_origin_iframes() {
+        const hiddenFrameUrls = await this.page
+            .locator('iframe')
+            .evaluateAll((elements) => elements
+            .filter((el) => {
+            const element = el;
+            const style = window.getComputedStyle(element);
+            const rect = element.getBoundingClientRect();
+            return (style.visibility === 'hidden' ||
+                style.display === 'none' ||
+                rect.width === 0 ||
+                rect.height === 0);
+        })
+            .map((el) => el.src));
+        const currentHost = this.safeHostname(this.getPageUrl());
+        return this.getFrames()
+            .map((frame) => this.getFrameUrl(frame))
+            .filter((url) => {
+            if (!url)
+                return false;
+            const host = this.safeHostname(url);
+            if (!host)
+                return false;
+            if (host === currentHost)
+                return false;
+            if (hiddenFrameUrls.includes(url))
+                return false;
+            return !this.isAdUrl(url);
+        });
+    }
+    // @ts-ignore - Decorator type mismatch with TypeScript strict mode
+    async _build_dom_tree(highlight_elements, focus_element, viewport_expansion) {
+        const canEvaluate = await this.page.evaluate(() => 1 + 1);
+        if (canEvaluate !== 2) {
+            throw new Error('The page cannot evaluate JavaScript code properly');
+        }
+        const pageUrl = this.getPageUrl();
+        if (is_new_tab_page(pageUrl) || pageUrl.startsWith('chrome://')) {
+            return [
+                new DOMElementNode(false, null, 'body', '', {}, []),
+                {},
+            ];
+        }
+        const args = {
+            doHighlightElements: highlight_elements,
+            focusHighlightIndex: focus_element,
+            viewportExpansion: viewport_expansion,
+            debugMode: this.isDebugEnabled(),
+        };
+        let eval_page;
+        try {
+            this.logger.debug(`🔧 Starting JavaScript DOM analysis for ${pageUrl.slice(0, 50)}...`);
+            eval_page = await this.page.evaluate(({ script, evaluateArgs }) => {
+                const fn = eval(script);
+                return fn(evaluateArgs);
+            }, { script: this.jsCode, evaluateArgs: args });
+            this.logger.debug('✅ JavaScript DOM analysis completed');
+        }
+        catch (error) {
+            this.logger.error(`Error evaluating DOMTree: ${error.message}`);
+            throw error;
+        }
+        if (args.debugMode && eval_page.perfMetrics) {
+            const perf = eval_page.perfMetrics;
+            const totalNodes = perf?.nodeMetrics?.totalNodes ?? 0;
+            let interactiveCount = 0;
+            if (eval_page.map) {
+                for (const node of Object.values(eval_page.map)) {
+                    if (node?.isInteractive) {
+                        interactiveCount += 1;
+                    }
+                }
+            }
+            this.logger.debug(`🔎 Ran buildDOMTree.js interactive element detection on: ${pageUrl.slice(0, 50)} interactive=${interactiveCount}/${totalNodes}`);
+        }
+        this.logger.debug('🔄 Starting DOM tree construction...');
+        const result = await this._construct_dom_tree(eval_page);
+        this.logger.debug('✅ DOM tree construction completed');
+        return result;
+    }
+    // @ts-ignore - Decorator type mismatch with TypeScript strict mode
+    async _construct_dom_tree(eval_page) {
+        const selector_map = {};
+        const node_map = new Map();
+        const child_index = new Map();
+        for (const [id, node_data] of Object.entries(eval_page.map)) {
+            const [node, children] = this._parse_node(node_data);
+            if (!node)
+                continue;
+            node_map.set(id, node);
+            child_index.set(id, children);
+            if (node instanceof DOMElementNode &&
+                node.highlight_index !== null &&
+                node.highlight_index !== undefined) {
+                selector_map[node.highlight_index] = node;
+            }
+        }
+        for (const [id, childrenIds] of child_index.entries()) {
+            const parentNode = node_map.get(id);
+            if (!(parentNode instanceof DOMElementNode))
+                continue;
+            for (const childId of childrenIds || []) {
+                const key = String(childId);
+                const childNode = node_map.get(key);
+                if (!childNode)
+                    continue;
+                childNode.parent = parentNode;
+                parentNode.children.push(childNode);
+            }
+        }
+        const rootNode = node_map.get(String(eval_page.rootId));
+        if (!(rootNode instanceof DOMElementNode)) {
+            throw new Error('Failed to construct DOM tree');
+        }
+        return [rootNode, selector_map];
+    }
+    _parse_node(node_data) {
+        if (!node_data) {
+            return [null, []];
+        }
+        if (node_data.type === 'TEXT_NODE') {
+            const textNode = new DOMTextNode(node_data.isVisible ?? false, null, node_data.text ?? '');
+            return [textNode, []];
+        }
+        const children = Array.isArray(node_data.children)
+            ? node_data.children
+            : [];
+        const tag = node_data.tagName ?? 'div';
+        const xpath = node_data.xpath ?? '';
+        const attributes = node_data.attributes ?? {};
+        const element = new DOMElementNode(node_data.isVisible ?? false, null, tag, xpath, attributes, []);
+        element.is_interactive = Boolean(node_data.isInteractive);
+        element.is_top_element = Boolean(node_data.isTopElement);
+        element.is_in_viewport = Boolean(node_data.isInViewport);
+        element.shadow_root = Boolean(node_data.shadowRoot);
+        element.highlight_index =
+            node_data.highlightIndex === undefined ||
+                node_data.highlightIndex === null
+                ? null
+                : Number(node_data.highlightIndex);
+        element.page_coordinates = node_data.pageCoordinates ?? null;
+        element.viewport_coordinates =
+            node_data.viewportCoordinates ?? null;
+        element.viewport_info = node_data.viewportInfo ?? null;
+        element.is_new = node_data.isNew ?? null;
+        return [element, children];
+    }
+    safeHostname(url) {
+        if (!url)
+            return '';
+        try {
+            return new URL(url).hostname;
+        }
+        catch {
+            return '';
+        }
+    }
+    getFrames() {
+        const frames = this.page.frames;
+        return typeof frames === 'function'
+            ? frames.call(this.page)
+            : (frames ?? []);
+    }
+    getFrameUrl(frame) {
+        return typeof frame.url === 'function' ? frame.url() : (frame.url ?? '');
+    }
+    isAdUrl(url) {
+        const host = this.safeHostname(url);
+        return ['doubleclick.net', 'adroll.com', 'googletagmanager.com'].some((domain) => host.endsWith(domain));
+    }
+    getPageUrl() {
+        return typeof this.page.url === 'function'
+            ? this.page.url()
+            : (this.page.url ?? '');
+    }
+    isDebugEnabled() {
+        return ((process.env.BROWSER_USE_LOGGING_LEVEL ?? '').toLowerCase() === 'debug');
+    }
+}
+__decorate([
+    observe_debug({
+        ignore_input: true,
+        ignore_output: true,
+        name: 'get_clickable_elements',
+    })
+    // @ts-ignore - Decorator type mismatch with TypeScript strict mode
+    ,
+    time_execution_async('--get_clickable_elements')
+], DomService.prototype, "get_clickable_elements", null);
+__decorate([
+    time_execution_async('--get_cross_origin_iframes')
+], DomService.prototype, "get_cross_origin_iframes", null);
+__decorate([
+    time_execution_async('--build_dom_tree')
+], DomService.prototype, "_build_dom_tree", null);
+__decorate([
+    time_execution_async('--construct_dom_tree')
+], DomService.prototype, "_construct_dom_tree", null);

package/dist/dom/utils.d.ts

@@ -0,0 +1 @@
+export declare const cap_text_length: (text: string, max_length: number) => string;

package/dist/dom/utils.js

@@ -0,0 +1,6 @@
+export const cap_text_length = (text, max_length) => {
+    if (text.length > max_length) {
+        return `${text.slice(0, max_length)}...`;
+    }
+    return text;
+};

package/dist/dom/views.d.ts

@@ -0,0 +1,61 @@
+import { CoordinateSet, HashedDomElement, ViewportInfo } from './history-tree-processor/view.js';
+export declare abstract class DOMBaseNode {
+    is_visible: boolean;
+    parent: DOMElementNode | null;
+    constructor(is_visible: boolean, parent?: DOMElementNode | null);
+    abstract toJSON(): Record<string, unknown>;
+}
+export declare class DOMTextNode extends DOMBaseNode {
+    text: string;
+    type: string;
+    constructor(is_visible: boolean, parent: DOMElementNode | null, text: string);
+    has_parent_with_highlight_index(): boolean;
+    is_parent_in_viewport(): boolean;
+    is_parent_top_element(): boolean;
+    toJSON(): {
+        text: string;
+        type: string;
+    };
+}
+export declare const DEFAULT_INCLUDE_ATTRIBUTES: string[];
+export declare class DOMElementNode extends DOMBaseNode {
+    tag_name: string;
+    xpath: string;
+    attributes: Record<string, string>;
+    children: DOMBaseNode[];
+    is_interactive: boolean;
+    is_top_element: boolean;
+    is_in_viewport: boolean;
+    shadow_root: boolean;
+    highlight_index: number | null;
+    viewport_coordinates: CoordinateSet | null;
+    page_coordinates: CoordinateSet | null;
+    viewport_info: ViewportInfo | null;
+    is_new: boolean | null;
+    private cached_hash;
+    constructor(is_visible: boolean, parent: DOMElementNode | null, tag_name: string, xpath: string, attributes: Record<string, string>, children: DOMBaseNode[]);
+    toJSON(): {
+        tag_name: string;
+        xpath: string;
+        attributes: Record<string, string>;
+        is_visible: boolean;
+        is_interactive: boolean;
+        is_top_element: boolean;
+        is_in_viewport: boolean;
+        shadow_root: boolean;
+        highlight_index: number | null;
+        viewport_coordinates: CoordinateSet | null;
+        page_coordinates: CoordinateSet | null;
+        children: Record<string, unknown>[];
+    };
+    toString(): string;
+    get hash(): HashedDomElement;
+    get_all_text_till_next_clickable_element(max_depth?: number): string;
+    clickable_elements_to_string(include_attributes?: string[]): string;
+}
+export type SelectorMap = Record<number, DOMElementNode>;
+export declare class DOMState {
+    element_tree: DOMElementNode;
+    selector_map: SelectorMap;
+    constructor(element_tree: DOMElementNode, selector_map: SelectorMap);
+}

package/dist/dom/views.js

@@ -0,0 +1,247 @@
+import { cap_text_length } from './utils.js';
+import { time_execution_sync } from '../utils.js';
+import { HistoryTreeProcessor } from './history-tree-processor/service.js';
+export class DOMBaseNode {
+    is_visible;
+    parent;
+    constructor(is_visible, parent = null) {
+        this.is_visible = is_visible;
+        this.parent = parent;
+    }
+}
+export class DOMTextNode extends DOMBaseNode {
+    text;
+    type = 'TEXT_NODE';
+    constructor(is_visible, parent, text) {
+        super(is_visible, parent);
+        this.text = text;
+    }
+    has_parent_with_highlight_index() {
+        let current = this.parent;
+        while (current) {
+            if (current.highlight_index !== null &&
+                current.highlight_index !== undefined) {
+                return true;
+            }
+            current = current.parent;
+        }
+        return false;
+    }
+    is_parent_in_viewport() {
+        return Boolean(this.parent?.is_in_viewport);
+    }
+    is_parent_top_element() {
+        return Boolean(this.parent?.is_top_element);
+    }
+    toJSON() {
+        return {
+            text: this.text,
+            type: this.type,
+        };
+    }
+}
+export const DEFAULT_INCLUDE_ATTRIBUTES = [
+    'title',
+    'type',
+    'checked',
+    'name',
+    'role',
+    'value',
+    'placeholder',
+    'data-date-format',
+    'alt',
+    'aria-label',
+    'aria-expanded',
+    'data-state',
+    'aria-checked',
+];
+export class DOMElementNode extends DOMBaseNode {
+    tag_name;
+    xpath;
+    attributes;
+    children;
+    is_interactive = false;
+    is_top_element = false;
+    is_in_viewport = false;
+    shadow_root = false;
+    highlight_index = null;
+    viewport_coordinates = null;
+    page_coordinates = null;
+    viewport_info = null;
+    is_new = null;
+    cached_hash = null;
+    constructor(is_visible, parent, tag_name, xpath, attributes, children) {
+        super(is_visible, parent);
+        this.tag_name = tag_name;
+        this.xpath = xpath;
+        this.attributes = attributes;
+        this.children = children;
+    }
+    toJSON() {
+        return {
+            tag_name: this.tag_name,
+            xpath: this.xpath,
+            attributes: this.attributes,
+            is_visible: this.is_visible,
+            is_interactive: this.is_interactive,
+            is_top_element: this.is_top_element,
+            is_in_viewport: this.is_in_viewport,
+            shadow_root: this.shadow_root,
+            highlight_index: this.highlight_index,
+            viewport_coordinates: this.viewport_coordinates,
+            page_coordinates: this.page_coordinates,
+            children: this.children.map((child) => child.toJSON()),
+        };
+    }
+    toString() {
+        let tag_str = `<${this.tag_name}`;
+        for (const [key, value] of Object.entries(this.attributes)) {
+            tag_str += ` ${key}="${value}"`;
+        }
+        tag_str += '>';
+        const extras = [];
+        if (this.is_interactive)
+            extras.push('interactive');
+        if (this.is_top_element)
+            extras.push('top');
+        if (this.shadow_root)
+            extras.push('shadow-root');
+        if (this.highlight_index !== null && this.highlight_index !== undefined) {
+            extras.push(`highlight:${this.highlight_index}`);
+        }
+        if (this.is_in_viewport)
+            extras.push('in-viewport');
+        if (extras.length) {
+            tag_str += ` [${extras.join(', ')}]`;
+        }
+        return tag_str;
+    }
+    get hash() {
+        if (!this.cached_hash) {
+            this.cached_hash = HistoryTreeProcessor._hash_dom_element(this);
+        }
+        return this.cached_hash;
+    }
+    get_all_text_till_next_clickable_element(max_depth = -1) {
+        const text_parts = [];
+        const collect_text = (node, current_depth) => {
+            if (max_depth !== -1 && current_depth > max_depth) {
+                return;
+            }
+            if (node instanceof DOMElementNode &&
+                node !== this &&
+                node.highlight_index !== null &&
+                node.highlight_index !== undefined) {
+                return;
+            }
+            if (node instanceof DOMTextNode) {
+                text_parts.push(node.text);
+            }
+            else if (node instanceof DOMElementNode) {
+                for (const child of node.children) {
+                    collect_text(child, current_depth + 1);
+                }
+            }
+        };
+        collect_text(this, 0);
+        return text_parts.join('\n').trim();
+    }
+    clickable_elements_to_string(include_attributes) {
+        return CLICKABLE_ELEMENTS_TO_STRING_IMPL.call(this, include_attributes);
+    }
+}
+const CLICKABLE_ELEMENTS_TO_STRING_IMPL = time_execution_sync('--clickable_elements_to_string')(function (include_attributes) {
+    const formatted_text = [];
+    const attributes_list = include_attributes ?? DEFAULT_INCLUDE_ATTRIBUTES;
+    const process_node = (node, depth) => {
+        const next_depth = depth;
+        const depth_str = '\t'.repeat(depth);
+        if (node instanceof DOMElementNode) {
+            let working_depth = next_depth;
+            if (node.highlight_index !== null && node.highlight_index !== undefined) {
+                working_depth += 1;
+                let text = node.get_all_text_till_next_clickable_element();
+                let attributes_html_str = null;
+                if (attributes_list.length) {
+                    const attributes_to_include = Object.fromEntries(Object.entries(node.attributes)
+                        .filter(([key, value]) => attributes_list.includes(key) && String(value).trim() !== '')
+                        .map(([key, value]) => [key, String(value).trim()]));
+                    const ordered_keys = attributes_list.filter((key) => key in attributes_to_include);
+                    if (ordered_keys.length > 1) {
+                        const keys_to_remove = new Set();
+                        const seen_values = {};
+                        for (const key of ordered_keys) {
+                            const value = attributes_to_include[key];
+                            if (value && value.length > 5) {
+                                if (seen_values[value]) {
+                                    keys_to_remove.add(key);
+                                }
+                                else {
+                                    seen_values[value] = key;
+                                }
+                            }
+                        }
+                        for (const key of keys_to_remove) {
+                            delete attributes_to_include[key];
+                        }
+                    }
+                    if (node.tag_name === attributes_to_include.role) {
+                        delete attributes_to_include.role;
+                    }
+                    for (const attr of ['aria-label', 'placeholder', 'title']) {
+                        if (attributes_to_include[attr] &&
+                            attributes_to_include[attr].trim().toLowerCase() ===
+                                text.trim().toLowerCase()) {
+                            delete attributes_to_include[attr];
+                        }
+                    }
+                    if (Object.entries(attributes_to_include).length) {
+                        attributes_html_str = Object.entries(attributes_to_include)
+                            .map(([key, value]) => `${key}=${cap_text_length(value, 15)}`)
+                            .join(' ');
+                    }
+                }
+                const highlight_indicator = node.is_new
+                    ? `*[${node.highlight_index}]`
+                    : `[${node.highlight_index}]`;
+                let line = `${depth_str}${highlight_indicator}<${node.tag_name}`;
+                if (attributes_html_str) {
+                    line += ` ${attributes_html_str}`;
+                }
+                if (text) {
+                    text = text.trim();
+                    if (!attributes_html_str) {
+                        line += ' ';
+                    }
+                    line += `>${text}`;
+                }
+                else if (!attributes_html_str) {
+                    line += ' ';
+                }
+                line += ' />';
+                formatted_text.push(line);
+            }
+            for (const child of node.children) {
+                process_node(child, working_depth);
+            }
+        }
+        else if (node instanceof DOMTextNode) {
+            if (node.has_parent_with_highlight_index()) {
+                return;
+            }
+            if (node.parent?.is_visible && node.parent.is_top_element) {
+                formatted_text.push(`${depth_str}${node.text}`);
+            }
+        }
+    };
+    process_node(this, 0);
+    return formatted_text.join('\n');
+});
+export class DOMState {
+    element_tree;
+    selector_map;
+    constructor(element_tree, selector_map) {
+        this.element_tree = element_tree;
+        this.selector_map = selector_map;
+    }
+}

package/dist/event-bus.d.ts

@@ -0,0 +1,11 @@
+export type EventHandler<T = any> = (event: T) => void | Promise<void>;
+export declare class EventBus {
+    readonly name: string;
+    private emitter;
+    constructor(name: string);
+    on<T = any>(eventType: string, handler: EventHandler<T>): void;
+    dispatch<T extends {
+        event_type?: string;
+    }>(event: T): void;
+    stop(): Promise<void>;
+}

package/dist/event-bus.js

@@ -0,0 +1,19 @@
+import EventEmitter from 'eventemitter3';
+export class EventBus {
+    name;
+    emitter = new EventEmitter();
+    constructor(name) {
+        this.name = name;
+    }
+    on(eventType, handler) {
+        this.emitter.on(eventType, handler);
+    }
+    dispatch(event) {
+        const eventType = event?.event_type ?? event?.constructor?.name ?? 'event';
+        this.emitter.emit(eventType, event);
+        this.emitter.emit('*', event);
+    }
+    async stop() {
+        this.emitter.removeAllListeners();
+    }
+}

package/dist/exceptions.d.ts

@@ -0,0 +1,10 @@
+export declare class LLMException extends Error {
+    readonly statusCode: number;
+    readonly detail: string;
+    constructor(statusCode: number, detail: string);
+}
+export declare class URLNotAllowedError extends Error {
+    readonly url: string;
+    readonly allowedDomains: string[];
+    constructor(url: string, allowedDomains: string[]);
+}

package/dist/exceptions.js

@@ -0,0 +1,22 @@
+export class LLMException extends Error {
+    statusCode;
+    detail;
+    constructor(statusCode, detail) {
+        super(`Error ${statusCode}: ${detail}`);
+        this.statusCode = statusCode;
+        this.detail = detail;
+        this.name = 'LLMException';
+    }
+}
+export class URLNotAllowedError extends Error {
+    url;
+    allowedDomains;
+    constructor(url, allowedDomains) {
+        super(`URL "${url}" is not allowed. ` +
+            `Only domains matching ${JSON.stringify(allowedDomains)} are permitted. ` +
+            `This is enforced because sensitive_data was provided to Agent.`);
+        this.url = url;
+        this.allowedDomains = allowedDomains;
+        this.name = 'URLNotAllowedError';
+    }
+}

package/dist/filesystem/file-system.d.ts

@@ -0,0 +1,68 @@
+export declare const INVALID_FILENAME_ERROR_MESSAGE = "Error: Invalid filename format. Must be alphanumeric with supported extension.";
+export declare const DEFAULT_FILE_SYSTEM_PATH = "browseruse_agent_data";
+export declare class FileSystemError extends Error {
+}
+declare abstract class BaseFile {
+    name: string;
+    protected content: string;
+    constructor(name: string, content?: string);
+    abstract get extension(): string;
+    get fullName(): string;
+    get size(): number;
+    get lineCount(): number;
+    protected writeFileContent(content: string): void;
+    protected appendFileContent(content: string): void;
+    read(): string;
+    syncToDisk(dir: string): Promise<void>;
+    syncToDiskSync(dir: string): void;
+    write(content: string, dir: string): Promise<void>;
+    writeSync(content: string, dir: string): void;
+    append(content: string, dir: string): Promise<void>;
+    appendSync(content: string, dir: string): void;
+    toJSON(): {
+        name: string;
+        content: string;
+    };
+}
+export interface FileState {
+    type: string;
+    data: {
+        name: string;
+        content: string;
+    };
+}
+export interface FileSystemState {
+    files: Record<string, FileState>;
+    base_dir: string;
+    extracted_content_count: number;
+}
+export declare class FileSystem {
+    private files;
+    private readonly defaultFiles;
+    private readonly baseDir;
+    readonly dataDir: string;
+    extractedContentCount: number;
+    constructor(baseDir: string, createDefaultFiles?: boolean);
+    private createDefaultFiles;
+    private isValidFilename;
+    private parseFilename;
+    private getFileClass;
+    private instantiateFile;
+    get_allowed_extensions(): string[];
+    get_dir(): string;
+    get_file(filename: string): BaseFile | null;
+    list_files(): string[];
+    display_file(filename: string): string | null;
+    read_file(filename: string, externalFile?: boolean): Promise<string>;
+    write_file(filename: string, content: string): Promise<string>;
+    append_file(filename: string, content: string): Promise<string>;
+    replace_file_str(filename: string, oldStr: string, newStr: string): Promise<string>;
+    save_extracted_content(content: string): Promise<string>;
+    describe(): string;
+    get_todo_contents(): string;
+    get_state(): FileSystemState;
+    nuke(): Promise<void>;
+    static from_state_sync(state: FileSystemState): FileSystem;
+    static from_state(state: FileSystemState): Promise<FileSystem>;
+}
+export {};