browser-use 0.2.0 → 0.4.0

package/dist/browser/views.d.ts

@@ -3,6 +3,8 @@ import type { DOMHistoryElement } from '../dom/history-tree-processor/view.js';
 export declare const PLACEHOLDER_4PX_SCREENSHOT = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII=";
 export interface TabInfo {
     page_id: number;
+    tab_id?: string;
+    target_id?: string;
     url: string;
     title: string;
     parent_page_id?: number | null;
@@ -19,6 +21,19 @@ export interface PageInfo {
     pixels_left: number;
     pixels_right: number;
 }
+export interface NetworkRequest {
+    url: string;
+    method?: string;
+    loading_duration_ms?: number;
+    resource_type?: string | null;
+}
+export interface PaginationButton {
+    button_type: string;
+    backend_node_id: number;
+    text: string;
+    selector: string;
+    is_disabled?: boolean;
+}
 interface BrowserStateSummaryInit {
     url: string;
     title: string;
@@ -30,6 +45,10 @@ interface BrowserStateSummaryInit {
     browser_errors?: string[];
     is_pdf_viewer?: boolean;
     loading_status?: string | null;
+    recent_events?: string | null;
+    pending_network_requests?: NetworkRequest[];
+    pagination_buttons?: PaginationButton[];
+    closed_popup_messages?: string[];
 }
 export declare class BrowserStateSummary extends DOMState {
     url: string;
@@ -42,6 +61,10 @@ export declare class BrowserStateSummary extends DOMState {
     browser_errors: string[];
     is_pdf_viewer: boolean;
     loading_status: string | null;
+    recent_events: string | null;
+    pending_network_requests: NetworkRequest[];
+    pagination_buttons: PaginationButton[];
+    closed_popup_messages: string[];
     constructor(dom_state: DOMState, init: BrowserStateSummaryInit);
 }
 export declare class BrowserStateHistory {
@@ -66,12 +89,28 @@ export declare class BrowserStateHistory {
             page_coordinates: import("../index.js").CoordinateSet | null;
             viewport_coordinates: import("../index.js").CoordinateSet | null;
             viewport_info: import("../index.js").ViewportInfo | null;
+            element_hash: string | null;
+            stable_hash: string | null;
+            ax_name: string | null;
         } | null)[];
         url: string;
         title: string;
     };
 }
+export interface BrowserErrorInit {
+    message: string;
+    short_term_memory?: string | null;
+    long_term_memory?: string | null;
+    details?: Record<string, unknown> | null;
+    event?: unknown;
+}
 export declare class BrowserError extends Error {
+    short_term_memory: string | null;
+    long_term_memory: string | null;
+    details: Record<string, unknown> | null;
+    while_handling_event: unknown;
+    constructor(messageOrInit: string | BrowserErrorInit, options?: Omit<BrowserErrorInit, 'message'>);
+    toString(): string;
 }
 export declare class URLNotAllowedError extends BrowserError {
 }

package/dist/browser/views.js

@@ -13,6 +13,10 @@ export class BrowserStateSummary extends DOMState {
     browser_errors;
     is_pdf_viewer;
     loading_status;
+    recent_events;
+    pending_network_requests;
+    pagination_buttons;
+    closed_popup_messages;
     constructor(dom_state, init) {
         super(dom_state.element_tree, dom_state.selector_map);
         this.url = init.url;
@@ -25,6 +29,10 @@ export class BrowserStateSummary extends DOMState {
         this.browser_errors = init.browser_errors ?? [];
         this.is_pdf_viewer = init.is_pdf_viewer ?? false;
         this.loading_status = init.loading_status ?? null;
+        this.recent_events = init.recent_events ?? null;
+        this.pending_network_requests = init.pending_network_requests ?? [];
+        this.pagination_buttons = init.pagination_buttons ?? [];
+        this.closed_popup_messages = init.closed_popup_messages ?? [];
     }
 }
 export class BrowserStateHistory {
@@ -67,6 +75,30 @@ export class BrowserStateHistory {
     }
 }
 export class BrowserError extends Error {
+    short_term_memory;
+    long_term_memory;
+    details;
+    while_handling_event;
+    constructor(messageOrInit, options) {
+        const init = typeof messageOrInit === 'string'
+            ? { message: messageOrInit, ...(options ?? {}) }
+            : messageOrInit;
+        super(init.message);
+        this.name = 'BrowserError';
+        this.short_term_memory = init.short_term_memory ?? null;
+        this.long_term_memory = init.long_term_memory ?? null;
+        this.details = init.details ?? null;
+        this.while_handling_event = init.event ?? null;
+    }
+    toString() {
+        if (this.details) {
+            return `${this.message} (${JSON.stringify(this.details)})`;
+        }
+        if (this.while_handling_event) {
+            return `${this.message} (while handling: ${String(this.while_handling_event)})`;
+        }
+        return this.message;
+    }
 }
 export class URLNotAllowedError extends BrowserError {
 }

package/dist/browser/watchdogs/aboutblank-watchdog.d.ts

@@ -0,0 +1,12 @@
+import { AboutBlankDVDScreensaverShownEvent, BrowserStopEvent, BrowserStoppedEvent, NavigateToUrlEvent, TabClosedEvent, TabCreatedEvent } from '../events.js';
+import { BaseWatchdog } from './base.js';
+export declare class AboutBlankWatchdog extends BaseWatchdog {
+    static LISTENS_TO: (typeof BrowserStopEvent | typeof BrowserStoppedEvent | typeof TabClosedEvent)[];
+    static EMITS: (typeof NavigateToUrlEvent | typeof AboutBlankDVDScreensaverShownEvent)[];
+    private _stopping;
+    on_BrowserStopEvent(): Promise<void>;
+    on_BrowserStoppedEvent(): Promise<void>;
+    on_TabClosedEvent(): Promise<void>;
+    on_TabCreatedEvent(event: TabCreatedEvent): Promise<void>;
+    private _injectDvdScreensaverOverlay;
+}

package/dist/browser/watchdogs/aboutblank-watchdog.js

@@ -0,0 +1,131 @@
+import { AboutBlankDVDScreensaverShownEvent, BrowserStopEvent, BrowserStoppedEvent, NavigateToUrlEvent, TabClosedEvent, TabCreatedEvent, } from '../events.js';
+import { BaseWatchdog } from './base.js';
+export class AboutBlankWatchdog extends BaseWatchdog {
+    static LISTENS_TO = [
+        BrowserStopEvent,
+        BrowserStoppedEvent,
+        TabClosedEvent,
+        TabCreatedEvent,
+    ];
+    static EMITS = [
+        NavigateToUrlEvent,
+        AboutBlankDVDScreensaverShownEvent,
+    ];
+    _stopping = false;
+    async on_BrowserStopEvent() {
+        this._stopping = true;
+    }
+    async on_BrowserStoppedEvent() {
+        this._stopping = true;
+    }
+    async on_TabClosedEvent() {
+        if (this._stopping) {
+            return;
+        }
+        if (this.browser_session.tabs.length > 0) {
+            return;
+        }
+        await this.event_bus.dispatch(new NavigateToUrlEvent({
+            url: 'about:blank',
+            new_tab: true,
+        }));
+    }
+    async on_TabCreatedEvent(event) {
+        if (this._stopping) {
+            return;
+        }
+        if (event.url !== 'about:blank') {
+            return;
+        }
+        let injectionError = null;
+        try {
+            await this._injectDvdScreensaverOverlay();
+        }
+        catch (error) {
+            injectionError =
+                error.message || 'DVD overlay injection failed';
+        }
+        await this.event_bus.dispatch(new AboutBlankDVDScreensaverShownEvent({
+            target_id: event.target_id,
+            error: injectionError,
+        }));
+    }
+    async _injectDvdScreensaverOverlay() {
+        const page = await this.browser_session.get_current_page();
+        if (!page?.evaluate || !page.url) {
+            return;
+        }
+        const currentUrl = page.url();
+        if (currentUrl !== 'about:blank') {
+            return;
+        }
+        await page.evaluate(() => {
+            if (window.__dvdAnimationRunning) {
+                return;
+            }
+            window.__dvdAnimationRunning = true;
+            const existing = document.getElementById('pretty-loading-animation');
+            if (existing) {
+                return;
+            }
+            const ensureBody = () => {
+                if (!document.body) {
+                    return false;
+                }
+                const overlay = document.createElement('div');
+                overlay.id = 'pretty-loading-animation';
+                overlay.style.position = 'fixed';
+                overlay.style.top = '0';
+                overlay.style.left = '0';
+                overlay.style.width = '100vw';
+                overlay.style.height = '100vh';
+                overlay.style.background = '#000';
+                overlay.style.zIndex = '99999';
+                overlay.style.overflow = 'hidden';
+                const img = document.createElement('img');
+                img.src = 'https://cf.browser-use.com/logo.svg';
+                img.alt = 'Browser-Use';
+                img.style.width = '180px';
+                img.style.height = 'auto';
+                img.style.position = 'absolute';
+                img.style.left = '0px';
+                img.style.top = '0px';
+                img.style.opacity = '0.85';
+                overlay.appendChild(img);
+                document.body.appendChild(overlay);
+                let x = Math.random() * Math.max(20, window.innerWidth - 200);
+                let y = Math.random() * Math.max(20, window.innerHeight - 120);
+                let dx = 1.4;
+                let dy = 1.2;
+                const animate = () => {
+                    if (!document.getElementById('pretty-loading-animation')) {
+                        window.__dvdAnimationRunning = false;
+                        return;
+                    }
+                    const imgWidth = img.offsetWidth || 180;
+                    const imgHeight = img.offsetHeight || 80;
+                    x += dx;
+                    y += dy;
+                    if (x <= 0 || x + imgWidth >= window.innerWidth) {
+                        dx = -dx;
+                        x = Math.max(0, Math.min(x, window.innerWidth - imgWidth));
+                    }
+                    if (y <= 0 || y + imgHeight >= window.innerHeight) {
+                        dy = -dy;
+                        y = Math.max(0, Math.min(y, window.innerHeight - imgHeight));
+                    }
+                    img.style.left = `${x}px`;
+                    img.style.top = `${y}px`;
+                    requestAnimationFrame(animate);
+                };
+                requestAnimationFrame(animate);
+                return true;
+            };
+            if (!ensureBody() && document.readyState === 'loading') {
+                document.addEventListener('DOMContentLoaded', ensureBody, {
+                    once: true,
+                });
+            }
+        });
+    }
+}

package/dist/browser/watchdogs/base.d.ts

@@ -0,0 +1,21 @@
+import type { EventBus, EventPayload, EventTypeReference } from '../../event-bus.js';
+import type { BrowserSession } from '../session.js';
+export interface BaseWatchdogInit {
+    browser_session: BrowserSession;
+    event_bus?: EventBus;
+}
+export declare abstract class BaseWatchdog {
+    static LISTENS_TO: EventTypeReference<EventPayload>[];
+    static EMITS: EventTypeReference<EventPayload>[];
+    protected readonly browser_session: BrowserSession;
+    protected readonly event_bus: EventBus;
+    private _attached;
+    private _registeredHandlers;
+    constructor(init: BaseWatchdogInit);
+    get is_attached(): boolean;
+    attach_to_session(): void;
+    detach_from_session(): void;
+    protected onAttached(): void;
+    protected onDetached(): void;
+    private _collectHandlerMethods;
+}

package/dist/browser/watchdogs/base.js

@@ -0,0 +1,81 @@
+const resolveEventType = (eventTypeRef) => typeof eventTypeRef === 'string' ? eventTypeRef : eventTypeRef.name;
+export class BaseWatchdog {
+    static LISTENS_TO = [];
+    static EMITS = [];
+    browser_session;
+    event_bus;
+    _attached = false;
+    _registeredHandlers = [];
+    constructor(init) {
+        this.browser_session = init.browser_session;
+        this.event_bus = init.event_bus ?? init.browser_session.event_bus;
+    }
+    get is_attached() {
+        return this._attached;
+    }
+    attach_to_session() {
+        if (this._attached) {
+            throw new Error(`[${this.constructor.name}] attach_to_session() called twice`);
+        }
+        const handlerMethods = this._collectHandlerMethods();
+        const declaredListenEvents = new Set(this.constructor.LISTENS_TO.map(resolveEventType));
+        const registeredEventTypes = new Set();
+        for (const methodName of handlerMethods) {
+            const event_type = methodName.slice(3);
+            if (declaredListenEvents.size > 0 &&
+                !declaredListenEvents.has(event_type)) {
+                throw new Error(`[${this.constructor.name}] Handler ${methodName} listens to ${event_type} but ${event_type} is not declared in LISTENS_TO`);
+            }
+            const handler_id = `${this.constructor.name}.${methodName}`;
+            const method = this[methodName];
+            if (typeof method !== 'function') {
+                continue;
+            }
+            const bound = method.bind(this);
+            this.event_bus.on(event_type, bound, { handler_id });
+            this._registeredHandlers.push({ event_type, handler_id });
+            registeredEventTypes.add(event_type);
+        }
+        if (declaredListenEvents.size > 0) {
+            const missing = [...declaredListenEvents].filter((eventType) => !registeredEventTypes.has(eventType));
+            if (missing.length > 0) {
+                throw new Error(`[${this.constructor.name}] LISTENS_TO declares ${missing.join(', ')} but no matching on_<EventName> handlers were found`);
+            }
+        }
+        this._attached = true;
+        this.onAttached();
+    }
+    detach_from_session() {
+        if (!this._attached) {
+            return;
+        }
+        for (const { event_type, handler_id } of this._registeredHandlers) {
+            this.event_bus.off(event_type, handler_id);
+        }
+        this._registeredHandlers = [];
+        this._attached = false;
+        this.onDetached();
+    }
+    onAttached() { }
+    onDetached() { }
+    _collectHandlerMethods() {
+        const methodNames = new Set();
+        let prototype = Object.getPrototypeOf(this);
+        while (prototype && prototype !== BaseWatchdog.prototype) {
+            for (const name of Object.getOwnPropertyNames(prototype)) {
+                if (!name.startsWith('on_')) {
+                    continue;
+                }
+                if (name.length <= 3) {
+                    continue;
+                }
+                const descriptor = Object.getOwnPropertyDescriptor(prototype, name);
+                if (typeof descriptor?.value === 'function') {
+                    methodNames.add(name);
+                }
+            }
+            prototype = Object.getPrototypeOf(prototype);
+        }
+        return [...methodNames];
+    }
+}

package/dist/browser/watchdogs/cdp-session-watchdog.d.ts

@@ -0,0 +1,14 @@
+import { BrowserConnectedEvent, BrowserStoppedEvent } from '../events.js';
+import { BaseWatchdog } from './base.js';
+export declare class CDPSessionWatchdog extends BaseWatchdog {
+    static LISTENS_TO: (typeof BrowserConnectedEvent | typeof BrowserStoppedEvent)[];
+    private _rootCdpSession;
+    private _listeners;
+    private _knownTargets;
+    on_BrowserConnectedEvent(): Promise<void>;
+    on_BrowserStoppedEvent(): Promise<void>;
+    protected onDetached(): void;
+    private _ensureCdpMonitoring;
+    private _teardownCdpMonitoring;
+    private _dispatchEventSafely;
+}

package/dist/browser/watchdogs/cdp-session-watchdog.js

@@ -0,0 +1,177 @@
+import { BrowserConnectedEvent, BrowserStoppedEvent, NavigationCompleteEvent, TabClosedEvent, TabCreatedEvent, } from '../events.js';
+import { BaseWatchdog } from './base.js';
+export class CDPSessionWatchdog extends BaseWatchdog {
+    static LISTENS_TO = [BrowserConnectedEvent, BrowserStoppedEvent];
+    _rootCdpSession = null;
+    _listeners = [];
+    _knownTargets = new Map();
+    async on_BrowserConnectedEvent() {
+        await this._ensureCdpMonitoring();
+    }
+    async on_BrowserStoppedEvent() {
+        await this._teardownCdpMonitoring();
+    }
+    onDetached() {
+        void this._teardownCdpMonitoring();
+    }
+    async _ensureCdpMonitoring() {
+        if (this._rootCdpSession) {
+            return;
+        }
+        if (!this.browser_session.browser_context?.newCDPSession) {
+            return;
+        }
+        const page = await this.browser_session.get_current_page();
+        if (!page) {
+            return;
+        }
+        try {
+            const cdpSession = (await this.browser_session.get_or_create_cdp_session(page));
+            this._rootCdpSession = cdpSession;
+            await cdpSession.send?.('Target.setDiscoverTargets', {
+                discover: true,
+                filter: [{ type: 'page' }, { type: 'iframe' }],
+            });
+            const targetsPayload = await cdpSession.send?.('Target.getTargets');
+            const targetInfos = Array.isArray(targetsPayload?.targetInfos)
+                ? targetsPayload.targetInfos
+                : [];
+            for (const targetInfo of targetInfos) {
+                const target_id = String(targetInfo?.targetId ?? '');
+                if (!target_id) {
+                    continue;
+                }
+                this.browser_session.session_manager.handle_target_info_changed({
+                    target_id,
+                    target_type: typeof targetInfo?.type === 'string' ? targetInfo.type : 'page',
+                    url: typeof targetInfo?.url === 'string' ? targetInfo.url : '',
+                    title: typeof targetInfo?.title === 'string' ? targetInfo.title : '',
+                });
+                this._knownTargets.set(target_id, {
+                    target_type: typeof targetInfo?.type === 'string' ? targetInfo.type : 'page',
+                    url: typeof targetInfo?.url === 'string' ? targetInfo.url : '',
+                });
+            }
+            const onAttached = (payload) => {
+                const targetInfo = payload?.targetInfo ?? {};
+                const target_id = String(targetInfo?.targetId ?? '');
+                if (!target_id) {
+                    return;
+                }
+                const target_type = typeof targetInfo?.type === 'string' ? targetInfo.type : 'page';
+                const url = typeof targetInfo?.url === 'string' ? targetInfo.url : '';
+                const isNewTarget = !this._knownTargets.has(target_id);
+                this._knownTargets.set(target_id, {
+                    target_type,
+                    url,
+                });
+                this.browser_session.session_manager.handle_target_attached({
+                    target_id,
+                    session_id: typeof payload?.sessionId === 'string' ? payload.sessionId : null,
+                    target_type,
+                    url,
+                    title: typeof targetInfo?.title === 'string' ? targetInfo.title : '',
+                });
+                if (isNewTarget && target_type === 'page') {
+                    this._dispatchEventSafely(new TabCreatedEvent({
+                        target_id,
+                        url,
+                    }));
+                }
+            };
+            const onDetached = (payload) => {
+                const target_id = String(payload?.targetId ?? '');
+                if (!target_id) {
+                    return;
+                }
+                const knownTarget = this._knownTargets.get(target_id);
+                this._knownTargets.delete(target_id);
+                this.browser_session.session_manager.handle_target_detached({
+                    target_id,
+                    session_id: typeof payload?.sessionId === 'string' ? payload.sessionId : null,
+                });
+                if (knownTarget?.target_type === 'page') {
+                    this._dispatchEventSafely(new TabClosedEvent({
+                        target_id,
+                    }));
+                }
+            };
+            const onTargetInfoChanged = (payload) => {
+                const targetInfo = payload?.targetInfo ?? {};
+                const target_id = String(targetInfo?.targetId ?? '');
+                if (!target_id) {
+                    return;
+                }
+                const knownTarget = this._knownTargets.get(target_id);
+                const target_type = typeof targetInfo?.type === 'string'
+                    ? targetInfo.type
+                    : (knownTarget?.target_type ?? 'page');
+                const url = typeof targetInfo?.url === 'string'
+                    ? targetInfo.url
+                    : (knownTarget?.url ?? '');
+                this._knownTargets.set(target_id, {
+                    target_type,
+                    url,
+                });
+                this.browser_session.session_manager.handle_target_info_changed({
+                    target_id,
+                    target_type,
+                    url,
+                    title: typeof targetInfo?.title === 'string' ? targetInfo.title : '',
+                });
+                if (!knownTarget && target_type === 'page') {
+                    this._dispatchEventSafely(new TabCreatedEvent({
+                        target_id,
+                        url,
+                    }));
+                    return;
+                }
+                if (knownTarget?.target_type === 'page' && knownTarget.url !== url) {
+                    this._dispatchEventSafely(new NavigationCompleteEvent({
+                        target_id,
+                        url,
+                        status: null,
+                        error_message: null,
+                        loading_status: null,
+                    }));
+                }
+            };
+            cdpSession.on?.('Target.attachedToTarget', onAttached);
+            cdpSession.on?.('Target.detachedFromTarget', onDetached);
+            cdpSession.on?.('Target.targetInfoChanged', onTargetInfoChanged);
+            this._listeners = [
+                { event: 'Target.attachedToTarget', handler: onAttached },
+                { event: 'Target.detachedFromTarget', handler: onDetached },
+                { event: 'Target.targetInfoChanged', handler: onTargetInfoChanged },
+            ];
+        }
+        catch (error) {
+            this.browser_session.logger.debug(`CDPSessionWatchdog monitoring unavailable: ${error.message}`);
+            await this._teardownCdpMonitoring();
+        }
+    }
+    async _teardownCdpMonitoring() {
+        if (!this._rootCdpSession) {
+            return;
+        }
+        for (const listener of this._listeners) {
+            this._rootCdpSession.off?.(listener.event, listener.handler);
+        }
+        this._listeners = [];
+        try {
+            await this._rootCdpSession.detach?.();
+        }
+        catch {
+            // Ignore CDP detach errors during shutdown.
+        }
+        finally {
+            this._rootCdpSession = null;
+            this._knownTargets.clear();
+        }
+    }
+    _dispatchEventSafely(event) {
+        void this.event_bus.dispatch(event).catch((error) => {
+            this.browser_session.logger.debug(`CDPSessionWatchdog failed to dispatch ${String(event?.event_name ?? 'event')}: ${error.message}`);
+        });
+    }
+}

package/dist/browser/watchdogs/crash-watchdog.d.ts

@@ -0,0 +1,38 @@
+import { BrowserConnectedEvent, BrowserErrorEvent, BrowserStoppedEvent, TabClosedEvent, TargetCrashedEvent } from '../events.js';
+import { BaseWatchdog } from './base.js';
+export declare class CrashWatchdog extends BaseWatchdog {
+    static LISTENS_TO: (typeof BrowserConnectedEvent | typeof BrowserStoppedEvent | typeof TabClosedEvent)[];
+    static EMITS: (typeof TargetCrashedEvent | typeof BrowserErrorEvent)[];
+    private _pageListeners;
+    private _pendingRequests;
+    private _requestIds;
+    private _requestCounter;
+    private _healthInterval;
+    private _networkTimeoutMs;
+    private _healthCheckIntervalMs;
+    private _consecutiveUnresponsiveChecks;
+    private _unresponsiveThreshold;
+    private _monitoringInProgress;
+    on_BrowserConnectedEvent(): Promise<void>;
+    on_TabCreatedEvent(): Promise<void>;
+    on_TabClosedEvent(): Promise<void>;
+    on_BrowserStoppedEvent(): Promise<void>;
+    protected onDetached(): void;
+    private _attachToKnownPages;
+    private _dropDetachedPages;
+    private _detachAllPages;
+    private _attachPage;
+    private _detachPageListeners;
+    private _getKnownPages;
+    private _handlePageCrash;
+    private _resolveTargetId;
+    private _safePageUrl;
+    private _normalizeCrashError;
+    private _trackRequestStart;
+    private _trackRequestDone;
+    private _startHealthMonitor;
+    private _stopHealthMonitor;
+    private _runHealthCheck;
+    private _checkNetworkTimeouts;
+    private _checkPageResponsiveness;
+}