browser-use 0.2.0 → 0.3.0

package/dist/actor/element.d.ts

@@ -0,0 +1,19 @@
+import type { BrowserSession } from '../browser/session.js';
+import type { DOMElementNode } from '../dom/views.js';
+export declare class Element {
+    private readonly browser_session;
+    readonly node: DOMElementNode;
+    constructor(browser_session: BrowserSession, node: DOMElementNode);
+    click(): Promise<string | null>;
+    fill(value: string, clear?: boolean): Promise<void>;
+    hover(): Promise<void>;
+    get_attribute(name: string): Promise<string>;
+    get_bounding_box(): Promise<{
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    } | null>;
+    select_option(values: string | string[]): Promise<void>;
+    evaluate(page_function: string, ...args: unknown[]): Promise<any>;
+}

package/dist/actor/element.js

@@ -0,0 +1,46 @@
+export class Element {
+    browser_session;
+    node;
+    constructor(browser_session, node) {
+        this.browser_session = browser_session;
+        this.node = node;
+    }
+    async click() {
+        return this.browser_session._click_element_node(this.node);
+    }
+    async fill(value, clear = true) {
+        return this.browser_session._input_text_element_node(this.node, value, {
+            clear,
+        });
+    }
+    async hover() {
+        const locator = await this.browser_session.get_locate_element(this.node);
+        if (!locator?.hover) {
+            return;
+        }
+        await locator.hover({ timeout: 5000 });
+    }
+    async get_attribute(name) {
+        return this.node.attributes?.[name] ?? null;
+    }
+    async get_bounding_box() {
+        const locator = await this.browser_session.get_locate_element(this.node);
+        if (!locator?.boundingBox) {
+            return null;
+        }
+        return locator.boundingBox();
+    }
+    async select_option(values) {
+        const list = Array.isArray(values) ? values : [values];
+        for (const value of list) {
+            await this.browser_session.select_dropdown_option(this.node, value);
+        }
+    }
+    async evaluate(page_function, ...args) {
+        const locator = await this.browser_session.get_locate_element(this.node);
+        if (!locator?.evaluate) {
+            throw new Error('Element evaluate is unavailable for this node');
+        }
+        return locator.evaluate(page_function, ...args);
+    }
+}

package/dist/actor/index.d.ts

@@ -0,0 +1,4 @@
+export * from './element.js';
+export * from './mouse.js';
+export * from './page.js';
+export * from './utils.js';

package/dist/actor/index.js

@@ -0,0 +1,4 @@
+export * from './element.js';
+export * from './mouse.js';
+export * from './page.js';
+export * from './utils.js';

package/dist/actor/mouse.d.ts

@@ -0,0 +1,19 @@
+import type { BrowserSession } from '../browser/session.js';
+import type { MouseButton } from '../browser/events.js';
+export declare class Mouse {
+    private readonly browser_session;
+    private readonly pageRef;
+    constructor(browser_session: BrowserSession, pageRef?: any | null);
+    private _page;
+    click(x: number, y: number, options?: {
+        button?: MouseButton;
+        click_count?: number;
+    }): Promise<void>;
+    move(x: number, y: number): Promise<void>;
+    down(options?: {
+        button?: MouseButton;
+    }): Promise<void>;
+    up(options?: {
+        button?: MouseButton;
+    }): Promise<void>;
+}

package/dist/actor/mouse.js

@@ -0,0 +1,39 @@
+export class Mouse {
+    browser_session;
+    pageRef;
+    constructor(browser_session, pageRef = null) {
+        this.browser_session = browser_session;
+        this.pageRef = pageRef;
+    }
+    async _page() {
+        if (this.pageRef) {
+            return this.pageRef;
+        }
+        return this.browser_session.get_current_page();
+    }
+    async click(x, y, options = {}) {
+        const button = options.button ?? 'left';
+        await this.browser_session.click_coordinates(x, y, { button });
+    }
+    async move(x, y) {
+        const page = await this._page();
+        if (!page?.mouse?.move) {
+            return;
+        }
+        await page.mouse.move(x, y);
+    }
+    async down(options = {}) {
+        const page = await this._page();
+        if (!page?.mouse?.down) {
+            return;
+        }
+        await page.mouse.down({ button: options.button ?? 'left' });
+    }
+    async up(options = {}) {
+        const page = await this._page();
+        if (!page?.mouse?.up) {
+            return;
+        }
+        await page.mouse.up({ button: options.button ?? 'left' });
+    }
+}

package/dist/actor/page.d.ts

@@ -0,0 +1,29 @@
+import type { WaitUntilState } from '../browser/events.js';
+import type { BrowserSession } from '../browser/session.js';
+import { Element } from './element.js';
+import { Mouse } from './mouse.js';
+export declare class Page {
+    private readonly browser_session;
+    private _mouse;
+    constructor(browser_session: BrowserSession);
+    get mouse(): Mouse;
+    _currentPage(): Promise<import("playwright-core").Page>;
+    get_url(): Promise<string>;
+    get_title(): Promise<string>;
+    goto(url: string, options?: {
+        wait_until?: WaitUntilState;
+        timeout_ms?: number | null;
+    }): Promise<void>;
+    navigate(url: string, options?: Parameters<Page['goto']>[1]): Promise<void>;
+    reload(): Promise<void>;
+    go_back(): Promise<void>;
+    go_forward(): Promise<void>;
+    evaluate(page_function: string | ((...args: unknown[]) => unknown), ...args: unknown[]): Promise<unknown>;
+    screenshot(options?: {
+        full_page?: boolean;
+    }): Promise<string | null>;
+    press(key: string): Promise<void>;
+    set_viewport_size(width: number, height: number): Promise<void>;
+    get_element_by_index(index: number): Promise<Element | null>;
+    must_get_element_by_index(index: number): Promise<Element>;
+}

package/dist/actor/page.js

@@ -0,0 +1,88 @@
+import { Element } from './element.js';
+import { Mouse } from './mouse.js';
+export class Page {
+    browser_session;
+    _mouse = null;
+    constructor(browser_session) {
+        this.browser_session = browser_session;
+    }
+    get mouse() {
+        if (!this._mouse) {
+            this._mouse = new Mouse(this.browser_session);
+        }
+        return this._mouse;
+    }
+    async _currentPage() {
+        const page = await this.browser_session.get_current_page();
+        if (!page) {
+            throw new Error('No active page available');
+        }
+        return page;
+    }
+    async get_url() {
+        const page = await this._currentPage();
+        return typeof page.url === 'function' ? page.url() : '';
+    }
+    async get_title() {
+        const page = await this._currentPage();
+        return typeof page.title === 'function' ? page.title() : '';
+    }
+    async goto(url, options = {}) {
+        await this.browser_session.navigate_to(url, {
+            wait_until: options.wait_until,
+            timeout_ms: options.timeout_ms,
+        });
+    }
+    async navigate(url, options = {}) {
+        await this.goto(url, options);
+    }
+    async reload() {
+        await this.browser_session.refresh();
+    }
+    async go_back() {
+        await this.browser_session.go_back();
+    }
+    async go_forward() {
+        await this.browser_session.go_forward();
+    }
+    async evaluate(page_function, ...args) {
+        const page = await this._currentPage();
+        if (typeof page_function === 'function') {
+            return page.evaluate(page_function, ...args);
+        }
+        if (args.length === 0) {
+            return page.evaluate(page_function);
+        }
+        const expression = `(${page_function})(${args
+            .map((arg) => JSON.stringify(arg))
+            .join(',')})`;
+        return page.evaluate(expression);
+    }
+    async screenshot(options = {}) {
+        return this.browser_session.take_screenshot(options.full_page ?? false);
+    }
+    async press(key) {
+        await this.browser_session.send_keys(key);
+    }
+    async set_viewport_size(width, height) {
+        const page = await this._currentPage();
+        if (!page.setViewportSize) {
+            return;
+        }
+        await page.setViewportSize({ width, height });
+    }
+    async get_element_by_index(index) {
+        const node = await this.browser_session.get_dom_element_by_index(index);
+        if (!node) {
+            return null;
+        }
+        return new Element(this.browser_session, node);
+    }
+    async must_get_element_by_index(index) {
+        const element = await this.get_element_by_index(index);
+        if (!element) {
+            throw new Error(`Element not found for index ${index}`);
+        }
+        return element;
+    }
+}

package/dist/actor/utils.d.ts

@@ -0,0 +1,4 @@
+export declare class Utils {
+    static get_key_info(key: string): [string, number | null];
+}
+export declare const get_key_info: (key: string) => [string, number | null];

package/dist/actor/utils.js

@@ -0,0 +1,35 @@
+export class Utils {
+    static get_key_info(key) {
+        const key_map = {
+            Backspace: ['Backspace', 8],
+            Tab: ['Tab', 9],
+            Enter: ['Enter', 13],
+            Escape: ['Escape', 27],
+            Space: ['Space', 32],
+            ArrowLeft: ['ArrowLeft', 37],
+            ArrowUp: ['ArrowUp', 38],
+            ArrowRight: ['ArrowRight', 39],
+            ArrowDown: ['ArrowDown', 40],
+            Delete: ['Delete', 46],
+            Shift: ['ShiftLeft', 16],
+            Control: ['ControlLeft', 17],
+            Alt: ['AltLeft', 18],
+            Meta: ['MetaLeft', 91],
+        };
+        const direct = key_map[key];
+        if (direct) {
+            return direct;
+        }
+        if (key.length === 1) {
+            if (/[a-z]/i.test(key)) {
+                const upper = key.toUpperCase();
+                return [`Key${upper}`, upper.charCodeAt(0)];
+            }
+            if (/[0-9]/.test(key)) {
+                return [`Digit${key}`, key.charCodeAt(0)];
+            }
+        }
+        return [key, null];
+    }
+}
+export const get_key_info = (key) => Utils.get_key_info(key);

package/dist/agent/cloud-events.d.ts

@@ -261,4 +261,22 @@ export declare class CreateAgentSessionEvent extends BaseEvent {
         device_id: string | null;
     };
 }
+export declare class UpdateAgentSessionEvent extends BaseEvent {
+    browser_session_stopped: boolean | null;
+    browser_session_stopped_at: Date | null;
+    end_reason: string | null;
+    constructor(init: Partial<UpdateAgentSessionEvent> & {
+        id: string;
+        user_id?: string;
+    });
+    toJSON(): {
+        browser_session_stopped: boolean | null;
+        browser_session_stopped_at: string | null;
+        end_reason: string | null;
+        event_type: string;
+        id: string;
+        user_id: string;
+        device_id: string | null;
+    };
+}
 export {};

package/dist/agent/cloud-events.js

@@ -1,11 +1,17 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { uuid7str } from '../utils.js';
+import { createLogger } from '../logging-config.js';
 const MAX_STRING_LENGTH = 100_000;
 const MAX_URL_LENGTH = 100_000;
 const MAX_TASK_LENGTH = 100_000;
 const MAX_COMMENT_LENGTH = 2_000;
 const MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024;
+const MAX_LLM_MODEL_LENGTH = 200;
+const MAX_END_REASON_LENGTH = 100;
+const logger = createLogger('browser_use.agent.cloud_events');
+const estimateBase64DecodedBytes = (value) => Math.floor((value.length * 3) / 4);
+const extractBase64Payload = (value) => value.includes(',') ? value.split(',').slice(1).join(',') : value;
 const getDeviceId = (agent) => agent.cloud_sync?.auth_client?.device_id ?? null;
 const getBrowserProfile = (agent) => agent.browser_profile ?? agent.browser_session?.browser_profile ?? null;
 const serializeAgentState = (agent) => {
@@ -105,7 +111,17 @@ export class CreateAgentOutputFileEvent extends BaseEvent {
         super('CreateAgentOutputFileEvent', init);
         this.task_id = init.task_id;
         this.file_name = init.file_name;
-        this.file_content = init.file_content ?? null;
+        if (init.file_content != null) {
+            const payload = extractBase64Payload(init.file_content);
+            const estimatedSize = estimateBase64DecodedBytes(payload);
+            if (estimatedSize > MAX_FILE_CONTENT_SIZE) {
+                throw new Error(`file_content exceeds maximum size of ${MAX_FILE_CONTENT_SIZE} bytes`);
+            }
+            this.file_content = init.file_content;
+        }
+        else {
+            this.file_content = null;
+        }
         this.content_type = init.content_type ?? null;
         this.created_at = init.created_at ?? new Date();
     }
@@ -156,6 +172,13 @@ export class CreateAgentStepEvent extends BaseEvent {
         this.memory = init.memory;
         this.next_goal = init.next_goal;
         this.actions = init.actions;
+        if (init.screenshot_url?.startsWith('data:')) {
+            const payload = extractBase64Payload(init.screenshot_url);
+            const estimatedSize = estimateBase64DecodedBytes(payload);
+            if (estimatedSize > MAX_FILE_CONTENT_SIZE) {
+                throw new Error(`screenshot_url exceeds maximum size of ${MAX_FILE_CONTENT_SIZE} bytes`);
+            }
+        }
         this.screenshot_url = init.screenshot_url ?? null;
         this.url = init.url;
     }
@@ -164,6 +187,12 @@ export class CreateAgentStepEvent extends BaseEvent {
         const screenshot = browser_state_summary.screenshot
             ? `data:image/png;base64,${browser_state_summary.screenshot}`
             : null;
+        if (browser_state_summary.screenshot) {
+            logger.debug(`Including screenshot in CreateAgentStepEvent, length: ${browser_state_summary.screenshot.length}`);
+        }
+        else {
+            logger.debug('No screenshot in browser_state_summary for CreateAgentStepEvent');
+        }
         return new CreateAgentStepEvent({
             device_id: getDeviceId(agent),
             agent_task_id: String(agent.task_id),
@@ -208,6 +237,12 @@ export class CreateAgentTaskEvent extends BaseEvent {
     constructor(init) {
         super('CreateAgentTaskEvent', init);
         this.agent_session_id = init.agent_session_id;
+        if (init.llm_model.length > MAX_LLM_MODEL_LENGTH) {
+            throw new Error(`llm_model exceeds maximum length of ${MAX_LLM_MODEL_LENGTH}`);
+        }
+        if (init.task.length > MAX_TASK_LENGTH) {
+            throw new Error(`task exceeds maximum length of ${MAX_TASK_LENGTH}`);
+        }
         this.llm_model = init.llm_model;
         this.task = init.task;
         this.stopped = init.stopped ?? false;
@@ -227,7 +262,7 @@ export class CreateAgentTaskEvent extends BaseEvent {
             id: String(agent.task_id),
             device_id: getDeviceId(agent),
             agent_session_id: String(agent.session_id),
-            task: agent.task.slice(0, MAX_TASK_LENGTH),
+            task: agent.task,
             llm_model: agent.llm.model_name || agent.llm.model || 'unknown',
             agent_state: serializeAgentState(agent),
             stopped: false,
@@ -316,3 +351,31 @@ export class CreateAgentSessionEvent extends BaseEvent {
         };
     }
 }
+export class UpdateAgentSessionEvent extends BaseEvent {
+    browser_session_stopped;
+    browser_session_stopped_at;
+    end_reason;
+    constructor(init) {
+        super('UpdateAgentSessionEvent', init);
+        this.browser_session_stopped = init.browser_session_stopped ?? null;
+        this.browser_session_stopped_at = init.browser_session_stopped_at ?? null;
+        if (init.end_reason != null) {
+            const endReason = String(init.end_reason);
+            if (endReason.length > MAX_END_REASON_LENGTH) {
+                throw new Error(`end_reason exceeds maximum length of ${MAX_END_REASON_LENGTH}`);
+            }
+            this.end_reason = endReason;
+        }
+        else {
+            this.end_reason = null;
+        }
+    }
+    toJSON() {
+        return {
+            ...super.toJSON(),
+            browser_session_stopped: this.browser_session_stopped,
+            browser_session_stopped_at: this.browser_session_stopped_at?.toISOString() ?? null,
+            end_reason: this.end_reason,
+        };
+    }
+}

package/dist/agent/gif.d.ts

@@ -1,5 +1,6 @@
 import type { AgentHistoryList } from './views.js';
 export declare const decode_unicode_escapes_to_utf8: (text: string) => string;
+export declare const is_valid_gif_screenshot_candidate: (screenshot: string | null | undefined, pageUrl: string | null | undefined) => boolean;
 export interface HistoryGifOptions {
     output_path?: string;
     duration?: number;

package/dist/agent/gif.js

@@ -4,6 +4,7 @@ import { createCanvas, loadImage } from 'canvas';
 import GIFEncoder from 'gif-encoder-2';
 import { createLogger } from '../logging-config.js';
 import { PLACEHOLDER_4PX_SCREENSHOT } from '../browser/views.js';
+import { is_new_tab_page } from '../utils.js';
 const logger = createLogger('browser_use.agent.gif');
 export const decode_unicode_escapes_to_utf8 = (text) => {
     if (!text.includes('\\u')) {
@@ -29,6 +30,8 @@ const loadScreenshot = async (screenshot) => {
     return loadImage(normalized);
 };
 const FONT_CANDIDATES = [
+    '"PingFang"',
+    '"STHeiti Medium"',
     '"Microsoft YaHei"',
     '"SimHei"',
     '"SimSun"',
@@ -38,6 +41,12 @@ const FONT_CANDIDATES = [
     '"sans-serif"',
 ];
 const pickFont = () => FONT_CANDIDATES.join(', ');
+export const is_valid_gif_screenshot_candidate = (screenshot, pageUrl) => {
+    if (!screenshot || screenshot === PLACEHOLDER_4PX_SCREENSHOT) {
+        return false;
+    }
+    return !is_new_tab_page(pageUrl ?? '');
+};
 const wrapText = (ctx, text, maxWidth) => {
     const words = decode_unicode_escapes_to_utf8(text).split(/\s+/);
     const lines = [];
@@ -158,9 +167,17 @@ export const create_history_gif = async (task, history, { output_path = 'agent_h
         return;
     }
     const screenshots = history.screenshots();
-    const firstRealScreenshot = screenshots.find((shot) => shot && shot !== PLACEHOLDER_4PX_SCREENSHOT);
+    let firstRealScreenshot = null;
+    for (let index = 0; index < screenshots.length; index += 1) {
+        const screenshot = screenshots[index];
+        const pageUrl = history.history[index]?.state?.url ?? null;
+        if (is_valid_gif_screenshot_candidate(screenshot, pageUrl)) {
+            firstRealScreenshot = screenshot;
+            break;
+        }
+    }
     if (!firstRealScreenshot) {
-        logger.warn('No valid screenshots found (all are placeholders)');
+        logger.warn('No valid screenshots found (all are placeholders or from new tab pages)');
         return;
     }
     const firstImage = await loadScreenshot(firstRealScreenshot);
@@ -187,6 +204,11 @@ export const create_history_gif = async (task, history, { output_path = 'agent_h
         encoder.addFrame(ctx);
     }
     for (let index = 0; index < screenshots.length; index += 1) {
+        const historyItem = history.history[index];
+        if (is_new_tab_page(historyItem?.state?.url ?? '')) {
+            logger.debug(`Skipping screenshot from new tab page (${historyItem?.state?.url ?? ''}) at step ${index}`);
+            continue;
+        }
         const screenshot = screenshots[index];
         if (!screenshot || screenshot === PLACEHOLDER_4PX_SCREENSHOT) {
             continue;

package/dist/agent/judge.d.ts

@@ -0,0 +1,17 @@
+import { type Message } from '../llm/messages.js';
+export interface ConstructJudgeMessagesOptions {
+    task: string;
+    final_result: string;
+    agent_steps: string[];
+    screenshot_paths: string[];
+    max_images?: number;
+    ground_truth?: string | null;
+    use_vision?: boolean | 'auto';
+}
+export interface ConstructSimpleJudgeMessagesOptions {
+    task: string;
+    final_result: string;
+    current_date?: string;
+}
+export declare const construct_judge_messages: (options: ConstructJudgeMessagesOptions) => Message[];
+export declare const construct_simple_judge_messages: (options: ConstructSimpleJudgeMessagesOptions) => Message[];