browser-use 0.0.1 → 0.0.2

package/dist/agent/prompts.js

@@ -0,0 +1,259 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { SystemMessage, UserMessage, ContentPartTextParam, ContentPartImageParam, ImageURL, } from '../llm/messages.js';
+import { observe_debug } from '../observability.js';
+import { is_new_tab_page } from '../utils.js';
+const readPromptTemplate = (filename) => {
+    const filePath = fileURLToPath(new URL(filename, import.meta.url));
+    return fs.readFileSync(filePath, 'utf-8');
+};
+export class SystemPrompt {
+    actionDescription;
+    maxActionsPerStep;
+    overrideSystemMessage;
+    extendSystemMessage;
+    useThinking;
+    flashMode;
+    promptTemplate = '';
+    systemMessage;
+    constructor(actionDescription, maxActionsPerStep = 10, overrideSystemMessage = null, extendSystemMessage = null, useThinking = true, flashMode = false) {
+        this.actionDescription = actionDescription;
+        this.maxActionsPerStep = maxActionsPerStep;
+        this.overrideSystemMessage = overrideSystemMessage;
+        this.extendSystemMessage = extendSystemMessage;
+        this.useThinking = useThinking;
+        this.flashMode = flashMode;
+        if (overrideSystemMessage) {
+            this.promptTemplate = overrideSystemMessage;
+        }
+        else {
+            this.loadPromptTemplate();
+        }
+        let prompt = this.promptTemplate.replace('{max_actions}', String(this.maxActionsPerStep));
+        if (this.extendSystemMessage) {
+            prompt += `\n${this.extendSystemMessage}`;
+        }
+        this.systemMessage = new SystemMessage(prompt);
+        this.systemMessage.cache = true;
+    }
+    loadPromptTemplate() {
+        const templateName = this.flashMode
+            ? './system_prompt_flash.md'
+            : this.useThinking
+                ? './system_prompt.md'
+                : './system_prompt_no_thinking.md';
+        try {
+            this.promptTemplate = readPromptTemplate(templateName);
+        }
+        catch (error) {
+            throw new Error(`Failed to load system prompt template: ${error.message}`);
+        }
+    }
+    get_system_message() {
+        return this.systemMessage;
+    }
+}
+export class AgentMessagePrompt {
+    browserState;
+    fileSystem;
+    agentHistoryDescription;
+    readStateDescription;
+    task;
+    includeAttributes;
+    stepInfo;
+    pageFilteredActions;
+    maxClickableElementsLength;
+    sensitiveData;
+    availableFilePaths;
+    screenshots;
+    visionDetailLevel;
+    constructor(init) {
+        this.browserState = init.browser_state_summary;
+        this.fileSystem = init.file_system;
+        this.agentHistoryDescription = init.agent_history_description ?? null;
+        this.readStateDescription = init.read_state_description ?? null;
+        this.task = init.task ?? null;
+        this.includeAttributes = init.include_attributes ?? null;
+        this.stepInfo = init.step_info ?? null;
+        this.pageFilteredActions = init.page_filtered_actions ?? null;
+        this.maxClickableElementsLength =
+            init.max_clickable_elements_length ?? 40000;
+        this.sensitiveData = init.sensitive_data ?? null;
+        this.availableFilePaths = init.available_file_paths ?? null;
+        this.screenshots = init.screenshots ?? [];
+        this.visionDetailLevel = init.vision_detail_level ?? 'auto';
+    }
+    browserStateDescription() {
+        let elementsText = this.browserState.element_tree.clickable_elements_to_string(this.includeAttributes ?? undefined);
+        let truncatedText = '';
+        if (elementsText.length > this.maxClickableElementsLength) {
+            elementsText = elementsText.slice(0, this.maxClickableElementsLength);
+            truncatedText = ` (truncated to ${this.maxClickableElementsLength} characters)`;
+        }
+        const hasContentAbove = (this.browserState.pixels_above ?? 0) > 0;
+        const hasContentBelow = (this.browserState.pixels_below ?? 0) > 0;
+        const pi = this.browserState.page_info;
+        let pageInfoText = '';
+        if (pi) {
+            const pagesAbove = pi.viewport_height > 0 ? pi.pixels_above / pi.viewport_height : 0;
+            const pagesBelow = pi.viewport_height > 0 ? pi.pixels_below / pi.viewport_height : 0;
+            const totalPages = pi.viewport_height > 0 ? pi.page_height / pi.viewport_height : 0;
+            const currentPosition = pi.scroll_y / Math.max(pi.page_height - pi.viewport_height, 1);
+            pageInfoText = `Page info: ${pi.viewport_width}x${pi.viewport_height}px viewport, ${pi.page_width}x${pi.page_height}px total page size, ${pagesAbove.toFixed(1)} pages above, ${pagesBelow.toFixed(1)} pages below, ${totalPages.toFixed(1)} total pages, at ${(currentPosition * 100).toFixed(0)}% of page`;
+        }
+        if (elementsText) {
+            if (hasContentAbove) {
+                if (pi) {
+                    const pagesAbove = pi.viewport_height > 0 ? pi.pixels_above / pi.viewport_height : 0;
+                    elementsText = `... ${this.browserState.pixels_above} pixels above (${pagesAbove.toFixed(1)} pages) - scroll to see more or extract structured data if you are looking for specific information ...\n${elementsText}`;
+                }
+                else {
+                    elementsText = `... ${this.browserState.pixels_above} pixels above - scroll to see more or extract structured data if you are looking for specific information ...\n${elementsText}`;
+                }
+            }
+            else {
+                elementsText = `[Start of page]\n${elementsText}`;
+            }
+            if (hasContentBelow) {
+                if (pi) {
+                    const pagesBelow = pi.viewport_height > 0 ? pi.pixels_below / pi.viewport_height : 0;
+                    elementsText = `${elementsText}\n... ${this.browserState.pixels_below} pixels below (${pagesBelow.toFixed(1)} pages) - scroll to see more or extract structured data if you are looking for specific information ...`;
+                }
+                else {
+                    elementsText = `${elementsText}\n... ${this.browserState.pixels_below} pixels below - scroll to see more or extract structured data if you are looking for specific information ...`;
+                }
+            }
+            else {
+                elementsText = `${elementsText}\n[End of page]`;
+            }
+        }
+        else {
+            elementsText = 'empty page';
+        }
+        let tabsText = '';
+        const currentTabCandidates = [];
+        for (const tab of this.browserState.tabs) {
+            if (tab.url === this.browserState.url &&
+                tab.title === this.browserState.title) {
+                currentTabCandidates.push(tab.page_id);
+            }
+        }
+        const currentTabId = currentTabCandidates.length === 1 ? currentTabCandidates[0] : null;
+        for (const tab of this.browserState.tabs) {
+            tabsText += `Tab ${tab.page_id}: ${tab.url} - ${tab.title.slice(0, 30)}\n`;
+        }
+        const currentTabText = currentTabId !== null ? `Current tab: ${currentTabId}` : '';
+        const pdfMessage = this.browserState.is_pdf_viewer
+            ? 'PDF viewer cannot be rendered. Do not use extract_structured_data here; use read_file on downloaded PDF via available_file_paths.\n\n'
+            : '';
+        return `${currentTabText}
+Available tabs:
+${tabsText}
+${pageInfoText}
+${pdfMessage}Interactive elements from top layer of the current page inside the viewport${truncatedText}:
+${elementsText}
+`;
+    }
+    agentStateDescription() {
+        const todoContents = this.fileSystem.get_todo_contents();
+        const todoText = todoContents ||
+            '[Current todo.md is empty, fill it with your plan when applicable]';
+        const timestamp = new Date().toISOString().slice(0, 16).replace('T', ' ');
+        const stepInfo = this.stepInfo != null
+            ? `Step ${this.stepInfo.step_number + 1} of ${this.stepInfo.max_steps}\n`
+            : '';
+        const stepInfoDescription = `${stepInfo}Current date and time: ${timestamp}`;
+        let agentState = `<user_request>
+${this.task ?? ''}
+</user_request>
+<file_system>
+${this.fileSystem.describe()}
+</file_system>
+<todo_contents>
+${todoText}
+</todo_contents>
+`;
+        if (this.sensitiveData) {
+            agentState += `<sensitive_data>
+${this.sensitiveData}
+</sensitive_data>
+`;
+        }
+        agentState += `<step_info>
+${stepInfoDescription}
+</step_info>
+`;
+        if (this.availableFilePaths?.length) {
+            agentState += `<available_file_paths>
+${this.availableFilePaths.join('\n')}
+</available_file_paths>
+`;
+        }
+        return agentState;
+    }
+    // @ts-ignore - Decorator type mismatch with TypeScript strict mode
+    get_user_message(use_vision = true) {
+        if (is_new_tab_page(this.browserState.url) &&
+            this.stepInfo &&
+            this.stepInfo.step_number === 0 &&
+            this.browserState.tabs.length === 1) {
+            use_vision = false;
+        }
+        let stateDescription = `<agent_history>
+${(this.agentHistoryDescription ?? '').trim()}
+</agent_history>
+`;
+        stateDescription += `<agent_state>
+${this.agentStateDescription().trim()}
+</agent_state>
+`;
+        stateDescription += `<browser_state>
+${this.browserStateDescription().trim()}
+</browser_state>
+`;
+        const readState = (this.readStateDescription ?? '').trim();
+        if (readState) {
+            stateDescription += `<read_state>
+${readState}
+</read_state>
+`;
+        }
+        if (this.pageFilteredActions) {
+            stateDescription += `<page_specific_actions>
+${this.pageFilteredActions}
+</page_specific_actions>
+`;
+        }
+        if (use_vision && this.screenshots.length > 0) {
+            const parts = [
+                new ContentPartTextParam(stateDescription),
+            ];
+            this.screenshots.forEach((shot, index) => {
+                const label = index === this.screenshots.length - 1
+                    ? 'Current screenshot:'
+                    : 'Previous screenshot:';
+                parts.push(new ContentPartTextParam(label));
+                parts.push(new ContentPartImageParam(new ImageURL(`data:image/png;base64,${shot}`, this.visionDetailLevel, 'image/png')));
+            });
+            const message = new UserMessage(parts);
+            message.cache = true;
+            return message;
+        }
+        const message = new UserMessage(stateDescription);
+        message.cache = true;
+        return message;
+    }
+}
+__decorate([
+    observe_debug({
+        name: 'agent_message_prompt:get_user_message',
+        ignore_input: true,
+        ignore_output: true,
+    })
+], AgentMessagePrompt.prototype, "get_user_message", null);

package/dist/agent/service.d.ts

@@ -0,0 +1,290 @@
+import { createLogger } from '../logging-config.js';
+import { EventBus } from '../event-bus.js';
+import type { Controller } from '../controller/service.js';
+import type { FileSystem } from '../filesystem/file-system.js';
+import { SystemPrompt } from './prompts.js';
+import { MessageManager } from './message-manager/service.js';
+import { BrowserStateSummary } from '../browser/views.js';
+import { BrowserSession } from '../browser/session.js';
+import { BrowserProfile } from '../browser/profile.js';
+import type { Browser, BrowserContext, Page } from '../browser/types.js';
+import type { BaseChatModel } from '../llm/base.js';
+import { ActionResult, AgentHistoryList, AgentOutput, AgentSettings, AgentState, AgentStepInfo, ActionModel } from './views.js';
+import type { StructuredOutputParser } from './views.js';
+import { ScreenshotService } from '../screenshots/service.js';
+import { ProductTelemetry } from '../telemetry/service.js';
+export declare const log_response: (response: AgentOutput, registry?: Controller<any>, logInstance?: import("../logging-config.js").Logger) => void;
+type ControllerContext = unknown;
+type AgentHookFunc<Context, AgentStructuredOutput> = (agent: Agent<Context, AgentStructuredOutput>) => Promise<void> | void;
+interface RerunHistoryOptions {
+    max_retries?: number;
+    skip_failures?: boolean;
+    delay_between_actions?: number;
+    signal?: AbortSignal | null;
+}
+interface AgentConstructorParams<Context, AgentStructuredOutput> {
+    task: string;
+    llm: BaseChatModel;
+    page?: Page | null;
+    browser?: Browser | BrowserSession | null;
+    browser_context?: BrowserContext | null;
+    browser_profile?: BrowserProfile | null;
+    browser_session?: BrowserSession | null;
+    controller?: Controller<Context> | null;
+    sensitive_data?: Record<string, string | Record<string, string>> | null;
+    initial_actions?: Array<Record<string, Record<string, unknown>>> | null;
+    register_new_step_callback?: ((summary: BrowserStateSummary, output: AgentOutput, step: number) => void | Promise<void>) | null;
+    register_done_callback?: ((history: AgentHistoryList<AgentStructuredOutput>) => void | Promise<void>) | null;
+    register_external_agent_status_raise_error_callback?: (() => Promise<boolean>) | null;
+    output_model_schema?: StructuredOutputParser<AgentStructuredOutput> | null;
+    use_vision?: boolean;
+    use_vision_for_planner?: boolean;
+    save_conversation_path?: string | null;
+    save_conversation_path_encoding?: BufferEncoding | null;
+    max_failures?: number;
+    retry_delay?: number;
+    override_system_message?: string | null;
+    extend_system_message?: string | null;
+    validate_output?: boolean;
+    generate_gif?: boolean | string;
+    available_file_paths?: string[] | null;
+    include_attributes?: string[];
+    max_actions_per_step?: number;
+    use_thinking?: boolean;
+    flash_mode?: boolean;
+    max_history_items?: number | null;
+    page_extraction_llm?: BaseChatModel | null;
+    planner_llm?: BaseChatModel | null;
+    planner_interval?: number;
+    is_planner_reasoning?: boolean;
+    extend_planner_system_message?: string | null;
+    injected_agent_state?: AgentState | null;
+    context?: Context | null;
+    source?: string | null;
+    file_system_path?: string | null;
+    task_id?: string | null;
+    cloud_sync?: any;
+    calculate_cost?: boolean;
+    display_files_in_done_text?: boolean;
+    include_tool_call_examples?: boolean;
+    vision_detail_level?: AgentSettings['vision_detail_level'];
+    session_attachment_mode?: AgentSettings['session_attachment_mode'];
+    llm_timeout?: number;
+    step_timeout?: number;
+}
+export declare class Agent<Context = ControllerContext, AgentStructuredOutput = unknown> {
+    private static _sharedSessionStepLocks;
+    static DEFAULT_AGENT_DATA_DIR: string;
+    browser_session: BrowserSession | null;
+    llm: BaseChatModel;
+    unfiltered_actions: string;
+    initial_actions: Array<Record<string, Record<string, unknown>>> | null;
+    register_new_step_callback: AgentConstructorParams<Context, AgentStructuredOutput>['register_new_step_callback'];
+    register_done_callback: AgentConstructorParams<Context, AgentStructuredOutput>['register_done_callback'];
+    register_external_agent_status_raise_error_callback: AgentConstructorParams<Context, AgentStructuredOutput>['register_external_agent_status_raise_error_callback'];
+    context: Context | null;
+    telemetry: ProductTelemetry;
+    eventbus: EventBus;
+    enable_cloud_sync: boolean;
+    cloud_sync: any;
+    file_system: FileSystem | null;
+    screenshot_service: ScreenshotService | null;
+    agent_directory: string;
+    private _current_screenshot_path;
+    has_downloads_path: boolean;
+    private _last_known_downloads;
+    version: string;
+    source: string;
+    step_start_time: number;
+    _external_pause_event: {
+        resolve: (() => void) | null;
+        promise: Promise<void>;
+    };
+    output_model_schema: StructuredOutputParser<AgentStructuredOutput> | null;
+    id: string;
+    task_id: string;
+    session_id: string;
+    task: string;
+    controller: Controller<Context>;
+    settings: AgentSettings;
+    token_cost_service: any;
+    state: AgentState;
+    history: AgentHistoryList<AgentStructuredOutput>;
+    _message_manager: MessageManager;
+    available_file_paths: string[];
+    sensitive_data: Record<string, string | Record<string, string>> | null;
+    _logger: ReturnType<typeof createLogger> | null;
+    _file_system_path: string | null;
+    agent_current_page: Page | null;
+    _session_start_time: number;
+    _task_start_time: number;
+    _force_exit_telemetry_logged: boolean;
+    private _closePromise;
+    private _hasBrowserSessionClaim;
+    private _sharedPinnedTabId;
+    private _enforceDoneOnlyForCurrentStep;
+    system_prompt_class: SystemPrompt;
+    ActionModel: typeof ActionModel;
+    AgentOutput: typeof AgentOutput;
+    DoneActionModel: typeof ActionModel;
+    DoneAgentOutput: typeof AgentOutput;
+    constructor(params: AgentConstructorParams<Context, AgentStructuredOutput>);
+    private _createSessionIdWithAgentSuffix;
+    private _copyBrowserProfile;
+    private _getBrowserContextFromPage;
+    private _claim_or_isolate_browser_session;
+    private _release_browser_session_claim;
+    private _has_any_browser_session_attachments;
+    private _is_shared_session_mode;
+    private _capture_shared_pinned_tab;
+    private _restore_shared_pinned_tab_if_needed;
+    private _run_with_shared_session_step_lock;
+    private _cleanup_shared_session_step_lock_if_unused;
+    private _init_browser_session;
+    private _sleep_blocking;
+    /**
+     * Convert dictionary-based actions to ActionModel instances
+     */
+    private _convertInitialActions;
+    /**
+     * Handle model-specific vision capabilities
+     * Some models like DeepSeek and Grok don't support vision yet
+     */
+    private _handleModelSpecificVision;
+    /**
+     * Verify that the LLM API keys are setup and the LLM API is responding properly.
+     * Also handles model capability detection.
+     */
+    private _verifyAndSetupLlm;
+    /**
+     * Validates security settings when sensitive_data is provided
+     * Checks if allowed_domains is properly configured to prevent credential leakage
+     */
+    private _validateSecuritySettings;
+    private _initFileSystem;
+    private _setScreenshotService;
+    get logger(): import("../logging-config.js").Logger;
+    get message_manager(): MessageManager;
+    /**
+     * Get the browser instance from the browser session
+     */
+    get browser(): Browser;
+    /**
+     * Get the browser context from the browser session
+     */
+    get browserContext(): BrowserContext;
+    /**
+     * Get the browser profile from the browser session
+     */
+    get browserProfile(): BrowserProfile;
+    /**
+     * Add a new task to the agent, keeping the same task_id as tasks are continuous
+     */
+    addNewTask(newTask: string): void;
+    /**
+     * Take a step and return whether the task is done and valid
+     * @returns Tuple of [is_done, is_valid]
+     */
+    takeStep(stepInfo?: AgentStepInfo): Promise<[boolean, boolean]>;
+    /**
+     * Remove think tags from text
+     */
+    private _removeThinkTags;
+    /**
+     * Log a comprehensive summary of the next action(s)
+     */
+    private _logNextActionSummary;
+    private _set_browser_use_version_and_source;
+    /**
+     * Setup dynamic action models from controller's registry
+     * Initially only include actions with no filters
+     */
+    private _setup_action_models;
+    /**
+     * Update action models with page-specific actions
+     * Called during each step to filter actions based on current page context
+     */
+    private _updateActionModelsForPage;
+    run(max_steps?: number, on_step_start?: AgentHookFunc<Context, AgentStructuredOutput> | null, on_step_end?: AgentHookFunc<Context, AgentStructuredOutput> | null): Promise<AgentHistoryList<AgentStructuredOutput>>;
+    private _executeWithTimeout;
+    _step(step_info?: AgentStepInfo | null, signal?: AbortSignal | null): Promise<void>;
+    private _prepare_context;
+    private _storeScreenshotForStep;
+    private _get_next_action;
+    private _execute_actions;
+    private _post_process;
+    multi_act(actions: Array<Record<string, Record<string, unknown>>>, options?: {
+        check_for_new_elements?: boolean;
+        signal?: AbortSignal | null;
+    }): Promise<ActionResult[]>;
+    rerun_history(history: AgentHistoryList, options?: RerunHistoryOptions): Promise<ActionResult[]>;
+    private _execute_history_step;
+    private _update_action_indices;
+    load_and_rerun(history_file?: string | null, options?: RerunHistoryOptions): Promise<ActionResult[]>;
+    save_history(file_path?: string | null): void;
+    private _coerceHistoryElement;
+    private _createAbortError;
+    private _throwIfAborted;
+    private _relayAbortSignal;
+    private _sleep;
+    wait_until_resumed(): Promise<void>;
+    log_completion(): Promise<void>;
+    pause(): void;
+    resume(): void;
+    stop(): void;
+    close(): Promise<void>;
+    /**
+     * Get the trace and trace_details objects for the agent
+     * Contains comprehensive metadata about the agent run for debugging and analysis
+     */
+    get_trace_object(): {
+        trace: Record<string, any>;
+        trace_details: Record<string, any>;
+    };
+    private _log_agent_run;
+    private _raise_if_stopped_or_paused;
+    private _handle_post_llm_processing;
+    /**
+     * Handle all types of errors that can occur during a step
+     * Implements comprehensive error categorization with:
+     * - Validation error hints
+     * - Rate limit auto-retry
+     * - Parse error guidance
+     * - Token limit warnings
+     * - Network error detection
+     * - Browser error handling
+     * - LLM-specific errors
+     */
+    private _handle_step_error;
+    /**
+     * Check if an error is a network error
+     */
+    private _isNetworkError;
+    /**
+     * Check if an error is a browser/Playwright error
+     */
+    private _isBrowserError;
+    /**
+     * Check if an error is a timeout error
+     */
+    private _isTimeoutError;
+    /**
+     * Check if an error is a rate limit error from various LLM providers
+     */
+    private _isRateLimitError;
+    private _finalize;
+    private _handle_final_step;
+    private _parseCompletionPayload;
+    private _isModelActionMissing;
+    private _get_model_output_with_retry;
+    private _validateAndNormalizeActions;
+    private _update_action_models_for_page;
+    private _check_and_update_downloads;
+    private _update_available_file_paths;
+    private _log_step_context;
+    private _log_step_completion_summary;
+    private _log_agent_event;
+    private _make_history_item;
+    save_file_system_state(): void;
+}
+export {};