browser-use 0.2.0 → 0.4.0

package/dist/agent/prompts.d.ts

@@ -1,17 +1,20 @@
-import { SystemMessage, UserMessage } from '../llm/messages.js';
+import { SystemMessage, UserMessage, ContentPartTextParam, ContentPartImageParam } from '../llm/messages.js';
 import type { AgentStepInfo } from './views.js';
 import type { BrowserStateSummary } from '../browser/views.js';
 import type { FileSystem } from '../filesystem/file-system.js';
 export declare class SystemPrompt {
-    private readonly actionDescription;
     private readonly maxActionsPerStep;
     private readonly overrideSystemMessage;
     private readonly extendSystemMessage;
     private readonly useThinking;
     private readonly flashMode;
+    private readonly isAnthropic;
+    private readonly isBrowserUseModel;
+    private readonly modelName;
     private promptTemplate;
     private systemMessage;
-    constructor(actionDescription: string, maxActionsPerStep?: number, overrideSystemMessage?: string | null, extendSystemMessage?: string | null, useThinking?: boolean, flashMode?: boolean);
+    constructor(maxActionsPerStep?: number, overrideSystemMessage?: string | null, extendSystemMessage?: string | null, useThinking?: boolean, flashMode?: boolean, isAnthropic?: boolean, isBrowserUseModel?: boolean, modelName?: string | null);
+    private isAnthropic45Model;
     private loadPromptTemplate;
     get_system_message(): SystemMessage;
 }
@@ -29,6 +32,12 @@ interface AgentMessagePromptInit {
     available_file_paths?: string[] | null;
     screenshots?: string[] | null;
     vision_detail_level?: 'auto' | 'low' | 'high';
+    include_recent_events?: boolean;
+    sample_images?: Array<ContentPartTextParam | ContentPartImageParam> | null;
+    read_state_images?: Array<Record<string, unknown>> | null;
+    llm_screenshot_size?: [number, number] | null;
+    unavailable_skills_info?: string | null;
+    plan_description?: string | null;
 }
 export declare class AgentMessagePrompt {
     private readonly browserState;
@@ -44,9 +53,21 @@ export declare class AgentMessagePrompt {
     private readonly availableFilePaths?;
     private readonly screenshots;
     private readonly visionDetailLevel;
+    private readonly includeRecentEvents;
+    private readonly sampleImages;
+    private readonly readStateImages;
+    private readonly llmScreenshotSize;
+    private readonly unavailableSkillsInfo;
+    private readonly planDescription;
     constructor(init: AgentMessagePromptInit);
+    private extractPageStatistics;
     private browserStateDescription;
     private agentStateDescription;
+    private resizeScreenshotForLlm;
     get_user_message(use_vision?: boolean): UserMessage;
 }
+export declare const get_rerun_summary_prompt: (originalTask: string, totalSteps: number, successCount: number, errorCount: number) => string;
+export declare const get_rerun_summary_message: (prompt: string, screenshotB64?: string | null) => UserMessage;
+export declare const get_ai_step_system_prompt: () => string;
+export declare const get_ai_step_user_prompt: (query: string, statsSummary: string, content: string) => string;
 export {};

package/dist/agent/prompts.js

@@ -6,30 +6,38 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 };
 import fs from 'node:fs';
 import { fileURLToPath } from 'node:url';
+import { Image, createCanvas } from 'canvas';
 import { SystemMessage, UserMessage, ContentPartTextParam, ContentPartImageParam, ImageURL, } from '../llm/messages.js';
 import { observe_debug } from '../observability.js';
-import { is_new_tab_page } from '../utils.js';
+import { is_new_tab_page, sanitize_surrogates } from '../utils.js';
+import { createLogger } from '../logging-config.js';
+import { DOMElementNode } from '../dom/views.js';
+const logger = createLogger('browser_use.agent.prompts');
 const readPromptTemplate = (filename) => {
     const filePath = fileURLToPath(new URL(filename, import.meta.url));
     return fs.readFileSync(filePath, 'utf-8');
 };
 export class SystemPrompt {
-    actionDescription;
     maxActionsPerStep;
     overrideSystemMessage;
     extendSystemMessage;
     useThinking;
     flashMode;
+    isAnthropic;
+    isBrowserUseModel;
+    modelName;
     promptTemplate = '';
     systemMessage;
-    constructor(actionDescription, maxActionsPerStep = 10, overrideSystemMessage = null, extendSystemMessage = null, useThinking = true, flashMode = false) {
-        this.actionDescription = actionDescription;
+    constructor(maxActionsPerStep = 3, overrideSystemMessage = null, extendSystemMessage = null, useThinking = true, flashMode = false, isAnthropic = false, isBrowserUseModel = false, modelName = null) {
         this.maxActionsPerStep = maxActionsPerStep;
         this.overrideSystemMessage = overrideSystemMessage;
         this.extendSystemMessage = extendSystemMessage;
         this.useThinking = useThinking;
         this.flashMode = flashMode;
-        if (overrideSystemMessage) {
+        this.isAnthropic = isAnthropic;
+        this.isBrowserUseModel = isBrowserUseModel;
+        this.modelName = modelName;
+        if (overrideSystemMessage !== null) {
             this.promptTemplate = overrideSystemMessage;
         }
         else {
@@ -42,17 +50,47 @@ export class SystemPrompt {
         this.systemMessage = new SystemMessage(prompt);
         this.systemMessage.cache = true;
     }
+    isAnthropic45Model() {
+        if (!this.modelName) {
+            return false;
+        }
+        const modelLower = this.modelName.toLowerCase();
+        const isOpus45 = modelLower.includes('opus') &&
+            (modelLower.includes('4.5') || modelLower.includes('4-5'));
+        const isHaiku45 = modelLower.includes('haiku') &&
+            (modelLower.includes('4.5') || modelLower.includes('4-5'));
+        return isOpus45 || isHaiku45;
+    }
     loadPromptTemplate() {
-        const templateName = this.flashMode
-            ? './system_prompt_flash.md'
-            : this.useThinking
-                ? './system_prompt.md'
-                : './system_prompt_no_thinking.md';
+        let templateName = './system_prompt.md';
+        if (this.isBrowserUseModel) {
+            if (this.flashMode) {
+                templateName = './system_prompt_browser_use_flash.md';
+            }
+            else if (this.useThinking) {
+                templateName = './system_prompt_browser_use.md';
+            }
+            else {
+                templateName = './system_prompt_browser_use_no_thinking.md';
+            }
+        }
+        else if (this.flashMode && this.isAnthropic45Model()) {
+            templateName = './system_prompt_anthropic_flash.md';
+        }
+        else if (this.flashMode && this.isAnthropic) {
+            templateName = './system_prompt_flash_anthropic.md';
+        }
+        else if (this.flashMode) {
+            templateName = './system_prompt_flash.md';
+        }
+        else if (!this.useThinking) {
+            templateName = './system_prompt_no_thinking.md';
+        }
         try {
             this.promptTemplate = readPromptTemplate(templateName);
         }
         catch (error) {
-            throw new Error(`Failed to load system prompt template: ${error.message}`);
+            throw new Error(`Failed to load system prompt template: ${error.message}`, { cause: error });
         }
     }
     get_system_message() {
@@ -73,6 +111,12 @@ export class AgentMessagePrompt {
     availableFilePaths;
     screenshots;
     visionDetailLevel;
+    includeRecentEvents;
+    sampleImages;
+    readStateImages;
+    llmScreenshotSize;
+    unavailableSkillsInfo;
+    planDescription;
     constructor(init) {
         this.browserState = init.browser_state_summary;
         this.fileSystem = init.file_system;
@@ -88,48 +132,97 @@ export class AgentMessagePrompt {
         this.availableFilePaths = init.available_file_paths ?? null;
         this.screenshots = init.screenshots ?? [];
         this.visionDetailLevel = init.vision_detail_level ?? 'auto';
+        this.includeRecentEvents = init.include_recent_events ?? false;
+        this.sampleImages = init.sample_images ?? [];
+        this.readStateImages = init.read_state_images ?? [];
+        this.llmScreenshotSize = init.llm_screenshot_size ?? null;
+        this.unavailableSkillsInfo = init.unavailable_skills_info ?? null;
+        this.planDescription = init.plan_description ?? null;
+    }
+    extractPageStatistics() {
+        const stats = {
+            links: 0,
+            iframes: 0,
+            shadow_open: 0,
+            shadow_closed: 0,
+            scroll_containers: 0,
+            images: 0,
+            interactive_elements: 0,
+            total_elements: 0,
+        };
+        const root = this.browserState.element_tree;
+        if (!root) {
+            return stats;
+        }
+        const traverseNode = (node) => {
+            stats.total_elements += 1;
+            const tag = String(node.tag_name ?? '').toLowerCase();
+            if (tag === 'a') {
+                stats.links += 1;
+            }
+            else if (tag === 'iframe' || tag === 'frame') {
+                stats.iframes += 1;
+            }
+            else if (tag === 'img') {
+                stats.images += 1;
+            }
+            if (node.is_interactive) {
+                stats.interactive_elements += 1;
+            }
+            if (node.shadow_root) {
+                // The TS DOM snapshot currently tracks presence of a shadow root, but
+                // does not expose open-vs-closed mode; count these as open for parity.
+                stats.shadow_open += 1;
+            }
+            for (const child of node.children) {
+                if (child instanceof DOMElementNode) {
+                    traverseNode(child);
+                }
+            }
+        };
+        traverseNode(root);
+        return stats;
     }
     browserStateDescription() {
-        let elementsText = this.browserState.element_tree.clickable_elements_to_string(this.includeAttributes ?? undefined);
+        const pageStats = this.extractPageStatistics();
+        let statsText = '<page_stats>';
+        if (pageStats.total_elements < 10) {
+            statsText += 'Page appears empty (SPA not loaded?) - ';
+        }
+        statsText += `${pageStats.links} links, ${pageStats.interactive_elements} interactive, ${pageStats.iframes} iframes`;
+        if (pageStats.shadow_open > 0 || pageStats.shadow_closed > 0) {
+            statsText += `, ${pageStats.shadow_open} shadow(open), ${pageStats.shadow_closed} shadow(closed)`;
+        }
+        if (pageStats.images > 0) {
+            statsText += `, ${pageStats.images} images`;
+        }
+        statsText += `, ${pageStats.total_elements} total elements`;
+        statsText += '</page_stats>\n';
+        let elementsText = this.browserState.llm_representation(this.includeAttributes ?? undefined);
         let truncatedText = '';
         if (elementsText.length > this.maxClickableElementsLength) {
             elementsText = elementsText.slice(0, this.maxClickableElementsLength);
             truncatedText = ` (truncated to ${this.maxClickableElementsLength} characters)`;
         }
-        const hasContentAbove = (this.browserState.pixels_above ?? 0) > 0;
-        const hasContentBelow = (this.browserState.pixels_below ?? 0) > 0;
+        let hasContentAbove = false;
+        let hasContentBelow = false;
         const pi = this.browserState.page_info;
         let pageInfoText = '';
         if (pi) {
             const pagesAbove = pi.viewport_height > 0 ? pi.pixels_above / pi.viewport_height : 0;
             const pagesBelow = pi.viewport_height > 0 ? pi.pixels_below / pi.viewport_height : 0;
-            const totalPages = pi.viewport_height > 0 ? pi.page_height / pi.viewport_height : 0;
-            const currentPosition = pi.scroll_y / Math.max(pi.page_height - pi.viewport_height, 1);
-            pageInfoText = `Page info: ${pi.viewport_width}x${pi.viewport_height}px viewport, ${pi.page_width}x${pi.page_height}px total page size, ${pagesAbove.toFixed(1)} pages above, ${pagesBelow.toFixed(1)} pages below, ${totalPages.toFixed(1)} total pages, at ${(currentPosition * 100).toFixed(0)}% of page`;
+            hasContentAbove = pagesAbove > 0;
+            hasContentBelow = pagesBelow > 0;
+            pageInfoText = '<page_info>';
+            pageInfoText += `${pagesAbove.toFixed(1)} above, `;
+            pageInfoText += `${pagesBelow.toFixed(1)} below `;
+            pageInfoText += '</page_info>\n';
         }
         if (elementsText) {
-            if (hasContentAbove) {
-                if (pi) {
-                    const pagesAbove = pi.viewport_height > 0 ? pi.pixels_above / pi.viewport_height : 0;
-                    elementsText = `... ${this.browserState.pixels_above} pixels above (${pagesAbove.toFixed(1)} pages) - scroll to see more or extract structured data if you are looking for specific information ...\n${elementsText}`;
-                }
-                else {
-                    elementsText = `... ${this.browserState.pixels_above} pixels above - scroll to see more or extract structured data if you are looking for specific information ...\n${elementsText}`;
-                }
-            }
-            else {
+            if (!hasContentAbove) {
                 elementsText = `[Start of page]\n${elementsText}`;
             }
-            if (hasContentBelow) {
-                if (pi) {
-                    const pagesBelow = pi.viewport_height > 0 ? pi.pixels_below / pi.viewport_height : 0;
-                    elementsText = `${elementsText}\n... ${this.browserState.pixels_below} pixels below (${pagesBelow.toFixed(1)} pages) - scroll to see more or extract structured data if you are looking for specific information ...`;
-                }
-                else {
-                    elementsText = `${elementsText}\n... ${this.browserState.pixels_below} pixels below - scroll to see more or extract structured data if you are looking for specific information ...`;
-                }
-            }
-            else {
+            if (!hasContentBelow) {
                 elementsText = `${elementsText}\n[End of page]`;
             }
         }
@@ -137,38 +230,54 @@ export class AgentMessagePrompt {
             elementsText = 'empty page';
         }
         let tabsText = '';
+        const resolveTabIdentifier = (tab) => typeof tab.tab_id === 'string' && tab.tab_id.trim()
+            ? tab.tab_id.trim().slice(-4)
+            : String(tab.page_id);
         const currentTabCandidates = [];
         for (const tab of this.browserState.tabs) {
             if (tab.url === this.browserState.url &&
                 tab.title === this.browserState.title) {
-                currentTabCandidates.push(tab.page_id);
+                currentTabCandidates.push(resolveTabIdentifier(tab));
             }
         }
         const currentTabId = currentTabCandidates.length === 1 ? currentTabCandidates[0] : null;
         for (const tab of this.browserState.tabs) {
-            tabsText += `Tab ${tab.page_id}: ${tab.url} - ${tab.title.slice(0, 30)}\n`;
+            tabsText += `Tab ${resolveTabIdentifier(tab)}: ${tab.url} - ${tab.title.slice(0, 30)}\n`;
         }
         const currentTabText = currentTabId !== null ? `Current tab: ${currentTabId}` : '';
         const pdfMessage = this.browserState.is_pdf_viewer
-            ? 'PDF viewer cannot be rendered. Do not use extract_structured_data here; use read_file on downloaded PDF via available_file_paths.\n\n'
+            ? 'PDF viewer cannot be rendered. In this page, DO NOT use the extract action as PDF content cannot be rendered. Use the read_file action on the downloaded PDF in available_file_paths to read the full text content.\n\n'
+            : '';
+        const recentEventsText = this.includeRecentEvents && this.browserState.recent_events
+            ? `Recent browser events: ${this.browserState.recent_events}\n`
             : '';
-        return `${currentTabText}
+        let closedPopupsText = '';
+        if (Array.isArray(this.browserState.closed_popup_messages) &&
+            this.browserState.closed_popup_messages.length > 0) {
+            closedPopupsText = 'Auto-closed JavaScript dialogs:\n';
+            for (const popupMessage of this.browserState.closed_popup_messages) {
+                closedPopupsText += `  - ${popupMessage}\n`;
+            }
+            closedPopupsText += '\n';
+        }
+        return `${statsText}${currentTabText}
 Available tabs:
 ${tabsText}
 ${pageInfoText}
-${pdfMessage}Interactive elements from top layer of the current page inside the viewport${truncatedText}:
+${recentEventsText}${closedPopupsText}${pdfMessage}Interactive elements${truncatedText}:
 ${elementsText}
 `;
     }
     agentStateDescription() {
         const todoContents = this.fileSystem.get_todo_contents();
-        const todoText = todoContents ||
-            '[Current todo.md is empty, fill it with your plan when applicable]';
-        const timestamp = new Date().toISOString().slice(0, 16).replace('T', ' ');
-        const stepInfo = this.stepInfo != null
-            ? `Step ${this.stepInfo.step_number + 1} of ${this.stepInfo.max_steps}\n`
+        const todoText = todoContents || '[empty todo.md, fill it when applicable]';
+        const now = new Date();
+        const pad = (value) => String(value).padStart(2, '0');
+        const dateString = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`;
+        let stepInfoDescription = this.stepInfo != null
+            ? `Step${this.stepInfo.step_number + 1} maximum:${this.stepInfo.max_steps}\n`
             : '';
-        const stepInfoDescription = `${stepInfo}Current date and time: ${timestamp}`;
+        stepInfoDescription += `Today:${dateString}`;
         let agentState = `<user_request>
 ${this.task ?? ''}
 </user_request>
@@ -179,24 +288,47 @@ ${this.fileSystem.describe()}
 ${todoText}
 </todo_contents>
 `;
+        if (this.planDescription) {
+            agentState += `<plan>
+${this.planDescription}
+</plan>
+`;
+        }
         if (this.sensitiveData) {
-            agentState += `<sensitive_data>
-${this.sensitiveData}
-</sensitive_data>
+            agentState += `<sensitive_data>${this.sensitiveData}</sensitive_data>
 `;
         }
-        agentState += `<step_info>
-${stepInfoDescription}
-</step_info>
+        agentState += `<step_info>${stepInfoDescription}</step_info>
 `;
         if (this.availableFilePaths?.length) {
-            agentState += `<available_file_paths>
-${this.availableFilePaths.join('\n')}
-</available_file_paths>
+            agentState += `<available_file_paths>${this.availableFilePaths.join('\n')}
+Use with absolute paths</available_file_paths>
 `;
         }
         return agentState;
     }
+    resizeScreenshotForLlm(screenshotB64) {
+        if (!this.llmScreenshotSize) {
+            return screenshotB64;
+        }
+        try {
+            const [targetWidth, targetHeight] = this.llmScreenshotSize;
+            const image = new Image();
+            image.src = Buffer.from(screenshotB64, 'base64');
+            if (image.width === targetWidth && image.height === targetHeight) {
+                return screenshotB64;
+            }
+            logger.info(`Resizing screenshot from ${image.width}x${image.height} to ${targetWidth}x${targetHeight} for LLM`);
+            const canvas = createCanvas(targetWidth, targetHeight);
+            const context = canvas.getContext('2d');
+            context.drawImage(image, 0, 0, targetWidth, targetHeight);
+            return canvas.toBuffer('image/png').toString('base64');
+        }
+        catch (error) {
+            logger.warning(`Failed to resize screenshot: ${error.message}, using original`);
+            return screenshotB64;
+        }
+    }
     // @ts-ignore - Decorator type mismatch with TypeScript strict mode
     get_user_message(use_vision = true) {
         if (is_new_tab_page(this.browserState.url) &&
@@ -208,6 +340,7 @@ ${this.availableFilePaths.join('\n')}
         let stateDescription = `<agent_history>
 ${(this.agentHistoryDescription ?? '').trim()}
 </agent_history>
+
 `;
         stateDescription += `<agent_state>
 ${this.agentStateDescription().trim()}
@@ -230,17 +363,37 @@ ${this.pageFilteredActions}
 </page_specific_actions>
 `;
         }
-        if (use_vision && this.screenshots.length > 0) {
+        if (this.unavailableSkillsInfo) {
+            stateDescription += `\n${this.unavailableSkillsInfo}\n`;
+        }
+        stateDescription = sanitize_surrogates(stateDescription);
+        const hasReadStateImages = this.readStateImages.length > 0;
+        if ((use_vision === true && this.screenshots.length > 0) ||
+            hasReadStateImages) {
             const parts = [
                 new ContentPartTextParam(stateDescription),
             ];
+            parts.push(...this.sampleImages);
             this.screenshots.forEach((shot, index) => {
                 const label = index === this.screenshots.length - 1
                     ? 'Current screenshot:'
                     : 'Previous screenshot:';
+                const processedScreenshot = this.resizeScreenshotForLlm(shot);
                 parts.push(new ContentPartTextParam(label));
-                parts.push(new ContentPartImageParam(new ImageURL(`data:image/png;base64,${shot}`, this.visionDetailLevel, 'image/png')));
+                parts.push(new ContentPartImageParam(new ImageURL(`data:image/png;base64,${processedScreenshot}`, this.visionDetailLevel, 'image/png')));
             });
+            for (const imageInfo of this.readStateImages) {
+                const imageName = typeof imageInfo.name === 'string' ? imageInfo.name : 'unknown';
+                const imageData = typeof imageInfo.data === 'string' ? imageInfo.data : null;
+                if (!imageData) {
+                    continue;
+                }
+                const mediaType = imageName.toLowerCase().endsWith('.png')
+                    ? 'image/png'
+                    : 'image/jpeg';
+                parts.push(new ContentPartTextParam(`Image from file: ${imageName}`));
+                parts.push(new ContentPartImageParam(new ImageURL(`data:${mediaType};base64,${imageData}`, this.visionDetailLevel, mediaType)));
+            }
             const message = new UserMessage(parts);
             message.cache = true;
             return message;
@@ -257,3 +410,65 @@ __decorate([
         ignore_output: true,
     })
 ], AgentMessagePrompt.prototype, "get_user_message", null);
+export const get_rerun_summary_prompt = (originalTask, totalSteps, successCount, errorCount) => `You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.
+
+Original task: ${originalTask}
+
+Execution statistics:
+- Total steps: ${totalSteps}
+- Successful steps: ${successCount}
+- Failed steps: ${errorCount}
+
+Analyze the screenshot to determine:
+1. Whether the task completed successfully
+2. What the final state shows
+3. Overall completion status (complete/partial/failed)
+
+Respond with:
+- summary: A clear, concise summary of what happened during the rerun
+- success: Whether the task completed successfully (true/false)
+- completion_status: One of "complete", "partial", or "failed"`;
+export const get_rerun_summary_message = (prompt, screenshotB64 = null) => {
+    if (screenshotB64) {
+        const parts = [
+            new ContentPartTextParam(prompt),
+            new ContentPartImageParam(new ImageURL(`data:image/png;base64,${screenshotB64}`)),
+        ];
+        return new UserMessage(parts);
+    }
+    return new UserMessage(prompt);
+};
+export const get_ai_step_system_prompt = () => `
+You are an expert at extracting data from webpages.
+
+<input>
+You will be given:
+1. A query describing what to extract
+2. The markdown of the webpage (filtered to remove noise)
+3. Optionally, a screenshot of the current page state
+</input>
+
+<instructions>
+- Extract information from the webpage that is relevant to the query
+- ONLY use the information available in the webpage - do not make up information
+- If the information is not available, mention that clearly
+- If the query asks for all items, list all of them
+</instructions>
+
+<output>
+- Present ALL relevant information in a concise way
+- Do not use conversational format - directly output the relevant information
+- If information is unavailable, state that clearly
+</output>
+`.trim();
+export const get_ai_step_user_prompt = (query, statsSummary, content) => `<query>
+${query}
+</query>
+
+<content_stats>
+${statsSummary}
+</content_stats>
+
+<webpage_content>
+${content}
+</webpage_content>`;