browser-use 0.2.0 → 0.4.0

package/dist/agent/judge.js

@@ -0,0 +1,197 @@
+import fs from 'node:fs';
+import { ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage, } from '../llm/messages.js';
+const truncateText = (text, maxLength, fromBeginning = false) => {
+    if (text.length <= maxLength) {
+        return text;
+    }
+    if (fromBeginning) {
+        return `...[text truncated]${text.slice(-(maxLength - 20))}`;
+    }
+    return `${text.slice(0, maxLength - 23)}...[text truncated]...`;
+};
+const encodeImage = (imagePath) => {
+    try {
+        if (!fs.existsSync(imagePath)) {
+            return null;
+        }
+        return fs.readFileSync(imagePath).toString('base64');
+    }
+    catch {
+        return null;
+    }
+};
+export const construct_judge_messages = (options) => {
+    const { task, final_result, agent_steps, screenshot_paths, max_images = 10, ground_truth = null, use_vision = true, } = options;
+    const task_truncated = truncateText(task, 40000);
+    const final_result_truncated = truncateText(final_result, 40000);
+    const steps_text_truncated = truncateText(agent_steps.join('\n'), 40000);
+    const encoded_images = [];
+    if (use_vision !== false) {
+        const selected = screenshot_paths.length > max_images
+            ? screenshot_paths.slice(-max_images)
+            : screenshot_paths;
+        for (const screenshotPath of selected) {
+            const encoded = encodeImage(screenshotPath);
+            if (!encoded) {
+                continue;
+            }
+            encoded_images.push(new ContentPartImageParam(new ImageURL(`data:image/png;base64,${encoded}`, 'auto', 'image/png')));
+        }
+    }
+    const ground_truth_section = ground_truth
+        ? `
+**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):**
+The <ground_truth> section contains verified correct information for this task. This can be:
+- Evaluation criteria: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items")
+- Factual answers: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris")
+- Expected outcomes: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded")
+
+The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false.
+`
+        : '';
+    const system_prompt = `You are an expert judge evaluating browser automation agent performance.
+
+<evaluation_framework>
+${ground_truth_section}
+**PRIMARY EVALUATION CRITERIA (in order of importance):**
+1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.
+2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?
+3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed?
+4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory.
+5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low.
+
+**VERDICT GUIDELINES:**
+- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information.
+- false: Task not completed, or only partially completed.
+
+**Examples of task completion verdict:**
+- If task asks for 10 items and agent finds 4 items correctly: false
+- If task completed to full user requirements but with some errors to improve in the trajectory: true
+- If task impossible due to captcha/login requirements: false
+- If the trajectory is ideal and the output is perfect: true
+- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false
+- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false
+- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false
+- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false
+- If the task asks to look for a section of a page that does not exist: false
+- If the agent concludes the task is impossible but it is not: false
+- If the agent concludes the task is impossible and it truly is impossible: false
+- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false
+
+**FAILURE CONDITIONS (automatically set verdict to false):**
+- Blocked by captcha or missing authentication
+- Output format completely wrong or missing
+- Infinite loops or severe technical failures
+- Critical user requirements ignored
+- Page not loaded
+- Browser crashed
+- Agent could not interact with required UI elements
+- The agent moved on from a important step in the task without completing it
+- The agent made up content that is not in the screenshot or the page state
+- The agent calls done action before completing all key points of the task
+
+**IMPOSSIBLE TASK DETECTION:**
+Set impossible_task to true when the task fundamentally could not be completed due to:
+- Vague or ambiguous task instructions that cannot be reasonably interpreted
+- Website genuinely broken or non-functional (be conservative - temporary issues don't count)
+- Required links/pages truly inaccessible (404, 403, etc.)
+- Task requires authentication/login but no credentials were provided
+- Task asks for functionality that doesn't exist on the target site
+- Other insurmountable external obstacles beyond the agent's control
+
+Do NOT mark as impossible if:
+- Agent made poor decisions but task was achievable
+- Temporary page loading issues that could be retried
+- Agent didn't try the right approach
+- Website works but agent struggled with it
+
+**CAPTCHA DETECTION:**
+Set reached_captcha to true if:
+- Screenshots show captcha challenges (reCAPTCHA, hCaptcha, etc.)
+- Agent reports being blocked by bot detection
+- Error messages indicate captcha/verification requirements
+- Any evidence the agent encountered anti-bot measures during execution
+
+**IMPORTANT EVALUATION NOTES:**
+- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false.
+- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there.
+- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information.
+- **ignore unexpected dates and times** - These agent traces are from varying dates, you can assume the dates the agent uses for search or filtering are correct.
+- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request.
+- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee.
+</evaluation_framework>
+
+<response_format>
+Respond with EXACTLY this JSON structure (no additional text before or after):
+
+{
+  "reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction.",
+  "verdict": true or false,
+  "failure_reason": "Max 5 sentences explanation of why the task was not completed successfully in case of failure. If verdict is true, use an empty string.",
+  "impossible_task": true or false,
+  "reached_captcha": true or false
+}
+</response_format>`;
+    const ground_truth_prompt = ground_truth
+        ? `
+<ground_truth>
+${ground_truth}
+</ground_truth>
+`
+        : '';
+    const user_prompt = `<task>
+${task_truncated || 'No task provided'}
+</task>
+${ground_truth_prompt}
+<agent_trajectory>
+${steps_text_truncated || 'No agent trajectory provided'}
+</agent_trajectory>
+
+<final_result>
+${final_result_truncated || 'No final result provided'}
+</final_result>
+
+${encoded_images.length} screenshots from execution are attached.
+
+Evaluate this agent execution given the criteria and respond with the exact JSON structure requested.`;
+    const content_parts = [
+        new ContentPartTextParam(user_prompt),
+    ];
+    content_parts.push(...encoded_images);
+    return [new SystemMessage(system_prompt), new UserMessage(content_parts)];
+};
+export const construct_simple_judge_messages = (options) => {
+    const task_truncated = truncateText(options.task, 20000);
+    const final_result_truncated = truncateText(options.final_result, 20000);
+    const current_date = options.current_date ?? new Date().toISOString().slice(0, 10);
+    const system_prompt = `You are a strict verifier checking whether a browser automation agent actually completed its task.
+
+Today's date is ${current_date}. The agent ran recently - dates near today are expected and NOT fabricated.
+
+Given the task and the agent's final response, determine if the response genuinely satisfies ALL requirements.
+
+Check for these common failure patterns:
+1. **Incorrect data**: Wrong number of items, missing filters/criteria, wrong format
+2. **Unverified actions**: Agent claims to have submitted a form, posted a comment, or saved a file but there's no evidence
+3. **Incomplete results**: Some requirements from the task are not addressed in the response
+4. **Fabricated content**: Data that looks plausible but wasn't actually extracted from any page. NOTE: dates and times close to today's date (${current_date}) are NOT fabricated - the agent browses live websites and extracts real-time content.
+5. **Partial completion reported as success**: Response acknowledges failure or blockers (captcha, access denied, etc.) but still claims success
+
+Respond with EXACTLY this JSON structure:
+{
+  "is_correct": true or false,
+  "reason": "Brief explanation if not correct, empty string if correct"
+}
+
+Be strict: if the response doesn't clearly satisfy every requirement, set is_correct to false.`;
+    const user_prompt = `<task>
+${task_truncated || 'No task provided'}
+</task>
+
+<agent_final_response>
+${final_result_truncated || 'No response provided'}
+</agent_final_response>
+
+Does the agent's response fully satisfy all requirements of the task? Respond with the JSON structure.`;
+    return [new SystemMessage(system_prompt), new UserMessage(user_prompt)];
+};

package/dist/agent/message-manager/service.d.ts

@@ -1,5 +1,6 @@
-import { SystemMessage, UserMessage, type Message } from '../../llm/messages.js';
-import { ActionResult, AgentOutput, AgentStepInfo } from '../views.js';
+import { ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, type Message } from '../../llm/messages.js';
+import type { BaseChatModel } from '../../llm/base.js';
+import { ActionResult, AgentOutput, AgentStepInfo, MessageCompactionSettings } from '../views.js';
 import { BrowserStateSummary } from '../../browser/views.js';
 import { FileSystem } from '../../filesystem/file-system.js';
 import { MessageManagerState } from './views.js';
@@ -11,20 +12,27 @@ export declare class MessageManager {
     private readonly maxHistoryItems;
     private readonly visionDetailLevel;
     private readonly includeToolCallExamples;
+    private readonly includeRecentEvents;
+    private readonly sampleImages;
+    private readonly llmScreenshotSize;
     private task;
     private systemPrompt;
     private sensitiveDataDescription;
     private lastInputMessages;
     private includeAttributes;
-    constructor(task: string, systemMessage: SystemMessage, fileSystem: FileSystem, state?: MessageManagerState, useThinking?: boolean, includeAttributes?: string[] | null, sensitiveData?: Record<string, string | Record<string, string>> | undefined, maxHistoryItems?: number | null, visionDetailLevel?: 'auto' | 'low' | 'high', includeToolCallExamples?: boolean);
+    last_state_message_text: string | null;
+    constructor(task: string, systemMessage: SystemMessage, fileSystem: FileSystem, state?: MessageManagerState, useThinking?: boolean, includeAttributes?: string[] | null, sensitiveData?: Record<string, string | Record<string, string>> | undefined, maxHistoryItems?: number | null, visionDetailLevel?: 'auto' | 'low' | 'high', includeToolCallExamples?: boolean, includeRecentEvents?: boolean, sampleImages?: Array<ContentPartTextParam | ContentPartImageParam> | null, llmScreenshotSize?: [number, number] | null);
     get agent_history_description(): string;
     add_new_task(new_task: string): void;
     private updateAgentHistoryDescription;
     private getSensitiveDataDescription;
-    create_state_messages(browser_state_summary: BrowserStateSummary, model_output?: AgentOutput | null, result?: ActionResult[] | null, step_info?: AgentStepInfo | null, use_vision?: boolean, page_filtered_actions?: string | null, sensitive_data?: Record<string, string | Record<string, string>> | null, available_file_paths?: string[] | null): void;
+    prepare_step_state(browser_state_summary: BrowserStateSummary, model_output?: AgentOutput | null, result?: ActionResult[] | null, step_info?: AgentStepInfo | null, sensitive_data?: Record<string, string | Record<string, string>> | null): void;
+    maybe_compact_messages(llm: BaseChatModel | null, settings: MessageCompactionSettings | null, step_info?: AgentStepInfo | null): Promise<boolean>;
+    create_state_messages(browser_state_summary: BrowserStateSummary, model_output?: AgentOutput | null, result?: ActionResult[] | null, step_info?: AgentStepInfo | null, use_vision?: boolean | 'auto', page_filtered_actions?: string | null, sensitive_data?: Record<string, string | Record<string, string>> | null, available_file_paths?: string[] | null, include_recent_events?: boolean | null, plan_description?: string | null, unavailable_skills_info?: string | null, skip_state_update?: boolean): void;
     get_messages(): Message[];
     private setMessageWithType;
     private addContextMessage;
     _add_context_message(message: SystemMessage | UserMessage): void;
+    private extractStateMessageText;
     private filterSensitiveData;
 }

package/dist/agent/message-manager/service.js

@@ -1,4 +1,4 @@
-import { ContentPartTextParam, } from '../../llm/messages.js';
+import { ContentPartTextParam, SystemMessage, UserMessage, } from '../../llm/messages.js';
 import { AgentMessagePrompt } from '../prompts.js';
 import { MessageManagerState, HistoryItem } from './views.js';
 import { match_url_with_domain_pattern } from '../../utils.js';
@@ -12,12 +12,16 @@ export class MessageManager {
     maxHistoryItems;
     visionDetailLevel;
     includeToolCallExamples;
+    includeRecentEvents;
+    sampleImages;
+    llmScreenshotSize;
     task;
     systemPrompt;
     sensitiveDataDescription = '';
     lastInputMessages = [];
     includeAttributes;
-    constructor(task, systemMessage, fileSystem, state = new MessageManagerState(), useThinking = true, includeAttributes = null, sensitiveData, maxHistoryItems = null, visionDetailLevel = 'auto', includeToolCallExamples = false) {
+    last_state_message_text = null;
+    constructor(task, systemMessage, fileSystem, state = new MessageManagerState(), useThinking = true, includeAttributes = null, sensitiveData, maxHistoryItems = null, visionDetailLevel = 'auto', includeToolCallExamples = false, includeRecentEvents = false, sampleImages = null, llmScreenshotSize = null) {
         this.fileSystem = fileSystem;
         this.state = state;
         this.useThinking = useThinking;
@@ -25,6 +29,12 @@ export class MessageManager {
         this.maxHistoryItems = maxHistoryItems;
         this.visionDetailLevel = visionDetailLevel;
         this.includeToolCallExamples = includeToolCallExamples;
+        this.includeRecentEvents = includeRecentEvents;
+        this.sampleImages = sampleImages;
+        this.llmScreenshotSize = llmScreenshotSize;
+        if (this.maxHistoryItems != null && this.maxHistoryItems <= 5) {
+            throw new Error('max_history_items must be null or greater than 5');
+        }
         this.task = task;
         this.systemPrompt = systemMessage;
         this.includeAttributes = includeAttributes ?? [];
@@ -33,16 +43,21 @@ export class MessageManager {
         }
     }
     get agent_history_description() {
+        const compactedPrefix = this.state.compacted_memory
+            ? `<compacted_memory>\n${this.state.compacted_memory}\n</compacted_memory>\n`
+            : '';
         if (this.maxHistoryItems == null) {
-            return this.state.agent_history_items
-                .map((item) => item.to_string())
-                .join('\n');
+            return (compactedPrefix +
+                this.state.agent_history_items
+                    .map((item) => item.to_string())
+                    .join('\n'));
         }
         const totalItems = this.state.agent_history_items.length;
         if (totalItems <= this.maxHistoryItems) {
-            return this.state.agent_history_items
-                .map((item) => item.to_string())
-                .join('\n');
+            return (compactedPrefix +
+                this.state.agent_history_items
+                    .map((item) => item.to_string())
+                    .join('\n'));
         }
         const omitted = totalItems - this.maxHistoryItems;
         const keepRecent = this.maxHistoryItems - 1;
@@ -52,53 +67,80 @@ export class MessageManager {
         parts.push(...this.state.agent_history_items
             .slice(-keepRecent)
             .map((item) => item.to_string()));
-        return parts.join('\n');
+        return compactedPrefix + parts.join('\n');
     }
     add_new_task(new_task) {
-        this.task = new_task;
-        this.state.agent_history_items.push(new HistoryItem(null, null, null, null, null, null, `User updated <user_request> to: ${new_task}`));
+        const normalizedTask = `<follow_up_user_request> ${new_task.trim()} </follow_up_user_request>`;
+        if (!this.task.includes('<initial_user_request>')) {
+            this.task = `<initial_user_request>${this.task}</initial_user_request>`;
+        }
+        this.task += `\n${normalizedTask}`;
+        this.state.agent_history_items.push(new HistoryItem(null, null, null, null, null, null, normalizedTask));
     }
     updateAgentHistoryDescription(model_output, result, step_info) {
         const results = result ?? [];
         const stepNumber = step_info?.step_number ?? null;
         this.state.read_state_description = '';
-        let actionText = '';
-        results.forEach((action, idx) => {
-            const suffix = `Action ${idx + 1}/${results.length}: `;
+        this.state.read_state_images = [];
+        let actionResults = '';
+        let readStateIndex = 0;
+        results.forEach((action) => {
             if (action.include_extracted_content_only_once &&
                 action.extracted_content) {
-                this.state.read_state_description += `${action.extracted_content}\n`;
+                this.state.read_state_description += `<read_state_${readStateIndex}>\n${action.extracted_content}\n</read_state_${readStateIndex}>\n`;
+                readStateIndex += 1;
+            }
+            if (Array.isArray(action.images) && action.images.length > 0) {
+                this.state.read_state_images.push(...action.images);
             }
             if (action.long_term_memory) {
-                actionText += `${suffix}${action.long_term_memory}\n`;
+                actionResults += `${action.long_term_memory}\n`;
             }
             else if (action.extracted_content &&
                 !action.include_extracted_content_only_once) {
-                actionText += `${suffix}${action.extracted_content}\n`;
+                actionResults += `${action.extracted_content}\n`;
             }
             if (action.error) {
                 const err = action.error.length > 200
                     ? `${action.error.slice(0, 100)}......${action.error.slice(-100)}`
                     : action.error;
-                actionText += `${suffix}${err}\n`;
+                actionResults += `${err}\n`;
             }
         });
-        const normalizedActionText = actionText ? actionText.trim() : null;
+        const MAX_CONTENT_SIZE = 60000;
+        if (this.state.read_state_description.length > MAX_CONTENT_SIZE) {
+            this.state.read_state_description = `${this.state.read_state_description.slice(0, MAX_CONTENT_SIZE)}\n... [Content truncated at 60k characters]`;
+        }
+        this.state.read_state_description =
+            this.state.read_state_description.trim();
+        let normalizedActionResults = actionResults
+            ? `Result\n${actionResults}`.trim()
+            : null;
+        if (normalizedActionResults) {
+            if (normalizedActionResults.length > MAX_CONTENT_SIZE) {
+                normalizedActionResults = `${normalizedActionResults.slice(0, MAX_CONTENT_SIZE)}\n... [Content truncated at 60k characters]`;
+            }
+        }
         if (!model_output) {
-            if (stepNumber != null && stepNumber > 0) {
-                this.state.agent_history_items.push(new HistoryItem(stepNumber, null, null, null, null, 'Agent failed to output in the right format.', null));
+            if (stepNumber != null) {
+                if (stepNumber === 0 && normalizedActionResults) {
+                    this.state.agent_history_items.push(new HistoryItem(stepNumber, null, null, null, normalizedActionResults, null, null));
+                }
+                else if (stepNumber > 0) {
+                    this.state.agent_history_items.push(new HistoryItem(stepNumber, null, null, null, null, 'Agent failed to output in the right format.', null));
+                }
             }
             return;
         }
         const brain = model_output.current_state;
-        this.state.agent_history_items.push(new HistoryItem(stepNumber, brain.evaluation_previous_goal, brain.memory, brain.next_goal, normalizedActionText, null, null));
+        this.state.agent_history_items.push(new HistoryItem(stepNumber, brain.evaluation_previous_goal, brain.memory, brain.next_goal, normalizedActionResults, null, null));
     }
-    getSensitiveDataDescription(currentUrl) {
+    getSensitiveDataDescription(currentUrl, sensitiveData = this.sensitiveData) {
         const placeholders = new Set();
-        if (!this.sensitiveData) {
+        if (!sensitiveData) {
             return '';
         }
-        for (const [key, value] of Object.entries(this.sensitiveData)) {
+        for (const [key, value] of Object.entries(sensitiveData)) {
             if (value && typeof value === 'object' && !Array.isArray(value)) {
                 if (currentUrl &&
                     match_url_with_domain_pattern(currentUrl, key, true)) {
@@ -112,18 +154,121 @@ export class MessageManager {
         if (!placeholders.size) {
             return '';
         }
-        return `Here are placeholders for sensitive data:\n${Array.from(placeholders).sort().join(', ')}\nTo use them, write <secret>the placeholder name</secret>`;
+        const placeholderList = `[${Array.from(placeholders)
+            .sort()
+            .map((placeholder) => `'${placeholder.replaceAll("'", "\\'")}'`)
+            .join(', ')}]`;
+        return `Here are placeholders for sensitive data:\n${placeholderList}\nTo use them, write <secret>the placeholder name</secret>`;
     }
-    create_state_messages(browser_state_summary, model_output = null, result = null, step_info = null, use_vision = true, page_filtered_actions = null, sensitive_data = null, available_file_paths = null) {
+    prepare_step_state(browser_state_summary, model_output = null, result = null, step_info = null, sensitive_data = null) {
         this.state.history.context_messages = [];
         this.updateAgentHistoryDescription(model_output, result, step_info);
-        if (sensitive_data) {
-            this.sensitiveDataDescription = this.getSensitiveDataDescription(browser_state_summary.url);
+        const effectiveSensitiveData = sensitive_data ?? this.sensitiveData;
+        if (effectiveSensitiveData) {
+            this.sensitiveDataDescription = this.getSensitiveDataDescription(browser_state_summary.url, effectiveSensitiveData);
+        }
+    }
+    async maybe_compact_messages(llm, settings, step_info = null) {
+        if (!settings || !settings.enabled || !llm || !step_info) {
+            return false;
+        }
+        const stepsSince = step_info.step_number - (this.state.last_compaction_step ?? 0);
+        if (stepsSince < settings.compact_every_n_steps) {
+            return false;
+        }
+        const historyItems = this.state.agent_history_items;
+        const fullHistoryText = historyItems
+            .map((item) => item.to_string())
+            .join('\n')
+            .trim();
+        const triggerCharCount = settings.trigger_char_count ?? 40000;
+        if (fullHistoryText.length < triggerCharCount) {
+            return false;
+        }
+        logger.debug(`Compacting message history (items=${historyItems.length}, chars=${fullHistoryText.length})`);
+        const compactionSections = [];
+        if (this.state.compacted_memory) {
+            compactionSections.push(`<previous_compacted_memory>\n${this.state.compacted_memory}\n</previous_compacted_memory>`);
+        }
+        compactionSections.push(`<agent_history>\n${fullHistoryText}\n</agent_history>`);
+        if (settings.include_read_state && this.state.read_state_description) {
+            compactionSections.push(`<read_state>\n${this.state.read_state_description}\n</read_state>`);
+        }
+        let compactionInput = compactionSections.join('\n\n');
+        if (this.sensitiveData) {
+            const filtered = this.filterSensitiveData(new UserMessage(compactionInput));
+            compactionInput = filtered.text;
+        }
+        let systemPrompt = 'You are summarizing an agent run for prompt compaction.\n' +
+            'Capture task requirements, key facts, decisions, partial progress, errors, and next steps.\n' +
+            'Preserve important entities, values, URLs, and file paths.\n' +
+            'Return plain text only. Do not include tool calls or JSON.';
+        if (settings.summary_max_chars) {
+            systemPrompt += ` Keep under ${settings.summary_max_chars} characters if possible.`;
+        }
+        let summary;
+        try {
+            const response = await llm.ainvoke([
+                new SystemMessage(systemPrompt),
+                new UserMessage(compactionInput),
+            ]);
+            summary = String(response?.completion ?? '').trim();
+        }
+        catch (error) {
+            logger.warning(`Failed to compact messages: ${error instanceof Error ? error.message : String(error)}`);
+            return false;
+        }
+        if (!summary) {
+            return false;
+        }
+        if (settings.summary_max_chars &&
+            summary.length > settings.summary_max_chars) {
+            summary = `${summary.slice(0, settings.summary_max_chars).trimEnd()}…`;
+        }
+        this.state.compacted_memory = summary;
+        this.state.compaction_count += 1;
+        this.state.last_compaction_step = step_info.step_number;
+        const keepLast = Math.max(0, settings.keep_last_items);
+        if (historyItems.length > keepLast + 1) {
+            if (keepLast === 0) {
+                this.state.agent_history_items = [historyItems[0]];
+            }
+            else {
+                this.state.agent_history_items = [
+                    historyItems[0],
+                    ...historyItems.slice(-keepLast),
+                ];
+            }
+        }
+        logger.debug(`Compaction complete (summary_chars=${summary.length}, history_items=${this.state.agent_history_items.length})`);
+        return true;
+    }
+    create_state_messages(browser_state_summary, model_output = null, result = null, step_info = null, use_vision = true, page_filtered_actions = null, sensitive_data = null, available_file_paths = null, include_recent_events = null, plan_description = null, unavailable_skills_info = null, skip_state_update = false) {
+        if (!skip_state_update) {
+            this.prepare_step_state(browser_state_summary, model_output, result, step_info, sensitive_data);
         }
         const screenshots = [];
-        if (browser_state_summary.screenshot) {
+        let includeScreenshotRequested = false;
+        if (result) {
+            for (const actionResult of result) {
+                if (actionResult.metadata?.include_screenshot) {
+                    includeScreenshotRequested = true;
+                    break;
+                }
+            }
+        }
+        let includeScreenshot = false;
+        if (use_vision === true) {
+            includeScreenshot = true;
+        }
+        else if (use_vision === 'auto') {
+            includeScreenshot = includeScreenshotRequested;
+        }
+        if (includeScreenshot && browser_state_summary.screenshot) {
             screenshots.push(browser_state_summary.screenshot);
         }
+        const effectiveUseVision = screenshots.length > 0;
+        const includeRecentEvents = include_recent_events ?? this.includeRecentEvents;
         const prompt = new AgentMessagePrompt({
             browser_state_summary,
             file_system: this.fileSystem,
@@ -137,8 +282,15 @@ export class MessageManager {
             available_file_paths,
             screenshots,
             vision_detail_level: this.visionDetailLevel,
+            include_recent_events: includeRecentEvents,
+            sample_images: this.sampleImages,
+            read_state_images: this.state.read_state_images,
+            llm_screenshot_size: this.llmScreenshotSize,
+            plan_description,
+            unavailable_skills_info,
         });
-        const message = prompt.get_user_message(use_vision);
+        const message = prompt.get_user_message(effectiveUseVision);
+        this.last_state_message_text = this.extractStateMessageText(message);
         this.setMessageWithType(message, 'state');
     }
     get_messages() {
@@ -147,25 +299,39 @@ export class MessageManager {
         return this.lastInputMessages;
     }
     setMessageWithType(message, messageType) {
-        const filtered = this.sensitiveData
-            ? this.filterSensitiveData(message)
-            : message;
         if (messageType === 'system') {
-            this.state.history.system_message = filtered;
+            this.state.history.system_message = message;
         }
         else {
+            const filtered = this.sensitiveData
+                ? this.filterSensitiveData(message)
+                : message;
             this.state.history.state_message = filtered;
         }
     }
     addContextMessage(message) {
-        const filtered = this.sensitiveData
-            ? this.filterSensitiveData(message)
-            : message;
-        this.state.history.context_messages.push(filtered);
+        this.state.history.context_messages.push(message);
     }
     _add_context_message(message) {
         this.addContextMessage(message);
     }
+    extractStateMessageText(message) {
+        if (typeof message.content === 'string') {
+            return message.content;
+        }
+        if (!Array.isArray(message.content)) {
+            return null;
+        }
+        return message.content
+            .map((part) => {
+            if (part instanceof ContentPartTextParam) {
+                return part.text;
+            }
+            return null;
+        })
+            .filter((part) => typeof part === 'string')
+            .join('\n');
+    }
     filterSensitiveData(message) {
         if (!this.sensitiveData) {
             return message;

package/dist/agent/message-manager/utils.js

@@ -29,7 +29,6 @@ const formatConversation = (messages, response) => {
             : (message.text ?? ''));
         lines.push('');
     });
-    lines.push(' RESPONSE');
     lines.push(serializeResponse(response));
     return lines.join('\n');
 };

package/dist/agent/message-manager/views.d.ts

@@ -21,6 +21,10 @@ export declare class MessageManagerState {
     tool_id: number;
     agent_history_items: HistoryItem[];
     read_state_description: string;
+    read_state_images: Array<Record<string, unknown>>;
+    compacted_memory: string | null;
+    compaction_count: number;
+    last_compaction_step: number | null;
     get historyMessages(): Message[];
     get_messages(): Message[];
 }

package/dist/agent/message-manager/views.js

@@ -19,28 +19,28 @@ export class HistoryItem {
         }
     }
     to_string() {
-        const stepStr = this.step_number != null ? `step_${this.step_number}` : 'step_unknown';
+        const stepStr = this.step_number != null ? 'step' : 'step_unknown';
         if (this.error) {
-            return `<${stepStr}>\n${this.error}\n</${stepStr}>`;
+            return `<${stepStr}>\n${this.error}`;
         }
         if (this.system_message) {
-            return `<sys>\n${this.system_message}\n</sys>`;
+            return this.system_message;
         }
         const parts = [];
         if (this.evaluation_previous_goal) {
-            parts.push(`Evaluation of Previous Step: ${this.evaluation_previous_goal}`);
+            parts.push(`${this.evaluation_previous_goal}`);
         }
         if (this.memory) {
-            parts.push(`Memory: ${this.memory}`);
+            parts.push(`${this.memory}`);
         }
         if (this.next_goal) {
-            parts.push(`Next Goal: ${this.next_goal}`);
+            parts.push(`${this.next_goal}`);
         }
         if (this.action_results) {
             parts.push(this.action_results);
         }
         const content = parts.join('\n');
-        return `<${stepStr}>\n${content}\n</${stepStr}>`;
+        return `<${stepStr}>\n${content}`;
     }
 }
 export class MessageHistory {
@@ -64,6 +64,10 @@ export class MessageManagerState {
         new HistoryItem(0, null, null, null, null, null, 'Agent initialized'),
     ];
     read_state_description = '';
+    read_state_images = [];
+    compacted_memory = null;
+    compaction_count = 0;
+    last_compaction_step = null;
     get historyMessages() {
         return this.history.get_messages();
     }