browser-use 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +761 -0
  3. package/dist/agent/cloud-events.d.ts +264 -0
  4. package/dist/agent/cloud-events.js +318 -0
  5. package/dist/agent/gif.d.ts +15 -0
  6. package/dist/agent/gif.js +215 -0
  7. package/dist/agent/index.d.ts +8 -0
  8. package/dist/agent/index.js +8 -0
  9. package/dist/agent/message-manager/service.d.ts +30 -0
  10. package/dist/agent/message-manager/service.js +208 -0
  11. package/dist/agent/message-manager/utils.d.ts +2 -0
  12. package/dist/agent/message-manager/utils.js +41 -0
  13. package/dist/agent/message-manager/views.d.ts +26 -0
  14. package/dist/agent/message-manager/views.js +73 -0
  15. package/dist/agent/prompts.d.ts +52 -0
  16. package/dist/agent/prompts.js +259 -0
  17. package/dist/agent/service.d.ts +290 -0
  18. package/dist/agent/service.js +2200 -0
  19. package/dist/agent/views.d.ts +741 -0
  20. package/dist/agent/views.js +537 -0
  21. package/dist/browser/browser.d.ts +7 -0
  22. package/dist/browser/browser.js +5 -0
  23. package/dist/browser/context.d.ts +8 -0
  24. package/dist/browser/context.js +4 -0
  25. package/dist/browser/dvd-screensaver.d.ts +101 -0
  26. package/dist/browser/dvd-screensaver.js +270 -0
  27. package/dist/browser/extensions.d.ts +63 -0
  28. package/dist/browser/extensions.js +359 -0
  29. package/dist/browser/index.d.ts +10 -0
  30. package/dist/browser/index.js +9 -0
  31. package/dist/browser/playwright-manager.d.ts +47 -0
  32. package/dist/browser/playwright-manager.js +146 -0
  33. package/dist/browser/profile.d.ts +196 -0
  34. package/dist/browser/profile.js +815 -0
  35. package/dist/browser/session.d.ts +505 -0
  36. package/dist/browser/session.js +3409 -0
  37. package/dist/browser/types.d.ts +1184 -0
  38. package/dist/browser/types.js +1 -0
  39. package/dist/browser/utils.d.ts +1 -0
  40. package/dist/browser/utils.js +19 -0
  41. package/dist/browser/views.d.ts +78 -0
  42. package/dist/browser/views.js +72 -0
  43. package/dist/cli.d.ts +2 -0
  44. package/dist/cli.js +44 -0
  45. package/dist/config.d.ts +108 -0
  46. package/dist/config.js +430 -0
  47. package/dist/controller/index.d.ts +3 -0
  48. package/dist/controller/index.js +3 -0
  49. package/dist/controller/registry/index.d.ts +2 -0
  50. package/dist/controller/registry/index.js +2 -0
  51. package/dist/controller/registry/service.d.ts +45 -0
  52. package/dist/controller/registry/service.js +184 -0
  53. package/dist/controller/registry/views.d.ts +55 -0
  54. package/dist/controller/registry/views.js +174 -0
  55. package/dist/controller/service.d.ts +49 -0
  56. package/dist/controller/service.js +1176 -0
  57. package/dist/controller/views.d.ts +241 -0
  58. package/dist/controller/views.js +88 -0
  59. package/dist/dom/clickable-element-processor/service.d.ts +11 -0
  60. package/dist/dom/clickable-element-processor/service.js +60 -0
  61. package/dist/dom/dom_tree/index.js +1400 -0
  62. package/dist/dom/history-tree-processor/service.d.ts +14 -0
  63. package/dist/dom/history-tree-processor/service.js +75 -0
  64. package/dist/dom/history-tree-processor/view.d.ts +54 -0
  65. package/dist/dom/history-tree-processor/view.js +56 -0
  66. package/dist/dom/playground/extraction.d.ts +19 -0
  67. package/dist/dom/playground/extraction.js +187 -0
  68. package/dist/dom/playground/process-dom.d.ts +1 -0
  69. package/dist/dom/playground/process-dom.js +5 -0
  70. package/dist/dom/playground/test-accessibility.d.ts +44 -0
  71. package/dist/dom/playground/test-accessibility.js +111 -0
  72. package/dist/dom/service.d.ts +19 -0
  73. package/dist/dom/service.js +227 -0
  74. package/dist/dom/utils.d.ts +1 -0
  75. package/dist/dom/utils.js +6 -0
  76. package/dist/dom/views.d.ts +61 -0
  77. package/dist/dom/views.js +247 -0
  78. package/dist/event-bus.d.ts +11 -0
  79. package/dist/event-bus.js +19 -0
  80. package/dist/exceptions.d.ts +10 -0
  81. package/dist/exceptions.js +22 -0
  82. package/dist/filesystem/file-system.d.ts +68 -0
  83. package/dist/filesystem/file-system.js +412 -0
  84. package/dist/filesystem/index.d.ts +1 -0
  85. package/dist/filesystem/index.js +1 -0
  86. package/dist/index.d.ts +31 -0
  87. package/dist/index.js +33 -0
  88. package/dist/integrations/gmail/actions.d.ts +12 -0
  89. package/dist/integrations/gmail/actions.js +113 -0
  90. package/dist/integrations/gmail/index.d.ts +2 -0
  91. package/dist/integrations/gmail/index.js +2 -0
  92. package/dist/integrations/gmail/service.d.ts +61 -0
  93. package/dist/integrations/gmail/service.js +260 -0
  94. package/dist/llm/anthropic/chat.d.ts +28 -0
  95. package/dist/llm/anthropic/chat.js +126 -0
  96. package/dist/llm/anthropic/index.d.ts +2 -0
  97. package/dist/llm/anthropic/index.js +2 -0
  98. package/dist/llm/anthropic/serializer.d.ts +68 -0
  99. package/dist/llm/anthropic/serializer.js +285 -0
  100. package/dist/llm/aws/chat-anthropic.d.ts +61 -0
  101. package/dist/llm/aws/chat-anthropic.js +176 -0
  102. package/dist/llm/aws/chat-bedrock.d.ts +15 -0
  103. package/dist/llm/aws/chat-bedrock.js +80 -0
  104. package/dist/llm/aws/index.d.ts +3 -0
  105. package/dist/llm/aws/index.js +3 -0
  106. package/dist/llm/aws/serializer.d.ts +5 -0
  107. package/dist/llm/aws/serializer.js +68 -0
  108. package/dist/llm/azure/chat.d.ts +15 -0
  109. package/dist/llm/azure/chat.js +83 -0
  110. package/dist/llm/azure/index.d.ts +1 -0
  111. package/dist/llm/azure/index.js +1 -0
  112. package/dist/llm/base.d.ts +16 -0
  113. package/dist/llm/base.js +1 -0
  114. package/dist/llm/deepseek/chat.d.ts +15 -0
  115. package/dist/llm/deepseek/chat.js +51 -0
  116. package/dist/llm/deepseek/index.d.ts +2 -0
  117. package/dist/llm/deepseek/index.js +2 -0
  118. package/dist/llm/deepseek/serializer.d.ts +6 -0
  119. package/dist/llm/deepseek/serializer.js +57 -0
  120. package/dist/llm/exceptions.d.ts +10 -0
  121. package/dist/llm/exceptions.js +18 -0
  122. package/dist/llm/google/chat.d.ts +20 -0
  123. package/dist/llm/google/chat.js +144 -0
  124. package/dist/llm/google/index.d.ts +2 -0
  125. package/dist/llm/google/index.js +2 -0
  126. package/dist/llm/google/serializer.d.ts +6 -0
  127. package/dist/llm/google/serializer.js +64 -0
  128. package/dist/llm/groq/chat.d.ts +15 -0
  129. package/dist/llm/groq/chat.js +52 -0
  130. package/dist/llm/groq/index.d.ts +3 -0
  131. package/dist/llm/groq/index.js +3 -0
  132. package/dist/llm/groq/parser.d.ts +32 -0
  133. package/dist/llm/groq/parser.js +189 -0
  134. package/dist/llm/groq/serializer.d.ts +6 -0
  135. package/dist/llm/groq/serializer.js +56 -0
  136. package/dist/llm/messages.d.ts +77 -0
  137. package/dist/llm/messages.js +157 -0
  138. package/dist/llm/ollama/chat.d.ts +15 -0
  139. package/dist/llm/ollama/chat.js +77 -0
  140. package/dist/llm/ollama/index.d.ts +2 -0
  141. package/dist/llm/ollama/index.js +2 -0
  142. package/dist/llm/ollama/serializer.d.ts +6 -0
  143. package/dist/llm/ollama/serializer.js +53 -0
  144. package/dist/llm/openai/chat.d.ts +38 -0
  145. package/dist/llm/openai/chat.js +174 -0
  146. package/dist/llm/openai/index.d.ts +3 -0
  147. package/dist/llm/openai/index.js +3 -0
  148. package/dist/llm/openai/like.d.ts +17 -0
  149. package/dist/llm/openai/like.js +19 -0
  150. package/dist/llm/openai/serializer.d.ts +6 -0
  151. package/dist/llm/openai/serializer.js +57 -0
  152. package/dist/llm/openrouter/chat.d.ts +15 -0
  153. package/dist/llm/openrouter/chat.js +74 -0
  154. package/dist/llm/openrouter/index.d.ts +2 -0
  155. package/dist/llm/openrouter/index.js +2 -0
  156. package/dist/llm/openrouter/serializer.d.ts +3 -0
  157. package/dist/llm/openrouter/serializer.js +3 -0
  158. package/dist/llm/schema.d.ts +6 -0
  159. package/dist/llm/schema.js +77 -0
  160. package/dist/llm/views.d.ts +15 -0
  161. package/dist/llm/views.js +12 -0
  162. package/dist/logging-config.d.ts +25 -0
  163. package/dist/logging-config.js +89 -0
  164. package/dist/mcp/client.d.ts +142 -0
  165. package/dist/mcp/client.js +638 -0
  166. package/dist/mcp/controller.d.ts +6 -0
  167. package/dist/mcp/controller.js +38 -0
  168. package/dist/mcp/index.d.ts +3 -0
  169. package/dist/mcp/index.js +3 -0
  170. package/dist/mcp/server.d.ts +134 -0
  171. package/dist/mcp/server.js +759 -0
  172. package/dist/observability-decorators.d.ts +158 -0
  173. package/dist/observability-decorators.js +286 -0
  174. package/dist/observability.d.ts +23 -0
  175. package/dist/observability.js +58 -0
  176. package/dist/screenshots/index.d.ts +1 -0
  177. package/dist/screenshots/index.js +1 -0
  178. package/dist/screenshots/service.d.ts +6 -0
  179. package/dist/screenshots/service.js +28 -0
  180. package/dist/sync/auth.d.ts +27 -0
  181. package/dist/sync/auth.js +205 -0
  182. package/dist/sync/index.d.ts +2 -0
  183. package/dist/sync/index.js +2 -0
  184. package/dist/sync/service.d.ts +21 -0
  185. package/dist/sync/service.js +146 -0
  186. package/dist/telemetry/index.d.ts +2 -0
  187. package/dist/telemetry/index.js +2 -0
  188. package/dist/telemetry/service.d.ts +12 -0
  189. package/dist/telemetry/service.js +85 -0
  190. package/dist/telemetry/views.d.ts +112 -0
  191. package/dist/telemetry/views.js +112 -0
  192. package/dist/tokens/index.d.ts +2 -0
  193. package/dist/tokens/index.js +2 -0
  194. package/dist/tokens/service.d.ts +35 -0
  195. package/dist/tokens/service.js +423 -0
  196. package/dist/tokens/views.d.ts +58 -0
  197. package/dist/tokens/views.js +1 -0
  198. package/dist/utils.d.ts +128 -0
  199. package/dist/utils.js +529 -0
  200. package/package.json +94 -5
@@ -0,0 +1,215 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { createCanvas, loadImage } from 'canvas';
4
+ import GIFEncoder from 'gif-encoder-2';
5
+ import { createLogger } from '../logging-config.js';
6
+ import { PLACEHOLDER_4PX_SCREENSHOT } from '../browser/views.js';
7
+ const logger = createLogger('browser_use.agent.gif');
8
+ export const decode_unicode_escapes_to_utf8 = (text) => {
9
+ if (!text.includes('\\u')) {
10
+ return text;
11
+ }
12
+ try {
13
+ return Buffer.from(text, 'latin1').toString('utf8');
14
+ }
15
+ catch {
16
+ return text;
17
+ }
18
+ };
19
+ const asDataUrl = (screenshot) => {
20
+ if (!screenshot) {
21
+ return '';
22
+ }
23
+ return screenshot.startsWith('data:')
24
+ ? screenshot
25
+ : `data:image/png;base64,${screenshot}`;
26
+ };
27
+ const loadScreenshot = async (screenshot) => {
28
+ const normalized = asDataUrl(screenshot);
29
+ return loadImage(normalized);
30
+ };
31
+ const FONT_CANDIDATES = [
32
+ '"Microsoft YaHei"',
33
+ '"SimHei"',
34
+ '"SimSun"',
35
+ '"Noto Sans CJK SC"',
36
+ '"Arial"',
37
+ '"Helvetica"',
38
+ '"sans-serif"',
39
+ ];
40
+ const pickFont = () => FONT_CANDIDATES.join(', ');
41
+ const wrapText = (ctx, text, maxWidth) => {
42
+ const words = decode_unicode_escapes_to_utf8(text).split(/\s+/);
43
+ const lines = [];
44
+ let currentLine = '';
45
+ for (const word of words) {
46
+ const testLine = currentLine ? `${currentLine} ${word}` : word;
47
+ const metrics = ctx.measureText(testLine);
48
+ if (metrics.width > maxWidth && currentLine) {
49
+ lines.push(currentLine);
50
+ currentLine = word;
51
+ }
52
+ else {
53
+ currentLine = testLine;
54
+ }
55
+ }
56
+ if (currentLine) {
57
+ lines.push(currentLine);
58
+ }
59
+ return lines;
60
+ };
61
+ const drawRoundedRect = (ctx, x, y, width, height, radius, fillStyle) => {
62
+ ctx.save();
63
+ ctx.fillStyle = fillStyle;
64
+ ctx.beginPath();
65
+ ctx.moveTo(x + radius, y);
66
+ ctx.lineTo(x + width - radius, y);
67
+ ctx.quadraticCurveTo(x + width, y, x + width, y + radius);
68
+ ctx.lineTo(x + width, y + height - radius);
69
+ ctx.quadraticCurveTo(x + width, y + height, x + width - radius, y + height);
70
+ ctx.lineTo(x + radius, y + height);
71
+ ctx.quadraticCurveTo(x, y + height, x, y + height - radius);
72
+ ctx.lineTo(x, y + radius);
73
+ ctx.quadraticCurveTo(x, y, x + radius, y);
74
+ ctx.closePath();
75
+ ctx.fill();
76
+ ctx.restore();
77
+ };
78
+ const addOverlayToContext = (ctx, width, height, stepNumber, goalText, fontFamily, titleFontSize) => {
79
+ const margin = 40;
80
+ const textColor = 'rgba(255,255,255,1)';
81
+ const boxColor = 'rgba(0,0,0,0.7)';
82
+ ctx.save();
83
+ ctx.fillStyle = textColor;
84
+ ctx.font = `${titleFontSize}px ${fontFamily}`;
85
+ ctx.textBaseline = 'top';
86
+ const stepText = String(stepNumber);
87
+ const stepMetrics = ctx.measureText(stepText);
88
+ const stepWidth = stepMetrics.width;
89
+ const stepHeight = titleFontSize;
90
+ const stepX = margin;
91
+ const stepY = height - stepHeight - margin - 10;
92
+ drawRoundedRect(ctx, stepX - 20, stepY - 20, stepWidth + 40, stepHeight + 40, 15, boxColor);
93
+ ctx.fillText(stepText, stepX, stepY);
94
+ const maxWidth = width - margin * 4;
95
+ const lines = wrapText(ctx, goalText, maxWidth);
96
+ const totalHeight = lines.length * (titleFontSize + 10);
97
+ const goalX = (width - maxWidth) / 2;
98
+ const goalY = stepY - totalHeight - 80;
99
+ drawRoundedRect(ctx, goalX - 20, goalY - 20, maxWidth + 40, totalHeight + 40, 15, boxColor);
100
+ lines.forEach((line, idx) => {
101
+ ctx.fillText(line, goalX, goalY + idx * (titleFontSize + 10));
102
+ });
103
+ ctx.restore();
104
+ };
105
+ const addLogo = (ctx, width, image) => {
106
+ if (!image)
107
+ return;
108
+ ctx.save();
109
+ const margin = 20;
110
+ const targetHeight = 150;
111
+ const aspect = image.width / image.height || 1;
112
+ const targetWidth = targetHeight * aspect;
113
+ ctx.globalAlpha = 0.9;
114
+ // @ts-ignore - drawImage signature compatibility
115
+ ctx.drawImage(image, width - targetWidth - margin, margin, targetWidth, targetHeight);
116
+ ctx.restore();
117
+ };
118
+ const loadLogo = async () => {
119
+ try {
120
+ const logoPath = path.resolve('static/browser-use.png');
121
+ await fs.promises.access(logoPath, fs.constants.F_OK);
122
+ return await loadImage(logoPath);
123
+ }
124
+ catch {
125
+ return null;
126
+ }
127
+ };
128
+ const renderTaskFrame = (ctx, width, height, task, fontFamily, fontSize, lineSpacing, logo) => {
129
+ ctx.save();
130
+ ctx.fillStyle = '#000';
131
+ ctx.fillRect(0, 0, width, height);
132
+ ctx.fillStyle = '#fff';
133
+ ctx.font = `${fontSize}px ${fontFamily}`;
134
+ ctx.textAlign = 'center';
135
+ ctx.textBaseline = 'top';
136
+ const maxWidth = width - 280;
137
+ const lines = wrapText(ctx, task, maxWidth);
138
+ const totalHeight = lines.length * fontSize * lineSpacing;
139
+ let y = height / 2 - totalHeight / 2;
140
+ lines.forEach((line) => {
141
+ ctx.fillText(line, width / 2, y);
142
+ y += fontSize * lineSpacing;
143
+ });
144
+ ctx.restore();
145
+ addLogo(ctx, width, logo);
146
+ };
147
+ const drawScreenshotFrame = (ctx, width, height, image, stepNumber, goalText, fontFamily, titleFontSize, logo) => {
148
+ // @ts-ignore - drawImage signature compatibility
149
+ ctx.drawImage(image, 0, 0, width, height);
150
+ if (goalText) {
151
+ addOverlayToContext(ctx, width, height, stepNumber, goalText, fontFamily, titleFontSize);
152
+ }
153
+ addLogo(ctx, width, logo);
154
+ };
155
+ export const create_history_gif = async (task, history, { output_path = 'agent_history.gif', duration = 3000, show_goals = true, show_task = true, show_logo = false, font_size = 40, title_font_size = 56, goal_font_size = 44, line_spacing = 1.5, } = {}) => {
156
+ if (!history.history.length) {
157
+ logger.warn('No history to create GIF from');
158
+ return;
159
+ }
160
+ const screenshots = history.screenshots();
161
+ const firstRealScreenshot = screenshots.find((shot) => shot && shot !== PLACEHOLDER_4PX_SCREENSHOT);
162
+ if (!firstRealScreenshot) {
163
+ logger.warn('No valid screenshots found (all are placeholders)');
164
+ return;
165
+ }
166
+ const firstImage = await loadScreenshot(firstRealScreenshot);
167
+ const width = firstImage.width;
168
+ const height = firstImage.height;
169
+ const canvas = createCanvas(width, height);
170
+ const ctx = canvas.getContext('2d');
171
+ const encoder = new GIFEncoder(width, height);
172
+ await fs.promises.mkdir(path.dirname(path.resolve(output_path)), {
173
+ recursive: true,
174
+ });
175
+ const writeStream = fs.createWriteStream(path.resolve(output_path));
176
+ encoder.createReadStream().pipe(writeStream);
177
+ encoder.start();
178
+ encoder.setRepeat(0);
179
+ encoder.setDelay(duration);
180
+ encoder.setQuality(10);
181
+ const fontFamily = pickFont();
182
+ const logo = show_logo ? await loadLogo() : null;
183
+ if (show_task && task) {
184
+ ctx.clearRect(0, 0, width, height);
185
+ // @ts-ignore - node-canvas types differ slightly from browser canvas
186
+ renderTaskFrame(ctx, width, height, task, fontFamily, font_size + 16, line_spacing, logo);
187
+ encoder.addFrame(ctx);
188
+ }
189
+ for (let index = 0; index < screenshots.length; index += 1) {
190
+ const screenshot = screenshots[index];
191
+ if (!screenshot || screenshot === PLACEHOLDER_4PX_SCREENSHOT) {
192
+ continue;
193
+ }
194
+ try {
195
+ const image = await loadScreenshot(screenshot);
196
+ ctx.clearRect(0, 0, width, height);
197
+ const goalText = show_goals &&
198
+ history.history[index]?.model_output?.current_state.next_goal
199
+ ? history.history[index].model_output.current_state.next_goal
200
+ : '';
201
+ // @ts-ignore - node-canvas types differ slightly from browser canvas
202
+ drawScreenshotFrame(ctx, width, height, image, index + 1, goalText, fontFamily, goal_font_size, logo);
203
+ encoder.addFrame(ctx);
204
+ }
205
+ catch (error) {
206
+ logger.warn(`Failed to process screenshot at step ${index + 1}: ${error.message}`);
207
+ }
208
+ }
209
+ encoder.finish();
210
+ await new Promise((resolve, reject) => {
211
+ writeStream.on('finish', resolve);
212
+ writeStream.on('error', reject);
213
+ });
214
+ logger.info(`Created GIF at ${output_path}`);
215
+ };
@@ -0,0 +1,8 @@
1
+ export * from './views.js';
2
+ export * from './message-manager/service.js';
3
+ export * from './message-manager/views.js';
4
+ export * from './message-manager/utils.js';
5
+ export * from './prompts.js';
6
+ export * from './cloud-events.js';
7
+ export * from './gif.js';
8
+ export * from './service.js';
@@ -0,0 +1,8 @@
1
+ export * from './views.js';
2
+ export * from './message-manager/service.js';
3
+ export * from './message-manager/views.js';
4
+ export * from './message-manager/utils.js';
5
+ export * from './prompts.js';
6
+ export * from './cloud-events.js';
7
+ export * from './gif.js';
8
+ export * from './service.js';
@@ -0,0 +1,30 @@
1
+ import { SystemMessage, UserMessage, type Message } from '../../llm/messages.js';
2
+ import { ActionResult, AgentOutput, AgentStepInfo } from '../views.js';
3
+ import { BrowserStateSummary } from '../../browser/views.js';
4
+ import { FileSystem } from '../../filesystem/file-system.js';
5
+ import { MessageManagerState } from './views.js';
6
+ export declare class MessageManager {
7
+ private readonly fileSystem;
8
+ private readonly state;
9
+ private readonly useThinking;
10
+ private readonly sensitiveData?;
11
+ private readonly maxHistoryItems;
12
+ private readonly visionDetailLevel;
13
+ private readonly includeToolCallExamples;
14
+ private task;
15
+ private systemPrompt;
16
+ private sensitiveDataDescription;
17
+ private lastInputMessages;
18
+ private includeAttributes;
19
+ constructor(task: string, systemMessage: SystemMessage, fileSystem: FileSystem, state?: MessageManagerState, useThinking?: boolean, includeAttributes?: string[] | null, sensitiveData?: Record<string, string | Record<string, string>> | undefined, maxHistoryItems?: number | null, visionDetailLevel?: 'auto' | 'low' | 'high', includeToolCallExamples?: boolean);
20
+ get agent_history_description(): string;
21
+ add_new_task(new_task: string): void;
22
+ private updateAgentHistoryDescription;
23
+ private getSensitiveDataDescription;
24
+ create_state_messages(browser_state_summary: BrowserStateSummary, model_output?: AgentOutput | null, result?: ActionResult[] | null, step_info?: AgentStepInfo | null, use_vision?: boolean, page_filtered_actions?: string | null, sensitive_data?: Record<string, string | Record<string, string>> | null, available_file_paths?: string[] | null): void;
25
+ get_messages(): Message[];
26
+ private setMessageWithType;
27
+ private addContextMessage;
28
+ _add_context_message(message: SystemMessage | UserMessage): void;
29
+ private filterSensitiveData;
30
+ }
@@ -0,0 +1,208 @@
1
+ import { ContentPartTextParam, } from '../../llm/messages.js';
2
+ import { AgentMessagePrompt } from '../prompts.js';
3
+ import { MessageManagerState, HistoryItem } from './views.js';
4
+ import { match_url_with_domain_pattern } from '../../utils.js';
5
+ import { createLogger } from '../../logging-config.js';
6
+ const logger = createLogger('browser_use.agent.message_manager');
7
+ export class MessageManager {
8
+ fileSystem;
9
+ state;
10
+ useThinking;
11
+ sensitiveData;
12
+ maxHistoryItems;
13
+ visionDetailLevel;
14
+ includeToolCallExamples;
15
+ task;
16
+ systemPrompt;
17
+ sensitiveDataDescription = '';
18
+ lastInputMessages = [];
19
+ includeAttributes;
20
+ constructor(task, systemMessage, fileSystem, state = new MessageManagerState(), useThinking = true, includeAttributes = null, sensitiveData, maxHistoryItems = null, visionDetailLevel = 'auto', includeToolCallExamples = false) {
21
+ this.fileSystem = fileSystem;
22
+ this.state = state;
23
+ this.useThinking = useThinking;
24
+ this.sensitiveData = sensitiveData;
25
+ this.maxHistoryItems = maxHistoryItems;
26
+ this.visionDetailLevel = visionDetailLevel;
27
+ this.includeToolCallExamples = includeToolCallExamples;
28
+ this.task = task;
29
+ this.systemPrompt = systemMessage;
30
+ this.includeAttributes = includeAttributes ?? [];
31
+ if (!this.state.history.system_message) {
32
+ this.setMessageWithType(this.systemPrompt, 'system');
33
+ }
34
+ }
35
+ get agent_history_description() {
36
+ if (this.maxHistoryItems == null) {
37
+ return this.state.agent_history_items
38
+ .map((item) => item.to_string())
39
+ .join('\n');
40
+ }
41
+ const totalItems = this.state.agent_history_items.length;
42
+ if (totalItems <= this.maxHistoryItems) {
43
+ return this.state.agent_history_items
44
+ .map((item) => item.to_string())
45
+ .join('\n');
46
+ }
47
+ const omitted = totalItems - this.maxHistoryItems;
48
+ const keepRecent = this.maxHistoryItems - 1;
49
+ const parts = [];
50
+ parts.push(this.state.agent_history_items[0].to_string());
51
+ parts.push(`<sys>[... ${omitted} previous steps omitted...]</sys>`);
52
+ parts.push(...this.state.agent_history_items
53
+ .slice(-keepRecent)
54
+ .map((item) => item.to_string()));
55
+ return parts.join('\n');
56
+ }
57
+ add_new_task(new_task) {
58
+ this.task = new_task;
59
+ this.state.agent_history_items.push(new HistoryItem(null, null, null, null, null, null, `User updated <user_request> to: ${new_task}`));
60
+ }
61
+ updateAgentHistoryDescription(model_output, result, step_info) {
62
+ const results = result ?? [];
63
+ const stepNumber = step_info?.step_number ?? null;
64
+ this.state.read_state_description = '';
65
+ let actionText = '';
66
+ results.forEach((action, idx) => {
67
+ const suffix = `Action ${idx + 1}/${results.length}: `;
68
+ if (action.include_extracted_content_only_once &&
69
+ action.extracted_content) {
70
+ this.state.read_state_description += `${action.extracted_content}\n`;
71
+ }
72
+ if (action.long_term_memory) {
73
+ actionText += `${suffix}${action.long_term_memory}\n`;
74
+ }
75
+ else if (action.extracted_content &&
76
+ !action.include_extracted_content_only_once) {
77
+ actionText += `${suffix}${action.extracted_content}\n`;
78
+ }
79
+ if (action.error) {
80
+ const err = action.error.length > 200
81
+ ? `${action.error.slice(0, 100)}......${action.error.slice(-100)}`
82
+ : action.error;
83
+ actionText += `${suffix}${err}\n`;
84
+ }
85
+ });
86
+ const normalizedActionText = actionText ? actionText.trim() : null;
87
+ if (!model_output) {
88
+ if (stepNumber != null && stepNumber > 0) {
89
+ this.state.agent_history_items.push(new HistoryItem(stepNumber, null, null, null, null, 'Agent failed to output in the right format.', null));
90
+ }
91
+ return;
92
+ }
93
+ const brain = model_output.current_state;
94
+ this.state.agent_history_items.push(new HistoryItem(stepNumber, brain.evaluation_previous_goal, brain.memory, brain.next_goal, normalizedActionText, null, null));
95
+ }
96
+ getSensitiveDataDescription(currentUrl) {
97
+ const placeholders = new Set();
98
+ if (!this.sensitiveData) {
99
+ return '';
100
+ }
101
+ for (const [key, value] of Object.entries(this.sensitiveData)) {
102
+ if (value && typeof value === 'object' && !Array.isArray(value)) {
103
+ if (currentUrl &&
104
+ match_url_with_domain_pattern(currentUrl, key, true)) {
105
+ Object.keys(value).forEach((entry) => placeholders.add(entry));
106
+ }
107
+ }
108
+ else if (typeof value === 'string') {
109
+ placeholders.add(key);
110
+ }
111
+ }
112
+ if (!placeholders.size) {
113
+ return '';
114
+ }
115
+ return `Here are placeholders for sensitive data:\n${Array.from(placeholders).sort().join(', ')}\nTo use them, write <secret>the placeholder name</secret>`;
116
+ }
117
+ create_state_messages(browser_state_summary, model_output = null, result = null, step_info = null, use_vision = true, page_filtered_actions = null, sensitive_data = null, available_file_paths = null) {
118
+ this.state.history.context_messages = [];
119
+ this.updateAgentHistoryDescription(model_output, result, step_info);
120
+ if (sensitive_data) {
121
+ this.sensitiveDataDescription = this.getSensitiveDataDescription(browser_state_summary.url);
122
+ }
123
+ const screenshots = [];
124
+ if (browser_state_summary.screenshot) {
125
+ screenshots.push(browser_state_summary.screenshot);
126
+ }
127
+ const prompt = new AgentMessagePrompt({
128
+ browser_state_summary,
129
+ file_system: this.fileSystem,
130
+ agent_history_description: this.agent_history_description,
131
+ read_state_description: this.state.read_state_description,
132
+ task: this.task,
133
+ include_attributes: this.includeAttributes,
134
+ step_info,
135
+ page_filtered_actions,
136
+ sensitive_data: this.sensitiveDataDescription,
137
+ available_file_paths,
138
+ screenshots,
139
+ vision_detail_level: this.visionDetailLevel,
140
+ });
141
+ const message = prompt.get_user_message(use_vision);
142
+ this.setMessageWithType(message, 'state');
143
+ }
144
+ get_messages() {
145
+ logger.debug('');
146
+ this.lastInputMessages = this.state.history.get_messages();
147
+ return this.lastInputMessages;
148
+ }
149
+ setMessageWithType(message, messageType) {
150
+ const filtered = this.sensitiveData
151
+ ? this.filterSensitiveData(message)
152
+ : message;
153
+ if (messageType === 'system') {
154
+ this.state.history.system_message = filtered;
155
+ }
156
+ else {
157
+ this.state.history.state_message = filtered;
158
+ }
159
+ }
160
+ addContextMessage(message) {
161
+ const filtered = this.sensitiveData
162
+ ? this.filterSensitiveData(message)
163
+ : message;
164
+ this.state.history.context_messages.push(filtered);
165
+ }
166
+ _add_context_message(message) {
167
+ this.addContextMessage(message);
168
+ }
169
+ filterSensitiveData(message) {
170
+ if (!this.sensitiveData) {
171
+ return message;
172
+ }
173
+ const replaceSensitive = (value) => {
174
+ const placeholders = {};
175
+ for (const [keyOrDomain, content] of Object.entries(this.sensitiveData)) {
176
+ if (content && typeof content === 'object' && !Array.isArray(content)) {
177
+ for (const [k, v] of Object.entries(content)) {
178
+ if (v)
179
+ placeholders[k] = v;
180
+ }
181
+ }
182
+ else if (typeof content === 'string') {
183
+ placeholders[keyOrDomain] = content;
184
+ }
185
+ }
186
+ if (!Object.keys(placeholders).length) {
187
+ return value;
188
+ }
189
+ let updated = value;
190
+ for (const [key, val] of Object.entries(placeholders)) {
191
+ updated = updated.replaceAll(val, `<secret>${key}</secret>`);
192
+ }
193
+ return updated;
194
+ };
195
+ if (typeof message.content === 'string') {
196
+ message.content = replaceSensitive(message.content);
197
+ }
198
+ else if (Array.isArray(message.content)) {
199
+ message.content = message.content.map((part) => {
200
+ if (part instanceof ContentPartTextParam) {
201
+ part.text = replaceSensitive(part.text);
202
+ }
203
+ return part;
204
+ });
205
+ }
206
+ return message;
207
+ }
208
+ }
@@ -0,0 +1,2 @@
1
+ import { Message } from '../../llm/messages.js';
2
+ export declare const saveConversation: (inputMessages: Message[], response: unknown, target: string, encoding?: BufferEncoding) => Promise<void>;
@@ -0,0 +1,41 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ const serializeResponse = (response) => {
4
+ if (!response) {
5
+ return '';
6
+ }
7
+ if (typeof response.model_dump_json === 'function') {
8
+ try {
9
+ const raw = response.model_dump_json({ exclude_unset: true });
10
+ return JSON.stringify(JSON.parse(raw), null, 2);
11
+ }
12
+ catch {
13
+ /* fall through */
14
+ }
15
+ }
16
+ try {
17
+ return JSON.stringify(response, null, 2);
18
+ }
19
+ catch {
20
+ return String(response);
21
+ }
22
+ };
23
+ const formatConversation = (messages, response) => {
24
+ const lines = [];
25
+ messages.forEach((message) => {
26
+ lines.push(` ${message.role} `);
27
+ lines.push(typeof message.text === 'function'
28
+ ? message.text()
29
+ : (message.text ?? ''));
30
+ lines.push('');
31
+ });
32
+ lines.push(' RESPONSE');
33
+ lines.push(serializeResponse(response));
34
+ return lines.join('\n');
35
+ };
36
+ export const saveConversation = async (inputMessages, response, target, encoding = 'utf-8') => {
37
+ const targetPath = path.resolve(target);
38
+ await fs.promises.mkdir(path.dirname(targetPath), { recursive: true });
39
+ const payload = formatConversation(inputMessages, response);
40
+ await fs.promises.writeFile(targetPath, payload, { encoding });
41
+ };
@@ -0,0 +1,26 @@
1
+ import type { Message } from '../../llm/messages.js';
2
+ export declare class HistoryItem {
3
+ step_number: number | null;
4
+ evaluation_previous_goal: string | null;
5
+ memory: string | null;
6
+ next_goal: string | null;
7
+ action_results: string | null;
8
+ error: string | null;
9
+ system_message: string | null;
10
+ constructor(step_number?: number | null, evaluation_previous_goal?: string | null, memory?: string | null, next_goal?: string | null, action_results?: string | null, error?: string | null, system_message?: string | null);
11
+ to_string(): string;
12
+ }
13
+ export declare class MessageHistory {
14
+ system_message: Message | null;
15
+ state_message: Message | null;
16
+ context_messages: Message[];
17
+ get_messages(): Message[];
18
+ }
19
+ export declare class MessageManagerState {
20
+ history: MessageHistory;
21
+ tool_id: number;
22
+ agent_history_items: HistoryItem[];
23
+ read_state_description: string;
24
+ get historyMessages(): Message[];
25
+ get_messages(): Message[];
26
+ }
@@ -0,0 +1,73 @@
1
+ export class HistoryItem {
2
+ step_number;
3
+ evaluation_previous_goal;
4
+ memory;
5
+ next_goal;
6
+ action_results;
7
+ error;
8
+ system_message;
9
+ constructor(step_number = null, evaluation_previous_goal = null, memory = null, next_goal = null, action_results = null, error = null, system_message = null) {
10
+ this.step_number = step_number;
11
+ this.evaluation_previous_goal = evaluation_previous_goal;
12
+ this.memory = memory;
13
+ this.next_goal = next_goal;
14
+ this.action_results = action_results;
15
+ this.error = error;
16
+ this.system_message = system_message;
17
+ if (this.error && this.system_message) {
18
+ throw new Error('Cannot have both error and system_message at the same time');
19
+ }
20
+ }
21
+ to_string() {
22
+ const stepStr = this.step_number != null ? `step_${this.step_number}` : 'step_unknown';
23
+ if (this.error) {
24
+ return `<${stepStr}>\n${this.error}\n</${stepStr}>`;
25
+ }
26
+ if (this.system_message) {
27
+ return `<sys>\n${this.system_message}\n</sys>`;
28
+ }
29
+ const parts = [];
30
+ if (this.evaluation_previous_goal) {
31
+ parts.push(`Evaluation of Previous Step: ${this.evaluation_previous_goal}`);
32
+ }
33
+ if (this.memory) {
34
+ parts.push(`Memory: ${this.memory}`);
35
+ }
36
+ if (this.next_goal) {
37
+ parts.push(`Next Goal: ${this.next_goal}`);
38
+ }
39
+ if (this.action_results) {
40
+ parts.push(this.action_results);
41
+ }
42
+ const content = parts.join('\n');
43
+ return `<${stepStr}>\n${content}\n</${stepStr}>`;
44
+ }
45
+ }
46
+ export class MessageHistory {
47
+ system_message = null;
48
+ state_message = null;
49
+ context_messages = [];
50
+ get_messages() {
51
+ const messages = [];
52
+ if (this.system_message)
53
+ messages.push(this.system_message);
54
+ if (this.state_message)
55
+ messages.push(this.state_message);
56
+ messages.push(...this.context_messages);
57
+ return messages;
58
+ }
59
+ }
60
+ export class MessageManagerState {
61
+ history = new MessageHistory();
62
+ tool_id = 1;
63
+ agent_history_items = [
64
+ new HistoryItem(0, null, null, null, null, null, 'Agent initialized'),
65
+ ];
66
+ read_state_description = '';
67
+ get historyMessages() {
68
+ return this.history.get_messages();
69
+ }
70
+ get_messages() {
71
+ return this.history.get_messages();
72
+ }
73
+ }
@@ -0,0 +1,52 @@
1
+ import { SystemMessage, UserMessage } from '../llm/messages.js';
2
+ import type { AgentStepInfo } from './views.js';
3
+ import type { BrowserStateSummary } from '../browser/views.js';
4
+ import type { FileSystem } from '../filesystem/file-system.js';
5
+ export declare class SystemPrompt {
6
+ private readonly actionDescription;
7
+ private readonly maxActionsPerStep;
8
+ private readonly overrideSystemMessage;
9
+ private readonly extendSystemMessage;
10
+ private readonly useThinking;
11
+ private readonly flashMode;
12
+ private promptTemplate;
13
+ private systemMessage;
14
+ constructor(actionDescription: string, maxActionsPerStep?: number, overrideSystemMessage?: string | null, extendSystemMessage?: string | null, useThinking?: boolean, flashMode?: boolean);
15
+ private loadPromptTemplate;
16
+ get_system_message(): SystemMessage;
17
+ }
18
+ interface AgentMessagePromptInit {
19
+ browser_state_summary: BrowserStateSummary;
20
+ file_system: FileSystem;
21
+ agent_history_description?: string | null;
22
+ read_state_description?: string | null;
23
+ task?: string | null;
24
+ include_attributes?: string[] | null;
25
+ step_info?: AgentStepInfo | null;
26
+ page_filtered_actions?: string | null;
27
+ max_clickable_elements_length?: number;
28
+ sensitive_data?: string | null;
29
+ available_file_paths?: string[] | null;
30
+ screenshots?: string[] | null;
31
+ vision_detail_level?: 'auto' | 'low' | 'high';
32
+ }
33
+ export declare class AgentMessagePrompt {
34
+ private readonly browserState;
35
+ private readonly fileSystem;
36
+ private readonly agentHistoryDescription;
37
+ private readonly readStateDescription;
38
+ private readonly task;
39
+ private readonly includeAttributes?;
40
+ private readonly stepInfo?;
41
+ private readonly pageFilteredActions;
42
+ private readonly maxClickableElementsLength;
43
+ private readonly sensitiveData;
44
+ private readonly availableFilePaths?;
45
+ private readonly screenshots;
46
+ private readonly visionDetailLevel;
47
+ constructor(init: AgentMessagePromptInit);
48
+ private browserStateDescription;
49
+ private agentStateDescription;
50
+ get_user_message(use_vision?: boolean): UserMessage;
51
+ }
52
+ export {};