browser-use 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/README.md +295 -686
  2. package/dist/actor/element.d.ts +19 -0
  3. package/dist/actor/element.js +46 -0
  4. package/dist/actor/index.d.ts +4 -0
  5. package/dist/actor/index.js +4 -0
  6. package/dist/actor/mouse.d.ts +19 -0
  7. package/dist/actor/mouse.js +39 -0
  8. package/dist/actor/page.d.ts +29 -0
  9. package/dist/actor/page.js +88 -0
  10. package/dist/actor/utils.d.ts +4 -0
  11. package/dist/actor/utils.js +35 -0
  12. package/dist/agent/cloud-events.d.ts +18 -0
  13. package/dist/agent/cloud-events.js +65 -2
  14. package/dist/agent/gif.d.ts +1 -0
  15. package/dist/agent/gif.js +24 -2
  16. package/dist/agent/judge.d.ts +17 -0
  17. package/dist/agent/judge.js +197 -0
  18. package/dist/agent/message-manager/service.d.ts +12 -4
  19. package/dist/agent/message-manager/service.js +205 -39
  20. package/dist/agent/message-manager/utils.js +0 -1
  21. package/dist/agent/message-manager/views.d.ts +4 -0
  22. package/dist/agent/message-manager/views.js +11 -7
  23. package/dist/agent/prompts.d.ts +24 -3
  24. package/dist/agent/prompts.js +274 -59
  25. package/dist/agent/service.d.ts +99 -41
  26. package/dist/agent/service.js +2266 -472
  27. package/dist/agent/variable-detector.d.ts +12 -0
  28. package/dist/agent/variable-detector.js +211 -0
  29. package/dist/agent/views.d.ts +237 -18
  30. package/dist/agent/views.js +446 -33
  31. package/dist/browser/cloud/cloud.d.ts +20 -0
  32. package/dist/browser/cloud/cloud.js +129 -0
  33. package/dist/browser/cloud/index.d.ts +2 -0
  34. package/dist/browser/cloud/index.js +2 -0
  35. package/dist/browser/cloud/views.d.ts +41 -0
  36. package/dist/browser/cloud/views.js +35 -0
  37. package/dist/browser/events.d.ts +345 -0
  38. package/dist/browser/events.js +566 -0
  39. package/dist/browser/extensions.js +17 -17
  40. package/dist/browser/index.d.ts +4 -0
  41. package/dist/browser/index.js +4 -0
  42. package/dist/browser/profile.d.ts +8 -2
  43. package/dist/browser/profile.js +79 -12
  44. package/dist/browser/session-manager.d.ts +85 -0
  45. package/dist/browser/session-manager.js +208 -0
  46. package/dist/browser/session.d.ts +100 -8
  47. package/dist/browser/session.js +1097 -58
  48. package/dist/browser/types.d.ts +0 -2
  49. package/dist/browser/views.d.ts +39 -0
  50. package/dist/browser/views.js +32 -0
  51. package/dist/browser/watchdogs/aboutblank-watchdog.d.ts +12 -0
  52. package/dist/browser/watchdogs/aboutblank-watchdog.js +131 -0
  53. package/dist/browser/watchdogs/base.d.ts +21 -0
  54. package/dist/browser/watchdogs/base.js +81 -0
  55. package/dist/browser/watchdogs/cdp-session-watchdog.d.ts +14 -0
  56. package/dist/browser/watchdogs/cdp-session-watchdog.js +177 -0
  57. package/dist/browser/watchdogs/crash-watchdog.d.ts +38 -0
  58. package/dist/browser/watchdogs/crash-watchdog.js +296 -0
  59. package/dist/browser/watchdogs/default-action-watchdog.d.ts +49 -0
  60. package/dist/browser/watchdogs/default-action-watchdog.js +212 -0
  61. package/dist/browser/watchdogs/dom-watchdog.d.ts +8 -0
  62. package/dist/browser/watchdogs/dom-watchdog.js +31 -0
  63. package/dist/browser/watchdogs/downloads-watchdog.d.ts +77 -0
  64. package/dist/browser/watchdogs/downloads-watchdog.js +409 -0
  65. package/dist/browser/watchdogs/har-recording-watchdog.d.ts +19 -0
  66. package/dist/browser/watchdogs/har-recording-watchdog.js +317 -0
  67. package/dist/browser/watchdogs/index.d.ts +15 -0
  68. package/dist/browser/watchdogs/index.js +15 -0
  69. package/dist/browser/watchdogs/local-browser-watchdog.d.ts +10 -0
  70. package/dist/browser/watchdogs/local-browser-watchdog.js +32 -0
  71. package/dist/browser/watchdogs/permissions-watchdog.d.ts +8 -0
  72. package/dist/browser/watchdogs/permissions-watchdog.js +73 -0
  73. package/dist/browser/watchdogs/popups-watchdog.d.ts +13 -0
  74. package/dist/browser/watchdogs/popups-watchdog.js +77 -0
  75. package/dist/browser/watchdogs/recording-watchdog.d.ts +27 -0
  76. package/dist/browser/watchdogs/recording-watchdog.js +249 -0
  77. package/dist/browser/watchdogs/screenshot-watchdog.d.ts +6 -0
  78. package/dist/browser/watchdogs/screenshot-watchdog.js +13 -0
  79. package/dist/browser/watchdogs/security-watchdog.d.ts +10 -0
  80. package/dist/browser/watchdogs/security-watchdog.js +84 -0
  81. package/dist/browser/watchdogs/storage-state-watchdog.d.ts +24 -0
  82. package/dist/browser/watchdogs/storage-state-watchdog.js +288 -0
  83. package/dist/cli.d.ts +7 -2
  84. package/dist/cli.js +182 -25
  85. package/dist/code-use/formatting.d.ts +3 -0
  86. package/dist/code-use/formatting.js +18 -0
  87. package/dist/code-use/index.d.ts +6 -0
  88. package/dist/code-use/index.js +6 -0
  89. package/dist/code-use/namespace.d.ts +5 -0
  90. package/dist/code-use/namespace.js +81 -0
  91. package/dist/code-use/notebook-export.d.ts +3 -0
  92. package/dist/code-use/notebook-export.js +56 -0
  93. package/dist/code-use/service.d.ts +24 -0
  94. package/dist/code-use/service.js +104 -0
  95. package/dist/code-use/utils.d.ts +4 -0
  96. package/dist/code-use/utils.js +98 -0
  97. package/dist/code-use/views.d.ts +108 -0
  98. package/dist/code-use/views.js +165 -0
  99. package/dist/config.d.ts +13 -0
  100. package/dist/config.js +69 -3
  101. package/dist/controller/registry/service.d.ts +10 -1
  102. package/dist/controller/registry/service.js +266 -10
  103. package/dist/controller/registry/views.d.ts +4 -1
  104. package/dist/controller/registry/views.js +25 -2
  105. package/dist/controller/service.d.ts +10 -1
  106. package/dist/controller/service.js +1807 -268
  107. package/dist/controller/views.d.ts +78 -155
  108. package/dist/controller/views.js +61 -12
  109. package/dist/dom/history-tree-processor/service.d.ts +5 -0
  110. package/dist/dom/history-tree-processor/service.js +169 -14
  111. package/dist/dom/history-tree-processor/view.d.ts +7 -1
  112. package/dist/dom/history-tree-processor/view.js +10 -1
  113. package/dist/dom/markdown-extractor.d.ts +37 -0
  114. package/dist/dom/markdown-extractor.js +345 -0
  115. package/dist/dom/service.d.ts +3 -1
  116. package/dist/dom/service.js +76 -0
  117. package/dist/dom/views.d.ts +1 -0
  118. package/dist/dom/views.js +45 -0
  119. package/dist/event-bus.d.ts +107 -7
  120. package/dist/event-bus.js +313 -10
  121. package/dist/exceptions.d.ts +0 -3
  122. package/dist/exceptions.js +0 -7
  123. package/dist/filesystem/file-system.d.ts +18 -0
  124. package/dist/filesystem/file-system.js +503 -42
  125. package/dist/index.d.ts +7 -0
  126. package/dist/index.js +6 -0
  127. package/dist/integrations/gmail/actions.d.ts +3 -3
  128. package/dist/integrations/gmail/actions.js +4 -4
  129. package/dist/llm/anthropic/chat.d.ts +18 -1
  130. package/dist/llm/anthropic/chat.js +123 -55
  131. package/dist/llm/anthropic/serializer.d.ts +2 -0
  132. package/dist/llm/anthropic/serializer.js +81 -9
  133. package/dist/llm/aws/chat-anthropic.d.ts +17 -0
  134. package/dist/llm/aws/chat-anthropic.js +126 -26
  135. package/dist/llm/aws/chat-bedrock.d.ts +28 -1
  136. package/dist/llm/aws/chat-bedrock.js +161 -34
  137. package/dist/llm/aws/serializer.d.ts +13 -1
  138. package/dist/llm/aws/serializer.js +56 -17
  139. package/dist/llm/azure/chat.d.ts +53 -2
  140. package/dist/llm/azure/chat.js +366 -54
  141. package/dist/llm/base.d.ts +2 -0
  142. package/dist/llm/browser-use/chat.d.ts +40 -0
  143. package/dist/llm/browser-use/chat.js +305 -0
  144. package/dist/llm/browser-use/index.d.ts +1 -0
  145. package/dist/llm/browser-use/index.js +1 -0
  146. package/dist/llm/cerebras/chat.d.ts +39 -0
  147. package/dist/llm/cerebras/chat.js +178 -0
  148. package/dist/llm/cerebras/index.d.ts +2 -0
  149. package/dist/llm/cerebras/index.js +2 -0
  150. package/dist/llm/cerebras/serializer.d.ts +7 -0
  151. package/dist/llm/cerebras/serializer.js +82 -0
  152. package/dist/llm/deepseek/chat.d.ts +19 -2
  153. package/dist/llm/deepseek/chat.js +138 -25
  154. package/dist/llm/google/chat.d.ts +46 -2
  155. package/dist/llm/google/chat.js +267 -64
  156. package/dist/llm/google/serializer.d.ts +9 -1
  157. package/dist/llm/google/serializer.js +141 -34
  158. package/dist/llm/groq/chat.d.ts +21 -2
  159. package/dist/llm/groq/chat.js +125 -26
  160. package/dist/llm/groq/parser.js +3 -1
  161. package/dist/llm/mistral/chat.d.ts +43 -0
  162. package/dist/llm/mistral/chat.js +154 -0
  163. package/dist/llm/mistral/index.d.ts +2 -0
  164. package/dist/llm/mistral/index.js +2 -0
  165. package/dist/llm/mistral/schema.d.ts +8 -0
  166. package/dist/llm/mistral/schema.js +27 -0
  167. package/dist/llm/models.d.ts +2 -0
  168. package/dist/llm/models.js +317 -0
  169. package/dist/llm/ollama/chat.d.ts +13 -1
  170. package/dist/llm/ollama/chat.js +110 -19
  171. package/dist/llm/ollama/serializer.d.ts +1 -0
  172. package/dist/llm/ollama/serializer.js +34 -12
  173. package/dist/llm/openai/chat.d.ts +16 -0
  174. package/dist/llm/openai/chat.js +94 -44
  175. package/dist/llm/openai/like.d.ts +5 -3
  176. package/dist/llm/openai/like.js +7 -3
  177. package/dist/llm/openai/responses-serializer.d.ts +18 -0
  178. package/dist/llm/openai/responses-serializer.js +72 -0
  179. package/dist/llm/openrouter/chat.d.ts +28 -2
  180. package/dist/llm/openrouter/chat.js +115 -29
  181. package/dist/llm/schema.d.ts +11 -1
  182. package/dist/llm/schema.js +81 -1
  183. package/dist/llm/vercel/chat.d.ts +50 -0
  184. package/dist/llm/vercel/chat.js +276 -0
  185. package/dist/llm/vercel/index.d.ts +1 -0
  186. package/dist/llm/vercel/index.js +1 -0
  187. package/dist/llm/vercel/serializer.d.ts +5 -0
  188. package/dist/llm/vercel/serializer.js +7 -0
  189. package/dist/llm/views.d.ts +2 -1
  190. package/dist/llm/views.js +3 -1
  191. package/dist/logging-config.d.ts +2 -0
  192. package/dist/logging-config.js +82 -29
  193. package/dist/mcp/client.d.ts +10 -5
  194. package/dist/mcp/client.js +14 -9
  195. package/dist/mcp/controller.d.ts +42 -3
  196. package/dist/mcp/controller.js +56 -31
  197. package/dist/mcp/server.d.ts +14 -0
  198. package/dist/mcp/server.js +255 -52
  199. package/dist/observability.js +10 -4
  200. package/dist/sandbox/index.d.ts +2 -0
  201. package/dist/sandbox/index.js +2 -0
  202. package/dist/sandbox/sandbox.d.ts +19 -0
  203. package/dist/sandbox/sandbox.js +140 -0
  204. package/dist/sandbox/views.d.ts +67 -0
  205. package/dist/sandbox/views.js +121 -0
  206. package/dist/skill-cli/index.d.ts +3 -0
  207. package/dist/skill-cli/index.js +3 -0
  208. package/dist/skill-cli/protocol.d.ts +30 -0
  209. package/dist/skill-cli/protocol.js +48 -0
  210. package/dist/skill-cli/server.d.ts +11 -0
  211. package/dist/skill-cli/server.js +85 -0
  212. package/dist/skill-cli/sessions.d.ts +24 -0
  213. package/dist/skill-cli/sessions.js +47 -0
  214. package/dist/skills/index.d.ts +3 -0
  215. package/dist/skills/index.js +3 -0
  216. package/dist/skills/service.d.ts +27 -0
  217. package/dist/skills/service.js +266 -0
  218. package/dist/skills/utils.d.ts +6 -0
  219. package/dist/skills/utils.js +53 -0
  220. package/dist/skills/views.d.ts +40 -0
  221. package/dist/skills/views.js +10 -0
  222. package/dist/sync/auth.js +8 -3
  223. package/dist/sync/service.d.ts +6 -6
  224. package/dist/sync/service.js +54 -89
  225. package/dist/telemetry/views.d.ts +20 -6
  226. package/dist/telemetry/views.js +23 -5
  227. package/dist/tokens/custom-pricing.d.ts +2 -0
  228. package/dist/tokens/custom-pricing.js +22 -0
  229. package/dist/tokens/index.d.ts +2 -0
  230. package/dist/tokens/index.js +2 -0
  231. package/dist/tokens/mappings.d.ts +1 -0
  232. package/dist/tokens/mappings.js +3 -0
  233. package/dist/tokens/service.js +27 -8
  234. package/dist/tools/extraction/index.d.ts +2 -0
  235. package/dist/tools/extraction/index.js +2 -0
  236. package/dist/tools/extraction/schema-utils.d.ts +6 -0
  237. package/dist/tools/extraction/schema-utils.js +237 -0
  238. package/dist/tools/extraction/views.d.ts +7 -0
  239. package/dist/tools/index.d.ts +5 -0
  240. package/dist/tools/index.js +5 -0
  241. package/dist/tools/registry/index.d.ts +2 -0
  242. package/dist/tools/registry/index.js +2 -0
  243. package/dist/tools/registry/service.d.ts +1 -0
  244. package/dist/tools/registry/service.js +1 -0
  245. package/dist/tools/registry/views.d.ts +1 -0
  246. package/dist/tools/registry/views.js +1 -0
  247. package/dist/tools/service.d.ts +2 -0
  248. package/dist/tools/service.js +1 -0
  249. package/dist/tools/utils.d.ts +2 -0
  250. package/dist/tools/utils.js +57 -0
  251. package/dist/tools/views.d.ts +1 -0
  252. package/dist/tools/views.js +1 -0
  253. package/dist/utils.d.ts +10 -1
  254. package/dist/utils.js +70 -3
  255. package/package.json +87 -26
  256. package/dist/dom/playground/process-dom.js +0 -5
  257. package/dist/dom/playground/test-accessibility.d.ts +0 -44
  258. package/dist/dom/playground/test-accessibility.js +0 -111
  259. /package/dist/{dom/playground/process-dom.d.ts → tools/extraction/views.js} +0 -0
@@ -1,8 +1,10 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
+ import { createHash } from 'node:crypto';
3
4
  import { ActionModel } from '../controller/registry/views.js';
4
5
  import { BrowserStateHistory } from '../browser/views.js';
5
6
  import { HistoryTreeProcessor } from '../dom/history-tree-processor/service.js';
7
+ import { DEFAULT_INCLUDE_ATTRIBUTES, } from '../dom/views.js';
6
8
  import { MessageManagerState } from './message-manager/views.js';
7
9
  // Re-export ActionModel for agent/service.ts
8
10
  export { ActionModel };
@@ -21,8 +23,11 @@ const parseStructuredOutput = (schema, value) => {
21
23
  export class ActionResult {
22
24
  is_done;
23
25
  success;
26
+ judgement;
24
27
  error;
25
28
  attachments;
29
+ images;
30
+ metadata;
26
31
  long_term_memory;
27
32
  extracted_content;
28
33
  include_extracted_content_only_once;
@@ -30,8 +35,11 @@ export class ActionResult {
30
35
  constructor(init = {}) {
31
36
  this.is_done = init.is_done ?? false;
32
37
  this.success = init.success ?? null;
38
+ this.judgement = init.judgement ?? null;
33
39
  this.error = init.error ?? null;
34
40
  this.attachments = init.attachments ?? null;
41
+ this.images = init.images ?? null;
42
+ this.metadata = init.metadata ?? null;
35
43
  this.long_term_memory = init.long_term_memory ?? null;
36
44
  this.extracted_content = init.extracted_content ?? null;
37
45
  this.include_extracted_content_only_once =
@@ -48,8 +56,11 @@ export class ActionResult {
48
56
  return {
49
57
  is_done: this.is_done,
50
58
  success: this.success,
59
+ judgement: this.judgement,
51
60
  error: this.error,
52
61
  attachments: this.attachments,
62
+ images: this.images,
63
+ metadata: this.metadata,
53
64
  long_term_memory: this.long_term_memory,
54
65
  extracted_content: this.extracted_content,
55
66
  include_extracted_content_only_once: this.include_extracted_content_only_once,
@@ -63,45 +74,229 @@ export class ActionResult {
63
74
  return JSON.stringify(this.toJSON());
64
75
  }
65
76
  }
77
+ export class PageFingerprint {
78
+ url;
79
+ element_count;
80
+ text_hash;
81
+ constructor(url, element_count, text_hash) {
82
+ this.url = url;
83
+ this.element_count = element_count;
84
+ this.text_hash = text_hash;
85
+ }
86
+ static from_browser_state(url, dom_text, element_count) {
87
+ const text_hash = createHash('sha256')
88
+ .update(dom_text, 'utf8')
89
+ .digest('hex')
90
+ .slice(0, 16);
91
+ return new PageFingerprint(url, element_count, text_hash);
92
+ }
93
+ equals(other) {
94
+ return (this.url === other.url &&
95
+ this.element_count === other.element_count &&
96
+ this.text_hash === other.text_hash);
97
+ }
98
+ }
99
+ const stableSerialize = (value) => {
100
+ if (value === null || value === undefined) {
101
+ return '';
102
+ }
103
+ if (typeof value !== 'object') {
104
+ return String(value);
105
+ }
106
+ if (Array.isArray(value)) {
107
+ return `[${value.map((item) => stableSerialize(item)).join(',')}]`;
108
+ }
109
+ const entries = Object.entries(value)
110
+ .filter(([, entryValue]) => entryValue !== null && entryValue !== undefined)
111
+ .sort(([a], [b]) => a.localeCompare(b));
112
+ return `{${entries
113
+ .map(([key, entryValue]) => `${key}:${stableSerialize(entryValue)}`)
114
+ .join(',')}}`;
115
+ };
116
+ const normalizeActionForHash = (action_name, params) => {
117
+ if (action_name === 'search' || action_name === 'search_google') {
118
+ const query = String(params.query ?? '');
119
+ const tokens = Array.from(new Set(query
120
+ .toLowerCase()
121
+ .replace(/[^\w\s]/g, ' ')
122
+ .split(/\s+/)
123
+ .filter(Boolean))).sort();
124
+ const engine = typeof params.engine === 'string' && params.engine.trim()
125
+ ? params.engine.trim().toLowerCase()
126
+ : action_name === 'search_google'
127
+ ? 'google'
128
+ : 'google';
129
+ return `search|${engine}|${tokens.join('|')}`;
130
+ }
131
+ if (action_name === 'click' ||
132
+ action_name === 'click_element' ||
133
+ action_name === 'click_element_by_index') {
134
+ return `click|${String(params.index ?? '')}`;
135
+ }
136
+ if (action_name === 'input' || action_name === 'input_text') {
137
+ const text = String(params.text ?? '')
138
+ .trim()
139
+ .toLowerCase();
140
+ return `input|${String(params.index ?? '')}|${text}`;
141
+ }
142
+ if (action_name === 'navigate' || action_name === 'go_to_url') {
143
+ return `navigate|${String(params.url ?? '')}`;
144
+ }
145
+ if (action_name.startsWith('scroll')) {
146
+ const direction = typeof params.down === 'boolean'
147
+ ? params.down
148
+ ? 'down'
149
+ : 'up'
150
+ : action_name.includes('up')
151
+ ? 'up'
152
+ : 'down';
153
+ const index = String(params.index ?? '');
154
+ return `scroll|${direction}|${index}`;
155
+ }
156
+ return `${action_name}|${stableSerialize(params)}`;
157
+ };
158
+ export const compute_action_hash = (action_name, params) => {
159
+ const normalized = normalizeActionForHash(action_name, params);
160
+ return createHash('sha256')
161
+ .update(normalized, 'utf8')
162
+ .digest('hex')
163
+ .slice(0, 12);
164
+ };
165
+ export class ActionLoopDetector {
166
+ window_size;
167
+ recent_action_hashes;
168
+ recent_page_fingerprints;
169
+ max_repetition_count;
170
+ most_repeated_hash;
171
+ consecutive_stagnant_pages;
172
+ constructor(init) {
173
+ this.window_size = init?.window_size ?? 20;
174
+ this.recent_action_hashes = init?.recent_action_hashes ?? [];
175
+ this.recent_page_fingerprints = init?.recent_page_fingerprints ?? [];
176
+ this.max_repetition_count = init?.max_repetition_count ?? 0;
177
+ this.most_repeated_hash = init?.most_repeated_hash ?? null;
178
+ this.consecutive_stagnant_pages = init?.consecutive_stagnant_pages ?? 0;
179
+ }
180
+ record_action(action_name, params) {
181
+ const hash = compute_action_hash(action_name, params);
182
+ this.recent_action_hashes.push(hash);
183
+ if (this.recent_action_hashes.length > this.window_size) {
184
+ this.recent_action_hashes = this.recent_action_hashes.slice(-this.window_size);
185
+ }
186
+ this.update_repetition_stats();
187
+ }
188
+ record_page_state(url, dom_text, element_count) {
189
+ const fp = PageFingerprint.from_browser_state(url, dom_text, element_count);
190
+ const last = this.recent_page_fingerprints.at(-1);
191
+ if (last && last.equals(fp)) {
192
+ this.consecutive_stagnant_pages += 1;
193
+ }
194
+ else {
195
+ this.consecutive_stagnant_pages = 0;
196
+ }
197
+ this.recent_page_fingerprints.push(fp);
198
+ if (this.recent_page_fingerprints.length > 5) {
199
+ this.recent_page_fingerprints = this.recent_page_fingerprints.slice(-5);
200
+ }
201
+ }
202
+ update_repetition_stats() {
203
+ if (!this.recent_action_hashes.length) {
204
+ this.max_repetition_count = 0;
205
+ this.most_repeated_hash = null;
206
+ return;
207
+ }
208
+ const counts = new Map();
209
+ for (const hash of this.recent_action_hashes) {
210
+ counts.set(hash, (counts.get(hash) ?? 0) + 1);
211
+ }
212
+ let maxHash = null;
213
+ let maxCount = 0;
214
+ for (const [hash, count] of counts.entries()) {
215
+ if (count > maxCount) {
216
+ maxHash = hash;
217
+ maxCount = count;
218
+ }
219
+ }
220
+ this.most_repeated_hash = maxHash;
221
+ this.max_repetition_count = maxCount;
222
+ }
223
+ get_nudge_message() {
224
+ const messages = [];
225
+ if (this.max_repetition_count >= 12) {
226
+ messages.push(`Heads up: you have repeated a similar action ${this.max_repetition_count} times in the last ${this.recent_action_hashes.length} actions. If you are making progress with each repetition, keep going. If not, a different approach might get you there faster.`);
227
+ }
228
+ else if (this.max_repetition_count >= 8) {
229
+ messages.push(`Heads up: you have repeated a similar action ${this.max_repetition_count} times in the last ${this.recent_action_hashes.length} actions. Are you still making progress with each attempt? If so, carry on. Otherwise, it might be worth trying a different approach.`);
230
+ }
231
+ else if (this.max_repetition_count >= 5) {
232
+ messages.push(`Heads up: you have repeated a similar action ${this.max_repetition_count} times in the last ${this.recent_action_hashes.length} actions. If this is intentional and making progress, carry on. If not, it might be worth reconsidering your approach.`);
233
+ }
234
+ if (this.consecutive_stagnant_pages >= 5) {
235
+ messages.push(`The page content has not changed across ${this.consecutive_stagnant_pages} consecutive actions. Your actions might not be having the intended effect. It could be worth trying a different element or approach.`);
236
+ }
237
+ if (!messages.length) {
238
+ return null;
239
+ }
240
+ return messages.join('\n\n');
241
+ }
242
+ }
243
+ export const defaultMessageCompactionSettings = () => ({
244
+ enabled: true,
245
+ compact_every_n_steps: 15,
246
+ trigger_char_count: 40000,
247
+ trigger_token_count: null,
248
+ chars_per_token: 4,
249
+ keep_last_items: 6,
250
+ summary_max_chars: 6000,
251
+ include_read_state: false,
252
+ compaction_llm: null,
253
+ });
254
+ export const normalizeMessageCompactionSettings = (settings) => {
255
+ const merged = {
256
+ ...defaultMessageCompactionSettings(),
257
+ ...settings,
258
+ };
259
+ if (merged.trigger_char_count != null && merged.trigger_token_count != null) {
260
+ throw new Error('Set trigger_char_count or trigger_token_count for message_compaction, not both.');
261
+ }
262
+ if (merged.trigger_token_count != null) {
263
+ merged.trigger_char_count = Math.floor(merged.trigger_token_count * merged.chars_per_token);
264
+ }
265
+ else if (merged.trigger_char_count == null) {
266
+ merged.trigger_char_count = 40000;
267
+ }
268
+ return merged;
269
+ };
66
270
  export const defaultAgentSettings = () => ({
67
271
  session_attachment_mode: 'copy',
68
- allow_insecure_sensitive_data: false,
69
272
  use_vision: true,
273
+ include_recent_events: false,
70
274
  vision_detail_level: 'auto',
71
- use_vision_for_planner: false,
72
275
  save_conversation_path: null,
73
276
  save_conversation_path_encoding: 'utf-8',
74
277
  max_failures: 3,
75
- retry_delay: 10,
76
- validate_output: false,
77
278
  generate_gif: false,
78
279
  override_system_message: null,
79
280
  extend_system_message: null,
80
- include_attributes: [
81
- 'title',
82
- 'type',
83
- 'name',
84
- 'role',
85
- 'tabindex',
86
- 'aria-label',
87
- 'placeholder',
88
- 'value',
89
- 'alt',
90
- 'aria-expanded',
91
- ],
92
- max_actions_per_step: 10,
281
+ include_attributes: [...DEFAULT_INCLUDE_ATTRIBUTES],
282
+ max_actions_per_step: 5,
93
283
  use_thinking: true,
94
284
  flash_mode: false,
285
+ use_judge: true,
286
+ ground_truth: null,
95
287
  max_history_items: null,
96
288
  page_extraction_llm: null,
97
- planner_llm: null,
98
- planner_interval: 1,
99
- is_planner_reasoning: false,
100
- extend_planner_system_message: null,
289
+ enable_planning: true,
290
+ planning_replan_on_stall: 3,
291
+ planning_exploration_limit: 5,
101
292
  calculate_cost: false,
102
293
  include_tool_call_examples: false,
103
294
  llm_timeout: 60,
104
295
  step_timeout: 180,
296
+ final_response_after_failure: true,
297
+ message_compaction: null,
298
+ loop_detection_window: 20,
299
+ loop_detection_enabled: true,
105
300
  });
106
301
  export class AgentState {
107
302
  agent_id;
@@ -109,20 +304,32 @@ export class AgentState {
109
304
  consecutive_failures;
110
305
  last_result;
111
306
  last_plan;
307
+ plan;
308
+ current_plan_item_index;
309
+ plan_generation_step;
112
310
  last_model_output;
113
311
  paused;
114
312
  stopped;
313
+ session_initialized;
314
+ follow_up_task;
115
315
  message_manager_state;
116
316
  file_system_state;
317
+ loop_detector;
117
318
  constructor(init) {
118
319
  this.agent_id = init?.agent_id ?? '';
119
320
  this.n_steps = init?.n_steps ?? 1;
120
321
  this.consecutive_failures = init?.consecutive_failures ?? 0;
121
322
  this.last_result = init?.last_result ?? null;
122
323
  this.last_plan = init?.last_plan ?? null;
324
+ this.plan =
325
+ init?.plan?.map((item) => item instanceof PlanItem ? item : new PlanItem(item)) ?? null;
326
+ this.current_plan_item_index = init?.current_plan_item_index ?? 0;
327
+ this.plan_generation_step = init?.plan_generation_step ?? null;
123
328
  this.last_model_output = init?.last_model_output ?? null;
124
329
  this.paused = init?.paused ?? false;
125
330
  this.stopped = init?.stopped ?? false;
331
+ this.session_initialized = init?.session_initialized ?? false;
332
+ this.follow_up_task = init?.follow_up_task ?? false;
126
333
  if (init?.message_manager_state instanceof MessageManagerState) {
127
334
  this.message_manager_state = init.message_manager_state;
128
335
  }
@@ -133,6 +340,15 @@ export class AgentState {
133
340
  this.message_manager_state = new MessageManagerState();
134
341
  }
135
342
  this.file_system_state = init?.file_system_state ?? null;
343
+ if (init?.loop_detector instanceof ActionLoopDetector) {
344
+ this.loop_detector = init.loop_detector;
345
+ }
346
+ else if (init?.loop_detector) {
347
+ this.loop_detector = Object.assign(new ActionLoopDetector(), init.loop_detector);
348
+ }
349
+ else {
350
+ this.loop_detector = new ActionLoopDetector();
351
+ }
136
352
  }
137
353
  model_dump() {
138
354
  return {
@@ -141,11 +357,17 @@ export class AgentState {
141
357
  consecutive_failures: this.consecutive_failures,
142
358
  last_result: this.last_result?.map((result) => result.model_dump()) ?? null,
143
359
  last_plan: this.last_plan,
360
+ plan: this.plan?.map((item) => item.model_dump()) ?? null,
361
+ current_plan_item_index: this.current_plan_item_index,
362
+ plan_generation_step: this.plan_generation_step,
144
363
  last_model_output: this.last_model_output?.model_dump() ?? null,
145
364
  paused: this.paused,
146
365
  stopped: this.stopped,
366
+ session_initialized: this.session_initialized,
367
+ follow_up_task: this.follow_up_task,
147
368
  message_manager_state: JSON.parse(JSON.stringify(this.message_manager_state)),
148
369
  file_system_state: this.file_system_state,
370
+ loop_detector: JSON.parse(JSON.stringify(this.loop_detector)),
149
371
  };
150
372
  }
151
373
  toJSON() {
@@ -167,26 +389,46 @@ export class StepMetadata {
167
389
  step_start_time;
168
390
  step_end_time;
169
391
  step_number;
170
- constructor(step_start_time, step_end_time, step_number) {
392
+ step_interval;
393
+ constructor(step_start_time, step_end_time, step_number, step_interval = null) {
171
394
  this.step_start_time = step_start_time;
172
395
  this.step_end_time = step_end_time;
173
396
  this.step_number = step_number;
397
+ this.step_interval = step_interval;
174
398
  }
175
399
  get duration_seconds() {
176
400
  return this.step_end_time - this.step_start_time;
177
401
  }
178
402
  }
403
+ export class PlanItem {
404
+ text;
405
+ status;
406
+ constructor(init) {
407
+ this.text = init?.text ?? '';
408
+ this.status = init?.status ?? 'pending';
409
+ }
410
+ model_dump() {
411
+ return {
412
+ text: this.text,
413
+ status: this.status,
414
+ };
415
+ }
416
+ }
179
417
  export class AgentOutput {
180
418
  thinking;
181
419
  evaluation_previous_goal;
182
420
  memory;
183
421
  next_goal;
422
+ current_plan_item;
423
+ plan_update;
184
424
  action;
185
425
  constructor(init) {
186
426
  this.thinking = init?.thinking ?? null;
187
427
  this.evaluation_previous_goal = init?.evaluation_previous_goal ?? null;
188
428
  this.memory = init?.memory ?? null;
189
429
  this.next_goal = init?.next_goal ?? null;
430
+ this.current_plan_item = init?.current_plan_item ?? null;
431
+ this.plan_update = init?.plan_update ?? null;
190
432
  this.action = (init?.action ?? []).map((entry) => entry instanceof ActionModel ? entry : new ActionModel(entry));
191
433
  }
192
434
  get current_state() {
@@ -203,6 +445,8 @@ export class AgentOutput {
203
445
  evaluation_previous_goal: this.evaluation_previous_goal,
204
446
  memory: this.memory,
205
447
  next_goal: this.next_goal,
448
+ current_plan_item: this.current_plan_item,
449
+ plan_update: this.plan_update,
206
450
  action: this.action.map((action) => action.model_dump?.() ?? action),
207
451
  };
208
452
  }
@@ -224,6 +468,12 @@ export class AgentOutput {
224
468
  evaluation_previous_goal: data.evaluation_previous_goal ?? null,
225
469
  memory: data.memory ?? null,
226
470
  next_goal: data.next_goal ?? null,
471
+ current_plan_item: typeof data.current_plan_item === 'number'
472
+ ? data.current_plan_item
473
+ : null,
474
+ plan_update: Array.isArray(data.plan_update)
475
+ ? data.plan_update.filter((item) => typeof item === 'string')
476
+ : null,
227
477
  action: actions,
228
478
  });
229
479
  }
@@ -255,6 +505,8 @@ export class AgentOutput {
255
505
  this.thinking = null;
256
506
  this.evaluation_previous_goal = null;
257
507
  this.next_goal = null;
508
+ this.current_plan_item = null;
509
+ this.plan_update = null;
258
510
  }
259
511
  };
260
512
  }
@@ -264,11 +516,13 @@ export class AgentHistory {
264
516
  result;
265
517
  state;
266
518
  metadata;
267
- constructor(model_output, result, state, metadata = null) {
519
+ state_message;
520
+ constructor(model_output, result, state, metadata = null, state_message = null) {
268
521
  this.model_output = model_output;
269
522
  this.result = result;
270
523
  this.state = state;
271
524
  this.metadata = metadata;
525
+ this.state_message = state_message;
272
526
  }
273
527
  static get_interacted_element(model_output, selector_map) {
274
528
  const elements = [];
@@ -284,9 +538,72 @@ export class AgentHistory {
284
538
  }
285
539
  return elements;
286
540
  }
287
- toJSON() {
541
+ static _filterSensitiveDataFromString(value, sensitive_data) {
542
+ if (!sensitive_data) {
543
+ return value;
544
+ }
545
+ const placeholders = {};
546
+ for (const [keyOrDomain, content] of Object.entries(sensitive_data)) {
547
+ if (typeof content === 'string' && content) {
548
+ placeholders[keyOrDomain] = content;
549
+ }
550
+ else if (content && typeof content === 'object') {
551
+ for (const [key, val] of Object.entries(content)) {
552
+ if (val) {
553
+ placeholders[key] = val;
554
+ }
555
+ }
556
+ }
557
+ }
558
+ if (!Object.keys(placeholders).length) {
559
+ return value;
560
+ }
561
+ let filtered = value;
562
+ for (const [key, secret] of Object.entries(placeholders)) {
563
+ filtered = filtered.split(secret).join(`<secret>${key}</secret>`);
564
+ }
565
+ return filtered;
566
+ }
567
+ static _filterSensitiveDataFromDict(data, sensitive_data) {
568
+ if (!sensitive_data) {
569
+ return data;
570
+ }
571
+ const filtered = {};
572
+ for (const [key, value] of Object.entries(data)) {
573
+ if (typeof value === 'string') {
574
+ filtered[key] = this._filterSensitiveDataFromString(value, sensitive_data);
575
+ }
576
+ else if (value && typeof value === 'object' && !Array.isArray(value)) {
577
+ filtered[key] = this._filterSensitiveDataFromDict(value, sensitive_data);
578
+ }
579
+ else if (Array.isArray(value)) {
580
+ filtered[key] = value.map((item) => {
581
+ if (typeof item === 'string') {
582
+ return this._filterSensitiveDataFromString(item, sensitive_data);
583
+ }
584
+ if (item && typeof item === 'object' && !Array.isArray(item)) {
585
+ return this._filterSensitiveDataFromDict(item, sensitive_data);
586
+ }
587
+ return item;
588
+ });
589
+ }
590
+ else {
591
+ filtered[key] = value;
592
+ }
593
+ }
594
+ return filtered;
595
+ }
596
+ toJSON(sensitive_data = null) {
597
+ let modelOutput = this.model_output?.toJSON() ?? null;
598
+ if (modelOutput &&
599
+ Array.isArray(modelOutput.action) &&
600
+ sensitive_data) {
601
+ modelOutput.action = modelOutput.action.map((action) => Object.prototype.hasOwnProperty.call(action, 'input')
602
+ ? AgentHistory._filterSensitiveDataFromDict(action, sensitive_data)
603
+ : action);
604
+ }
288
605
  return {
289
- model_output: this.model_output?.toJSON() ?? null,
606
+ model_output: modelOutput,
290
607
  result: this.result.map((r) => r.toJSON()),
291
608
  state: this.state.to_dict(),
292
609
  metadata: this.metadata
@@ -294,8 +611,10 @@ export class AgentHistory {
294
611
  step_start_time: this.metadata.step_start_time,
295
612
  step_end_time: this.metadata.step_end_time,
296
613
  step_number: this.metadata.step_number,
614
+ step_interval: this.metadata.step_interval,
297
615
  }
298
616
  : null,
617
+ state_message: this.state_message,
299
618
  };
300
619
  }
301
620
  }
@@ -360,6 +679,27 @@ export class AgentHistoryList {
360
679
  }
361
680
  return null;
362
681
  }
682
+ judgement() {
683
+ if (!this.history.length) {
684
+ return null;
685
+ }
686
+ const last = this.history[this.history.length - 1];
687
+ const result = last.result[last.result.length - 1];
688
+ if (result?.judgement && typeof result.judgement === 'object') {
689
+ return result.judgement;
690
+ }
691
+ return null;
692
+ }
693
+ is_judged() {
694
+ return this.judgement() != null;
695
+ }
696
+ is_validated() {
697
+ const judgement = this.judgement();
698
+ if (!judgement) {
699
+ return null;
700
+ }
701
+ return judgement.verdict === true;
702
+ }
363
703
  has_errors() {
364
704
  return this.errors().some((error) => error != null);
365
705
  }
@@ -481,6 +821,32 @@ export class AgentHistoryList {
481
821
  number_of_steps() {
482
822
  return this.history.length;
483
823
  }
824
+ agent_steps() {
825
+ const steps = [];
826
+ for (let stepIndex = 0; stepIndex < this.history.length; stepIndex += 1) {
827
+ const historyItem = this.history[stepIndex];
828
+ let stepText = `Step ${stepIndex + 1}:\n`;
829
+ if (historyItem.model_output?.action?.length) {
830
+ const actions = historyItem.model_output.action.map((action) => typeof action?.model_dump === 'function'
831
+ ? action.model_dump()
832
+ : action);
833
+ stepText += `Actions: ${JSON.stringify(actions, null, 1)}\n`;
834
+ }
835
+ if (historyItem.result?.length) {
836
+ for (let resultIndex = 0; resultIndex < historyItem.result.length; resultIndex += 1) {
837
+ const result = historyItem.result[resultIndex];
838
+ if (result?.extracted_content) {
839
+ stepText += `Result ${resultIndex + 1}: ${String(result.extracted_content)}\n`;
840
+ }
841
+ if (result?.error) {
842
+ stepText += `Error ${resultIndex + 1}: ${String(result.error)}\n`;
843
+ }
844
+ }
845
+ }
846
+ steps.push(stepText);
847
+ }
848
+ return steps;
849
+ }
484
850
  get structured_output() {
485
851
  const final_result = this.final_result();
486
852
  if (!final_result || !this._output_model_schema) {
@@ -488,14 +854,24 @@ export class AgentHistoryList {
488
854
  }
489
855
  return parseStructuredOutput(this._output_model_schema, final_result);
490
856
  }
491
- save_to_file(filepath) {
857
+ get_structured_output(outputModel) {
858
+ const finalResult = this.final_result();
859
+ if (!finalResult) {
860
+ return null;
861
+ }
862
+ return parseStructuredOutput(outputModel, finalResult);
863
+ }
864
+ save_to_file(filepath, sensitive_data = null) {
492
865
  const dir = path.dirname(filepath);
493
866
  fs.mkdirSync(dir, { recursive: true });
494
- fs.writeFileSync(filepath, JSON.stringify(this.toJSON(), null, 2), 'utf-8');
867
+ fs.writeFileSync(filepath, JSON.stringify(this.toJSON(sensitive_data), null, 2), 'utf-8');
495
868
  }
496
869
  static load_from_file(filepath, outputModel) {
497
870
  const content = fs.readFileSync(filepath, 'utf-8');
498
871
  const payload = JSON.parse(content);
872
+ return AgentHistoryList.load_from_dict(payload, outputModel);
873
+ }
874
+ static load_from_dict(payload, outputModel) {
499
875
  const historyItems = (payload.history ?? []).map((entry) => {
500
876
  const modelOutput = entry.model_output
501
877
  ? outputModel.fromJSON(entry.model_output)
@@ -503,20 +879,45 @@ export class AgentHistoryList {
503
879
  const result = (entry.result ?? []).map((item) => new ActionResult(item));
504
880
  const state = new BrowserStateHistory(entry.state?.url ?? '', entry.state?.title ?? '', entry.state?.tabs ?? [], entry.state?.interacted_element ?? [], entry.state?.screenshot_path ?? null);
505
881
  const metadata = entry.metadata
506
- ? new StepMetadata(entry.metadata.step_start_time, entry.metadata.step_end_time, entry.metadata.step_number)
882
+ ? new StepMetadata(entry.metadata.step_start_time, entry.metadata.step_end_time, entry.metadata.step_number, entry.metadata.step_interval ?? null)
507
883
  : null;
508
- return new AgentHistory(modelOutput, result, state, metadata);
884
+ return new AgentHistory(modelOutput, result, state, metadata, entry.state_message ?? null);
509
885
  });
510
886
  return new AgentHistoryList(historyItems);
511
887
  }
512
- toJSON() {
888
+ toJSON(sensitive_data = null) {
513
889
  return {
514
- history: this.history.map((item) => item.toJSON()),
515
- usage: this.usage,
890
+ history: this.history.map((item) => item.toJSON(sensitive_data)),
516
891
  };
517
892
  }
893
+ model_dump(sensitive_data = null) {
894
+ return this.toJSON(sensitive_data);
895
+ }
896
+ }
897
+ export class DetectedVariable {
898
+ name;
899
+ original_value;
900
+ type;
901
+ format;
902
+ constructor(name, original_value, type = 'string', format = null) {
903
+ this.name = name;
904
+ this.original_value = original_value;
905
+ this.type = type;
906
+ this.format = format;
907
+ }
518
908
  model_dump() {
519
- return this.toJSON();
909
+ return {
910
+ name: this.name,
911
+ original_value: this.original_value,
912
+ type: this.type,
913
+ format: this.format,
914
+ };
915
+ }
916
+ }
917
+ export class VariableMetadata {
918
+ detected_variables;
919
+ constructor(detected_variables = {}) {
920
+ this.detected_variables = detected_variables;
520
921
  }
521
922
  }
522
923
  export class AgentError extends Error {
@@ -530,6 +931,18 @@ export class AgentError extends Error {
530
931
  if (error.name === 'RateLimitError') {
531
932
  return AgentError.RATE_LIMIT_ERROR;
532
933
  }
934
+ const errorStr = error.message ?? String(error);
935
+ if (errorStr.includes('LLM response missing required fields') ||
936
+ errorStr.includes('Expected format: AgentOutput')) {
937
+ const [mainError] = errorStr.split('\n');
938
+ let helpfulMessage = `${mainError}\n\n` +
939
+ 'The previous response had an invalid output structure. ' +
940
+ 'Please stick to the required output format. \n\n';
941
+ if (include_trace && error?.stack) {
942
+ helpfulMessage += `\n\nFull stacktrace:\n${error.stack}`;
943
+ }
944
+ return helpfulMessage;
945
+ }
533
946
  if (include_trace && error?.stack) {
534
947
  return `${error.message}\nStacktrace:\n${error.stack}`;
535
948
  }