@agentium/browser 3.1.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,66 +1,185 @@
1
- "use strict";
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __export = (target, all) => {
9
- for (var name in all)
10
- __defProp(target, name, { get: all[name], enumerable: true });
11
- };
12
- var __copyProps = (to, from, except, desc) => {
13
- if (from && typeof from === "object" || typeof from === "function") {
14
- for (let key of __getOwnPropNames(from))
15
- if (!__hasOwnProp.call(to, key) && key !== except)
16
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
- }
18
- return to;
19
- };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
-
30
- // src/index.ts
31
- var index_exports = {};
32
- __export(index_exports, {
33
- BrowserAgent: () => BrowserAgent,
34
- BrowserProvider: () => BrowserProvider,
35
- CredentialVault: () => CredentialVault
36
- });
37
- module.exports = __toCommonJS(index_exports);
38
-
39
- // src/browser-agent.ts
40
- var import_core = require("@agentium/core");
41
- var import_zod = require("zod");
42
-
43
- // src/stealth.ts
44
- var REALISTIC_USER_AGENTS = [
45
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
46
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
47
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
48
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
49
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
50
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15"
1
+ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
+ let _agentium_core = require("@agentium/core");
3
+ let zod_v3 = require("zod/v3");
4
+ function searchUrl(query, engine = "duckduckgo") {
5
+ const q = encodeURIComponent(query.trim());
6
+ switch (engine) {
7
+ case "google": return `https://www.google.com/search?q=${q}`;
8
+ case "bing": return `https://www.bing.com/search?q=${q}`;
9
+ default: return `https://html.duckduckgo.com/html/?q=${q}`;
10
+ }
11
+ }
12
+ function isSearchResultsUrl(url) {
13
+ try {
14
+ const u = new URL(url);
15
+ const host = u.hostname.replace(/^www\./, "");
16
+ if (host === "duckduckgo.com" || host.endsWith(".duckduckgo.com")) return u.searchParams.has("q") || u.pathname.includes("/html");
17
+ if (host === "google.com" || host.endsWith(".google.com")) return u.pathname.startsWith("/search");
18
+ if (host === "bing.com" || host.endsWith(".bing.com")) return u.pathname.startsWith("/search");
19
+ return false;
20
+ } catch {
21
+ return false;
22
+ }
23
+ }
24
+ /** Bot-block / challenge pages (DDG 418, Google sorry, Cloudflare, …). */
25
+ function isBlockedPageUrl(url) {
26
+ return /418\.html|\/sorry\/|captcha|challenge|cf-browser-verification/i.test(url);
27
+ }
28
+ function isBotChallengeText(text) {
29
+ return /unfortunately, bots use|select all squares containing a duck|unusual traffic from your computer|are you a robot|verify you are human|detected unusual traffic/i.test(text);
30
+ }
31
+ const CHROME_LABEL = /^(email us|feedback|privacy|terms|settings|facebook|twitter|reddit|about|help|sign in|log in|login|all regions|any time|more results|next|previous|images|videos|news|maps|shopping|lite|duckduckgo|menu|home|cookies|advertising|github|wikipedia|instagram|youtube)$/i;
32
+ function isResultTitle(label) {
33
+ const t = label.replace(/\s+/g, " ").trim();
34
+ if (t.length < 16) return false;
35
+ if (CHROME_LABEL.test(t)) return false;
36
+ if (/^(https?:|javascript:|mailto:)/i.test(t)) return false;
37
+ return true;
38
+ }
39
+ function looksLikeResultList(text) {
40
+ const lines = text.split("\n").map((l) => l.replace(/^\d+\.\s*/, "").trim()).filter(Boolean);
41
+ return lines.length >= 2 && lines.filter(isResultTitle).length >= 2;
42
+ }
43
+ /** Longest non-chrome link labels — search result titles, not "email us". */
44
+ function titlesFromElements(elements, max = 5) {
45
+ const seen = /* @__PURE__ */ new Set();
46
+ const labels = [];
47
+ for (const e of elements) {
48
+ if (e.tag !== "a" && e.role !== "link" && e.role !== "heading") continue;
49
+ const label = e.label?.replace(/\s+/g, " ").trim() ?? "";
50
+ if (!isResultTitle(label) || seen.has(label.toLowerCase())) continue;
51
+ seen.add(label.toLowerCase());
52
+ labels.push(label);
53
+ }
54
+ labels.sort((a, b) => b.length - a.length);
55
+ return labels.slice(0, max).map((t, i) => `${i + 1}. ${t}`).join("\n");
56
+ }
57
+ const SERP_TITLE_SELECTORS = [
58
+ "a.result__a",
59
+ "a.result-link",
60
+ "#links .result__a",
61
+ "h2 a",
62
+ "h3 a",
63
+ "a > h3"
64
+ ];
65
+ function guessSearchQuery(task) {
66
+ const quoted = task.match(/["']([^"']{1,200})["']/);
67
+ if (quoted?.[1]) return quoted[1].trim();
68
+ return task.replace(/^(please\s+)?(search for|google|bing|find|look up)\s+/i, "").slice(0, 200).trim();
69
+ }
70
+ /**
71
+ * Closed action list for this frame: chrome (back, scroll, done, …) plus
72
+ * `click_N` / `type_N` from the DOM snapshot. Used by the Jev planner and
73
+ * as documentation of what `executeAction` can run without free-form JSON.
74
+ */
75
+ function buildActionSpace(elements, tabs = [], opts) {
76
+ const max = opts?.max ?? 40;
77
+ const criteria = {
78
+ back: "Go to the previous page",
79
+ screenshot: "Take a fresh screenshot next step",
80
+ fail: "Give up — the task cannot be completed",
81
+ new_tab: "Open a blank new tab"
82
+ };
83
+ if (opts?.allowWait !== false) criteria.wait = "Wait for the page to settle. Use at most once.";
84
+ if (opts?.allowDone !== false) criteria.done = "The task is finished — return the result titles you already found";
85
+ if (opts?.allowSearch !== false) criteria.search = "Open a NEW web search. Do not pick this if results are already on screen.";
86
+ if ((opts?.pagesBelow ?? 1) > 0) criteria.scroll_down = "Scroll down one viewport";
87
+ if ((opts?.pagesAbove ?? 1) > 0) criteria.scroll_up = "Scroll up one viewport";
88
+ const slice = elements.slice(0, max);
89
+ for (const e of slice) {
90
+ const label = e.label?.trim() || e.role || e.tag;
91
+ criteria[`click_${e.index}`] = `Click [${e.index}] ${e.role}: ${label}`;
92
+ if (e.isInput) criteria[`type_${e.index}`] = `Type into [${e.index}] ${label}`;
93
+ }
94
+ for (const tab of tabs) {
95
+ if (!tab.active) criteria[`switch_tab_${tab.id}`] = `Switch to tab ${tab.id} (${tab.url || "blank"})`;
96
+ if (tabs.length > 1) criteria[`close_tab_${tab.id}`] = `Close tab ${tab.id}`;
97
+ }
98
+ return {
99
+ criteria,
100
+ toAction(label) {
101
+ return labelToAction(label);
102
+ }
103
+ };
104
+ }
105
+ function labelToAction(label, extras) {
106
+ if (label === "back") return { action: "back" };
107
+ if (label === "scroll_down") return {
108
+ action: "scroll",
109
+ direction: "down"
110
+ };
111
+ if (label === "scroll_up") return {
112
+ action: "scroll",
113
+ direction: "up"
114
+ };
115
+ if (label === "wait") return {
116
+ action: "wait",
117
+ ms: 1500
118
+ };
119
+ if (label === "screenshot") return { action: "screenshot" };
120
+ if (label === "done") return {
121
+ action: "done",
122
+ result: extras?.doneResult ?? "Done"
123
+ };
124
+ if (label === "fail") return {
125
+ action: "fail",
126
+ reason: "Jev chose fail"
127
+ };
128
+ if (label === "search") return {
129
+ action: "search",
130
+ query: extras?.searchQuery ?? ""
131
+ };
132
+ if (label === "new_tab") return { action: "new_tab" };
133
+ if (label === "none") return void 0;
134
+ const click = /^click_(\d+)$/.exec(label);
135
+ if (click) return {
136
+ action: "click",
137
+ index: Number(click[1])
138
+ };
139
+ const type = /^type_(\d+)$/.exec(label);
140
+ if (type) return {
141
+ action: "type",
142
+ index: Number(type[1]),
143
+ text: extras?.typeText ?? ""
144
+ };
145
+ const sw = /^switch_tab_(.+)$/.exec(label);
146
+ if (sw) return {
147
+ action: "switch_tab",
148
+ tabId: sw[1]
149
+ };
150
+ const cl = /^close_tab_(.+)$/.exec(label);
151
+ if (cl) return {
152
+ action: "close_tab",
153
+ tabId: cl[1]
154
+ };
155
+ }
156
+ //#endregion
157
+ //#region src/stealth.ts
158
+ const REALISTIC_USER_AGENTS = [
159
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
160
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
161
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
162
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
163
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
164
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15"
51
165
  ];
166
+ /** Pick a random realistic user-agent string. */
52
167
  function pickUserAgent() {
53
- return REALISTIC_USER_AGENTS[Math.floor(Math.random() * REALISTIC_USER_AGENTS.length)];
168
+ return REALISTIC_USER_AGENTS[Math.floor(Math.random() * REALISTIC_USER_AGENTS.length)];
54
169
  }
170
+ /**
171
+ * JavaScript that runs inside every page to patch common bot-detection vectors.
172
+ * Injected via Playwright's `context.addInitScript()`.
173
+ */
55
174
  function getStealthScript() {
56
- return `
57
- // \u2500\u2500 navigator.webdriver \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
175
+ return `
176
+ // ── navigator.webdriver ──────────────────────────────────────────
58
177
  Object.defineProperty(navigator, 'webdriver', {
59
178
  get: () => undefined,
60
179
  configurable: true,
61
180
  });
62
181
 
63
- // \u2500\u2500 navigator.plugins \u2014 appear non-empty \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
182
+ // ── navigator.plugins — appear non-empty ─────────────────────────
64
183
  Object.defineProperty(navigator, 'plugins', {
65
184
  get: () => {
66
185
  const plugins = [
@@ -74,13 +193,13 @@ function getStealthScript() {
74
193
  configurable: true,
75
194
  });
76
195
 
77
- // \u2500\u2500 navigator.languages \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
196
+ // ── navigator.languages ──────────────────────────────────────────
78
197
  Object.defineProperty(navigator, 'languages', {
79
198
  get: () => ['en-US', 'en'],
80
199
  configurable: true,
81
200
  });
82
201
 
83
- // \u2500\u2500 navigator.permissions.query \u2014 hide "denied" for notifications \u2500
202
+ // ── navigator.permissions.query — hide "denied" for notifications ─
84
203
  const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions);
85
204
  window.navigator.permissions.query = (params) => {
86
205
  if (params.name === 'notifications') {
@@ -89,7 +208,7 @@ function getStealthScript() {
89
208
  return originalQuery(params);
90
209
  };
91
210
 
92
- // \u2500\u2500 chrome runtime \u2014 make it look like a real Chrome \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
211
+ // ── chrome runtime — make it look like a real Chrome ─────────────
93
212
  if (!window.chrome) {
94
213
  window.chrome = {};
95
214
  }
@@ -100,7 +219,7 @@ function getStealthScript() {
100
219
  };
101
220
  }
102
221
 
103
- // \u2500\u2500 WebGL renderer \u2014 mask headless indicators \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
222
+ // ── WebGL renderer — mask headless indicators ────────────────────
104
223
  const getParameterOrig = WebGLRenderingContext.prototype.getParameter;
105
224
  WebGLRenderingContext.prototype.getParameter = function(param) {
106
225
  if (param === 37445) return 'Intel Inc.';
@@ -108,7 +227,7 @@ function getStealthScript() {
108
227
  return getParameterOrig.call(this, param);
109
228
  };
110
229
 
111
- // \u2500\u2500 WebGL2 renderer \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
230
+ // ── WebGL2 renderer ──────────────────────────────────────────────
112
231
  if (typeof WebGL2RenderingContext !== 'undefined') {
113
232
  const getParam2Orig = WebGL2RenderingContext.prototype.getParameter;
114
233
  WebGL2RenderingContext.prototype.getParameter = function(param) {
@@ -118,8 +237,8 @@ function getStealthScript() {
118
237
  };
119
238
  }
120
239
 
121
- // \u2500\u2500 Remove "cdc_" Playwright/ChromeDriver markers from new elements \u2500
122
- // Only watches childList additions, NOT every attribute mutation \u2014
240
+ // ── Remove "cdc_" Playwright/ChromeDriver markers from new elements ─
241
+ // Only watches childList additions, NOT every attribute mutation —
123
242
  // attributes:true subtree:true is O(every DOM write) on heavy SPAs
124
243
  // like FreightOS and noticeably slows the page.
125
244
  try {
@@ -148,580 +267,582 @@ function getStealthScript() {
148
267
  // An earlier version returned the parent window, which breaks legitimate
149
268
  // page JS on any site that uses iframes (checkout widgets, embedded
150
269
  // videos, A/B test scripts, analytics). The "anti-detection" value was
151
- // minimal \u2014 modern detectors don't rely on that property anyway \u2014 and
270
+ // minimal — modern detectors don't rely on that property anyway — and
152
271
  // the breakage caused pages to fail to boot, leaving font-test residue
153
272
  // on screen.
154
273
  `;
155
274
  }
275
+ /**
276
+ * Build Playwright context options from a StealthConfig.
277
+ */
156
278
  function buildStealthContextOpts(config, viewport) {
157
- const opts = {
158
- viewport,
159
- userAgent: config.userAgent ?? pickUserAgent(),
160
- locale: config.locale ?? "en-US",
161
- timezoneId: config.timezone ?? "America/New_York",
162
- colorScheme: "light",
163
- // Default to 1 — matches the most common real-user setup and avoids
164
- // visual zoom/stretch in headed mode on non-Retina displays (the host
165
- // OS would downsample a 2× rendered surface). Users on Retina-only
166
- // deployments can opt back into DPR=2 for sharper screenshots.
167
- deviceScaleFactor: config.deviceScaleFactor ?? 1,
168
- hasTouch: false,
169
- javaScriptEnabled: true,
170
- ignoreHTTPSErrors: config.ignoreHTTPSErrors ?? false
171
- };
172
- if (config.geolocation) {
173
- opts.geolocation = config.geolocation;
174
- opts.permissions = ["geolocation"];
175
- }
176
- return opts;
279
+ const opts = {
280
+ viewport,
281
+ userAgent: config.userAgent ?? pickUserAgent(),
282
+ locale: config.locale ?? "en-US",
283
+ timezoneId: config.timezone ?? "America/New_York",
284
+ colorScheme: "light",
285
+ deviceScaleFactor: config.deviceScaleFactor ?? 1,
286
+ hasTouch: false,
287
+ javaScriptEnabled: true,
288
+ ignoreHTTPSErrors: config.ignoreHTTPSErrors ?? false
289
+ };
290
+ if (config.geolocation) {
291
+ opts.geolocation = config.geolocation;
292
+ opts.permissions = ["geolocation"];
293
+ }
294
+ return opts;
177
295
  }
296
+ /**
297
+ * Build Playwright launch options for stealth.
298
+ */
178
299
  function buildStealthLaunchArgs(config) {
179
- const args = [
180
- "--disable-blink-features=AutomationControlled",
181
- "--disable-infobars",
182
- "--disable-dev-shm-usage",
183
- "--no-first-run",
184
- "--no-default-browser-check"
185
- ];
186
- const proxy = config.proxy ? { server: config.proxy.server, username: config.proxy.username, password: config.proxy.password } : void 0;
187
- return { args, proxy };
300
+ return {
301
+ args: [
302
+ "--disable-blink-features=AutomationControlled",
303
+ "--disable-infobars",
304
+ "--disable-dev-shm-usage",
305
+ "--no-first-run",
306
+ "--no-default-browser-check"
307
+ ],
308
+ proxy: config.proxy ? {
309
+ server: config.proxy.server,
310
+ username: config.proxy.username,
311
+ password: config.proxy.password
312
+ } : void 0
313
+ };
188
314
  }
189
-
190
- // src/browser-provider.ts
315
+ //#endregion
316
+ //#region src/browser-provider.ts
317
+ /**
318
+ * Playwright wrapper with stealth anti-detection, human-like behavior,
319
+ * indexed DOM element resolution, and a rich action vocabulary.
320
+ *
321
+ * The `BrowserProvider` is intentionally LLM-agnostic — it exposes the
322
+ * primitives that `BrowserAgent` orchestrates via vision+DOM reasoning.
323
+ */
191
324
  var BrowserProvider = class {
192
- browser = null;
193
- context = null;
194
- page = null;
195
- pages = /* @__PURE__ */ new Map();
196
- activeTabId = "tab-0";
197
- tabCounter = 0;
198
- _viewport;
199
- _videoDir;
200
- _humanize;
201
- /**
202
- * Most recent DOM snapshot (one per `extractDOM` call). Indexed actions
203
- * (`clickByIndex`, `inputByIndex`, …) resolve their `index` against this.
204
- */
205
- _lastDom = [];
206
- /** True if we connected over CDP (don't tear down the browser on close). */
207
- _attached = false;
208
- constructor() {
209
- this._viewport = { width: 1280, height: 720 };
210
- }
211
- // ── Lifecycle ────────────────────────────────────────────────────────
212
- async launch(opts) {
213
- const pw = await import("playwright");
214
- const chromium = pw.chromium;
215
- this._viewport = opts?.viewport ?? { width: 1280, height: 720 };
216
- if (opts?.humanize) {
217
- const h = typeof opts.humanize === "object" ? opts.humanize : {};
218
- this._humanize = {
219
- typingDelay: h.typingDelay ?? [40, 120],
220
- clickJitter: h.clickJitter ?? 3,
221
- actionDelay: h.actionDelay ?? [200, 800],
222
- mouseMovement: h.mouseMovement ?? true
223
- };
224
- }
225
- if (opts?.cdpUrl) {
226
- this.browser = await chromium.connectOverCDP(opts.cdpUrl);
227
- this._attached = true;
228
- const contexts = this.browser.contexts();
229
- this.context = contexts.length > 0 ? contexts[0] : await this.browser.newContext({ viewport: this._viewport });
230
- const existingPages = this.context.pages();
231
- this.page = existingPages.length > 0 ? existingPages[0] : await this.context.newPage();
232
- this.tabCounter = 0;
233
- this.activeTabId = "tab-0";
234
- this.pages.set("tab-0", this.page);
235
- return;
236
- }
237
- const stealthEnabled = !!opts?.stealth;
238
- const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
239
- const headless = opts?.headless ?? true;
240
- const launchOpts = { headless };
241
- const args = [];
242
- if (stealthEnabled) {
243
- const stealth = buildStealthLaunchArgs(stealthCfg);
244
- args.push(...stealth.args);
245
- if (stealth.proxy) launchOpts.proxy = stealth.proxy;
246
- }
247
- if (!headless) {
248
- const CHROME_VERTICAL_CHROME_PX = 140;
249
- args.push(`--window-size=${this._viewport.width},${this._viewport.height + CHROME_VERTICAL_CHROME_PX}`);
250
- args.push("--window-position=0,0");
251
- }
252
- if (args.length) launchOpts.args = args;
253
- this.browser = await chromium.launch(launchOpts);
254
- let contextOpts;
255
- if (stealthEnabled) {
256
- contextOpts = buildStealthContextOpts(stealthCfg, this._viewport);
257
- } else {
258
- contextOpts = {
259
- viewport: this._viewport,
260
- userAgent: pickUserAgent()
261
- };
262
- }
263
- if (opts?.storageState) {
264
- contextOpts.storageState = opts.storageState;
265
- }
266
- if (opts?.recordVideo) {
267
- const dir = typeof opts.recordVideo === "object" ? opts.recordVideo.dir : "./browser-videos";
268
- contextOpts.recordVideo = { dir, size: this._viewport };
269
- this._videoDir = dir;
270
- }
271
- this.context = await this.browser.newContext(contextOpts);
272
- if (stealthEnabled && stealthCfg.patchFingerprint !== false) {
273
- await this.context.addInitScript(getStealthScript());
274
- }
275
- this.page = await this.context.newPage();
276
- this.tabCounter = 0;
277
- this.activeTabId = "tab-0";
278
- this.pages.set("tab-0", this.page);
279
- }
280
- // ── Cookie / Auth Persistence ────────────────────────────────────────
281
- async saveStorageState(path) {
282
- this.ensureContext();
283
- await this.context.storageState({ path });
284
- }
285
- // ── Navigation ───────────────────────────────────────────────────────
286
- async navigate(url) {
287
- this.ensurePage();
288
- if (!/^https?:\/\//i.test(url)) {
289
- throw new Error(`Invalid URL scheme: only http:// and https:// are allowed`);
290
- }
291
- await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
292
- await this.waitForStable(500);
293
- }
294
- async back() {
295
- this.ensurePage();
296
- await this.page.goBack({ waitUntil: "domcontentloaded", timeout: 15e3 });
297
- }
298
- // ── Screenshot ───────────────────────────────────────────────────────
299
- async screenshot() {
300
- this.ensurePage();
301
- return await this.page.screenshot({
302
- type: "png",
303
- fullPage: false,
304
- scale: "css"
305
- });
306
- }
307
- /** Viewport size in CSS pixels (matches screenshot dimensions). */
308
- get viewport() {
309
- return this._viewport;
310
- }
311
- /** Most recent DOM snapshot. Each entry has a stable `index`. */
312
- get lastDom() {
313
- return this._lastDom;
314
- }
315
- // ── Coordinate-based Interaction ─────────────────────────────────────
316
- async click(x, y) {
317
- this.ensurePage();
318
- const [cx, cy] = this.clampToViewport(x, y);
319
- const [fx, fy] = this.jitter(cx, cy);
320
- if (this._humanize?.mouseMovement) {
321
- await this.humanMouseMove(fx, fy);
322
- }
323
- await this.page.mouse.click(fx, fy);
324
- await this.humanPause();
325
- }
326
- async type(text) {
327
- this.ensurePage();
328
- const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
329
- await this.page.keyboard.type(text, { delay });
330
- await this.humanPause();
331
- }
332
- async clickAndType(x, y, text) {
333
- const [cx, cy] = this.clampToViewport(x, y);
334
- await this.click(cx, cy);
335
- await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
336
- const [fx, fy] = this.jitter(cx, cy);
337
- await this.page.mouse.click(fx, fy, { clickCount: 3 });
338
- await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
339
- await this.type(text);
340
- }
341
- async pressKey(key) {
342
- this.ensurePage();
343
- await this.page.keyboard.press(key);
344
- }
345
- /**
346
- * Send arbitrary keyboard keys / shortcuts. Accepts a single
347
- * Playwright key spec (`"Enter"`, `"Control+l"`, `"Shift+ArrowDown"`)
348
- * or a space-separated sequence (`"Tab Tab Enter"`).
349
- */
350
- async sendKeys(keys) {
351
- this.ensurePage();
352
- const tokens = keys.trim().split(/\s+/).filter(Boolean);
353
- for (const token of tokens) {
354
- await this.page.keyboard.press(token);
355
- await this.sleep(this._humanize ? this.randInt(40, 120) : 30);
356
- }
357
- await this.humanPause();
358
- }
359
- async scroll(direction, amount) {
360
- this.ensurePage();
361
- const base = amount ?? 400;
362
- const jittered = this._humanize ? base + this.randInt(-40, 40) : base;
363
- const scrollY = direction === "down" ? jittered : -jittered;
364
- if (this._humanize) {
365
- const steps = this.randInt(2, 4);
366
- const perStep = scrollY / steps;
367
- for (let i = 0; i < steps; i++) {
368
- await this.page.mouse.wheel(0, perStep);
369
- await this.sleep(this.randInt(30, 80));
370
- }
371
- } else {
372
- await this.page.mouse.wheel(0, scrollY);
373
- }
374
- await this.humanPause();
375
- }
376
- // ── Indexed Interaction (preferred path) ─────────────────────────────
377
- /**
378
- * Build a Playwright locator for a DOM-snapshot index. Each `extractDOM`
379
- * call tags surviving elements with `data-bua-idx="<n>"`; we resolve by
380
- * that attribute. Returns null if the index is unknown.
381
- */
382
- locatorForIndex(index) {
383
- if (!this.page) return null;
384
- if (!this._lastDom.find((e) => e.index === index)) return null;
385
- return this.page.locator(`[data-bua-idx="${index}"]`).first();
386
- }
387
- /**
388
- * Click an element by its DOM-snapshot index. The most reliable click
389
- * path on dynamic pages — survives layout shifts and DPR oddities.
390
- */
391
- async clickByIndex(index, opts) {
392
- this.ensurePage();
393
- const loc = this.locatorForIndex(index);
394
- if (!loc) return false;
395
- try {
396
- await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {
397
- });
398
- await loc.click({ timeout: opts?.timeout ?? 5e3 });
399
- await this.humanPause();
400
- return true;
401
- } catch {
402
- return false;
403
- }
404
- }
405
- /**
406
- * Focus an indexed input, optionally clear it, and type. Returns false
407
- * if the index couldn't be resolved or the input couldn't be focused.
408
- */
409
- async inputByIndex(index, text, opts) {
410
- this.ensurePage();
411
- const loc = this.locatorForIndex(index);
412
- if (!loc) return false;
413
- try {
414
- const timeout = opts?.timeout ?? 5e3;
415
- await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {
416
- });
417
- if (opts?.clear !== false) {
418
- await loc.fill("", { timeout });
419
- }
420
- const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
421
- await loc.click({ timeout });
422
- await loc.pressSequentially(text, { delay });
423
- if (opts?.submit) {
424
- await this.page.keyboard.press("Enter");
425
- }
426
- await this.humanPause();
427
- return true;
428
- } catch {
429
- return false;
430
- }
431
- }
432
- async uploadFileByIndex(index, path) {
433
- this.ensurePage();
434
- const loc = this.locatorForIndex(index);
435
- if (!loc) return false;
436
- try {
437
- await loc.setInputFiles(path, { timeout: 5e3 });
438
- await this.humanPause();
439
- return true;
440
- } catch {
441
- return false;
442
- }
443
- }
444
- /** Scroll the indexed element into view (no click). */
445
- async scrollIntoViewByIndex(index) {
446
- this.ensurePage();
447
- const loc = this.locatorForIndex(index);
448
- if (!loc) return false;
449
- try {
450
- await loc.scrollIntoViewIfNeeded({ timeout: 3e3 });
451
- return true;
452
- } catch {
453
- return false;
454
- }
455
- }
456
- // ── Text-based Interaction ───────────────────────────────────────────
457
- /**
458
- * Deterministic, DOM-based click using Playwright's text locator.
459
- *
460
- * Returns `true` if a matching, visible, clickable element was found and
461
- * clicked within `timeout` ms; `false` otherwise (so the caller can fall
462
- * back to coordinate clicking). Substring-matches by default — e.g.
463
- * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
464
- */
465
- async clickByText(keyword, opts) {
466
- this.ensurePage();
467
- const timeout = opts?.timeout ?? 3e3;
468
- try {
469
- const locator = this.page.locator(`text=${keyword}`).first();
470
- await locator.click({ timeout });
471
- await this.humanPause();
472
- return true;
473
- } catch {
474
- return false;
475
- }
476
- }
477
- /**
478
- * Scroll the first occurrence of `text` into view. Returns false if no
479
- * match was found within the timeout.
480
- */
481
- async findText(text, opts) {
482
- this.ensurePage();
483
- const timeout = opts?.timeout ?? 3e3;
484
- try {
485
- const locator = this.page.getByText(text).first();
486
- await locator.scrollIntoViewIfNeeded({ timeout });
487
- return true;
488
- } catch {
489
- return false;
490
- }
491
- }
492
- // ── Dropdowns ────────────────────────────────────────────────────────
493
- /**
494
- * Read the options of a native `<select>` at the given DOM-snapshot
495
- * index. Returns `[]` if the element is not a `<select>`.
496
- */
497
- async dropdownOptions(index) {
498
- this.ensurePage();
499
- const loc = this.locatorForIndex(index);
500
- if (!loc) return [];
501
- try {
502
- return await loc.evaluate((el) => {
503
- if (!el || el.tagName !== "SELECT") return [];
504
- return Array.from(el.options).map((opt) => ({
505
- value: opt.value,
506
- label: (opt.label || opt.textContent || "").trim(),
507
- selected: !!opt.selected
508
- }));
509
- });
510
- } catch {
511
- return [];
512
- }
513
- }
514
- /**
515
- * Select an option in a native `<select>` by its visible text or value.
516
- * Returns false if the element isn't a `<select>` or no option matched.
517
- */
518
- async selectDropdown(index, text) {
519
- this.ensurePage();
520
- const loc = this.locatorForIndex(index);
521
- if (!loc) return false;
522
- try {
523
- await loc.selectOption({ label: text }, { timeout: 3e3 });
524
- await this.humanPause();
525
- return true;
526
- } catch {
527
- try {
528
- await loc.selectOption({ value: text }, { timeout: 3e3 });
529
- await this.humanPause();
530
- return true;
531
- } catch {
532
- return false;
533
- }
534
- }
535
- }
536
- // ── JS Evaluation ────────────────────────────────────────────────────
537
- /**
538
- * Run arbitrary JS in the page context. The caller is responsible for
539
- * gating this behind a config flag — the BrowserAgent only routes the
540
- * `evaluate` action here when `allowEvaluate: true`.
541
- *
542
- * The code is wrapped in `(async () => { ... })()` and the return value
543
- * is coerced to a string for the model.
544
- */
545
- async evaluate(code) {
546
- this.ensurePage();
547
- try {
548
- const result = await this.page.evaluate(
549
- // eslint-disable-next-line @typescript-eslint/no-implied-eval
550
- new Function("return (async () => { " + code + " })()")
551
- );
552
- if (result === void 0) return "undefined";
553
- if (result === null) return "null";
554
- if (typeof result === "string") return result;
555
- try {
556
- return JSON.stringify(result);
557
- } catch {
558
- return String(result);
559
- }
560
- } catch (e) {
561
- throw new Error(`evaluate failed: ${e?.message ?? e}`);
562
- }
563
- }
564
- // ── Page text (for `extract`) ────────────────────────────────────────
565
- /**
566
- * Returns a clean text representation of the visible page body, with
567
- * optional link extraction. Used by the BrowserAgent's `extract` action
568
- * — the text is passed to a (usually cheap) LLM with the user's query.
569
- */
570
- async pageText(opts) {
571
- this.ensurePage();
572
- const maxChars = opts?.maxChars ?? 2e4;
573
- const extractLinks = !!opts?.extractLinks;
574
- const raw = await this.page.evaluate((withLinks) => {
575
- const doc = globalThis.document;
576
- const SKIP = /* @__PURE__ */ new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "SVG"]);
577
- const win = globalThis.window;
578
- const lines = [];
579
- function visit(node) {
580
- if (!node) return;
581
- if (node.nodeType === 3) {
582
- const t = (node.nodeValue || "").trim();
583
- if (t) lines.push(t);
584
- return;
585
- }
586
- if (node.nodeType !== 1) return;
587
- const tag = node.tagName.toUpperCase();
588
- if (SKIP.has(tag)) return;
589
- const style = win.getComputedStyle(node);
590
- if (style && (style.visibility === "hidden" || style.display === "none")) return;
591
- if (tag === "A" && withLinks) {
592
- const href = node.getAttribute("href") || "";
593
- const txt = (node.innerText || node.textContent || "").trim();
594
- if (txt && href) {
595
- lines.push(`[${txt}](${href})`);
596
- return;
597
- }
598
- }
599
- for (const child of node.childNodes) visit(child);
600
- }
601
- visit(doc.body);
602
- return lines.join("\n");
603
- }, extractLinks);
604
- const collapsed = raw.replace(/\n{3,}/g, "\n\n").trim();
605
- return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}
606
- \u2026[truncated]` : collapsed;
607
- }
608
- // ── DOM Extraction (with index tagging) ──────────────────────────────
609
- /**
610
- * Snapshot the interactive elements visible in the viewport, tag each
611
- * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
612
- * return:
613
- * - `text`: a human-readable string fed to the model
614
- * - `elements`: the structured list with stable indices
615
- * - `scroll`: spatial context (pages above/below, hidden interactive count)
616
- *
617
- * Five properties matter for accuracy:
618
- * - **Hit-tested**: each listed coordinate / index actually reaches the
619
- * labeled element (overlays / occlusion skip the entry).
620
- * - **Visibility filtered (with parent chain)**: an element is dropped
621
- * if itself OR any ancestor is `display:none`, `visibility:hidden`,
622
- * `pointer-events:none`, or near-zero opacity.
623
- * - **Shadow DOM piercing**: traverses open shadow roots so custom
624
- * elements / web components are visible to the agent.
625
- * - **Same-origin iframes**: walks into each accessible iframe and
626
- * includes its interactive elements (offset by the iframe's screen
627
- * position so the coordinates the model sees are still viewport-
628
- * relative).
629
- * - **`cursor: pointer` fallback pass**: catches custom React widgets
630
- * that have no semantic role/href/onclick but are clickable.
631
- */
632
- async extractDOM(opts) {
633
- this.ensurePage();
634
- const max = opts?.maxElements ?? 200;
635
- const mainResult = await this.page.evaluate((limit) => {
636
- return globalThis.__buaExtract(limit, 0, 0, "main");
637
- }, max).catch(async () => {
638
- await this.installExtractorScript();
639
- return await this.page.evaluate((limit) => {
640
- return globalThis.__buaExtract(limit, 0, 0, "main");
641
- }, max);
642
- });
643
- const collected = mainResult.elements;
644
- const scroll = mainResult.scroll;
645
- try {
646
- const frames = this.page.frames();
647
- for (const frame of frames) {
648
- if (frame === this.page.mainFrame()) continue;
649
- if (collected.length >= max) break;
650
- let bbox = null;
651
- try {
652
- const owner = await frame.frameElement();
653
- if (owner) {
654
- const rect = await owner.boundingBox();
655
- if (rect) bbox = { x: rect.x, y: rect.y };
656
- }
657
- } catch {
658
- continue;
659
- }
660
- if (!bbox) continue;
661
- let iframeRes = null;
662
- try {
663
- iframeRes = await frame.evaluate(
664
- (args) => {
665
- return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
666
- },
667
- { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
668
- );
669
- } catch {
670
- try {
671
- await frame.evaluate(this.extractorScriptSource());
672
- iframeRes = await frame.evaluate(
673
- (args) => {
674
- return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
675
- },
676
- { limit: max - collected.length, ox: bbox.x, oy: bbox.y, frame: frame.url() || "(iframe)" }
677
- );
678
- } catch {
679
- }
680
- }
681
- if (iframeRes?.elements?.length) {
682
- const offset = collected.length;
683
- for (let i = 0; i < iframeRes.elements.length; i++) {
684
- const e = iframeRes.elements[i];
685
- e.index = offset + i + 1;
686
- collected.push(e);
687
- if (collected.length >= max) break;
688
- }
689
- }
690
- }
691
- } catch {
692
- }
693
- this._lastDom = collected;
694
- const lines = collected.map((e) => {
695
- const typeSuffix = e.type ? `(${e.type})` : "";
696
- const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
697
- return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
698
- });
699
- return { text: lines.join("\n"), elements: collected, scroll };
700
- }
701
- /**
702
- * Install the `__buaExtract` global on the main page. Idempotent —
703
- * subsequent calls are no-ops.
704
- */
705
- async installExtractorScript() {
706
- if (!this.page) return;
707
- await this.page.evaluate(this.extractorScriptSource());
708
- }
709
- /**
710
- * The extractor source. Lives in its own method so we can also inject
711
- * it into iframes that haven't yet had it loaded.
712
- *
713
- * This function intentionally runs entirely in the page context. It:
714
- * - traverses the regular DOM + open shadow roots (deep)
715
- * - applies a parent-chain visibility filter
716
- * - applies a `cursor:pointer` second pass for custom widgets
717
- * - hit-tests each candidate at its center to avoid overlay collisions
718
- * - returns scroll context (pages above/below, hidden counts)
719
- * - tags survivors with `data-bua-idx` for indexed actions
720
- */
721
- extractorScriptSource() {
722
- return (
723
- /* js */
724
- `
325
+ browser = null;
326
+ context = null;
327
+ page = null;
328
+ pages = /* @__PURE__ */ new Map();
329
+ activeTabId = "tab-0";
330
+ tabCounter = 0;
331
+ _viewport;
332
+ _videoDir;
333
+ _humanize;
334
+ /**
335
+ * Most recent DOM snapshot (one per `extractDOM` call). Indexed actions
336
+ * (`clickByIndex`, `inputByIndex`, …) resolve their `index` against this.
337
+ */
338
+ _lastDom = [];
339
+ /** True if we connected over CDP (don't tear down the browser on close). */
340
+ _attached = false;
341
+ constructor() {
342
+ this._viewport = {
343
+ width: 1280,
344
+ height: 720
345
+ };
346
+ }
347
+ async launch(opts) {
348
+ const chromium = (await import("playwright")).chromium;
349
+ this._viewport = opts?.viewport ?? {
350
+ width: 1280,
351
+ height: 720
352
+ };
353
+ if (opts?.humanize) {
354
+ const h = typeof opts.humanize === "object" ? opts.humanize : {};
355
+ this._humanize = {
356
+ typingDelay: h.typingDelay ?? [40, 120],
357
+ clickJitter: h.clickJitter ?? 3,
358
+ actionDelay: h.actionDelay ?? [200, 800],
359
+ mouseMovement: h.mouseMovement ?? true
360
+ };
361
+ }
362
+ if (opts?.cdpUrl) {
363
+ this.browser = await chromium.connectOverCDP(opts.cdpUrl);
364
+ this._attached = true;
365
+ const contexts = this.browser.contexts();
366
+ this.context = contexts.length > 0 ? contexts[0] : await this.browser.newContext({ viewport: this._viewport });
367
+ const existingPages = this.context.pages();
368
+ this.page = existingPages.length > 0 ? existingPages[0] : await this.context.newPage();
369
+ this.tabCounter = 0;
370
+ this.activeTabId = "tab-0";
371
+ this.pages.set("tab-0", this.page);
372
+ return;
373
+ }
374
+ const stealthEnabled = !!opts?.stealth;
375
+ const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
376
+ const headless = opts?.headless ?? true;
377
+ const launchOpts = { headless };
378
+ const args = [];
379
+ if (stealthEnabled) {
380
+ const stealth = buildStealthLaunchArgs(stealthCfg);
381
+ args.push(...stealth.args);
382
+ if (stealth.proxy) launchOpts.proxy = stealth.proxy;
383
+ }
384
+ if (!headless) {
385
+ args.push(`--window-size=${this._viewport.width},${this._viewport.height + 140}`);
386
+ args.push("--window-position=0,0");
387
+ }
388
+ if (args.length) launchOpts.args = args;
389
+ this.browser = await chromium.launch(launchOpts);
390
+ let contextOpts;
391
+ if (stealthEnabled) contextOpts = buildStealthContextOpts(stealthCfg, this._viewport);
392
+ else contextOpts = {
393
+ viewport: this._viewport,
394
+ userAgent: pickUserAgent()
395
+ };
396
+ if (opts?.storageState) contextOpts.storageState = opts.storageState;
397
+ if (opts?.recordVideo) {
398
+ const dir = typeof opts.recordVideo === "object" ? opts.recordVideo.dir : "./browser-videos";
399
+ contextOpts.recordVideo = {
400
+ dir,
401
+ size: this._viewport
402
+ };
403
+ this._videoDir = dir;
404
+ }
405
+ this.context = await this.browser.newContext(contextOpts);
406
+ if (stealthEnabled && stealthCfg.patchFingerprint !== false) await this.context.addInitScript(getStealthScript());
407
+ this.page = await this.context.newPage();
408
+ this.tabCounter = 0;
409
+ this.activeTabId = "tab-0";
410
+ this.pages.set("tab-0", this.page);
411
+ }
412
+ async saveStorageState(path) {
413
+ this.ensureContext();
414
+ await this.context.storageState({ path });
415
+ }
416
+ async navigate(url) {
417
+ this.ensurePage();
418
+ if (!/^https?:\/\//i.test(url)) throw new Error(`Invalid URL scheme: only http:// and https:// are allowed`);
419
+ await this.page.goto(url, {
420
+ waitUntil: "domcontentloaded",
421
+ timeout: 3e4
422
+ });
423
+ await this.waitForStable(500);
424
+ }
425
+ async back() {
426
+ this.ensurePage();
427
+ await this.page.goBack({
428
+ waitUntil: "domcontentloaded",
429
+ timeout: 15e3
430
+ });
431
+ }
432
+ async screenshot() {
433
+ this.ensurePage();
434
+ return await this.page.screenshot({
435
+ type: "png",
436
+ fullPage: false,
437
+ scale: "css"
438
+ });
439
+ }
440
+ /** Viewport size in CSS pixels (matches screenshot dimensions). */
441
+ get viewport() {
442
+ return this._viewport;
443
+ }
444
+ /** Most recent DOM snapshot. Each entry has a stable `index`. */
445
+ get lastDom() {
446
+ return this._lastDom;
447
+ }
448
+ async click(x, y) {
449
+ this.ensurePage();
450
+ const [cx, cy] = this.clampToViewport(x, y);
451
+ const [fx, fy] = this.jitter(cx, cy);
452
+ if (this._humanize?.mouseMovement) await this.humanMouseMove(fx, fy);
453
+ await this.page.mouse.click(fx, fy);
454
+ await this.humanPause();
455
+ }
456
+ async type(text) {
457
+ this.ensurePage();
458
+ const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
459
+ await this.page.keyboard.type(text, { delay });
460
+ await this.humanPause();
461
+ }
462
+ async clickAndType(x, y, text) {
463
+ const [cx, cy] = this.clampToViewport(x, y);
464
+ await this.click(cx, cy);
465
+ await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
466
+ const [fx, fy] = this.jitter(cx, cy);
467
+ await this.page.mouse.click(fx, fy, { clickCount: 3 });
468
+ await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
469
+ await this.type(text);
470
+ }
471
+ async pressKey(key) {
472
+ this.ensurePage();
473
+ await this.page.keyboard.press(key);
474
+ }
475
+ /**
476
+ * Send arbitrary keyboard keys / shortcuts. Accepts a single
477
+ * Playwright key spec (`"Enter"`, `"Control+l"`, `"Shift+ArrowDown"`)
478
+ * or a space-separated sequence (`"Tab Tab Enter"`).
479
+ */
480
+ async sendKeys(keys) {
481
+ this.ensurePage();
482
+ const tokens = keys.trim().split(/\s+/).filter(Boolean);
483
+ for (const token of tokens) {
484
+ await this.page.keyboard.press(token);
485
+ await this.sleep(this._humanize ? this.randInt(40, 120) : 30);
486
+ }
487
+ await this.humanPause();
488
+ }
489
+ async scroll(direction, amount) {
490
+ this.ensurePage();
491
+ const base = amount ?? 400;
492
+ const jittered = this._humanize ? base + this.randInt(-40, 40) : base;
493
+ const scrollY = direction === "down" ? jittered : -jittered;
494
+ if (this._humanize) {
495
+ const steps = this.randInt(2, 4);
496
+ const perStep = scrollY / steps;
497
+ for (let i = 0; i < steps; i++) {
498
+ await this.page.mouse.wheel(0, perStep);
499
+ await this.sleep(this.randInt(30, 80));
500
+ }
501
+ } else await this.page.mouse.wheel(0, scrollY);
502
+ await this.humanPause();
503
+ }
504
+ /**
505
+ * Build a Playwright locator for a DOM-snapshot index. Each `extractDOM`
506
+ * call tags surviving elements with `data-bua-idx="<n>"`; we resolve by
507
+ * that attribute. Returns null if the index is unknown.
508
+ */
509
+ locatorForIndex(index) {
510
+ if (!this.page) return null;
511
+ if (!this._lastDom.find((e) => e.index === index)) return null;
512
+ return this.page.locator(`[data-bua-idx="${index}"]`).first();
513
+ }
514
+ /**
515
+ * Click an element by its DOM-snapshot index. The most reliable click
516
+ * path on dynamic pages — survives layout shifts and DPR oddities.
517
+ */
518
+ async clickByIndex(index, opts) {
519
+ this.ensurePage();
520
+ const loc = this.locatorForIndex(index);
521
+ if (!loc) return false;
522
+ try {
523
+ await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {});
524
+ await loc.click({ timeout: opts?.timeout ?? 5e3 });
525
+ await this.humanPause();
526
+ return true;
527
+ } catch {
528
+ return false;
529
+ }
530
+ }
531
+ /**
532
+ * Focus an indexed input, optionally clear it, and type. Returns false
533
+ * if the index couldn't be resolved or the input couldn't be focused.
534
+ */
535
+ async inputByIndex(index, text, opts) {
536
+ this.ensurePage();
537
+ const loc = this.locatorForIndex(index);
538
+ if (!loc) return false;
539
+ try {
540
+ const timeout = opts?.timeout ?? 5e3;
541
+ await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {});
542
+ if (opts?.clear !== false) await loc.fill("", { timeout });
543
+ const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
544
+ await loc.click({ timeout });
545
+ await loc.pressSequentially(text, { delay });
546
+ if (opts?.submit) await this.page.keyboard.press("Enter");
547
+ await this.humanPause();
548
+ return true;
549
+ } catch {
550
+ return false;
551
+ }
552
+ }
553
+ async uploadFileByIndex(index, path) {
554
+ this.ensurePage();
555
+ const loc = this.locatorForIndex(index);
556
+ if (!loc) return false;
557
+ try {
558
+ await loc.setInputFiles(path, { timeout: 5e3 });
559
+ await this.humanPause();
560
+ return true;
561
+ } catch {
562
+ return false;
563
+ }
564
+ }
565
+ /** Scroll the indexed element into view (no click). */
566
+ async scrollIntoViewByIndex(index) {
567
+ this.ensurePage();
568
+ const loc = this.locatorForIndex(index);
569
+ if (!loc) return false;
570
+ try {
571
+ await loc.scrollIntoViewIfNeeded({ timeout: 3e3 });
572
+ return true;
573
+ } catch {
574
+ return false;
575
+ }
576
+ }
577
+ /**
578
+ * Deterministic, DOM-based click using Playwright's text locator.
579
+ *
580
+ * Returns `true` if a matching, visible, clickable element was found and
581
+ * clicked within `timeout` ms; `false` otherwise (so the caller can fall
582
+ * back to coordinate clicking). Substring-matches by default — e.g.
583
+ * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
584
+ */
585
+ async clickByText(keyword, opts) {
586
+ this.ensurePage();
587
+ const timeout = opts?.timeout ?? 3e3;
588
+ try {
589
+ await this.page.locator(`text=${keyword}`).first().click({ timeout });
590
+ await this.humanPause();
591
+ return true;
592
+ } catch {
593
+ return false;
594
+ }
595
+ }
596
+ /**
597
+ * Scroll the first occurrence of `text` into view. Returns false if no
598
+ * match was found within the timeout.
599
+ */
600
+ async findText(text, opts) {
601
+ this.ensurePage();
602
+ const timeout = opts?.timeout ?? 3e3;
603
+ try {
604
+ await this.page.getByText(text).first().scrollIntoViewIfNeeded({ timeout });
605
+ return true;
606
+ } catch {
607
+ return false;
608
+ }
609
+ }
610
+ /**
611
+ * Read the options of a native `<select>` at the given DOM-snapshot
612
+ * index. Returns `[]` if the element is not a `<select>`.
613
+ */
614
+ async dropdownOptions(index) {
615
+ this.ensurePage();
616
+ const loc = this.locatorForIndex(index);
617
+ if (!loc) return [];
618
+ try {
619
+ return await loc.evaluate((el) => {
620
+ if (el?.tagName !== "SELECT") return [];
621
+ return Array.from(el.options).map((opt) => ({
622
+ value: opt.value,
623
+ label: (opt.label || opt.textContent || "").trim(),
624
+ selected: !!opt.selected
625
+ }));
626
+ });
627
+ } catch {
628
+ return [];
629
+ }
630
+ }
631
+ /**
632
+ * Select an option in a native `<select>` by its visible text or value.
633
+ * Returns false if the element isn't a `<select>` or no option matched.
634
+ */
635
+ async selectDropdown(index, text) {
636
+ this.ensurePage();
637
+ const loc = this.locatorForIndex(index);
638
+ if (!loc) return false;
639
+ try {
640
+ await loc.selectOption({ label: text }, { timeout: 3e3 });
641
+ await this.humanPause();
642
+ return true;
643
+ } catch {
644
+ try {
645
+ await loc.selectOption({ value: text }, { timeout: 3e3 });
646
+ await this.humanPause();
647
+ return true;
648
+ } catch {
649
+ return false;
650
+ }
651
+ }
652
+ }
653
+ /**
654
+ * Run arbitrary JS in the page context. The caller is responsible for
655
+ * gating this behind a config flag — the BrowserAgent only routes the
656
+ * `evaluate` action here when `allowEvaluate: true`.
657
+ *
658
+ * The code is wrapped in `(async () => { ... })()` and the return value
659
+ * is coerced to a string for the model.
660
+ */
661
+ async evaluate(code) {
662
+ this.ensurePage();
663
+ try {
664
+ const result = await this.page.evaluate(new Function(`return (async () => { ${code} })()`));
665
+ if (result === void 0) return "undefined";
666
+ if (result === null) return "null";
667
+ if (typeof result === "string") return result;
668
+ try {
669
+ return JSON.stringify(result);
670
+ } catch {
671
+ return String(result);
672
+ }
673
+ } catch (e) {
674
+ throw new Error(`evaluate failed: ${e?.message ?? e}`);
675
+ }
676
+ }
677
+ /**
678
+ * Returns a clean text representation of the visible page body, with
679
+ * optional link extraction. Used by the BrowserAgent's `extract` action
680
+ * — the text is passed to a (usually cheap) LLM with the user's query.
681
+ */
682
+ async pageText(opts) {
683
+ this.ensurePage();
684
+ const maxChars = opts?.maxChars ?? 2e4;
685
+ const extractLinks = !!opts?.extractLinks;
686
+ const collapsed = (await this.page.evaluate((withLinks) => {
687
+ const doc = globalThis.document;
688
+ const SKIP = /* @__PURE__ */ new Set([
689
+ "SCRIPT",
690
+ "STYLE",
691
+ "NOSCRIPT",
692
+ "TEMPLATE",
693
+ "SVG"
694
+ ]);
695
+ const win = globalThis.window;
696
+ const lines = [];
697
+ function visit(node) {
698
+ if (!node) return;
699
+ if (node.nodeType === 3) {
700
+ const t = (node.nodeValue || "").trim();
701
+ if (t) lines.push(t);
702
+ return;
703
+ }
704
+ if (node.nodeType !== 1) return;
705
+ const tag = node.tagName.toUpperCase();
706
+ if (SKIP.has(tag)) return;
707
+ const style = win.getComputedStyle(node);
708
+ if (style && (style.visibility === "hidden" || style.display === "none")) return;
709
+ if (tag === "A" && withLinks) {
710
+ const href = node.getAttribute("href") || "";
711
+ const txt = (node.innerText || node.textContent || "").trim();
712
+ if (txt && href) {
713
+ lines.push(`[${txt}](${href})`);
714
+ return;
715
+ }
716
+ }
717
+ for (const child of node.childNodes) visit(child);
718
+ }
719
+ visit(doc.body);
720
+ return lines.join("\n");
721
+ }, extractLinks)).replace(/\n{3,}/g, "\n\n").trim();
722
+ return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}\n…[truncated]` : collapsed;
723
+ }
724
+ /**
725
+ * Snapshot the interactive elements visible in the viewport, tag each
726
+ * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
727
+ * return:
728
+ * - `text`: a human-readable string fed to the model
729
+ * - `elements`: the structured list with stable indices
730
+ * - `scroll`: spatial context (pages above/below, hidden interactive count)
731
+ *
732
+ * Five properties matter for accuracy:
733
+ * - **Hit-tested**: each listed coordinate / index actually reaches the
734
+ * labeled element (overlays / occlusion skip the entry).
735
+ * - **Visibility filtered (with parent chain)**: an element is dropped
736
+ * if itself OR any ancestor is `display:none`, `visibility:hidden`,
737
+ * `pointer-events:none`, or near-zero opacity.
738
+ * - **Shadow DOM piercing**: traverses open shadow roots so custom
739
+ * elements / web components are visible to the agent.
740
+ * - **Same-origin iframes**: walks into each accessible iframe and
741
+ * includes its interactive elements (offset by the iframe's screen
742
+ * position so the coordinates the model sees are still viewport-
743
+ * relative).
744
+ * - **`cursor: pointer` fallback pass**: catches custom React widgets
745
+ * that have no semantic role/href/onclick but are clickable.
746
+ */
747
+ async extractDOM(opts) {
748
+ this.ensurePage();
749
+ const max = opts?.maxElements ?? 200;
750
+ const mainResult = await this.page.evaluate((limit) => {
751
+ return globalThis.__buaExtract(limit, 0, 0, "main");
752
+ }, max).catch(async () => {
753
+ await this.installExtractorScript();
754
+ return await this.page.evaluate((limit) => {
755
+ return globalThis.__buaExtract(limit, 0, 0, "main");
756
+ }, max);
757
+ });
758
+ const collected = mainResult.elements;
759
+ const scroll = mainResult.scroll;
760
+ try {
761
+ const frames = this.page.frames();
762
+ for (const frame of frames) {
763
+ if (frame === this.page.mainFrame()) continue;
764
+ if (collected.length >= max) break;
765
+ let bbox = null;
766
+ try {
767
+ const owner = await frame.frameElement();
768
+ if (owner) {
769
+ const rect = await owner.boundingBox();
770
+ if (rect) bbox = {
771
+ x: rect.x,
772
+ y: rect.y
773
+ };
774
+ }
775
+ } catch {
776
+ continue;
777
+ }
778
+ if (!bbox) continue;
779
+ let iframeRes = null;
780
+ try {
781
+ iframeRes = await frame.evaluate((args) => {
782
+ return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
783
+ }, {
784
+ limit: max - collected.length,
785
+ ox: bbox.x,
786
+ oy: bbox.y,
787
+ frame: frame.url() || "(iframe)"
788
+ });
789
+ } catch {
790
+ try {
791
+ await frame.evaluate(this.extractorScriptSource());
792
+ iframeRes = await frame.evaluate((args) => {
793
+ return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
794
+ }, {
795
+ limit: max - collected.length,
796
+ ox: bbox.x,
797
+ oy: bbox.y,
798
+ frame: frame.url() || "(iframe)"
799
+ });
800
+ } catch {}
801
+ }
802
+ if (iframeRes?.elements?.length) {
803
+ const offset = collected.length;
804
+ for (let i = 0; i < iframeRes.elements.length; i++) {
805
+ const e = iframeRes.elements[i];
806
+ e.index = offset + i + 1;
807
+ collected.push(e);
808
+ if (collected.length >= max) break;
809
+ }
810
+ }
811
+ }
812
+ } catch {}
813
+ this._lastDom = collected;
814
+ return {
815
+ text: collected.map((e) => {
816
+ const typeSuffix = e.type ? `(${e.type})` : "";
817
+ const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
818
+ return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
819
+ }).join("\n"),
820
+ elements: collected,
821
+ scroll
822
+ };
823
+ }
824
+ /**
825
+ * Install the `__buaExtract` global on the main page. Idempotent —
826
+ * subsequent calls are no-ops.
827
+ */
828
+ async installExtractorScript() {
829
+ if (!this.page) return;
830
+ await this.page.evaluate(this.extractorScriptSource());
831
+ }
832
+ /**
833
+ * The extractor source. Lives in its own method so we can also inject
834
+ * it into iframes that haven't yet had it loaded.
835
+ *
836
+ * This function intentionally runs entirely in the page context. It:
837
+ * - traverses the regular DOM + open shadow roots (deep)
838
+ * - applies a parent-chain visibility filter
839
+ * - applies a `cursor:pointer` second pass for custom widgets
840
+ * - hit-tests each candidate at its center to avoid overlay collisions
841
+ * - returns scroll context (pages above/below, hidden counts)
842
+ * - tags survivors with `data-bua-idx` for indexed actions
843
+ */
844
+ extractorScriptSource() {
845
+ return `
725
846
  (function () {
726
847
  if (typeof window.__buaExtract === "function") return;
727
848
 
@@ -842,8 +963,10 @@ var BrowserProvider = class {
842
963
  var title = el.getAttribute("title") || "";
843
964
  var name = el.getAttribute("name") || "";
844
965
  var href = el.getAttribute("href") || "";
845
- var value = el.value || "";
846
- var label = ariaLabel || text || placeholder || title || value || name;
966
+ // Field contents are never labels. Password, OTP and payment values
967
+ // must not enter model observations or recorded DOM snapshots.
968
+ var isField = tag === "input" || tag === "textarea" || el.isContentEditable === true;
969
+ var label = ariaLabel || (isField ? "" : text) || placeholder || title || name;
847
970
  if (!label && href) label = href.slice(0, 60);
848
971
  if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
849
972
 
@@ -887,1425 +1010,1638 @@ var BrowserProvider = class {
887
1010
  };
888
1011
  };
889
1012
  })();
890
- `
891
- );
892
- }
893
- // ── Page Info ────────────────────────────────────────────────────────
894
- async getPageInfo() {
895
- this.ensurePage();
896
- const url = this.page.url();
897
- let title = "";
898
- try {
899
- title = await this.page.title();
900
- } catch (err) {
901
- console.warn("[agentium/browser] Error getting page title:", err instanceof Error ? err.message : err);
902
- }
903
- return { url, title, viewportSize: this._viewport };
904
- }
905
- async waitForStable(minWait = 300) {
906
- this.ensurePage();
907
- await this.sleep(minWait);
908
- try {
909
- await this.page.waitForLoadState("networkidle", { timeout: 5e3 });
910
- } catch (err) {
911
- console.warn("[agentium/browser] Error waiting for stable:", err instanceof Error ? err.message : err);
912
- }
913
- }
914
- // ── Multi-Tab / Parallel Browsing ────────────────────────────────────
915
- async newTab(url) {
916
- this.ensureContext();
917
- const newPage = await this.context.newPage();
918
- this.tabCounter++;
919
- const tabId = `tab-${this.tabCounter}`;
920
- this.pages.set(tabId, newPage);
921
- if (url) {
922
- await newPage.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
923
- }
924
- return tabId;
925
- }
926
- async switchTab(tabId) {
927
- const targetPage = this.pages.get(tabId);
928
- if (!targetPage) throw new Error(`Tab "${tabId}" not found`);
929
- this.page = targetPage;
930
- this.activeTabId = tabId;
931
- await this.page.bringToFront();
932
- }
933
- async closeTab(tabId) {
934
- if (this.pages.size <= 1) throw new Error("Cannot close the last tab");
935
- const targetPage = this.pages.get(tabId);
936
- if (!targetPage) throw new Error(`Tab "${tabId}" not found`);
937
- await targetPage.close();
938
- this.pages.delete(tabId);
939
- if (this.activeTabId === tabId) {
940
- const firstRemaining = this.pages.entries().next().value;
941
- if (firstRemaining) {
942
- this.activeTabId = firstRemaining[0];
943
- this.page = firstRemaining[1];
944
- }
945
- }
946
- }
947
- listTabs() {
948
- const tabs = [];
949
- for (const [id, pg] of this.pages) {
950
- tabs.push({ id, url: pg.url(), active: id === this.activeTabId });
951
- }
952
- return tabs;
953
- }
954
- get currentTabId() {
955
- return this.activeTabId;
956
- }
957
- // ── Video Recording ──────────────────────────────────────────────────
958
- async getVideoPath(tabId) {
959
- const targetPage = tabId ? this.pages.get(tabId) : this.page;
960
- if (!targetPage) return null;
961
- try {
962
- const video = targetPage.video();
963
- if (!video) return null;
964
- return await video.path();
965
- } catch (err) {
966
- console.warn("[agentium/browser] Error getting video path:", err instanceof Error ? err.message : err);
967
- return null;
968
- }
969
- }
970
- get videoDir() {
971
- return this._videoDir;
972
- }
973
- // ── Cleanup ──────────────────────────────────────────────────────────
974
- async close() {
975
- try {
976
- if (this.context && !this._attached) await this.context.close();
977
- } catch (err) {
978
- console.warn("[agentium/browser] Error closing context:", err instanceof Error ? err.message : err);
979
- }
980
- try {
981
- if (this.browser && !this._attached) await this.browser.close();
982
- else if (this.browser && this._attached) {
983
- try {
984
- await this.browser.close();
985
- } catch {
986
- }
987
- }
988
- } catch (err) {
989
- console.warn("[agentium/browser] Error closing browser:", err instanceof Error ? err.message : err);
990
- }
991
- this.page = null;
992
- this.context = null;
993
- this.browser = null;
994
- this.pages.clear();
995
- this._lastDom = [];
996
- }
997
- // ── Private: Humanize helpers ────────────────────────────────────────
998
- /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
999
- jitter(x, y) {
1000
- if (!this._humanize) return [x, y];
1001
- const j = this._humanize.clickJitter;
1002
- return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
1003
- }
1004
- /**
1005
- * Safety net: clamp coordinates returned by the vision model to the actual
1006
- * viewport. If a model occasionally returns image-space coordinates from a
1007
- * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
1008
- * from clicking at e.g. (2200, 1300) and either erroring or landing on a
1009
- * random off-screen element.
1010
- */
1011
- clampToViewport(x, y) {
1012
- const cx = Math.max(0, Math.min(this._viewport.width - 1, Math.round(x)));
1013
- const cy = Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)));
1014
- return [cx, cy];
1015
- }
1016
- /**
1017
- * Simulate human mouse movement using smoothstep interpolation.
1018
- */
1019
- async humanMouseMove(targetX, targetY) {
1020
- const steps = this.randInt(5, 12);
1021
- const startX = this._viewport.width / 2;
1022
- const startY = this._viewport.height / 2;
1023
- for (let i = 1; i <= steps; i++) {
1024
- const t = i / steps;
1025
- const ease = t * t * (3 - 2 * t);
1026
- const cx = startX + (targetX - startX) * ease + this.randInt(-2, 2);
1027
- const cy = startY + (targetY - startY) * ease + this.randInt(-2, 2);
1028
- await this.page.mouse.move(cx, cy);
1029
- await this.sleep(this.randInt(5, 20));
1030
- }
1031
- await this.page.mouse.move(targetX, targetY);
1032
- }
1033
- /** Small random pause after an interaction. */
1034
- async humanPause() {
1035
- if (!this._humanize) return;
1036
- const [min, max] = this._humanize.actionDelay;
1037
- await this.sleep(this.randInt(min, max));
1038
- }
1039
- randInt(min, max) {
1040
- return Math.floor(Math.random() * (max - min + 1)) + min;
1041
- }
1042
- ensurePage() {
1043
- if (!this.page) throw new Error("Browser not launched. Call launch() first.");
1044
- }
1045
- ensureContext() {
1046
- if (!this.context) throw new Error("Browser not launched. Call launch() first.");
1047
- }
1048
- sleep(ms) {
1049
- return new Promise((resolve) => setTimeout(resolve, ms));
1050
- }
1013
+ `;
1014
+ }
1015
+ async getPageInfo() {
1016
+ this.ensurePage();
1017
+ const url = this.page.url();
1018
+ let title = "";
1019
+ try {
1020
+ title = await this.page.title();
1021
+ } catch (err) {
1022
+ console.warn("[agentium/browser] Error getting page title:", err instanceof Error ? err.message : err);
1023
+ }
1024
+ return {
1025
+ url,
1026
+ title,
1027
+ viewportSize: this._viewport
1028
+ };
1029
+ }
1030
+ async waitForStable(minWait = 300) {
1031
+ this.ensurePage();
1032
+ await this.sleep(minWait);
1033
+ try {
1034
+ await this.page.waitForLoadState("networkidle", { timeout: 5e3 });
1035
+ } catch (err) {
1036
+ console.warn("[agentium/browser] Error waiting for stable:", err instanceof Error ? err.message : err);
1037
+ }
1038
+ }
1039
+ async newTab(url) {
1040
+ this.ensureContext();
1041
+ const newPage = await this.context.newPage();
1042
+ this.tabCounter++;
1043
+ const tabId = `tab-${this.tabCounter}`;
1044
+ this.pages.set(tabId, newPage);
1045
+ if (url) await newPage.goto(url, {
1046
+ waitUntil: "domcontentloaded",
1047
+ timeout: 3e4
1048
+ });
1049
+ return tabId;
1050
+ }
1051
+ async switchTab(tabId) {
1052
+ const targetPage = this.pages.get(tabId);
1053
+ if (!targetPage) throw new Error(`Tab "${tabId}" not found`);
1054
+ this.page = targetPage;
1055
+ this.activeTabId = tabId;
1056
+ await this.page.bringToFront();
1057
+ }
1058
+ async closeTab(tabId) {
1059
+ if (this.pages.size <= 1) throw new Error("Cannot close the last tab");
1060
+ const targetPage = this.pages.get(tabId);
1061
+ if (!targetPage) throw new Error(`Tab "${tabId}" not found`);
1062
+ await targetPage.close();
1063
+ this.pages.delete(tabId);
1064
+ if (this.activeTabId === tabId) {
1065
+ const firstRemaining = this.pages.entries().next().value;
1066
+ if (firstRemaining) {
1067
+ this.activeTabId = firstRemaining[0];
1068
+ this.page = firstRemaining[1];
1069
+ }
1070
+ }
1071
+ }
1072
+ async visibleText(maxChars = 2e3) {
1073
+ this.ensurePage();
1074
+ return this.page.evaluate((n) => (globalThis.document?.body?.innerText ?? "").slice(0, n), maxChars);
1075
+ }
1076
+ /**
1077
+ * Grep visible page text. No LLM. Used by `search_page`.
1078
+ */
1079
+ async searchPage(opts) {
1080
+ this.ensurePage();
1081
+ const pattern = opts.pattern;
1082
+ const useRegex = !!opts.regex;
1083
+ const caseSensitive = !!opts.caseSensitive;
1084
+ const maxResults = Math.min(Math.max(opts.maxResults ?? 20, 1), 50);
1085
+ const contextChars = opts.contextChars ?? 100;
1086
+ return this.page.evaluate(({ pattern, useRegex, caseSensitive, maxResults, contextChars }) => {
1087
+ const text = globalThis.document?.body?.innerText ?? "";
1088
+ const src = String(pattern);
1089
+ const flags = caseSensitive ? "g" : "gi";
1090
+ let re;
1091
+ try {
1092
+ re = useRegex ? new RegExp(src, flags) : new RegExp(src.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), flags);
1093
+ } catch {
1094
+ return [];
1095
+ }
1096
+ const matches = [];
1097
+ let m = re.exec(text);
1098
+ while (m && matches.length < Number(maxResults)) {
1099
+ const start = Math.max(0, m.index - Number(contextChars));
1100
+ const end = Math.min(text.length, m.index + m[0].length + Number(contextChars));
1101
+ matches.push({
1102
+ match: m[0],
1103
+ context: text.slice(start, end),
1104
+ index: m.index
1105
+ });
1106
+ if (!re.global) break;
1107
+ m = re.exec(text);
1108
+ }
1109
+ return matches;
1110
+ }, {
1111
+ pattern,
1112
+ useRegex,
1113
+ caseSensitive,
1114
+ maxResults,
1115
+ contextChars
1116
+ });
1117
+ }
1118
+ /**
1119
+ * querySelectorAll over the page. No LLM. Used by `find_elements`.
1120
+ */
1121
+ async findElements(selector, opts) {
1122
+ this.ensurePage();
1123
+ const maxResults = Math.min(Math.max(opts?.maxResults ?? 50, 1), 80);
1124
+ return this.page.evaluate(({ selector, maxResults }) => {
1125
+ const doc = globalThis.document;
1126
+ let nodes;
1127
+ try {
1128
+ nodes = Array.from(doc.querySelectorAll(selector)).slice(0, maxResults);
1129
+ } catch {
1130
+ return [];
1131
+ }
1132
+ return nodes.map((el) => ({
1133
+ tag: String(el.tagName ?? "").toLowerCase(),
1134
+ text: String(el.textContent ?? "").trim().slice(0, 200),
1135
+ href: el.href ? String(el.href) : void 0
1136
+ }));
1137
+ }, {
1138
+ selector,
1139
+ maxResults
1140
+ });
1141
+ }
1142
+ listTabs() {
1143
+ const tabs = [];
1144
+ for (const [id, pg] of this.pages) tabs.push({
1145
+ id,
1146
+ url: pg.url(),
1147
+ active: id === this.activeTabId
1148
+ });
1149
+ return tabs;
1150
+ }
1151
+ get currentTabId() {
1152
+ return this.activeTabId;
1153
+ }
1154
+ async getVideoPath(tabId) {
1155
+ const targetPage = tabId ? this.pages.get(tabId) : this.page;
1156
+ if (!targetPage) return null;
1157
+ try {
1158
+ const video = targetPage.video();
1159
+ if (!video) return null;
1160
+ return await video.path();
1161
+ } catch (err) {
1162
+ console.warn("[agentium/browser] Error getting video path:", err instanceof Error ? err.message : err);
1163
+ return null;
1164
+ }
1165
+ }
1166
+ get videoDir() {
1167
+ return this._videoDir;
1168
+ }
1169
+ async close() {
1170
+ try {
1171
+ if (this.context && !this._attached) await this.context.close();
1172
+ } catch (err) {
1173
+ console.warn("[agentium/browser] Error closing context:", err instanceof Error ? err.message : err);
1174
+ }
1175
+ try {
1176
+ if (this.browser && !this._attached) await this.browser.close();
1177
+ else if (this.browser && this._attached) try {
1178
+ await this.browser.close();
1179
+ } catch {}
1180
+ } catch (err) {
1181
+ console.warn("[agentium/browser] Error closing browser:", err instanceof Error ? err.message : err);
1182
+ }
1183
+ this.page = null;
1184
+ this.context = null;
1185
+ this.browser = null;
1186
+ this.pages.clear();
1187
+ this._lastDom = [];
1188
+ }
1189
+ /** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
1190
+ jitter(x, y) {
1191
+ if (!this._humanize) return [x, y];
1192
+ const j = this._humanize.clickJitter;
1193
+ return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
1194
+ }
1195
+ /**
1196
+ * Safety net: clamp coordinates returned by the vision model to the actual
1197
+ * viewport. If a model occasionally returns image-space coordinates from a
1198
+ * 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
1199
+ * from clicking at e.g. (2200, 1300) and either erroring or landing on a
1200
+ * random off-screen element.
1201
+ */
1202
+ clampToViewport(x, y) {
1203
+ return [Math.max(0, Math.min(this._viewport.width - 1, Math.round(x))), Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)))];
1204
+ }
1205
+ /**
1206
+ * Simulate human mouse movement using smoothstep interpolation.
1207
+ */
1208
+ async humanMouseMove(targetX, targetY) {
1209
+ const steps = this.randInt(5, 12);
1210
+ const startX = this._viewport.width / 2;
1211
+ const startY = this._viewport.height / 2;
1212
+ for (let i = 1; i <= steps; i++) {
1213
+ const t = i / steps;
1214
+ const ease = t * t * (3 - 2 * t);
1215
+ const cx = startX + (targetX - startX) * ease + this.randInt(-2, 2);
1216
+ const cy = startY + (targetY - startY) * ease + this.randInt(-2, 2);
1217
+ await this.page.mouse.move(cx, cy);
1218
+ await this.sleep(this.randInt(5, 20));
1219
+ }
1220
+ await this.page.mouse.move(targetX, targetY);
1221
+ }
1222
+ /** Small random pause after an interaction. */
1223
+ async humanPause() {
1224
+ if (!this._humanize) return;
1225
+ const [min, max] = this._humanize.actionDelay;
1226
+ await this.sleep(this.randInt(min, max));
1227
+ }
1228
+ randInt(min, max) {
1229
+ return Math.floor(Math.random() * (max - min + 1)) + min;
1230
+ }
1231
+ ensurePage() {
1232
+ if (!this.page) throw new Error("Browser not launched. Call launch() first.");
1233
+ }
1234
+ ensureContext() {
1235
+ if (!this.context) throw new Error("Browser not launched. Call launch() first.");
1236
+ }
1237
+ sleep(ms) {
1238
+ return new Promise((resolve) => setTimeout(resolve, ms));
1239
+ }
1051
1240
  };
1052
-
1053
- // src/loop-detector.ts
1241
+ //#endregion
1242
+ //#region src/loop-detector.ts
1054
1243
  var LoopDetector = class {
1055
- actionWindow = [];
1056
- pageHistory = [];
1057
- actionWindowSize;
1058
- pageHistorySize;
1059
- /** Thresholds: repeat counts at which each severity fires. */
1060
- actionThresholds;
1061
- /** Thresholds: consecutive unchanged-page counts at which each fires. */
1062
- pageThresholds;
1063
- constructor(opts) {
1064
- this.actionWindowSize = opts?.actionWindowSize ?? 20;
1065
- this.pageHistorySize = opts?.pageHistorySize ?? 10;
1066
- this.actionThresholds = {
1067
- warn: opts?.actionThresholds?.warn ?? 5,
1068
- escalate: opts?.actionThresholds?.escalate ?? 8,
1069
- abort: opts?.actionThresholds?.abort ?? 12
1070
- };
1071
- this.pageThresholds = {
1072
- warn: opts?.pageThresholds?.warn ?? 5,
1073
- escalate: opts?.pageThresholds?.escalate ?? 8,
1074
- abort: opts?.pageThresholds?.abort ?? 12
1075
- };
1076
- }
1077
- /**
1078
- * Record an action and return advice for the runtime. `severity` of
1079
- * `abort` means the loop should terminate now.
1080
- */
1081
- recordAction(action) {
1082
- const key = normalizeAction(action);
1083
- this.actionWindow.push(key);
1084
- if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
1085
- let count = 0;
1086
- for (const k of this.actionWindow) if (k === key) count++;
1087
- if (count >= this.actionThresholds.abort) {
1088
- return {
1089
- severity: "abort",
1090
- message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
1091
- };
1092
- }
1093
- if (count >= this.actionThresholds.escalate) {
1094
- return {
1095
- severity: "escalate",
1096
- message: `You have repeated essentially the same action ${count} times. The current approach is NOT working \u2014 pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
1097
- };
1098
- }
1099
- if (count >= this.actionThresholds.warn) {
1100
- return {
1101
- severity: "warn",
1102
- message: `You've repeated this action ${count} times \u2014 if the page hasn't changed, try a different approach.`
1103
- };
1104
- }
1105
- return { severity: "none" };
1106
- }
1107
- /**
1108
- * Record a page fingerprint and return advice if the page has been
1109
- * stagnant. Counts consecutive identical fingerprints at the END of
1110
- * the history (so a single navigation resets the counter).
1111
- */
1112
- recordPage(fp) {
1113
- this.pageHistory.push(fp);
1114
- if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
1115
- let stagnantCount = 1;
1116
- for (let i = this.pageHistory.length - 2; i >= 0; i--) {
1117
- const prev = this.pageHistory[i];
1118
- if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) {
1119
- stagnantCount++;
1120
- } else {
1121
- break;
1122
- }
1123
- }
1124
- if (stagnantCount >= this.pageThresholds.abort) {
1125
- return {
1126
- severity: "abort",
1127
- message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
1128
- };
1129
- }
1130
- if (stagnantCount >= this.pageThresholds.escalate) {
1131
- return {
1132
- severity: "escalate",
1133
- message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress \u2014 try dismissing popups, navigating elsewhere, or scrolling.`
1134
- };
1135
- }
1136
- if (stagnantCount >= this.pageThresholds.warn) {
1137
- return {
1138
- severity: "warn",
1139
- message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
1140
- };
1141
- }
1142
- return { severity: "none" };
1143
- }
1144
- /** Combine two advices, returning the more severe one. */
1145
- static combine(a, b) {
1146
- const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
1147
- return rank[a.severity] >= rank[b.severity] ? a : b;
1148
- }
1244
+ actionWindow = [];
1245
+ pageHistory = [];
1246
+ actionWindowSize;
1247
+ pageHistorySize;
1248
+ /** Thresholds: repeat counts at which each severity fires. */
1249
+ actionThresholds;
1250
+ /** Thresholds: consecutive unchanged-page counts at which each fires. */
1251
+ pageThresholds;
1252
+ constructor(opts) {
1253
+ this.actionWindowSize = opts?.actionWindowSize ?? 20;
1254
+ this.pageHistorySize = opts?.pageHistorySize ?? 10;
1255
+ this.actionThresholds = {
1256
+ warn: opts?.actionThresholds?.warn ?? 5,
1257
+ escalate: opts?.actionThresholds?.escalate ?? 8,
1258
+ abort: opts?.actionThresholds?.abort ?? 12
1259
+ };
1260
+ this.pageThresholds = {
1261
+ warn: opts?.pageThresholds?.warn ?? 5,
1262
+ escalate: opts?.pageThresholds?.escalate ?? 8,
1263
+ abort: opts?.pageThresholds?.abort ?? 12
1264
+ };
1265
+ }
1266
+ /**
1267
+ * Record an action and return advice for the runtime. `severity` of
1268
+ * `abort` means the loop should terminate now.
1269
+ */
1270
+ recordAction(action) {
1271
+ const key = normalizeAction(action);
1272
+ this.actionWindow.push(key);
1273
+ if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
1274
+ let count = 0;
1275
+ for (const k of this.actionWindow) if (k === key) count++;
1276
+ if (count >= this.actionThresholds.abort) return {
1277
+ severity: "abort",
1278
+ message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
1279
+ };
1280
+ if (count >= this.actionThresholds.escalate) return {
1281
+ severity: "escalate",
1282
+ message: `You have repeated essentially the same action ${count} times. The current approach is NOT working — pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
1283
+ };
1284
+ if (count >= this.actionThresholds.warn) return {
1285
+ severity: "warn",
1286
+ message: `You've repeated this action ${count} times — if the page hasn't changed, try a different approach.`
1287
+ };
1288
+ return { severity: "none" };
1289
+ }
1290
+ /**
1291
+ * Record a page fingerprint and return advice if the page has been
1292
+ * stagnant. Counts consecutive identical fingerprints at the END of
1293
+ * the history (so a single navigation resets the counter).
1294
+ */
1295
+ recordPage(fp) {
1296
+ this.pageHistory.push(fp);
1297
+ if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
1298
+ let stagnantCount = 1;
1299
+ for (let i = this.pageHistory.length - 2; i >= 0; i--) {
1300
+ const prev = this.pageHistory[i];
1301
+ if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) stagnantCount++;
1302
+ else break;
1303
+ }
1304
+ if (stagnantCount >= this.pageThresholds.abort) return {
1305
+ severity: "abort",
1306
+ message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
1307
+ };
1308
+ if (stagnantCount >= this.pageThresholds.escalate) return {
1309
+ severity: "escalate",
1310
+ message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress — try dismissing popups, navigating elsewhere, or scrolling.`
1311
+ };
1312
+ if (stagnantCount >= this.pageThresholds.warn) return {
1313
+ severity: "warn",
1314
+ message: `Page hasn't changed in ${stagnantCount} steps — verify your action is taking effect.`
1315
+ };
1316
+ return { severity: "none" };
1317
+ }
1318
+ /** Combine two advices, returning the more severe one. */
1319
+ static combine(a, b) {
1320
+ const rank = {
1321
+ none: 0,
1322
+ warn: 1,
1323
+ escalate: 2,
1324
+ abort: 3
1325
+ };
1326
+ return rank[a.severity] >= rank[b.severity] ? a : b;
1327
+ }
1149
1328
  };
1329
+ /**
1330
+ * Normalize an action for loop comparison. The goal is to canonicalize
1331
+ * tiny variations the LLM produces (e.g. different `description`s for
1332
+ * the same click) into a single key while keeping actions that are
1333
+ * fundamentally different (different `index`, `text`, `url`, …) distinct.
1334
+ */
1150
1335
  function normalizeAction(action) {
1151
- switch (action.action) {
1152
- case "click":
1153
- if (typeof action.index === "number") return `click:${action.index}`;
1154
- if (typeof action.x === "number" && typeof action.y === "number") {
1155
- return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
1156
- }
1157
- return `click:?`;
1158
- case "type":
1159
- if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
1160
- return `type:?:${truncate(action.text, 40)}`;
1161
- case "scroll":
1162
- return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
1163
- case "navigate":
1164
- return `navigate:${action.url}`;
1165
- case "back":
1166
- return "back";
1167
- case "wait":
1168
- return `wait:${Math.round(action.ms / 1e3)}`;
1169
- case "screenshot":
1170
- return "screenshot";
1171
- case "send_keys":
1172
- return `send_keys:${action.keys}`;
1173
- case "find_text":
1174
- return `find_text:${truncate(action.text, 40)}`;
1175
- case "evaluate":
1176
- return `evaluate:${truncate(action.code, 80)}`;
1177
- case "dropdown_options":
1178
- return `dropdown_options:${action.index}`;
1179
- case "select_dropdown":
1180
- return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
1181
- case "upload_file":
1182
- return `upload_file:${action.index}:${action.path}`;
1183
- case "extract":
1184
- return `extract:${truncate(action.query, 80)}`;
1185
- case "tool":
1186
- return `tool:${action.name}`;
1187
- case "done":
1188
- case "fail":
1189
- return action.action;
1190
- }
1336
+ switch (action.action) {
1337
+ case "click":
1338
+ if (typeof action.index === "number") return `click:${action.index}`;
1339
+ if (typeof action.x === "number" && typeof action.y === "number") return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
1340
+ return `click:?`;
1341
+ case "type":
1342
+ if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
1343
+ return `type:?:${truncate(action.text, 40)}`;
1344
+ case "scroll": return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
1345
+ case "navigate": return `navigate:${action.url}${action.newTab ? ":new" : ""}`;
1346
+ case "search": return `search:${truncate(action.query, 40)}:${action.engine ?? "default"}`;
1347
+ case "new_tab": return `new_tab:${action.url ?? ""}`;
1348
+ case "switch_tab": return `switch_tab:${action.tabId}`;
1349
+ case "close_tab": return `close_tab:${action.tabId}`;
1350
+ case "search_page": return `search_page:${truncate(action.pattern, 40)}`;
1351
+ case "find_elements": return `find_elements:${truncate(action.selector, 40)}`;
1352
+ case "back": return "back";
1353
+ case "wait": return `wait:${Math.round(action.ms / 1e3)}`;
1354
+ case "screenshot": return "screenshot";
1355
+ case "send_keys": return `send_keys:${action.keys}`;
1356
+ case "find_text": return `find_text:${truncate(action.text, 40)}`;
1357
+ case "evaluate": return `evaluate:${truncate(action.code, 80)}`;
1358
+ case "dropdown_options": return `dropdown_options:${action.index}`;
1359
+ case "select_dropdown": return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
1360
+ case "upload_file": return `upload_file:${action.index}:${action.path}`;
1361
+ case "extract": return `extract:${truncate(action.query, 80)}`;
1362
+ case "tool": return `tool:${action.name}`;
1363
+ case "done":
1364
+ case "fail": return action.action;
1365
+ }
1191
1366
  }
1367
+ /**
1368
+ * Tiny non-cryptographic string hash, stable across runs. Used for the
1369
+ * DOM-text portion of `PageFingerprint`.
1370
+ */
1192
1371
  function fnvHash(s) {
1193
- let h = 2166136261;
1194
- for (let i = 0; i < s.length; i++) {
1195
- h ^= s.charCodeAt(i);
1196
- h = h * 16777619 >>> 0;
1197
- }
1198
- return h;
1372
+ let h = 2166136261;
1373
+ for (let i = 0; i < s.length; i++) {
1374
+ h ^= s.charCodeAt(i);
1375
+ h = h * 16777619 >>> 0;
1376
+ }
1377
+ return h;
1199
1378
  }
1200
1379
  function truncate(s, n) {
1201
- if (!s) return "";
1202
- return s.length <= n ? s : `${s.slice(0, n)}\u2026`;
1380
+ if (!s) return "";
1381
+ return s.length <= n ? s : `${s.slice(0, n)}…`;
1203
1382
  }
1204
-
1205
- // src/prompts.ts
1383
+ //#endregion
1384
+ //#region src/prompts.ts
1206
1385
  function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
1207
- const maxActions = options?.maxActionsPerStep ?? 3;
1208
- const allowEvaluate = !!options?.allowEvaluate;
1209
- const tools = options?.tools ?? [];
1210
- const useVision = options?.useVision ?? "auto";
1211
- const useThinking = options?.useThinking ?? true;
1212
- if (options?.overrideSystemMessage) {
1213
- const lines2 = [options.overrideSystemMessage];
1214
- appendCredentials(lines2, credentialKeys);
1215
- lines2.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
1216
- return lines2.join("\n");
1217
- }
1218
- const lines = [];
1219
- lines.push(
1220
- `You are a browser automation agent. Your job is to complete the user's task by interacting with a real web page.`,
1221
- ``,
1222
- `## What you receive each step`,
1223
- `- The current URL and page title`,
1224
- `- A numbered list of interactive elements visible in the viewport (the "DOM snapshot"). Each entry looks like \`[idx] [cx,cy] role(type): "label"\` where \`idx\` is a stable per-step index, \`cx,cy\` are the CSS-pixel center coordinates, and \`label\` is the visible text / aria-label / placeholder. **Use the index to act on elements** \u2014 it is the most reliable handle.`
1225
- );
1226
- if (useVision !== false) {
1227
- lines.push(
1228
- `- A PNG screenshot of the same viewport at exactly ${viewport.width}\xD7${viewport.height} pixels. Use it for visual context (layout, icons, charts) when the DOM snapshot is ambiguous.`
1229
- );
1230
- } else {
1231
- lines.push(`- (Vision is disabled \u2014 no screenshot. Rely entirely on the DOM snapshot and page text.)`);
1232
- }
1233
- lines.push(
1234
- `- A short list of your previous actions (so you don't repeat yourself).`,
1235
- ``,
1236
- `## Coordinate System`,
1237
- `The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels, top-left origin. Coordinates outside that range are rejected.`,
1238
- ``,
1239
- `## Available Actions`,
1240
- `Respond with a JSON object \u2014 or, when several independent actions can be safely batched (e.g. filling multiple form fields), an array of up to ${maxActions} JSON objects. The runtime executes them in order and stops early if the page navigates.`,
1241
- ``,
1242
- `### click \u2014 by index (preferred)`,
1243
- `\`{ "action": "click", "index": <n>, "description": "<what you are clicking, ideally with the visible text in quotes>" }\``,
1244
- `If you cannot find a matching index (e.g. an element is partly off-screen and not in the snapshot), you may fall back to coordinates: \`{ "action": "click", "x": <n>, "y": <n>, "description": "Click on 'Cheapest' tab" }\`. The runtime will additionally try a Playwright text locator using the quoted phrase.`,
1245
- ``,
1246
- `### type \u2014 by index (preferred)`,
1247
- `Type into the element at \`index\`. The field is cleared first by default. Set \`"submit": true\` to press Enter after typing (e.g. submit a search).`,
1248
- `\`{ "action": "type", "index": <n>, "text": "<text>", "clear": true, "submit": false }\``,
1249
- `Fallback: \`{ "action": "type", "text": "<text>", "x": <n>, "y": <n> }\` clicks the coordinates first, then types.`,
1250
- ``,
1251
- `### scroll`,
1252
- `\`{ "action": "scroll", "direction": "down"|"up", "amount": 400 }\` or \`{ "action": "scroll", "index": <n> }\` to scroll a specific element into view.`,
1253
- ``,
1254
- `### find_text`,
1255
- `Scroll the first occurrence of a phrase into the viewport. Use this instead of repeated \`scroll\` actions when you know what you're hunting for.`,
1256
- `\`{ "action": "find_text", "text": "Annual report 2024" }\``,
1257
- ``,
1258
- `### send_keys`,
1259
- `Send arbitrary keys / shortcuts. Single key, combo (with \`+\`), or space-separated sequence.`,
1260
- `\`{ "action": "send_keys", "keys": "Tab Tab Enter" }\` \xB7 \`{ "action": "send_keys", "keys": "Control+l" }\` \xB7 \`{ "action": "send_keys", "keys": "Escape" }\``,
1261
- ``,
1262
- `### dropdown_options / select_dropdown`,
1263
- `Inspect or set a native \`<select>\` by index. **Do not click into native selects** \u2014 the OS overlay is not part of the DOM.`,
1264
- `\`{ "action": "dropdown_options", "index": <n> }\` returns the option list in the next observation.`,
1265
- `\`{ "action": "select_dropdown", "index": <n>, "text": "United States" }\` picks the matching option.`,
1266
- ``,
1267
- `### upload_file`,
1268
- `Set a file on an \`<input type="file">\` by index. \`path\` must be a path the runtime can read; the agent does NOT have a filesystem of its own.`,
1269
- `\`{ "action": "upload_file", "index": <n>, "path": "/abs/path/to/file.pdf" }\``,
1270
- ``,
1271
- `### navigate / back`,
1272
- `\`{ "action": "navigate", "url": "<full URL>" }\` \xB7 \`{ "action": "back" }\``,
1273
- ``,
1274
- `### wait / screenshot`,
1275
- `\`{ "action": "wait", "ms": <ms \u2264 10000> }\` \xB7 \`{ "action": "screenshot" }\` (request a fresh image on the next step \u2014 only useful when vision mode is "auto").`,
1276
- ``,
1277
- `### extract`,
1278
- `Extract structured information from the current page using a secondary LLM. Cheaper than reasoning over multiple screenshots yourself when you just need facts/text. The extracted result is returned in the next observation as \`Last extract result:\`.`,
1279
- `\`{ "action": "extract", "query": "List the top 5 result titles and their prices", "extractLinks": false }\``
1280
- );
1281
- if (allowEvaluate) {
1282
- lines.push(
1283
- ``,
1284
- `### evaluate (JS escape hatch)`,
1285
- `Run arbitrary JavaScript in the page context. Use this ONLY when no other action fits \u2014 e.g. shadow DOM access, complex selectors, custom widget state. The return value is stringified and returned in the next observation.`,
1286
- `\`{ "action": "evaluate", "code": "return document.title" }\``
1287
- );
1288
- }
1289
- if (tools.length > 0) {
1290
- lines.push(``, `### tool \u2014 invoke a custom tool`);
1291
- lines.push(
1292
- `In addition to browser actions, you have access to the following custom tools. Invoke them with \`{ "action": "tool", "name": "<tool>", "args": { ... } }\`. The result is returned in the next observation.`
1293
- );
1294
- for (const t of tools) {
1295
- lines.push(`- **${t.name}** \u2014 ${t.description}`);
1296
- }
1297
- }
1298
- lines.push(
1299
- ``,
1300
- `### done / fail`,
1301
- `\`{ "action": "done", "result": "<comprehensive summary of what was accomplished and any data the user asked for>" }\` \u2014 use when the task is complete.`,
1302
- `\`{ "action": "fail", "reason": "<why>" }\` \u2014 use when the task cannot be completed even after several attempts.`,
1303
- ``,
1304
- `## Rules`,
1305
- `1. ALWAYS check the DOM snapshot first. If the target element is listed, use its \`index\`. Coordinates are a fallback.`,
1306
- `2. For text-bearing targets, ALSO include the visible label in quotes in \`description\` \u2014 the runtime will use it as a third-tier fallback if the locator fails.`,
1307
- `3. Dismiss cookie banners, consent dialogs, and modal popups FIRST \u2014 they intercept clicks on elements behind them.`,
1308
- `4. If a previous action clearly hit the wrong thing, do NOT repeat it. Pick a different element or a different action.`,
1309
- `5. Batch independent actions when safe (e.g. filling 3 form fields). Don't batch when an action navigates or substantially changes the DOM.`,
1310
- `6. NEVER hallucinate. Only report data you can actually see in the snapshot, screenshot, or an extract result.`,
1311
- `7. When the task is fully complete, return \`done\` IMMEDIATELY with a thorough result \u2014 don't add extra confirmation steps.`,
1312
- `8. If after several attempts you cannot make progress, return \`fail\` with a clear reason.`
1313
- );
1314
- appendCredentials(lines, credentialKeys);
1315
- if (extraInstructions) {
1316
- lines.push(``, `## Additional Instructions`, extraInstructions);
1317
- }
1318
- lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
1319
- return lines.join("\n");
1386
+ const maxActions = options?.maxActionsPerStep ?? 3;
1387
+ const allowEvaluate = !!options?.allowEvaluate;
1388
+ const tools = options?.tools ?? [];
1389
+ const useVision = options?.useVision ?? "auto";
1390
+ const useThinking = options?.useThinking ?? true;
1391
+ if (options?.overrideSystemMessage) {
1392
+ const lines = [options.overrideSystemMessage];
1393
+ appendCredentials(lines, credentialKeys);
1394
+ lines.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
1395
+ return lines.join("\n");
1396
+ }
1397
+ const lines = [];
1398
+ lines.push(`You are a browser automation agent. Your job is to complete the user's task by interacting with a real web page.`, ``, `## What you receive each step`, `- The current URL and page title`, `- A numbered list of interactive elements visible in the viewport (the "DOM snapshot"). Each entry looks like \`[idx] [cx,cy] role(type): "label"\` where \`idx\` is a stable per-step index, \`cx,cy\` are the CSS-pixel center coordinates, and \`label\` is the visible text / aria-label / placeholder. **Use the index to act on elements** — it is the most reliable handle.`);
1399
+ if (useVision !== false) lines.push(`- A PNG screenshot of the same viewport at exactly ${viewport.width}×${viewport.height} pixels. Use it for visual context (layout, icons, charts) when the DOM snapshot is ambiguous.`);
1400
+ else lines.push(`- (Vision is disabled — no screenshot. Rely entirely on the DOM snapshot and page text.)`);
1401
+ lines.push(`- A short list of your previous actions (so you don't repeat yourself).`, ``, `## Coordinate System`, `The browser viewport is ${viewport.width}×${viewport.height} CSS pixels, top-left origin. Coordinates outside that range are rejected.`, ``, `## Available Actions`, `Respond with a JSON object — or, when several independent actions can be safely batched (e.g. filling multiple form fields), an array of up to ${maxActions} JSON objects. The runtime executes them in order and stops early if the page navigates.`, ``, `### click — by index (preferred)`, `\`{ "action": "click", "index": <n>, "description": "<what you are clicking, ideally with the visible text in quotes>" }\``, `If you cannot find a matching index (e.g. an element is partly off-screen and not in the snapshot), you may fall back to coordinates: \`{ "action": "click", "x": <n>, "y": <n>, "description": "Click on 'Cheapest' tab" }\`. The runtime will additionally try a Playwright text locator using the quoted phrase.`, ``, `### type — by index (preferred)`, `Type into the element at \`index\`. The field is cleared first by default. Set \`"submit": true\` to press Enter after typing (e.g. submit a search).`, `\`{ "action": "type", "index": <n>, "text": "<text>", "clear": true, "submit": false }\``, `Fallback: \`{ "action": "type", "text": "<text>", "x": <n>, "y": <n> }\` clicks the coordinates first, then types.`, ``, `### scroll`, `\`{ "action": "scroll", "direction": "down"|"up", "amount": 400 }\` or \`{ "action": "scroll", "index": <n> }\` to scroll a specific element into view.`, ``, `### find_text`, `Scroll the first occurrence of a phrase into the viewport. Use this instead of repeated \`scroll\` actions when you know what you're hunting for.`, `\`{ "action": "find_text", "text": "Annual report 2024" }\``, ``, `### send_keys`, `Send arbitrary keys / shortcuts. Single key, combo (with \`+\`), or space-separated sequence.`, `\`{ "action": "send_keys", "keys": "Tab Tab Enter" }\` · \`{ "action": "send_keys", "keys": "Control+l" }\` · \`{ "action": "send_keys", "keys": "Escape" }\``, ``, `### dropdown_options / select_dropdown`, `Inspect or set a native \`<select>\` by index. **Do not click into native selects** — the OS overlay is not part of the DOM.`, `\`{ "action": "dropdown_options", "index": <n> }\` returns the option list in the next observation.`, `\`{ "action": "select_dropdown", "index": <n>, "text": "United States" }\` picks the matching option.`, ``, `### upload_file`, `Set a file on an \`<input type="file">\` by index. \`path\` must be a path the runtime can read; the agent does NOT have a filesystem of its own.`, `\`{ "action": "upload_file", "index": <n>, "path": "/abs/path/to/file.pdf" }\``, ``, `### navigate / back / search`, `\`{ "action": "navigate", "url": "<full URL>" }\` · \`{ "action": "navigate", "url": "<full URL>", "newTab": true }\` · \`{ "action": "back" }\``, `\`{ "action": "search", "query": "<query>", "engine": "duckduckgo"|"google"|"bing" }\` opens a search-engine results page (default engine: duckduckgo).`, ``, `### tabs`, `Tab ids come from the **Open tabs** list in the observation.`, `\`{ "action": "new_tab", "url": "<optional URL>" }\` · \`{ "action": "switch_tab", "tabId": "tab-2" }\` · \`{ "action": "close_tab", "tabId": "tab-1" }\``, ``, `### search_page / find_elements`, `Zero-LLM inspect. Use these instead of \`extract\` when you just need to grep visible text or run a CSS selector.`, `\`{ "action": "search_page", "pattern": "Total", "regex": false, "caseSensitive": false }\``, `\`{ "action": "find_elements", "selector": "a.storylink" }\``, ``, `### wait / screenshot`, `\`{ "action": "wait", "ms": <ms ≤ 10000> }\` · \`{ "action": "screenshot" }\` (request a fresh image on the next step — only useful when vision mode is "auto").`, ``, `### extract`, `Extract structured information from the current page using a secondary LLM. Cheaper than reasoning over multiple screenshots yourself when you just need facts/text. The extracted result is returned in the next observation as \`Last extract result:\`.`, `\`{ "action": "extract", "query": "List the top 5 result titles and their prices", "extractLinks": false }\``);
1402
+ if (allowEvaluate) lines.push(``, `### evaluate (JS escape hatch)`, `Run arbitrary JavaScript in the page context. Use this ONLY when no other action fits — e.g. shadow DOM access, complex selectors, custom widget state. The return value is stringified and returned in the next observation.`, `\`{ "action": "evaluate", "code": "return document.title" }\``);
1403
+ if (tools.length > 0) {
1404
+ lines.push(``, `### tool — invoke a custom tool`);
1405
+ lines.push(`In addition to browser actions, you have access to the following custom tools. Invoke them with \`{ "action": "tool", "name": "<tool>", "args": { ... } }\`. The result is returned in the next observation.`);
1406
+ for (const t of tools) lines.push(`- **${t.name}** — ${t.description}`);
1407
+ }
1408
+ lines.push(``, `### done / fail`, `\`{ "action": "done", "result": "<comprehensive summary of what was accomplished and any data the user asked for>" }\` — use when the task is complete.`, `\`{ "action": "fail", "reason": "<why>" }\` — use when the task cannot be completed even after several attempts.`, ``, `## Rules`, `1. ALWAYS check the DOM snapshot first. If the target element is listed, use its \`index\`. Coordinates are a fallback.`, `2. For text-bearing targets, ALSO include the visible label in quotes in \`description\` — the runtime will use it as a third-tier fallback if the locator fails.`, `3. Dismiss cookie banners, consent dialogs, and modal popups FIRST — they intercept clicks on elements behind them.`, `4. If a previous action clearly hit the wrong thing, do NOT repeat it. Pick a different element or a different action.`, `5. Batch independent actions when safe (e.g. filling 3 form fields). Don't batch when an action navigates or substantially changes the DOM.`, `6. NEVER hallucinate. Only report data you can actually see in the snapshot, screenshot, or an extract result.`, `7. When the task is fully complete, return \`done\` IMMEDIATELY with a thorough result — don't add extra confirmation steps.`, `8. If after several attempts you cannot make progress, return \`fail\` with a clear reason.`);
1409
+ appendCredentials(lines, credentialKeys);
1410
+ if (extraInstructions) lines.push(``, `## Additional Instructions`, extraInstructions);
1411
+ lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
1412
+ return lines.join("\n");
1320
1413
  }
1321
1414
  function appendCredentials(lines, credentialKeys) {
1322
- if (!credentialKeys || credentialKeys.length === 0) return;
1323
- lines.push(
1324
- ``,
1325
- `## Secure Credentials`,
1326
- `The following credential placeholders are available for use in "type" actions:`,
1327
- ...credentialKeys.map((k) => `- \`{{${k}}}\``),
1328
- ``,
1329
- `When you need to fill in a login form or any field requiring these credentials,`,
1330
- `use the EXACT placeholder (e.g. \`{{email}}\`) as the "text" value in a type action.`,
1331
- `The system will securely replace them with real values at execution time.`,
1332
- `NEVER guess, invent, or ask the user for the actual credential values.`,
1333
- `NEVER include real credential values in "done" or "fail" results.`
1334
- );
1415
+ if (!credentialKeys || credentialKeys.length === 0) return;
1416
+ lines.push(``, `## Secure Credentials`, `The following credential placeholders are available for use in "type" actions:`, ...credentialKeys.map((k) => `- \`{{${k}}}\``), ``, `When you need to fill in a login form or any field requiring these credentials,`, `use the EXACT placeholder (e.g. \`{{email}}\`) as the "text" value in a type action.`, `The system will securely replace them with real values at execution time.`, `NEVER guess, invent, or ask the user for the actual credential values.`, `NEVER include real credential values in "done" or "fail" results.`);
1335
1417
  }
1336
1418
  function responseFormatLines(maxActions, useThinking) {
1337
- if (!useThinking) {
1338
- return [
1339
- `Respond with ONLY valid JSON. No markdown, no commentary.`,
1340
- `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`
1341
- ];
1342
- }
1343
- return [
1344
- `Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
1345
- ``,
1346
- `Use this exact shape:`,
1347
- "```json",
1348
- `{`,
1349
- ` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
1350
- ` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
1351
- ` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, \u2026)",`,
1352
- ` "next_goal": "What I want to accomplish in THIS step",`,
1353
- ` "action": <single action object OR array of up to ${maxActions} action objects>`,
1354
- `}`,
1355
- "```",
1356
- ``,
1357
- `On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
1358
- `Keep each text field to one short paragraph or a few lines. Be concrete.`,
1359
- `Only the "action" field is executed; the others help you self-correct over multiple steps.`
1360
- ];
1419
+ if (!useThinking) return [`Respond with ONLY valid JSON. No markdown, no commentary.`, `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`];
1420
+ return [
1421
+ `Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
1422
+ ``,
1423
+ `Use this exact shape:`,
1424
+ "```json",
1425
+ `{`,
1426
+ ` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
1427
+ ` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
1428
+ ` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, …)",`,
1429
+ ` "next_goal": "What I want to accomplish in THIS step",`,
1430
+ ` "action": <single action object OR array of up to ${maxActions} action objects>`,
1431
+ `}`,
1432
+ "```",
1433
+ ``,
1434
+ `On the very first step, set "evaluation_previous_goal" to "n/a — first step".`,
1435
+ `Keep each text field to one short paragraph or a few lines. Be concrete.`,
1436
+ `Only the "action" field is executed; the others help you self-correct over multiple steps.`
1437
+ ];
1361
1438
  }
1362
- function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget) {
1363
- const lines = [];
1364
- lines.push(`**Task:** ${task}`);
1365
- lines.push(`**Current URL:** ${pageUrl}`);
1366
- if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
1367
- if (stepBudget) {
1368
- lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
1369
- } else {
1370
- lines.push(`**Step:** ${stepIndex + 1}`);
1371
- }
1372
- if (scroll) {
1373
- const parts = [];
1374
- parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
1375
- if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
1376
- if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
1377
- if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
1378
- lines.push(`**Page stats:** ${parts.join(" \xB7 ")}`);
1379
- }
1380
- if (domSnapshot !== void 0) {
1381
- if (domSnapshot.trim().length === 0) {
1382
- lines.push(``);
1383
- lines.push(
1384
- `**\u26A0 Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`
1385
- );
1386
- } else {
1387
- lines.push(``);
1388
- lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
1389
- lines.push(domSnapshot);
1390
- }
1391
- }
1392
- if (lastExtract) {
1393
- lines.push(``);
1394
- lines.push(`**Last extract result:**`);
1395
- lines.push(lastExtract);
1396
- }
1397
- if (actionHistory.length > 0) {
1398
- lines.push(``);
1399
- lines.push(`**Previous actions:**`);
1400
- for (const entry of actionHistory.slice(-10)) {
1401
- lines.push(`- ${entry}`);
1402
- }
1403
- }
1404
- if (nudge) {
1405
- lines.push(``);
1406
- lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
1407
- }
1408
- lines.push(``);
1409
- lines.push(`Decide the next action(s) to complete the task.`);
1410
- return lines.join("\n");
1439
+ function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget, tabs) {
1440
+ const lines = [];
1441
+ lines.push(`**Task:** ${task}`);
1442
+ lines.push(`**Current URL:** ${pageUrl}`);
1443
+ if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
1444
+ if (stepBudget) lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
1445
+ else lines.push(`**Step:** ${stepIndex + 1}`);
1446
+ if (scroll) {
1447
+ const parts = [];
1448
+ parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
1449
+ if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
1450
+ if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
1451
+ if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
1452
+ lines.push(`**Page stats:** ${parts.join(" · ")}`);
1453
+ }
1454
+ if (tabs && tabs.length > 0) {
1455
+ lines.push(`**Open tabs:**`);
1456
+ for (const tab of tabs) {
1457
+ const mark = tab.active ? " (active)" : "";
1458
+ lines.push(`- ${tab.id}${mark}: ${tab.url || "blank"}`);
1459
+ }
1460
+ }
1461
+ if (domSnapshot !== void 0) {
1462
+ if (domSnapshot.trim().length === 0) {
1463
+ lines.push(``);
1464
+ lines.push(`**⚠ Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`);
1465
+ } else {
1466
+ lines.push(``);
1467
+ lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
1468
+ lines.push(domSnapshot);
1469
+ }
1470
+ }
1471
+ if (lastExtract) {
1472
+ lines.push(``);
1473
+ lines.push(`**Last extract result:**`);
1474
+ lines.push(lastExtract);
1475
+ }
1476
+ if (actionHistory.length > 0) {
1477
+ lines.push(``);
1478
+ lines.push(`**Previous actions:**`);
1479
+ for (const entry of actionHistory.slice(-10)) lines.push(`- ${entry}`);
1480
+ }
1481
+ if (nudge) {
1482
+ lines.push(``);
1483
+ lines.push(`**⚠ Runtime hint:** ${nudge}`);
1484
+ }
1485
+ lines.push(``);
1486
+ lines.push(`Decide the next action(s) to complete the task.`);
1487
+ return lines.join("\n");
1411
1488
  }
1412
1489
  function summarizeAction(action) {
1413
- switch (action.action) {
1414
- case "click":
1415
- if (typeof action.index === "number") return `Clicked [${action.index}]: ${action.description ?? ""}`.trim();
1416
- return `Clicked at (${action.x}, ${action.y}): ${action.description ?? ""}`.trim();
1417
- case "type":
1418
- if (typeof action.index === "number") return `Typed into [${action.index}]: "${action.text}"`;
1419
- return action.x != null ? `Clicked (${action.x}, ${action.y}) and typed "${action.text}"` : `Typed "${action.text}"`;
1420
- case "scroll":
1421
- if (typeof action.index === "number") return `Scrolled [${action.index}] into view`;
1422
- return `Scrolled ${action.direction}${action.amount ? ` ${action.amount}px` : ""}`;
1423
- case "navigate":
1424
- return `Navigated to ${action.url}`;
1425
- case "back":
1426
- return `Went back to previous page`;
1427
- case "wait":
1428
- return `Waited ${action.ms}ms`;
1429
- case "screenshot":
1430
- return `Requested a fresh screenshot`;
1431
- case "send_keys":
1432
- return `Sent keys: ${action.keys}`;
1433
- case "find_text":
1434
- return `Scrolled to text: "${action.text}"`;
1435
- case "evaluate":
1436
- return `Evaluated JS`;
1437
- case "dropdown_options":
1438
- return `Read dropdown options at [${action.index}]`;
1439
- case "select_dropdown":
1440
- return `Selected "${action.text}" in dropdown [${action.index}]`;
1441
- case "upload_file":
1442
- return `Uploaded file to [${action.index}]: ${action.path}`;
1443
- case "extract":
1444
- return `Extracted: "${action.query}"`;
1445
- case "tool":
1446
- return `Called tool "${action.name}"`;
1447
- case "done":
1448
- return `Done: ${action.result}`;
1449
- case "fail":
1450
- return `Failed: ${action.reason}`;
1451
- default:
1452
- return JSON.stringify(action);
1453
- }
1490
+ switch (action.action) {
1491
+ case "click":
1492
+ if (typeof action.index === "number") return `Clicked [${action.index}]: ${action.description ?? ""}`.trim();
1493
+ return `Clicked at (${action.x}, ${action.y}): ${action.description ?? ""}`.trim();
1494
+ case "type":
1495
+ if (typeof action.index === "number") return `Typed into [${action.index}]: "${action.text}"`;
1496
+ return action.x != null ? `Clicked (${action.x}, ${action.y}) and typed "${action.text}"` : `Typed "${action.text}"`;
1497
+ case "scroll":
1498
+ if (typeof action.index === "number") return `Scrolled [${action.index}] into view`;
1499
+ return `Scrolled ${action.direction}${action.amount ? ` ${action.amount}px` : ""}`;
1500
+ case "navigate": return action.newTab ? `Opened ${action.url} in a new tab` : `Navigated to ${action.url}`;
1501
+ case "search": return `Searched ${action.engine ?? "web"} for "${action.query}"`;
1502
+ case "new_tab": return action.url ? `Opened new tab: ${action.url}` : `Opened new tab`;
1503
+ case "switch_tab": return `Switched to ${action.tabId}`;
1504
+ case "close_tab": return `Closed ${action.tabId}`;
1505
+ case "search_page": return `Searched page for "${action.pattern}"`;
1506
+ case "find_elements": return `Found elements: ${action.selector}`;
1507
+ case "back": return `Went back to previous page`;
1508
+ case "wait": return `Waited ${action.ms}ms`;
1509
+ case "screenshot": return `Requested a fresh screenshot`;
1510
+ case "send_keys": return `Sent keys: ${action.keys}`;
1511
+ case "find_text": return `Scrolled to text: "${action.text}"`;
1512
+ case "evaluate": return `Evaluated JS`;
1513
+ case "dropdown_options": return `Read dropdown options at [${action.index}]`;
1514
+ case "select_dropdown": return `Selected "${action.text}" in dropdown [${action.index}]`;
1515
+ case "upload_file": return `Uploaded file to [${action.index}]: ${action.path}`;
1516
+ case "extract": return `Extracted: "${action.query}"`;
1517
+ case "tool": return `Called tool "${action.name}"`;
1518
+ case "done": return `Done: ${action.result}`;
1519
+ case "fail": return `Failed: ${action.reason}`;
1520
+ default: return JSON.stringify(action);
1521
+ }
1454
1522
  }
1455
-
1456
- // src/browser-agent.ts
1523
+ //#endregion
1524
+ //#region src/browser-agent.ts
1457
1525
  var BrowserAgent = class {
1458
- name;
1459
- eventBus;
1460
- model;
1461
- pageExtractionLLM;
1462
- fallbackModel;
1463
- useThinking;
1464
- historyWindow;
1465
- instructions;
1466
- extendSystemMessage;
1467
- overrideSystemMessage;
1468
- maxSteps;
1469
- maxFailures;
1470
- maxActionsPerStep;
1471
- initialActions;
1472
- useVision;
1473
- directlyOpenUrl;
1474
- headless;
1475
- viewport;
1476
- defaultStartUrl;
1477
- waitAfterAction;
1478
- maxRepeats;
1479
- useDOM;
1480
- allowEvaluate;
1481
- allowedDomains;
1482
- prohibitedDomains;
1483
- storageState;
1484
- cdpUrl;
1485
- recordVideo;
1486
- credentials;
1487
- stealth;
1488
- humanize;
1489
- tools;
1490
- costTracker;
1491
- memoryManager = null;
1492
- logger;
1493
- /** Access the MemoryManager (if memory is configured). */
1494
- get memory() {
1495
- return this.memoryManager;
1496
- }
1497
- constructor(config) {
1498
- this.name = config.name;
1499
- this.model = config.model;
1500
- this.pageExtractionLLM = config.pageExtractionLLM ?? null;
1501
- this.fallbackModel = config.fallbackModel ?? null;
1502
- this.useThinking = config.useThinking ?? true;
1503
- this.historyWindow = Math.max(0, config.historyWindow ?? 6);
1504
- this.instructions = config.instructions;
1505
- this.extendSystemMessage = config.extendSystemMessage;
1506
- this.overrideSystemMessage = config.overrideSystemMessage;
1507
- this.maxSteps = config.maxSteps ?? 30;
1508
- this.maxFailures = config.maxFailures ?? 3;
1509
- this.maxActionsPerStep = Math.max(1, config.maxActionsPerStep ?? 3);
1510
- this.initialActions = config.initialActions ?? [];
1511
- this.useVision = config.useVision ?? "auto";
1512
- this.directlyOpenUrl = config.directlyOpenUrl ?? true;
1513
- this.headless = config.headless ?? true;
1514
- this.viewport = config.viewport ?? { width: 1280, height: 720 };
1515
- this.defaultStartUrl = config.startUrl;
1516
- this.waitAfterAction = config.waitAfterAction ?? 1500;
1517
- this.maxRepeats = config.maxRepeats ?? 3;
1518
- this.useDOM = config.useDOM ?? true;
1519
- this.allowEvaluate = config.allowEvaluate ?? false;
1520
- this.allowedDomains = config.allowedDomains;
1521
- this.prohibitedDomains = config.prohibitedDomains;
1522
- this.storageState = config.storageState;
1523
- this.cdpUrl = config.cdpUrl;
1524
- this.recordVideo = config.recordVideo;
1525
- this.credentials = config.credentials;
1526
- this.stealth = config.stealth;
1527
- this.humanize = config.humanize;
1528
- this.tools = config.tools ?? [];
1529
- this.costTracker = config.costTracker ?? null;
1530
- this.eventBus = config.eventBus ?? new import_core.EventBus();
1531
- this.logger = new import_core.Logger({
1532
- prefix: `BrowserAgent:${config.name}`,
1533
- level: config.logLevel ?? "silent"
1534
- });
1535
- if (config.memory) {
1536
- this.memoryManager = new import_core.MemoryManager(config.memory);
1537
- }
1538
- }
1539
- async run(task, opts) {
1540
- const startTime = Date.now();
1541
- const maxSteps = opts?.maxSteps ?? this.maxSteps;
1542
- const sessionId = opts?.sessionId ?? `browser_${Date.now()}`;
1543
- const userId = opts?.userId;
1544
- const browser = new BrowserProvider();
1545
- const steps = [];
1546
- const actionHistory = [];
1547
- const extractedContent = [];
1548
- let lastExtractResult;
1549
- let consecutiveFailures = 0;
1550
- let lastActionWasScreenshot = false;
1551
- const loop = new LoopDetector();
1552
- const historyTurns = [];
1553
- const baseExtra = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
1554
- let extraInstructions = baseExtra;
1555
- if (this.memoryManager) {
1556
- await this.memoryManager.ensureReady();
1557
- const memoryContext = await this.memoryManager.buildContext(sessionId, userId, task, this.name);
1558
- if (memoryContext) {
1559
- extraInstructions = extraInstructions ? `${extraInstructions}
1560
-
1561
- ${memoryContext}` : memoryContext;
1562
- }
1563
- }
1564
- const credentialKeys = this.credentials?.keys();
1565
- const systemPrompt = buildSystemPrompt(this.viewport, extraInstructions || void 0, credentialKeys, {
1566
- overrideSystemMessage: this.overrideSystemMessage,
1567
- maxActionsPerStep: this.maxActionsPerStep,
1568
- allowEvaluate: this.allowEvaluate,
1569
- tools: this.tools,
1570
- useVision: this.useVision,
1571
- useDOM: this.useDOM,
1572
- useThinking: this.useThinking
1573
- });
1574
- try {
1575
- this.logger.info("Launching browser", {
1576
- headless: this.headless,
1577
- viewport: this.viewport,
1578
- useDOM: this.useDOM,
1579
- useVision: this.useVision,
1580
- useThinking: this.useThinking,
1581
- cdpUrl: this.cdpUrl ?? void 0,
1582
- recordVideo: !!this.recordVideo,
1583
- stealth: !!this.stealth,
1584
- humanize: !!this.humanize
1585
- });
1586
- await browser.launch({
1587
- headless: this.headless,
1588
- viewport: this.viewport,
1589
- storageState: this.storageState,
1590
- recordVideo: this.recordVideo,
1591
- stealth: this.stealth,
1592
- humanize: this.humanize,
1593
- cdpUrl: this.cdpUrl
1594
- });
1595
- const startUrl = opts?.startUrl ?? this.defaultStartUrl ?? this.detectUrlInTask(task);
1596
- if (startUrl) {
1597
- this.logger.info("Navigating to start URL", { url: startUrl });
1598
- this.assertDomainAllowed(startUrl);
1599
- await browser.navigate(startUrl);
1600
- await this.navigationHealthCheck(browser, startUrl);
1601
- }
1602
- for (const ia of this.initialActions) {
1603
- try {
1604
- await this.executeAction(browser, ia, actionHistory, extractedContent);
1605
- } catch (e) {
1606
- this.logger.warn("initialAction failed", { action: ia, error: e?.message });
1607
- }
1608
- await this.sleep(this.waitAfterAction);
1609
- }
1610
- for (let step = 0; step < maxSteps; step++) {
1611
- const pageInfo = await browser.getPageInfo();
1612
- let domSnapshot;
1613
- let scrollCtx;
1614
- if (this.useDOM) {
1615
- const dom = await browser.extractDOM();
1616
- domSnapshot = dom.text;
1617
- scrollCtx = dom.scroll;
1618
- }
1619
- const pageAdvice = loop.recordPage({
1620
- url: pageInfo.url,
1621
- interactiveCount: scrollCtx?.totalInteractive ?? 0,
1622
- textHash: fnvHash(domSnapshot ?? "")
1623
- });
1624
- if (pageAdvice.severity === "abort") {
1625
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1626
- result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
1627
- success: false
1628
- });
1629
- }
1630
- const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
1631
- const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
1632
- if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
1633
- const isLastStep = step === maxSteps - 1;
1634
- const nudgeParts = [];
1635
- if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
1636
- if (isLastStep) {
1637
- nudgeParts.push(
1638
- "This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial."
1639
- );
1640
- }
1641
- const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
1642
- const userText = buildUserMessage(
1643
- task,
1644
- pageInfo.url,
1645
- pageInfo.title,
1646
- step,
1647
- actionHistory,
1648
- domSnapshot,
1649
- lastExtractResult,
1650
- scrollCtx,
1651
- nudge,
1652
- { current: step, max: maxSteps }
1653
- );
1654
- const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
1655
- this.logger.debug("Calling model", { step, url: pageInfo.url, vision: wantVision });
1656
- const { response, modelUsed } = await this.callModelWithFallback(messages, opts?.apiKey);
1657
- if (!response) {
1658
- consecutiveFailures++;
1659
- actionHistory.push(`(model call failed \u2014 retrying, ${consecutiveFailures}/${this.maxFailures})`);
1660
- if (consecutiveFailures > this.maxFailures) {
1661
- return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
1662
- }
1663
- continue;
1664
- }
1665
- if (this.costTracker && response.usage) {
1666
- this.costTracker.track({
1667
- runId: sessionId,
1668
- agentName: this.name,
1669
- modelId: modelUsed.modelId,
1670
- usage: response.usage,
1671
- sessionId,
1672
- userId
1673
- });
1674
- }
1675
- const raw = typeof response.message.content === "string" ? response.message.content : "";
1676
- const envelope = this.parseEnvelope(raw);
1677
- if (!envelope) {
1678
- consecutiveFailures++;
1679
- this.logger.warn("Failed to parse model response", { raw: raw.slice(0, 200), consecutiveFailures });
1680
- if (consecutiveFailures > this.maxFailures) {
1681
- return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
1682
- }
1683
- actionHistory.push("(invalid JSON response \u2014 retrying)");
1684
- continue;
1685
- }
1686
- consecutiveFailures = 0;
1687
- historyTurns.push({ userText, hasScreenshot: wantVision, envelope });
1688
- if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) {
1689
- historyTurns.splice(0, historyTurns.length - this.historyWindow);
1690
- }
1691
- const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(
1692
- 0,
1693
- this.maxActionsPerStep
1694
- );
1695
- let didTerminate = null;
1696
- let didNavigate = false;
1697
- lastActionWasScreenshot = false;
1698
- for (let ai = 0; ai < actions.length; ai++) {
1699
- const action = actions[ai];
1700
- let summary = summarizeAction(action);
1701
- if (this.credentials) summary = this.credentials.mask(summary);
1702
- const advice = loop.recordAction(action);
1703
- if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
1704
- this.logger.warn("Loop detector aborting run", { advice });
1705
- actionHistory.push(`\u26A0 ${advice.message ?? "loop detected \u2014 auto-stopping"}`);
1706
- didTerminate = {
1707
- result: advice.message ?? "Stuck in a loop \u2014 auto-stopped.",
1708
- success: false
1709
- };
1710
- break;
1711
- }
1712
- actionHistory.push(summary);
1713
- this.logger.info(`Step ${step + 1}.${ai + 1}: ${summary}`);
1714
- this.eventBus.emit("browser.action", { action });
1715
- if (action.action === "done") {
1716
- const result = this.credentials ? this.credentials.mask(action.result) : action.result;
1717
- didTerminate = { result, success: true };
1718
- break;
1719
- }
1720
- if (action.action === "fail") {
1721
- const result = this.credentials ? this.credentials.mask(action.reason) : action.reason;
1722
- didTerminate = { result, success: false };
1723
- break;
1724
- }
1725
- let stepOk = true;
1726
- let stepOutput;
1727
- try {
1728
- const exec = await this.executeAction(browser, action, actionHistory, extractedContent);
1729
- stepOutput = exec?.output;
1730
- if (exec?.didNavigate) {
1731
- didNavigate = true;
1732
- await this.navigationHealthCheck(browser, pageInfo.url);
1733
- }
1734
- if (action.action === "extract" && exec?.output) lastExtractResult = exec.output;
1735
- if (action.action === "screenshot") lastActionWasScreenshot = true;
1736
- } catch (e) {
1737
- stepOk = false;
1738
- consecutiveFailures++;
1739
- this.logger.warn("Action failed", { action: action.action, error: e?.message, consecutiveFailures });
1740
- actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
1741
- if (consecutiveFailures > this.maxFailures) {
1742
- didTerminate = {
1743
- result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
1744
- success: false
1745
- };
1746
- break;
1747
- }
1748
- }
1749
- steps.push({
1750
- index: steps.length,
1751
- action,
1752
- screenshot,
1753
- pageUrl: pageInfo.url,
1754
- pageTitle: pageInfo.title,
1755
- timestamp: /* @__PURE__ */ new Date(),
1756
- dom: domSnapshot,
1757
- output: stepOutput,
1758
- ok: stepOk,
1759
- thinking: envelope.thinking,
1760
- evaluationPreviousGoal: envelope.evaluationPreviousGoal,
1761
- memory: envelope.memory,
1762
- nextGoal: envelope.nextGoal
1763
- });
1764
- this.eventBus.emit("browser.step", {
1765
- index: steps.length - 1,
1766
- action,
1767
- pageUrl: pageInfo.url,
1768
- screenshot
1769
- });
1770
- await this.sleep(this.waitAfterAction);
1771
- if (didNavigate) break;
1772
- }
1773
- if (didTerminate) {
1774
- return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
1775
- }
1776
- }
1777
- this.logger.warn("Max steps reached without explicit done", { maxSteps });
1778
- return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
1779
- } catch (error) {
1780
- this.logger.error("Browser agent error", { error: error.message });
1781
- this.eventBus.emit("browser.error", { error });
1782
- await browser.close();
1783
- return {
1784
- result: `Error: ${error.message}`,
1785
- success: false,
1786
- steps,
1787
- finalUrl: "",
1788
- finalScreenshot: Buffer.alloc(0),
1789
- durationMs: Date.now() - startTime,
1790
- extractedContent
1791
- };
1792
- }
1793
- }
1794
- /**
1795
- * Returns a ToolDef that lets a regular Agent delegate browser tasks
1796
- * to this BrowserAgent.
1797
- */
1798
- asTool(config) {
1799
- return {
1800
- name: config?.name ?? "browse_web",
1801
- description: config?.description ?? "Open a browser and autonomously complete a task on a website. Provide a clear task description and optionally a starting URL.",
1802
- parameters: import_zod.z.object({
1803
- task: import_zod.z.string().describe("What to do in the browser (e.g., 'Search for X and return the top 3 results')"),
1804
- startUrl: import_zod.z.string().optional().describe("URL to start at (e.g., 'https://www.google.com')")
1805
- }),
1806
- execute: async (args) => {
1807
- const result = await this.run(args.task, { startUrl: args.startUrl });
1808
- return result.result;
1809
- }
1810
- };
1811
- }
1812
- // ── Private helpers ──────────────────────────────────────────────────
1813
- shouldCaptureVision(step, lastActionWasScreenshot) {
1814
- if (this.useVision === false) return false;
1815
- if (this.useVision === true) return true;
1816
- if (step === 0) return true;
1817
- if (!this.useDOM) return true;
1818
- return lastActionWasScreenshot;
1819
- }
1820
- detectUrlInTask(task) {
1821
- if (!this.directlyOpenUrl) return void 0;
1822
- const match = task.match(/https?:\/\/[^\s)<>"']+/);
1823
- return match ? match[0] : void 0;
1824
- }
1825
- assertDomainAllowed(url) {
1826
- if (!this.allowedDomains?.length && !this.prohibitedDomains?.length) return;
1827
- let host;
1828
- try {
1829
- host = new URL(url).hostname.toLowerCase();
1830
- } catch {
1831
- throw new Error(`Cannot parse URL for domain check: ${url}`);
1832
- }
1833
- if (this.allowedDomains?.length && !this.allowedDomains.some((p) => matchDomain(host, p))) {
1834
- throw new Error(`Navigation blocked: ${host} is not in allowedDomains`);
1835
- }
1836
- if (this.prohibitedDomains?.length && this.prohibitedDomains.some((p) => matchDomain(host, p))) {
1837
- throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
1838
- }
1839
- }
1840
- /**
1841
- * Build the message array sent to the model: system prompt + a compact
1842
- * summary of older turns (if any) + the most recent `historyWindow`
1843
- * turns verbatim + the current step's user message.
1844
- *
1845
- * Inspired by browser-use's history compaction. Keeps tokens bounded
1846
- * while giving the model meaningful context about what it already
1847
- * tried.
1848
- */
1849
- buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
1850
- const msgs = [{ role: "system", content: systemPrompt }];
1851
- if (this.historyWindow > 0 && historyTurns.length > 0) {
1852
- for (const turn of historyTurns) {
1853
- const env = turn.envelope;
1854
- const replay = [];
1855
- if (env.thinking) replay.push(`thinking: ${env.thinking}`);
1856
- if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
1857
- if (env.memory) replay.push(`memory: ${env.memory}`);
1858
- if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
1859
- replay.push(`action: ${JSON.stringify(env.action)}`);
1860
- msgs.push({ role: "assistant", content: replay.join("\n") });
1861
- }
1862
- }
1863
- const content = [];
1864
- if (currentScreenshot && currentScreenshot.length > 0) {
1865
- content.push({
1866
- type: "image",
1867
- data: currentScreenshot.toString("base64"),
1868
- mimeType: "image/png"
1869
- });
1870
- }
1871
- content.push({ type: "text", text: currentUserText });
1872
- msgs.push({ role: "user", content });
1873
- return msgs;
1874
- }
1875
- /**
1876
- * Call the primary model, retrying once with `fallbackModel` on
1877
- * transient errors (5xx, 429, network). Returns the response or
1878
- * `null` if both models failed.
1879
- */
1880
- async callModelWithFallback(messages, apiKey) {
1881
- const reqOpts = { temperature: 0.1, maxTokens: 1024, apiKey, responseFormat: "json" };
1882
- try {
1883
- const r = await this.model.generate(messages, reqOpts);
1884
- return { response: r, modelUsed: this.model };
1885
- } catch (e) {
1886
- if (this.fallbackModel && this.isTransientError(e)) {
1887
- this.logger.warn("Primary model failed; trying fallbackModel", {
1888
- error: e?.message,
1889
- primary: this.model.modelId,
1890
- fallback: this.fallbackModel.modelId
1891
- });
1892
- try {
1893
- const r = await this.fallbackModel.generate(messages, reqOpts);
1894
- return { response: r, modelUsed: this.fallbackModel };
1895
- } catch (e2) {
1896
- this.logger.warn("Fallback model also failed", { error: e2?.message });
1897
- return { response: null, modelUsed: this.model };
1898
- }
1899
- }
1900
- this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
1901
- return { response: null, modelUsed: this.model };
1902
- }
1903
- }
1904
- isTransientError(e) {
1905
- const msg = String(e?.message ?? e).toLowerCase();
1906
- if (msg.includes("rate limit") || msg.includes("429")) return true;
1907
- if (msg.includes("timeout") || msg.includes("etimedout")) return true;
1908
- if (msg.includes("econnreset") || msg.includes("network")) return true;
1909
- if (/\b5\d\d\b/.test(msg)) return true;
1910
- if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
1911
- return false;
1912
- }
1913
- /**
1914
- * Parse the model's raw response into an `AgentOutput`. Tolerant to
1915
- * three shapes:
1916
- * - Full envelope: { thinking, evaluation_previous_goal, action, ... }
1917
- * - Raw action object (legacy / `useThinking: false`)
1918
- * - Raw action array
1919
- *
1920
- * Also strips ```json fences the model occasionally adds.
1921
- */
1922
- parseEnvelope(raw) {
1923
- if (!raw) return null;
1924
- let s = raw.trim();
1925
- const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
1926
- if (fence) s = fence[1].trim();
1927
- let json;
1928
- try {
1929
- json = JSON.parse(s);
1930
- } catch {
1931
- const m = s.match(/[[{][\s\S]*[\]}]/);
1932
- if (!m) return null;
1933
- try {
1934
- json = JSON.parse(m[0]);
1935
- } catch {
1936
- return null;
1937
- }
1938
- }
1939
- if (Array.isArray(json)) return { action: json };
1940
- if (json && typeof json === "object" && typeof json.action === "string") {
1941
- return { action: json };
1942
- }
1943
- if (json && typeof json === "object" && "action" in json) {
1944
- const out = { action: json.action };
1945
- out.thinking = json.thinking ?? json.reasoning ?? void 0;
1946
- out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
1947
- out.memory = json.memory ?? void 0;
1948
- out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
1949
- return out;
1950
- }
1951
- return null;
1952
- }
1953
- /**
1954
- * Quick post-navigation health check. If the page came back blank
1955
- * (no body text and no interactive elements), reload once and wait
1956
- * for stable. This catches the "FreightOS half-loaded font test" class
1957
- * of failure before the LLM ever sees it.
1958
- */
1959
- async navigationHealthCheck(browser, url) {
1960
- try {
1961
- const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
1962
- if (ok && ok.trim().length > 5) return;
1963
- this.logger.warn("Navigation produced an empty page; reloading once", { url });
1964
- await this.sleep(800);
1965
- try {
1966
- await browser.navigate(url);
1967
- } catch {
1968
- }
1969
- } catch {
1970
- }
1971
- }
1972
- /**
1973
- * Force-finalize the run with whatever partial data the agent has.
1974
- * Called when:
1975
- * - `maxSteps` is exhausted without an explicit `done`,
1976
- * - `maxFailures` is exceeded,
1977
- * - the model can't produce parseable JSON enough times to make
1978
- * forward progress.
1979
- * The result is composed from `extractedContent` + the last few
1980
- * action summaries so the caller gets something useful instead of
1981
- * just a one-line error.
1982
- */
1983
- async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
1984
- const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
1985
- const tail = actionHistory.slice(-5).join(" \u2192 ");
1986
- const extracts = extractedContent.length > 0 ? `
1987
-
1988
- Extracts so far:
1989
- ${extractedContent.join("\n---\n")}` : "";
1990
- const result = `${reasonLine}
1991
- Last actions: ${tail || "(none)"}${extracts}`;
1992
- return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1993
- result,
1994
- success: false
1995
- });
1996
- }
1997
- async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
1998
- let finalScreenshot;
1999
- let finalInfo;
2000
- try {
2001
- finalScreenshot = await browser.screenshot();
2002
- finalInfo = await browser.getPageInfo();
2003
- } catch {
2004
- finalScreenshot = Buffer.alloc(0);
2005
- finalInfo = { url: "", title: "" };
2006
- }
2007
- if (opts?.saveStorageState) {
2008
- try {
2009
- await browser.saveStorageState(opts.saveStorageState);
2010
- this.logger.info("Storage state saved", { path: opts.saveStorageState });
2011
- } catch (e) {
2012
- this.logger.warn("Failed to save storage state", { error: e.message });
2013
- }
2014
- }
2015
- let videoPath;
2016
- if (this.recordVideo) {
2017
- videoPath = await browser.getVideoPath() ?? void 0;
2018
- }
2019
- await browser.close();
2020
- const output = {
2021
- result: outcome.result,
2022
- success: outcome.success,
2023
- steps,
2024
- finalUrl: finalInfo.url,
2025
- finalScreenshot,
2026
- durationMs: Date.now() - startTime,
2027
- videoPath,
2028
- extractedContent
2029
- };
2030
- if (this.memoryManager) {
2031
- const sessionId = opts?.sessionId ?? `browser_${startTime}`;
2032
- const userId = opts?.userId;
2033
- const actionSummary = steps.map((s) => summarizeAction(s.action)).join("; ");
2034
- const messages = [
2035
- { role: "user", content: `Task: ${outcome.result}` },
2036
- { role: "assistant", content: `Actions: ${actionSummary}. Result: ${outcome.result}` }
2037
- ];
2038
- this.memoryManager.appendMessages(sessionId, messages, this.model).catch((e) => this.logger.warn("Memory persist failed", { error: String(e) }));
2039
- try {
2040
- this.memoryManager.afterRun(sessionId, userId, messages, this.model, this.name);
2041
- } catch (e) {
2042
- this.logger.warn("Memory afterRun failed", { error: String(e) });
2043
- }
2044
- }
2045
- this.eventBus.emit("browser.done", {
2046
- result: output.result,
2047
- success: outcome.success,
2048
- steps
2049
- });
2050
- return output;
2051
- }
2052
- /**
2053
- * Execute a single action. Returns `{ output?, didNavigate? }` for the
2054
- * caller's state tracking. May throw — the loop handles failure-budget
2055
- * accounting in that case.
2056
- */
2057
- async executeAction(browser, action, _actionHistory, extractedContent) {
2058
- switch (action.action) {
2059
- case "click": {
2060
- if (typeof action.index === "number") {
2061
- const ok = await browser.clickByIndex(action.index);
2062
- if (ok) return {};
2063
- }
2064
- const keyword = this.extractClickKeyword(action.description);
2065
- if (keyword && await browser.clickByText(keyword)) {
2066
- this.logger.debug("Clicked by text", { keyword });
2067
- return {};
2068
- }
2069
- if (typeof action.x === "number" && typeof action.y === "number") {
2070
- await browser.click(action.x, action.y);
2071
- return {};
2072
- }
2073
- throw new Error(`click action has no resolvable target (index/text/coordinates)`);
2074
- }
2075
- case "type": {
2076
- const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
2077
- const submit = action.submit ?? resolvedText.includes("\n");
2078
- const cleanText = submit ? resolvedText.replace(/\n+$/, "") : resolvedText;
2079
- if (typeof action.index === "number") {
2080
- const ok = await browser.inputByIndex(action.index, cleanText, {
2081
- clear: action.clear,
2082
- submit
2083
- });
2084
- if (ok) return {};
2085
- }
2086
- if (typeof action.x === "number" && typeof action.y === "number") {
2087
- await browser.clickAndType(action.x, action.y, cleanText);
2088
- if (submit) await browser.pressKey("Enter");
2089
- return {};
2090
- }
2091
- await browser.type(cleanText);
2092
- if (submit) await browser.pressKey("Enter");
2093
- return {};
2094
- }
2095
- case "scroll": {
2096
- if (typeof action.index === "number") {
2097
- await browser.scrollIntoViewByIndex(action.index);
2098
- return {};
2099
- }
2100
- await browser.scroll(action.direction, action.amount);
2101
- return {};
2102
- }
2103
- case "navigate": {
2104
- this.assertDomainAllowed(action.url);
2105
- await browser.navigate(action.url);
2106
- return { didNavigate: true };
2107
- }
2108
- case "back":
2109
- await browser.back();
2110
- return { didNavigate: true };
2111
- case "wait":
2112
- await this.sleep(Math.min(action.ms, 1e4));
2113
- return {};
2114
- case "screenshot":
2115
- return {};
2116
- case "send_keys":
2117
- await browser.sendKeys(action.keys);
2118
- return {};
2119
- case "find_text":
2120
- await browser.findText(action.text);
2121
- return {};
2122
- case "evaluate": {
2123
- if (!this.allowEvaluate) {
2124
- throw new Error("evaluate is disabled. Set allowEvaluate: true on BrowserAgent to enable.");
2125
- }
2126
- const out = await browser.evaluate(action.code);
2127
- return { output: `evaluate \u2192 ${out}` };
2128
- }
2129
- case "dropdown_options": {
2130
- const options = await browser.dropdownOptions(action.index);
2131
- const formatted = options.map((o, i) => `${i + 1}. "${o.label}" (value="${o.value}")${o.selected ? " [selected]" : ""}`).join("\n");
2132
- return { output: `Dropdown [${action.index}] options:
2133
- ${formatted || "(no options found)"}` };
2134
- }
2135
- case "select_dropdown": {
2136
- const ok = await browser.selectDropdown(action.index, action.text);
2137
- if (!ok) throw new Error(`Could not select "${action.text}" on dropdown [${action.index}]`);
2138
- return {};
2139
- }
2140
- case "upload_file": {
2141
- const ok = await browser.uploadFileByIndex(action.index, action.path);
2142
- if (!ok) throw new Error(`Could not upload "${action.path}" to [${action.index}]`);
2143
- return {};
2144
- }
2145
- case "extract": {
2146
- const pageText = await browser.pageText({ extractLinks: action.extractLinks });
2147
- const model = this.pageExtractionLLM ?? this.model;
2148
- const messages = [
2149
- {
2150
- role: "system",
2151
- content: "You extract information from web pages. Use ONLY the page content provided. If the requested information is not present, say so explicitly. Be concise and structured (lists/tables) when appropriate."
2152
- },
2153
- {
2154
- role: "user",
2155
- content: `Query: ${action.query}
2156
-
2157
- Page content:
2158
- ${pageText}`
2159
- }
2160
- ];
2161
- const response = await model.generate(messages, { temperature: 0, maxTokens: 2048 });
2162
- const out = typeof response.message.content === "string" ? response.message.content : "";
2163
- const masked = this.credentials ? this.credentials.mask(out) : out;
2164
- extractedContent.push(masked);
2165
- return { output: masked };
2166
- }
2167
- case "tool": {
2168
- const tool = this.tools.find((t) => t.name === action.name);
2169
- if (!tool) throw new Error(`Tool "${action.name}" is not registered on this BrowserAgent`);
2170
- const ctx = new import_core.RunContext({
2171
- sessionId: `browser_${this.name}_${Date.now()}`,
2172
- eventBus: this.eventBus
2173
- });
2174
- const result = await tool.execute(action.args ?? {}, ctx);
2175
- const out = typeof result === "string" ? result : JSON.stringify(result);
2176
- return { output: `${action.name} \u2192 ${out}` };
2177
- }
2178
- // `done` and `fail` are intercepted by the run loop before we get
2179
- // here. Listing them keeps the discriminated union exhaustive.
2180
- case "done":
2181
- case "fail":
2182
- return {};
2183
- }
2184
- }
2185
- sleep(ms) {
2186
- return new Promise((resolve) => setTimeout(resolve, ms));
2187
- }
2188
- /**
2189
- * Parse a quoted target keyword from a click action's `description`.
2190
- * Returns `undefined` for generic / ambiguous labels (login buttons,
2191
- * close, OK, etc.) where a substring text match could fire on the
2192
- * wrong element.
2193
- */
2194
- extractClickKeyword(description) {
2195
- if (!description) return void 0;
2196
- const match = description.match(
2197
- /['"\u2018\u2019\u201C\u201D]([^'"\u2018\u2019\u201C\u201D]{1,80})['"\u2018\u2019\u201C\u201D]/
2198
- );
2199
- if (!match) return void 0;
2200
- const keyword = match[1].trim();
2201
- if (!keyword || keyword.length < 2) return void 0;
2202
- const skip = /* @__PURE__ */ new Set([
2203
- "log in",
2204
- "login",
2205
- "sign in",
2206
- "sign up",
2207
- "submit",
2208
- "close",
2209
- "ok",
2210
- "okay",
2211
- "cancel",
2212
- "yes",
2213
- "no",
2214
- "x",
2215
- "continue",
2216
- "next",
2217
- "back",
2218
- "accept",
2219
- "dismiss",
2220
- "got it",
2221
- "agree",
2222
- "i agree",
2223
- "allow",
2224
- "deny"
2225
- ]);
2226
- if (skip.has(keyword.toLowerCase())) return void 0;
2227
- return keyword;
2228
- }
1526
+ name;
1527
+ eventBus;
1528
+ model;
1529
+ pageExtractionLLM;
1530
+ fallbackModel;
1531
+ useThinking;
1532
+ historyWindow;
1533
+ instructions;
1534
+ extendSystemMessage;
1535
+ overrideSystemMessage;
1536
+ maxSteps;
1537
+ maxFailures;
1538
+ maxActionsPerStep;
1539
+ initialActions;
1540
+ useVision;
1541
+ directlyOpenUrl;
1542
+ headless;
1543
+ viewport;
1544
+ defaultStartUrl;
1545
+ waitAfterAction;
1546
+ maxRepeats;
1547
+ useDOM;
1548
+ allowEvaluate;
1549
+ allowedDomains;
1550
+ prohibitedDomains;
1551
+ storageState;
1552
+ cdpUrl;
1553
+ recordVideo;
1554
+ credentials;
1555
+ stealth;
1556
+ humanize;
1557
+ tools;
1558
+ approvalManager;
1559
+ executionPolicy;
1560
+ planner;
1561
+ jevModel;
1562
+ maxActionChoices;
1563
+ searchEngine;
1564
+ jevProvider = null;
1565
+ costTracker;
1566
+ memoryManager = null;
1567
+ logger;
1568
+ /** Access the MemoryManager (if memory is configured). */
1569
+ get memory() {
1570
+ return this.memoryManager;
1571
+ }
1572
+ constructor(config) {
1573
+ this.name = config.name;
1574
+ this.model = config.model;
1575
+ this.pageExtractionLLM = config.pageExtractionLLM ?? null;
1576
+ this.fallbackModel = config.fallbackModel ?? null;
1577
+ this.useThinking = config.useThinking ?? true;
1578
+ this.historyWindow = Math.max(0, config.historyWindow ?? 6);
1579
+ this.instructions = config.instructions;
1580
+ this.extendSystemMessage = config.extendSystemMessage;
1581
+ this.overrideSystemMessage = config.overrideSystemMessage;
1582
+ this.maxSteps = config.maxSteps ?? 30;
1583
+ this.maxFailures = config.maxFailures ?? 3;
1584
+ this.maxActionsPerStep = Math.max(1, config.maxActionsPerStep ?? 3);
1585
+ this.initialActions = config.initialActions ?? [];
1586
+ this.useVision = config.useVision ?? "auto";
1587
+ this.directlyOpenUrl = config.directlyOpenUrl ?? true;
1588
+ this.headless = config.headless ?? true;
1589
+ this.viewport = config.viewport ?? {
1590
+ width: 1280,
1591
+ height: 720
1592
+ };
1593
+ this.defaultStartUrl = config.startUrl;
1594
+ this.waitAfterAction = config.waitAfterAction ?? 1500;
1595
+ this.maxRepeats = config.maxRepeats ?? 3;
1596
+ this.useDOM = config.useDOM ?? true;
1597
+ this.allowEvaluate = config.allowEvaluate ?? false;
1598
+ this.allowedDomains = config.allowedDomains;
1599
+ this.prohibitedDomains = config.prohibitedDomains;
1600
+ this.storageState = config.storageState;
1601
+ this.cdpUrl = config.cdpUrl;
1602
+ this.recordVideo = config.recordVideo;
1603
+ this.credentials = config.credentials;
1604
+ this.stealth = config.stealth;
1605
+ this.humanize = config.humanize;
1606
+ this.eventBus = config.eventBus ?? new _agentium_core.EventBus();
1607
+ this.tools = config.tools ?? [];
1608
+ this.executionPolicy = config.executionPolicy;
1609
+ this.approvalManager = config.approvalManager ?? (config.approval ? new _agentium_core.ApprovalManager({
1610
+ ...config.approval,
1611
+ eventBus: this.eventBus
1612
+ }) : null);
1613
+ this.planner = config.planner ?? "vision";
1614
+ this.jevModel = config.jevModel ?? "jev-latest";
1615
+ this.maxActionChoices = config.maxActionChoices ?? 40;
1616
+ this.searchEngine = config.searchEngine ?? "duckduckgo";
1617
+ this.costTracker = config.costTracker ?? null;
1618
+ this.logger = new _agentium_core.Logger({
1619
+ prefix: `BrowserAgent:${config.name}`,
1620
+ level: config.logLevel ?? "silent"
1621
+ });
1622
+ if (config.memory) this.memoryManager = new _agentium_core.MemoryManager(config.memory);
1623
+ }
1624
+ async run(task, opts) {
1625
+ const startTime = Date.now();
1626
+ const maxSteps = opts?.maxSteps ?? this.maxSteps;
1627
+ const sessionId = opts?.context?.sessionId ?? opts?.sessionId ?? `browser_${startTime}`;
1628
+ const userId = opts?.context?.userId ?? opts?.userId;
1629
+ const ctx = opts?.context ?? new _agentium_core.RunContext({
1630
+ sessionId,
1631
+ userId,
1632
+ tenantId: opts?.tenantId,
1633
+ signal: opts?.signal,
1634
+ runMode: opts?.runMode,
1635
+ executionPolicy: this.executionPolicy,
1636
+ eventBus: this.eventBus
1637
+ });
1638
+ const executor = new _agentium_core.ToolExecutor(this.tools, {
1639
+ approvalManager: this.approvalManager ?? void 0,
1640
+ executionPolicy: this.executionPolicy,
1641
+ agentName: this.name
1642
+ });
1643
+ const browser = new BrowserProvider();
1644
+ const steps = [];
1645
+ const actionHistory = [];
1646
+ const extractedContent = [];
1647
+ let lastExtractResult;
1648
+ let consecutiveFailures = 0;
1649
+ let lastActionWasScreenshot = false;
1650
+ const loop = new LoopDetector();
1651
+ /**
1652
+ * Rolling conversation history. Each entry stores the structured
1653
+ * agent envelope so we can replay model_thoughts to the model on
1654
+ * subsequent steps — this is what `historyWindow > 0` buys us.
1655
+ */
1656
+ const historyTurns = [];
1657
+ let extraInstructions = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
1658
+ if (this.memoryManager) {
1659
+ await this.memoryManager.ensureReady();
1660
+ const memoryContext = await this.memoryManager.buildContext(sessionId, userId, task, this.name);
1661
+ if (memoryContext) extraInstructions = extraInstructions ? `${extraInstructions}\n\n${memoryContext}` : memoryContext;
1662
+ }
1663
+ const credentialKeys = this.credentials?.keys();
1664
+ const systemPrompt = buildSystemPrompt(this.viewport, extraInstructions || void 0, credentialKeys, {
1665
+ overrideSystemMessage: this.overrideSystemMessage,
1666
+ maxActionsPerStep: this.maxActionsPerStep,
1667
+ allowEvaluate: this.allowEvaluate,
1668
+ tools: this.tools,
1669
+ useVision: this.useVision,
1670
+ useDOM: this.useDOM,
1671
+ useThinking: this.useThinking
1672
+ });
1673
+ try {
1674
+ if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
1675
+ if (ctx.runMode === "plan") throw new Error("Native browser operations are not supported in plan mode");
1676
+ this.logger.info("Launching browser", {
1677
+ headless: this.headless,
1678
+ viewport: this.viewport,
1679
+ useDOM: this.useDOM,
1680
+ useVision: this.useVision,
1681
+ useThinking: this.useThinking,
1682
+ cdpUrl: this.cdpUrl ?? void 0,
1683
+ recordVideo: !!this.recordVideo,
1684
+ stealth: !!this.stealth,
1685
+ humanize: !!this.humanize
1686
+ });
1687
+ await browser.launch({
1688
+ headless: this.headless,
1689
+ viewport: this.viewport,
1690
+ storageState: this.storageState,
1691
+ recordVideo: this.recordVideo,
1692
+ stealth: this.stealth,
1693
+ humanize: this.humanize,
1694
+ cdpUrl: this.cdpUrl
1695
+ });
1696
+ const startUrl = opts?.startUrl ?? this.defaultStartUrl ?? this.detectUrlInTask(task);
1697
+ if (startUrl) {
1698
+ this.logger.info("Navigating to start URL", { url: startUrl });
1699
+ this.assertDomainAllowed(startUrl);
1700
+ await browser.navigate(startUrl);
1701
+ await this.navigationHealthCheck(browser, startUrl);
1702
+ }
1703
+ for (const ia of this.initialActions) {
1704
+ try {
1705
+ await this.executeAction(browser, ia, actionHistory, extractedContent, ctx, executor);
1706
+ } catch (e) {
1707
+ this.logger.warn("initialAction failed", {
1708
+ action: ia,
1709
+ error: e?.message
1710
+ });
1711
+ }
1712
+ await this.sleep(this.waitAfterAction);
1713
+ }
1714
+ for (let step = 0; step < maxSteps; step++) {
1715
+ if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
1716
+ const pageInfo = await browser.getPageInfo();
1717
+ let domSnapshot;
1718
+ let scrollCtx;
1719
+ let elements = [];
1720
+ if (this.useDOM) {
1721
+ const dom = await browser.extractDOM();
1722
+ domSnapshot = dom.text;
1723
+ scrollCtx = dom.scroll;
1724
+ elements = dom.elements;
1725
+ }
1726
+ const tabs = browser.listTabs();
1727
+ const pageAdvice = loop.recordPage({
1728
+ url: pageInfo.url,
1729
+ interactiveCount: scrollCtx?.totalInteractive ?? 0,
1730
+ textHash: fnvHash(domSnapshot ?? "")
1731
+ });
1732
+ if (pageAdvice.severity === "abort") return await this.finalize(browser, steps, startTime, opts, extractedContent, {
1733
+ result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
1734
+ success: false
1735
+ });
1736
+ const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
1737
+ const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
1738
+ if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
1739
+ const isLastStep = step === maxSteps - 1;
1740
+ const nudgeParts = [];
1741
+ if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
1742
+ if (isLastStep) nudgeParts.push("This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial.");
1743
+ const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
1744
+ const userText = buildUserMessage(task, pageInfo.url, pageInfo.title, step, actionHistory, domSnapshot, lastExtractResult, scrollCtx, nudge, {
1745
+ current: step,
1746
+ max: maxSteps
1747
+ }, tabs);
1748
+ this.logger.debug("Calling planner", {
1749
+ step,
1750
+ url: pageInfo.url,
1751
+ planner: this.planner,
1752
+ vision: wantVision
1753
+ });
1754
+ let envelope = null;
1755
+ let modelUsed = this.model;
1756
+ if (this.planner === "jev") {
1757
+ const planned = await this.planWithJev({
1758
+ task,
1759
+ url: pageInfo.url,
1760
+ title: pageInfo.title,
1761
+ elements,
1762
+ tabs,
1763
+ actionHistory,
1764
+ lastExtract: lastExtractResult,
1765
+ pagesBelow: scrollCtx?.pagesBelow,
1766
+ pagesAbove: scrollCtx?.pagesAbove,
1767
+ apiKey: opts?.apiKey,
1768
+ signal: ctx.signal
1769
+ });
1770
+ envelope = planned.envelope;
1771
+ modelUsed = planned.modelUsed;
1772
+ if (this.costTracker && planned.usage) this.costTracker.track({
1773
+ runId: sessionId,
1774
+ agentName: this.name,
1775
+ modelId: modelUsed.modelId,
1776
+ usage: planned.usage,
1777
+ sessionId,
1778
+ userId
1779
+ });
1780
+ } else {
1781
+ const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
1782
+ const { response, modelUsed: used } = await this.callModelWithFallback(messages, opts?.apiKey, ctx.signal);
1783
+ modelUsed = used;
1784
+ if (!response) {
1785
+ consecutiveFailures++;
1786
+ actionHistory.push(`(model call failed — retrying, ${consecutiveFailures}/${this.maxFailures})`);
1787
+ if (consecutiveFailures > this.maxFailures) return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
1788
+ continue;
1789
+ }
1790
+ if (this.costTracker && response.usage) this.costTracker.track({
1791
+ runId: sessionId,
1792
+ agentName: this.name,
1793
+ modelId: modelUsed.modelId,
1794
+ usage: response.usage,
1795
+ sessionId,
1796
+ userId
1797
+ });
1798
+ const raw = typeof response.message.content === "string" ? response.message.content : "";
1799
+ envelope = this.parseEnvelope(raw);
1800
+ }
1801
+ if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
1802
+ if (!envelope) {
1803
+ consecutiveFailures++;
1804
+ this.logger.warn("Failed to parse planner response", {
1805
+ planner: this.planner,
1806
+ consecutiveFailures
1807
+ });
1808
+ if (consecutiveFailures > this.maxFailures) return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
1809
+ actionHistory.push("(invalid JSON response — retrying)");
1810
+ continue;
1811
+ }
1812
+ consecutiveFailures = 0;
1813
+ historyTurns.push({
1814
+ userText,
1815
+ hasScreenshot: wantVision,
1816
+ envelope
1817
+ });
1818
+ if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) historyTurns.splice(0, historyTurns.length - this.historyWindow);
1819
+ const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(0, this.maxActionsPerStep);
1820
+ let didTerminate = null;
1821
+ let didNavigate = false;
1822
+ lastActionWasScreenshot = false;
1823
+ for (let ai = 0; ai < actions.length; ai++) {
1824
+ if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
1825
+ const action = actions[ai];
1826
+ let summary = summarizeAction(action);
1827
+ if (this.credentials) summary = this.credentials.mask(summary);
1828
+ const advice = loop.recordAction(action);
1829
+ if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
1830
+ this.logger.warn("Loop detector aborting run", { advice });
1831
+ actionHistory.push(`⚠ ${advice.message ?? "loop detected — auto-stopping"}`);
1832
+ didTerminate = {
1833
+ result: advice.message ?? "Stuck in a loop — auto-stopped.",
1834
+ success: false
1835
+ };
1836
+ break;
1837
+ }
1838
+ actionHistory.push(summary);
1839
+ this.logger.info(`Step ${step + 1}.${ai + 1}: ${summary}`);
1840
+ this.eventBus.emit("browser.action", { action });
1841
+ if (action.action === "done") {
1842
+ didTerminate = {
1843
+ result: this.credentials ? this.credentials.mask(action.result) : action.result,
1844
+ success: true
1845
+ };
1846
+ break;
1847
+ }
1848
+ if (action.action === "fail") {
1849
+ didTerminate = {
1850
+ result: this.credentials ? this.credentials.mask(action.reason) : action.reason,
1851
+ success: false
1852
+ };
1853
+ break;
1854
+ }
1855
+ let stepOk = true;
1856
+ let stepOutput;
1857
+ try {
1858
+ const exec = await this.executeAction(browser, action, actionHistory, extractedContent, ctx, executor);
1859
+ stepOutput = exec?.output;
1860
+ if (exec?.didNavigate) {
1861
+ didNavigate = true;
1862
+ await this.navigationHealthCheck(browser, pageInfo.url);
1863
+ }
1864
+ if (exec?.output && (action.action === "extract" || action.action === "search" || action.action === "find_elements" || action.action === "search_page")) lastExtractResult = exec.output;
1865
+ if (action.action === "screenshot") lastActionWasScreenshot = true;
1866
+ } catch (e) {
1867
+ stepOk = false;
1868
+ consecutiveFailures++;
1869
+ this.logger.warn("Action failed", {
1870
+ action: action.action,
1871
+ error: e?.message,
1872
+ consecutiveFailures
1873
+ });
1874
+ actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
1875
+ if (consecutiveFailures > this.maxFailures) {
1876
+ didTerminate = {
1877
+ result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
1878
+ success: false
1879
+ };
1880
+ break;
1881
+ }
1882
+ }
1883
+ steps.push({
1884
+ index: steps.length,
1885
+ action,
1886
+ screenshot,
1887
+ pageUrl: pageInfo.url,
1888
+ pageTitle: pageInfo.title,
1889
+ timestamp: /* @__PURE__ */ new Date(),
1890
+ dom: domSnapshot,
1891
+ output: stepOutput,
1892
+ ok: stepOk,
1893
+ thinking: envelope.thinking,
1894
+ evaluationPreviousGoal: envelope.evaluationPreviousGoal,
1895
+ memory: envelope.memory,
1896
+ nextGoal: envelope.nextGoal
1897
+ });
1898
+ this.eventBus.emit("browser.step", {
1899
+ index: steps.length - 1,
1900
+ action,
1901
+ pageUrl: pageInfo.url,
1902
+ screenshot
1903
+ });
1904
+ await this.sleep(this.waitAfterAction);
1905
+ if (didNavigate) break;
1906
+ }
1907
+ if (didTerminate) return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
1908
+ }
1909
+ this.logger.warn("Max steps reached without explicit done", { maxSteps });
1910
+ return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
1911
+ } catch (error) {
1912
+ this.logger.error("Browser agent error", { error: error.message });
1913
+ this.eventBus.emit("browser.error", { error });
1914
+ await browser.close();
1915
+ return {
1916
+ result: `Error: ${error.message}`,
1917
+ success: false,
1918
+ steps,
1919
+ finalUrl: "",
1920
+ finalScreenshot: Buffer.alloc(0),
1921
+ durationMs: Date.now() - startTime,
1922
+ extractedContent
1923
+ };
1924
+ } finally {
1925
+ if (!opts?.context) this.approvalManager?.cancelRun(ctx.runId);
1926
+ }
1927
+ }
1928
+ /**
1929
+ * Returns a ToolDef that lets a regular Agent delegate browser tasks
1930
+ * to this BrowserAgent.
1931
+ */
1932
+ asTool(config) {
1933
+ return {
1934
+ name: config?.name ?? "browse_web",
1935
+ description: config?.description ?? "Open a browser and autonomously complete a task on a website. Provide a clear task description and optionally a starting URL.",
1936
+ parameters: zod_v3.z.object({
1937
+ task: zod_v3.z.string().describe("What to do in the browser (e.g., 'Search for X and return the top 3 results')"),
1938
+ startUrl: zod_v3.z.string().optional().describe("URL to start at (e.g., 'https://www.google.com')")
1939
+ }),
1940
+ execute: async (args, ctx) => {
1941
+ return (await this.run(args.task, {
1942
+ startUrl: args.startUrl,
1943
+ context: ctx
1944
+ })).result;
1945
+ }
1946
+ };
1947
+ }
1948
+ shouldCaptureVision(step, lastActionWasScreenshot) {
1949
+ if (this.planner === "jev" && this.useVision !== true) return false;
1950
+ if (this.useVision === false) return false;
1951
+ if (this.useVision === true) return true;
1952
+ if (step === 0) return true;
1953
+ if (!this.useDOM) return true;
1954
+ return lastActionWasScreenshot;
1955
+ }
1956
+ detectUrlInTask(task) {
1957
+ if (!this.directlyOpenUrl) return void 0;
1958
+ const match = task.match(/https?:\/\/[^\s)<>"']+/);
1959
+ return match ? match[0] : void 0;
1960
+ }
1961
+ assertDomainAllowed(url) {
1962
+ if (!this.allowedDomains?.length && !this.prohibitedDomains?.length) return;
1963
+ let host;
1964
+ try {
1965
+ host = new URL(url).hostname.toLowerCase();
1966
+ } catch {
1967
+ throw new Error(`Cannot parse URL for domain check: ${url}`);
1968
+ }
1969
+ if (this.allowedDomains?.length && !this.allowedDomains.some((p) => matchDomain(host, p))) throw new Error(`Navigation blocked: ${host} is not in allowedDomains`);
1970
+ if (this.prohibitedDomains?.length && this.prohibitedDomains.some((p) => matchDomain(host, p))) throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
1971
+ }
1972
+ /**
1973
+ * Build the message array sent to the model: system prompt + a compact
1974
+ * summary of older turns (if any) + the most recent `historyWindow`
1975
+ * turns verbatim + the current step's user message.
1976
+ *
1977
+ * Inspired by browser-use's history compaction. Keeps tokens bounded
1978
+ * while giving the model meaningful context about what it already
1979
+ * tried.
1980
+ */
1981
+ buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
1982
+ const msgs = [{
1983
+ role: "system",
1984
+ content: systemPrompt
1985
+ }];
1986
+ if (this.historyWindow > 0 && historyTurns.length > 0) for (const turn of historyTurns) {
1987
+ const env = turn.envelope;
1988
+ const replay = [];
1989
+ if (env.thinking) replay.push(`thinking: ${env.thinking}`);
1990
+ if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
1991
+ if (env.memory) replay.push(`memory: ${env.memory}`);
1992
+ if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
1993
+ replay.push(`action: ${JSON.stringify(env.action)}`);
1994
+ msgs.push({
1995
+ role: "assistant",
1996
+ content: replay.join("\n")
1997
+ });
1998
+ }
1999
+ const content = [];
2000
+ if (currentScreenshot && currentScreenshot.length > 0) content.push({
2001
+ type: "image",
2002
+ data: currentScreenshot.toString("base64"),
2003
+ mimeType: "image/png"
2004
+ });
2005
+ content.push({
2006
+ type: "text",
2007
+ text: currentUserText
2008
+ });
2009
+ msgs.push({
2010
+ role: "user",
2011
+ content
2012
+ });
2013
+ return msgs;
2014
+ }
2015
+ /**
2016
+ * Call the primary model, retrying once with `fallbackModel` on
2017
+ * transient errors (5xx, 429, network). Returns the response or
2018
+ * `null` if both models failed.
2019
+ */
2020
+ async callModelWithFallback(messages, apiKey, signal) {
2021
+ const reqOpts = {
2022
+ temperature: .1,
2023
+ maxTokens: 1024,
2024
+ apiKey,
2025
+ signal,
2026
+ responseFormat: "json"
2027
+ };
2028
+ try {
2029
+ signal?.throwIfAborted();
2030
+ const r = await this.model.generate(messages, reqOpts);
2031
+ signal?.throwIfAborted();
2032
+ return {
2033
+ response: r,
2034
+ modelUsed: this.model
2035
+ };
2036
+ } catch (e) {
2037
+ signal?.throwIfAborted();
2038
+ if (this.fallbackModel && this.isTransientError(e)) {
2039
+ this.logger.warn("Primary model failed; trying fallbackModel", {
2040
+ error: e?.message,
2041
+ primary: this.model.modelId,
2042
+ fallback: this.fallbackModel.modelId
2043
+ });
2044
+ try {
2045
+ const r = await this.fallbackModel.generate(messages, reqOpts);
2046
+ signal?.throwIfAborted();
2047
+ return {
2048
+ response: r,
2049
+ modelUsed: this.fallbackModel
2050
+ };
2051
+ } catch (e2) {
2052
+ signal?.throwIfAborted();
2053
+ this.logger.warn("Fallback model also failed", { error: e2?.message });
2054
+ return {
2055
+ response: null,
2056
+ modelUsed: this.model
2057
+ };
2058
+ }
2059
+ }
2060
+ this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
2061
+ return {
2062
+ response: null,
2063
+ modelUsed: this.model
2064
+ };
2065
+ }
2066
+ }
2067
+ isTransientError(e) {
2068
+ const msg = String(e?.message ?? e).toLowerCase();
2069
+ if (msg.includes("rate limit") || msg.includes("429")) return true;
2070
+ if (msg.includes("timeout") || msg.includes("etimedout")) return true;
2071
+ if (msg.includes("econnreset") || msg.includes("network")) return true;
2072
+ if (/\b5\d\d\b/.test(msg)) return true;
2073
+ if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
2074
+ return false;
2075
+ }
2076
+ getJev() {
2077
+ if (!this.jevProvider) this.jevProvider = (0, _agentium_core.jev)(this.jevModel);
2078
+ return this.jevProvider;
2079
+ }
2080
+ /**
2081
+ * Ask Jev to pick one label from this frame's action space, then map it
2082
+ * to a BrowserAction. Type/search strings come from a text model or the task.
2083
+ */
2084
+ async planWithJev(args) {
2085
+ const alreadySearched = args.actionHistory.some((h) => /^Searched /.test(h));
2086
+ const alreadyWaited = args.actionHistory.some((h) => /^Waited /.test(h));
2087
+ const foundTitles = args.lastExtract ?? titlesFromElements(args.elements);
2088
+ const space = buildActionSpace(args.elements, args.tabs, {
2089
+ max: this.maxActionChoices,
2090
+ pagesBelow: args.pagesBelow,
2091
+ pagesAbove: args.pagesAbove,
2092
+ allowSearch: !alreadySearched && !isSearchResultsUrl(args.url) && !isBlockedPageUrl(args.url),
2093
+ allowDone: looksLikeResultList(foundTitles),
2094
+ allowWait: !alreadyWaited
2095
+ });
2096
+ const provider = this.getJev();
2097
+ args.signal?.throwIfAborted();
2098
+ try {
2099
+ const response = await provider.generate([{
2100
+ role: "user",
2101
+ content: JSON.stringify({
2102
+ task: args.task,
2103
+ url: args.url,
2104
+ title: args.title,
2105
+ lastActions: args.actionHistory.slice(-8),
2106
+ lastExtract: args.lastExtract,
2107
+ visibleTitles: titlesFromElements(args.elements, 8),
2108
+ tabs: args.tabs,
2109
+ hint: looksLikeResultList(foundTitles) ? "Result titles are in lastExtract. Pick done." : alreadySearched ? "A search already ran. Wait or click a result title. Do not search again." : void 0
2110
+ })
2111
+ }], {
2112
+ questions: { action: (0, _agentium_core.choice)("Which browser action next? Do not repeat lastActions. If search results are already visible, click a result or done.", space.criteria) },
2113
+ apiKey: args.apiKey,
2114
+ signal: args.signal
2115
+ });
2116
+ args.signal?.throwIfAborted();
2117
+ const raw = typeof response.message.content === "string" ? response.message.content : "";
2118
+ let pick;
2119
+ try {
2120
+ const answers = JSON.parse(raw);
2121
+ pick = typeof answers.action === "string" ? answers.action : answers.action?.choice;
2122
+ } catch {
2123
+ pick = void 0;
2124
+ }
2125
+ if (!pick) return {
2126
+ envelope: null,
2127
+ modelUsed: provider,
2128
+ usage: response.usage
2129
+ };
2130
+ const extras = {
2131
+ searchQuery: pick === "search" ? guessSearchQuery(args.task) : void 0,
2132
+ typeText: pick.startsWith("type_") ? await this.inferTypeText(args.task, pick, args.signal) : void 0,
2133
+ doneResult: pick === "done" ? looksLikeResultList(args.lastExtract ?? "") ? args.lastExtract : titlesFromElements(args.elements) || args.lastExtract || "Done" : void 0
2134
+ };
2135
+ const action = labelToAction(pick, extras);
2136
+ if (!action) return {
2137
+ envelope: null,
2138
+ modelUsed: provider,
2139
+ usage: response.usage
2140
+ };
2141
+ return {
2142
+ envelope: {
2143
+ action,
2144
+ nextGoal: pick
2145
+ },
2146
+ modelUsed: provider,
2147
+ usage: response.usage
2148
+ };
2149
+ } catch (e) {
2150
+ args.signal?.throwIfAborted();
2151
+ this.logger.warn("Jev planner failed", { error: e?.message });
2152
+ return {
2153
+ envelope: null,
2154
+ modelUsed: provider
2155
+ };
2156
+ }
2157
+ }
2158
+ async pageLooksBlocked(browser) {
2159
+ if (isBlockedPageUrl((await browser.getPageInfo()).url)) return true;
2160
+ try {
2161
+ return isBotChallengeText(await browser.visibleText());
2162
+ } catch {
2163
+ return false;
2164
+ }
2165
+ }
2166
+ async scrapeSerpTitles(browser) {
2167
+ for (const selector of SERP_TITLE_SELECTORS) {
2168
+ const found = await browser.findElements(selector, { maxResults: 8 });
2169
+ const labels = [];
2170
+ const seen = /* @__PURE__ */ new Set();
2171
+ for (const f of found) {
2172
+ const text = f.text.replace(/\s+/g, " ").trim();
2173
+ if (!isResultTitle(text) || seen.has(text.toLowerCase())) continue;
2174
+ seen.add(text.toLowerCase());
2175
+ labels.push(text);
2176
+ }
2177
+ if (labels.length >= 2) return labels.slice(0, 5).map((t, i) => `${i + 1}. ${t}`).join("\n");
2178
+ }
2179
+ return "";
2180
+ }
2181
+ async inferTypeText(task, pick, signal) {
2182
+ signal?.throwIfAborted();
2183
+ const model = this.pageExtractionLLM ?? (this.model.providerId === "jev" ? null : this.model);
2184
+ if (!model) return guessSearchQuery(task);
2185
+ try {
2186
+ const response = await model.generate([{
2187
+ role: "user",
2188
+ content: `Task: ${task}\nThe next browser action is ${pick}. Reply with ONLY the text to type into that field. No quotes.`
2189
+ }], { signal });
2190
+ signal?.throwIfAborted();
2191
+ return (typeof response.message.content === "string" ? response.message.content.trim() : "").replace(/^["']|["']$/g, "").slice(0, 500);
2192
+ } catch {
2193
+ signal?.throwIfAborted();
2194
+ return guessSearchQuery(task);
2195
+ }
2196
+ }
2197
+ /**
2198
+ * Parse the model's raw response into an `AgentOutput`. Tolerant to
2199
+ * three shapes:
2200
+ * - Full envelope: { thinking, evaluation_previous_goal, action, ... }
2201
+ * - Raw action object (legacy / `useThinking: false`)
2202
+ * - Raw action array
2203
+ *
2204
+ * Also strips ```json fences the model occasionally adds.
2205
+ */
2206
+ parseEnvelope(raw) {
2207
+ if (!raw) return null;
2208
+ let s = raw.trim();
2209
+ const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
2210
+ if (fence) s = fence[1].trim();
2211
+ let json;
2212
+ try {
2213
+ json = JSON.parse(s);
2214
+ } catch {
2215
+ const m = s.match(/[[{][\s\S]*[\]}]/);
2216
+ if (!m) return null;
2217
+ try {
2218
+ json = JSON.parse(m[0]);
2219
+ } catch {
2220
+ return null;
2221
+ }
2222
+ }
2223
+ if (Array.isArray(json)) return { action: json };
2224
+ if (json && typeof json === "object" && typeof json.action === "string") return { action: json };
2225
+ if (json && typeof json === "object" && "action" in json) {
2226
+ const out = { action: json.action };
2227
+ out.thinking = json.thinking ?? json.reasoning ?? void 0;
2228
+ out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
2229
+ out.memory = json.memory ?? void 0;
2230
+ out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
2231
+ return out;
2232
+ }
2233
+ return null;
2234
+ }
2235
+ /**
2236
+ * Quick post-navigation health check. If the page came back blank
2237
+ * (no body text and no interactive elements), reload once and wait
2238
+ * for stable. This catches the "FreightOS half-loaded font test" class
2239
+ * of failure before the LLM ever sees it.
2240
+ */
2241
+ async navigationHealthCheck(browser, url) {
2242
+ try {
2243
+ const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
2244
+ if (ok && ok.trim().length > 5) return;
2245
+ this.logger.warn("Navigation produced an empty page; reloading once", { url });
2246
+ await this.sleep(800);
2247
+ try {
2248
+ await browser.navigate(url);
2249
+ } catch {}
2250
+ } catch {}
2251
+ }
2252
+ /**
2253
+ * Force-finalize the run with whatever partial data the agent has.
2254
+ * Called when:
2255
+ * - `maxSteps` is exhausted without an explicit `done`,
2256
+ * - `maxFailures` is exceeded,
2257
+ * - the model can't produce parseable JSON enough times to make
2258
+ * forward progress.
2259
+ * The result is composed from `extractedContent` + the last few
2260
+ * action summaries so the caller gets something useful instead of
2261
+ * just a one-line error.
2262
+ */
2263
+ async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
2264
+ const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
2265
+ const tail = actionHistory.slice(-5).join(" → ");
2266
+ const extracts = extractedContent.length > 0 ? `\n\nExtracts so far:\n${extractedContent.join("\n---\n")}` : "";
2267
+ const result = `${reasonLine}\nLast actions: ${tail || "(none)"}${extracts}`;
2268
+ return await this.finalize(browser, steps, startTime, opts, extractedContent, {
2269
+ result,
2270
+ success: false
2271
+ });
2272
+ }
2273
+ async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
2274
+ let finalScreenshot;
2275
+ let finalInfo;
2276
+ try {
2277
+ finalScreenshot = await browser.screenshot();
2278
+ finalInfo = await browser.getPageInfo();
2279
+ } catch {
2280
+ finalScreenshot = Buffer.alloc(0);
2281
+ finalInfo = {
2282
+ url: "",
2283
+ title: ""
2284
+ };
2285
+ }
2286
+ if (opts?.saveStorageState) try {
2287
+ await browser.saveStorageState(opts.saveStorageState);
2288
+ this.logger.info("Storage state saved", { path: opts.saveStorageState });
2289
+ } catch (e) {
2290
+ this.logger.warn("Failed to save storage state", { error: e.message });
2291
+ }
2292
+ let videoPath;
2293
+ if (this.recordVideo) videoPath = await browser.getVideoPath() ?? void 0;
2294
+ await browser.close();
2295
+ const output = {
2296
+ result: outcome.result,
2297
+ success: outcome.success,
2298
+ steps,
2299
+ finalUrl: finalInfo.url,
2300
+ finalScreenshot,
2301
+ durationMs: Date.now() - startTime,
2302
+ videoPath,
2303
+ extractedContent
2304
+ };
2305
+ if (this.memoryManager) {
2306
+ const sessionId = opts?.context?.sessionId ?? opts?.sessionId ?? `browser_${startTime}`;
2307
+ const userId = opts?.context?.userId ?? opts?.userId;
2308
+ const actionSummary = steps.map((s) => summarizeAction(s.action)).join("; ");
2309
+ const messages = [{
2310
+ role: "user",
2311
+ content: `Task: ${outcome.result}`
2312
+ }, {
2313
+ role: "assistant",
2314
+ content: `Actions: ${actionSummary}. Result: ${outcome.result}`
2315
+ }];
2316
+ this.memoryManager.appendMessages(sessionId, messages, this.model).catch((e) => this.logger.warn("Memory persist failed", { error: String(e) }));
2317
+ try {
2318
+ this.memoryManager.afterRun(sessionId, userId, messages, this.model, this.name);
2319
+ } catch (e) {
2320
+ this.logger.warn("Memory afterRun failed", { error: String(e) });
2321
+ }
2322
+ }
2323
+ this.eventBus.emit("browser.done", {
2324
+ result: output.result,
2325
+ success: outcome.success,
2326
+ steps
2327
+ });
2328
+ return output;
2329
+ }
2330
+ /**
2331
+ * Execute a single action. Returns `{ output?, didNavigate? }` for the
2332
+ * caller's state tracking. May throw — the loop handles failure-budget
2333
+ * accounting in that case.
2334
+ */
2335
+ async executeAction(browser, action, _actionHistory, extractedContent, ctx, executor) {
2336
+ if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
2337
+ switch (action.action) {
2338
+ case "click": {
2339
+ if (typeof action.index === "number") {
2340
+ if (await browser.clickByIndex(action.index)) return {};
2341
+ }
2342
+ const keyword = this.extractClickKeyword(action.description);
2343
+ if (keyword && await browser.clickByText(keyword)) {
2344
+ this.logger.debug("Clicked by text", { keyword });
2345
+ return {};
2346
+ }
2347
+ if (typeof action.x === "number" && typeof action.y === "number") {
2348
+ await browser.click(action.x, action.y);
2349
+ return {};
2350
+ }
2351
+ throw new Error(`click action has no resolvable target (index/text/coordinates)`);
2352
+ }
2353
+ case "type": {
2354
+ const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
2355
+ const submit = action.submit ?? resolvedText.includes("\n");
2356
+ const cleanText = submit ? resolvedText.replace(/\n+$/, "") : resolvedText;
2357
+ if (typeof action.index === "number") {
2358
+ if (await browser.inputByIndex(action.index, cleanText, {
2359
+ clear: action.clear,
2360
+ submit
2361
+ })) return {};
2362
+ }
2363
+ if (typeof action.x === "number" && typeof action.y === "number") {
2364
+ await browser.clickAndType(action.x, action.y, cleanText);
2365
+ if (submit) await browser.pressKey("Enter");
2366
+ return {};
2367
+ }
2368
+ await browser.type(cleanText);
2369
+ if (submit) await browser.pressKey("Enter");
2370
+ return {};
2371
+ }
2372
+ case "scroll":
2373
+ if (typeof action.index === "number") {
2374
+ await browser.scrollIntoViewByIndex(action.index);
2375
+ return {};
2376
+ }
2377
+ await browser.scroll(action.direction, action.amount);
2378
+ return {};
2379
+ case "navigate":
2380
+ this.assertDomainAllowed(action.url);
2381
+ if (action.newTab) {
2382
+ const tabId = await browser.newTab(action.url);
2383
+ await browser.switchTab(tabId);
2384
+ return {
2385
+ didNavigate: true,
2386
+ output: `Opened ${action.url} in ${tabId}`
2387
+ };
2388
+ }
2389
+ await browser.navigate(action.url);
2390
+ return { didNavigate: true };
2391
+ case "search": {
2392
+ const engine = action.engine ?? this.searchEngine;
2393
+ const url = searchUrl(action.query, engine);
2394
+ this.assertDomainAllowed(url);
2395
+ await browser.navigate(url);
2396
+ let used = engine;
2397
+ if (await this.pageLooksBlocked(browser)) {
2398
+ if (engine === "duckduckgo") {
2399
+ const fallback = searchUrl(action.query, "bing");
2400
+ this.assertDomainAllowed(fallback);
2401
+ await browser.navigate(fallback);
2402
+ used = "bing";
2403
+ } else return {
2404
+ didNavigate: true,
2405
+ output: `Search blocked on ${engine}. Pick fail or navigate elsewhere.`
2406
+ };
2407
+ }
2408
+ await browser.waitForStable(400);
2409
+ return {
2410
+ didNavigate: true,
2411
+ output: await this.scrapeSerpTitles(browser) || `Searched ${used} for "${action.query}"`
2412
+ };
2413
+ }
2414
+ case "new_tab": {
2415
+ if (action.url) this.assertDomainAllowed(action.url);
2416
+ const tabId = await browser.newTab(action.url);
2417
+ await browser.switchTab(tabId);
2418
+ return {
2419
+ didNavigate: !!action.url,
2420
+ output: `Opened ${tabId}`
2421
+ };
2422
+ }
2423
+ case "switch_tab":
2424
+ await browser.switchTab(action.tabId);
2425
+ return {
2426
+ didNavigate: true,
2427
+ output: `Switched to ${action.tabId}`
2428
+ };
2429
+ case "close_tab":
2430
+ await browser.closeTab(action.tabId);
2431
+ return { output: `Closed ${action.tabId}` };
2432
+ case "search_page": {
2433
+ const hits = await browser.searchPage({
2434
+ pattern: action.pattern,
2435
+ regex: action.regex,
2436
+ caseSensitive: action.caseSensitive,
2437
+ maxResults: action.maxResults
2438
+ });
2439
+ const lines = hits.map((h, i) => `${i + 1}. "${h.match}" — …${h.context}…`);
2440
+ return { output: hits.length ? `search_page:\n${lines.join("\n")}` : "search_page: no matches" };
2441
+ }
2442
+ case "find_elements": {
2443
+ const found = await browser.findElements(action.selector, { maxResults: action.maxResults });
2444
+ const lines = found.map((el, i) => `${i + 1}. <${el.tag}> ${el.text}${el.href ? ` ${el.href}` : ""}`);
2445
+ return { output: found.length ? `find_elements:\n${lines.join("\n")}` : "find_elements: none" };
2446
+ }
2447
+ case "back":
2448
+ await browser.back();
2449
+ return { didNavigate: true };
2450
+ case "wait":
2451
+ await this.sleep(Math.min(action.ms, 1e4));
2452
+ return {};
2453
+ case "screenshot": return {};
2454
+ case "send_keys":
2455
+ await browser.sendKeys(action.keys);
2456
+ return {};
2457
+ case "find_text":
2458
+ await browser.findText(action.text);
2459
+ return {};
2460
+ case "evaluate":
2461
+ if (!this.allowEvaluate) throw new Error("evaluate is disabled. Set allowEvaluate: true on BrowserAgent to enable.");
2462
+ return { output: `evaluate → ${await browser.evaluate(action.code)}` };
2463
+ case "dropdown_options": {
2464
+ const formatted = (await browser.dropdownOptions(action.index)).map((o, i) => `${i + 1}. "${o.label}" (value="${o.value}")${o.selected ? " [selected]" : ""}`).join("\n");
2465
+ return { output: `Dropdown [${action.index}] options:\n${formatted || "(no options found)"}` };
2466
+ }
2467
+ case "select_dropdown":
2468
+ if (!await browser.selectDropdown(action.index, action.text)) throw new Error(`Could not select "${action.text}" on dropdown [${action.index}]`);
2469
+ return {};
2470
+ case "upload_file":
2471
+ if (!await browser.uploadFileByIndex(action.index, action.path)) throw new Error(`Could not upload "${action.path}" to [${action.index}]`);
2472
+ return {};
2473
+ case "extract": {
2474
+ const pageText = await browser.pageText({ extractLinks: action.extractLinks });
2475
+ const model = this.pageExtractionLLM ?? this.model;
2476
+ const messages = [{
2477
+ role: "system",
2478
+ content: "You extract information from web pages. Use ONLY the page content provided. If the requested information is not present, say so explicitly. Be concise and structured (lists/tables) when appropriate."
2479
+ }, {
2480
+ role: "user",
2481
+ content: `Query: ${action.query}\n\nPage content:\n${pageText}`
2482
+ }];
2483
+ const response = await model.generate(messages, {
2484
+ temperature: 0,
2485
+ maxTokens: 2048,
2486
+ signal: ctx.signal
2487
+ });
2488
+ ctx.signal?.throwIfAborted();
2489
+ const out = typeof response.message.content === "string" ? response.message.content : "";
2490
+ const masked = this.credentials ? this.credentials.mask(out) : out;
2491
+ extractedContent.push(masked);
2492
+ return { output: masked };
2493
+ }
2494
+ case "tool": {
2495
+ const tool = this.tools.find((t) => t.name === action.name);
2496
+ if (!tool) throw new Error(`Tool "${action.name}" is not registered on this BrowserAgent`);
2497
+ const [result] = await executor.executeAll([{
2498
+ id: `browser-tool-${ctx.runId}-${Date.now()}`,
2499
+ name: tool.name,
2500
+ arguments: action.args ?? {}
2501
+ }], ctx);
2502
+ if (result.error) throw new Error(result.error);
2503
+ const out = typeof result.result === "string" ? result.result : result.result.content;
2504
+ return { output: `${action.name} → ${out}` };
2505
+ }
2506
+ case "done":
2507
+ case "fail": return {};
2508
+ }
2509
+ }
2510
+ sleep(ms) {
2511
+ return new Promise((resolve) => setTimeout(resolve, ms));
2512
+ }
2513
+ /**
2514
+ * Parse a quoted target keyword from a click action's `description`.
2515
+ * Returns `undefined` for generic / ambiguous labels (login buttons,
2516
+ * close, OK, etc.) where a substring text match could fire on the
2517
+ * wrong element.
2518
+ */
2519
+ extractClickKeyword(description) {
2520
+ if (!description) return void 0;
2521
+ const match = description.match(/['"\u2018\u2019\u201C\u201D]([^'"\u2018\u2019\u201C\u201D]{1,80})['"\u2018\u2019\u201C\u201D]/);
2522
+ if (!match) return void 0;
2523
+ const keyword = match[1].trim();
2524
+ if (!keyword || keyword.length < 2) return void 0;
2525
+ if ((/* @__PURE__ */ new Set([
2526
+ "log in",
2527
+ "login",
2528
+ "sign in",
2529
+ "sign up",
2530
+ "submit",
2531
+ "close",
2532
+ "ok",
2533
+ "okay",
2534
+ "cancel",
2535
+ "yes",
2536
+ "no",
2537
+ "x",
2538
+ "continue",
2539
+ "next",
2540
+ "back",
2541
+ "accept",
2542
+ "dismiss",
2543
+ "got it",
2544
+ "agree",
2545
+ "i agree",
2546
+ "allow",
2547
+ "deny"
2548
+ ])).has(keyword.toLowerCase())) return void 0;
2549
+ return keyword;
2550
+ }
2229
2551
  };
2552
+ /**
2553
+ * Domain wildcard matcher. Supports:
2554
+ * "example.com" — exact match
2555
+ * "*.example.com" — example.com and any subdomain
2556
+ * "*" — anything
2557
+ * Case-insensitive.
2558
+ */
2230
2559
  function matchDomain(host, pattern) {
2231
- const h = host.toLowerCase();
2232
- let p = pattern.toLowerCase().replace(/^https?\*?:\/\//, "").replace(/^\/+/, "");
2233
- if (p.includes("/")) p = p.split("/")[0];
2234
- if (p === "*") return true;
2235
- if (p === h) return true;
2236
- if (p.startsWith("*.")) {
2237
- const base = p.slice(2);
2238
- return h === base || h.endsWith(`.${base}`);
2239
- }
2240
- return false;
2560
+ const h = host.toLowerCase();
2561
+ let p = pattern.toLowerCase().replace(/^https?\*?:\/\//, "").replace(/^\/+/, "");
2562
+ if (p.includes("/")) p = p.split("/")[0];
2563
+ if (p === "*") return true;
2564
+ if (p === h) return true;
2565
+ if (p.startsWith("*.")) {
2566
+ const base = p.slice(2);
2567
+ return h === base || h.endsWith(`.${base}`);
2568
+ }
2569
+ return false;
2241
2570
  }
2242
-
2243
- // src/credential-vault.ts
2571
+ //#endregion
2572
+ //#region src/credential-vault.ts
2573
+ /**
2574
+ * Secure credential store for BrowserAgent.
2575
+ *
2576
+ * Secrets are stored in memory and NEVER sent to the LLM.
2577
+ * The model works with placeholders (e.g. `{{email}}`, `{{password}}`),
2578
+ * and the agent resolves them to real values only at execution time.
2579
+ */
2244
2580
  var CredentialVault = class {
2245
- secrets = /* @__PURE__ */ new Map();
2246
- constructor(initial) {
2247
- if (initial) {
2248
- for (const [key, value] of Object.entries(initial)) {
2249
- this.set(key, value);
2250
- }
2251
- }
2252
- }
2253
- /** Store a credential. Key names become the placeholder: `{{key}}`. */
2254
- set(key, value) {
2255
- this.secrets.set(key.toLowerCase(), value);
2256
- return this;
2257
- }
2258
- /** Retrieve a credential value. Returns undefined if not found. */
2259
- get(key) {
2260
- return this.secrets.get(key.toLowerCase());
2261
- }
2262
- has(key) {
2263
- return this.secrets.has(key.toLowerCase());
2264
- }
2265
- /** List available placeholder names (never exposes values). */
2266
- keys() {
2267
- return [...this.secrets.keys()];
2268
- }
2269
- /**
2270
- * Load credentials from environment variables.
2271
- * Maps env var names to placeholder keys.
2272
- *
2273
- * @example
2274
- * vault.fromEnv({ email: "LOGIN_EMAIL", password: "LOGIN_PASS" });
2275
- */
2276
- fromEnv(mapping) {
2277
- for (const [key, envVar] of Object.entries(mapping)) {
2278
- const value = process.env[envVar];
2279
- if (value) this.set(key, value);
2280
- }
2281
- return this;
2282
- }
2283
- /**
2284
- * Replace `{{key}}` placeholders in text with actual credential values.
2285
- * Used internally by BrowserAgent right before executing a type action.
2286
- */
2287
- resolve(text) {
2288
- return text.replace(/\{\{(\w+)\}\}/g, (_match, key) => {
2289
- return this.get(key) ?? `{{${key}}}`;
2290
- });
2291
- }
2292
- /**
2293
- * Replace any occurrence of real credential values in text with
2294
- * their `{{key}}` placeholder. Used to sanitize logs and action history.
2295
- */
2296
- mask(text) {
2297
- let masked = text;
2298
- for (const [key, value] of this.secrets) {
2299
- if (value && masked.includes(value)) {
2300
- masked = masked.split(value).join(`{{${key}}}`);
2301
- }
2302
- }
2303
- return masked;
2304
- }
2581
+ secrets = /* @__PURE__ */ new Map();
2582
+ constructor(initial) {
2583
+ if (initial) for (const [key, value] of Object.entries(initial)) this.set(key, value);
2584
+ }
2585
+ /** Store a credential. Key names become the placeholder: `{{key}}`. */
2586
+ set(key, value) {
2587
+ this.secrets.set(key.toLowerCase(), value);
2588
+ return this;
2589
+ }
2590
+ /** Retrieve a credential value. Returns undefined if not found. */
2591
+ get(key) {
2592
+ return this.secrets.get(key.toLowerCase());
2593
+ }
2594
+ has(key) {
2595
+ return this.secrets.has(key.toLowerCase());
2596
+ }
2597
+ /** List available placeholder names (never exposes values). */
2598
+ keys() {
2599
+ return [...this.secrets.keys()];
2600
+ }
2601
+ /**
2602
+ * Load credentials from environment variables.
2603
+ * Maps env var names to placeholder keys.
2604
+ *
2605
+ * @example
2606
+ * vault.fromEnv({ email: "LOGIN_EMAIL", password: "LOGIN_PASS" });
2607
+ */
2608
+ fromEnv(mapping) {
2609
+ for (const [key, envVar] of Object.entries(mapping)) {
2610
+ const value = process.env[envVar];
2611
+ if (value) this.set(key, value);
2612
+ }
2613
+ return this;
2614
+ }
2615
+ /**
2616
+ * Replace `{{key}}` placeholders in text with actual credential values.
2617
+ * Used internally by BrowserAgent right before executing a type action.
2618
+ */
2619
+ resolve(text) {
2620
+ return text.replace(/\{\{(\w+)\}\}/g, (_match, key) => {
2621
+ return this.get(key) ?? `{{${key}}}`;
2622
+ });
2623
+ }
2624
+ /**
2625
+ * Replace any occurrence of real credential values in text with
2626
+ * their `{{key}}` placeholder. Used to sanitize logs and action history.
2627
+ */
2628
+ mask(text) {
2629
+ let masked = text;
2630
+ for (const [key, value] of this.secrets) if (value && masked.includes(value)) masked = masked.split(value).join(`{{${key}}}`);
2631
+ return masked;
2632
+ }
2305
2633
  };
2306
- // Annotate the CommonJS export names for ESM import in node:
2307
- 0 && (module.exports = {
2308
- BrowserAgent,
2309
- BrowserProvider,
2310
- CredentialVault
2311
- });
2634
+ //#endregion
2635
+ exports.BrowserAgent = BrowserAgent;
2636
+ exports.BrowserProvider = BrowserProvider;
2637
+ exports.CredentialVault = CredentialVault;
2638
+ exports.buildActionSpace = buildActionSpace;
2639
+ exports.guessSearchQuery = guessSearchQuery;
2640
+ exports.isBlockedPageUrl = isBlockedPageUrl;
2641
+ exports.isBotChallengeText = isBotChallengeText;
2642
+ exports.isResultTitle = isResultTitle;
2643
+ exports.isSearchResultsUrl = isSearchResultsUrl;
2644
+ exports.labelToAction = labelToAction;
2645
+ exports.looksLikeResultList = looksLikeResultList;
2646
+ exports.searchUrl = searchUrl;
2647
+ exports.titlesFromElements = titlesFromElements;