@agentium/browser 3.1.2 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/browser-agent.d.ts +3 -1
- package/dist/browser-agent.d.ts.map +1 -1
- package/dist/browser-provider.d.ts.map +1 -1
- package/dist/index.cjs +2341 -2486
- package/dist/index.js +2328 -2437
- package/dist/types.d.ts +10 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +10 -7
package/dist/index.js
CHANGED
|
@@ -1,162 +1,184 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import { ApprovalManager, EventBus, Logger, MemoryManager, RunContext, ToolExecutor, choice, jev } from "@agentium/core";
|
|
2
|
+
import { z } from "zod/v3";
|
|
3
3
|
function searchUrl(query, engine = "duckduckgo") {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
default:
|
|
11
|
-
return `https://html.duckduckgo.com/html/?q=${q}`;
|
|
12
|
-
}
|
|
4
|
+
const q = encodeURIComponent(query.trim());
|
|
5
|
+
switch (engine) {
|
|
6
|
+
case "google": return `https://www.google.com/search?q=${q}`;
|
|
7
|
+
case "bing": return `https://www.bing.com/search?q=${q}`;
|
|
8
|
+
default: return `https://html.duckduckgo.com/html/?q=${q}`;
|
|
9
|
+
}
|
|
13
10
|
}
|
|
14
11
|
function isSearchResultsUrl(url) {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return false;
|
|
26
|
-
}
|
|
12
|
+
try {
|
|
13
|
+
const u = new URL(url);
|
|
14
|
+
const host = u.hostname.replace(/^www\./, "");
|
|
15
|
+
if (host === "duckduckgo.com" || host.endsWith(".duckduckgo.com")) return u.searchParams.has("q") || u.pathname.includes("/html");
|
|
16
|
+
if (host === "google.com" || host.endsWith(".google.com")) return u.pathname.startsWith("/search");
|
|
17
|
+
if (host === "bing.com" || host.endsWith(".bing.com")) return u.pathname.startsWith("/search");
|
|
18
|
+
return false;
|
|
19
|
+
} catch {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
27
22
|
}
|
|
23
|
+
/** Bot-block / challenge pages (DDG 418, Google sorry, Cloudflare, …). */
|
|
28
24
|
function isBlockedPageUrl(url) {
|
|
29
|
-
|
|
25
|
+
return /418\.html|\/sorry\/|captcha|challenge|cf-browser-verification/i.test(url);
|
|
30
26
|
}
|
|
31
27
|
function isBotChallengeText(text) {
|
|
32
|
-
|
|
33
|
-
text
|
|
34
|
-
);
|
|
28
|
+
return /unfortunately, bots use|select all squares containing a duck|unusual traffic from your computer|are you a robot|verify you are human|detected unusual traffic/i.test(text);
|
|
35
29
|
}
|
|
36
|
-
|
|
30
|
+
const CHROME_LABEL = /^(email us|feedback|privacy|terms|settings|facebook|twitter|reddit|about|help|sign in|log in|login|all regions|any time|more results|next|previous|images|videos|news|maps|shopping|lite|duckduckgo|menu|home|cookies|advertising|github|wikipedia|instagram|youtube)$/i;
|
|
37
31
|
function isResultTitle(label) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
32
|
+
const t = label.replace(/\s+/g, " ").trim();
|
|
33
|
+
if (t.length < 16) return false;
|
|
34
|
+
if (CHROME_LABEL.test(t)) return false;
|
|
35
|
+
if (/^(https?:|javascript:|mailto:)/i.test(t)) return false;
|
|
36
|
+
return true;
|
|
43
37
|
}
|
|
44
38
|
function looksLikeResultList(text) {
|
|
45
|
-
|
|
46
|
-
|
|
39
|
+
const lines = text.split("\n").map((l) => l.replace(/^\d+\.\s*/, "").trim()).filter(Boolean);
|
|
40
|
+
return lines.length >= 2 && lines.filter(isResultTitle).length >= 2;
|
|
47
41
|
}
|
|
42
|
+
/** Longest non-chrome link labels — search result titles, not "email us". */
|
|
48
43
|
function titlesFromElements(elements, max = 5) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
44
|
+
const seen = /* @__PURE__ */ new Set();
|
|
45
|
+
const labels = [];
|
|
46
|
+
for (const e of elements) {
|
|
47
|
+
if (e.tag !== "a" && e.role !== "link" && e.role !== "heading") continue;
|
|
48
|
+
const label = e.label?.replace(/\s+/g, " ").trim() ?? "";
|
|
49
|
+
if (!isResultTitle(label) || seen.has(label.toLowerCase())) continue;
|
|
50
|
+
seen.add(label.toLowerCase());
|
|
51
|
+
labels.push(label);
|
|
52
|
+
}
|
|
53
|
+
labels.sort((a, b) => b.length - a.length);
|
|
54
|
+
return labels.slice(0, max).map((t, i) => `${i + 1}. ${t}`).join("\n");
|
|
60
55
|
}
|
|
61
|
-
|
|
56
|
+
const SERP_TITLE_SELECTORS = [
|
|
57
|
+
"a.result__a",
|
|
58
|
+
"a.result-link",
|
|
59
|
+
"#links .result__a",
|
|
60
|
+
"h2 a",
|
|
61
|
+
"h3 a",
|
|
62
|
+
"a > h3"
|
|
63
|
+
];
|
|
62
64
|
function guessSearchQuery(task) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
65
|
+
const quoted = task.match(/["']([^"']{1,200})["']/);
|
|
66
|
+
if (quoted?.[1]) return quoted[1].trim();
|
|
67
|
+
return task.replace(/^(please\s+)?(search for|google|bing|find|look up)\s+/i, "").slice(0, 200).trim();
|
|
66
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Closed action list for this frame: chrome (back, scroll, done, …) plus
|
|
71
|
+
* `click_N` / `type_N` from the DOM snapshot. Used by the Jev planner and
|
|
72
|
+
* as documentation of what `executeAction` can run without free-form JSON.
|
|
73
|
+
*/
|
|
67
74
|
function buildActionSpace(elements, tabs = [], opts) {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
for (const tab of tabs) {
|
|
99
|
-
if (!tab.active) {
|
|
100
|
-
criteria[`switch_tab_${tab.id}`] = `Switch to tab ${tab.id} (${tab.url || "blank"})`;
|
|
101
|
-
}
|
|
102
|
-
if (tabs.length > 1) {
|
|
103
|
-
criteria[`close_tab_${tab.id}`] = `Close tab ${tab.id}`;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
return {
|
|
107
|
-
criteria,
|
|
108
|
-
toAction(label) {
|
|
109
|
-
return labelToAction(label);
|
|
110
|
-
}
|
|
111
|
-
};
|
|
75
|
+
const max = opts?.max ?? 40;
|
|
76
|
+
const criteria = {
|
|
77
|
+
back: "Go to the previous page",
|
|
78
|
+
screenshot: "Take a fresh screenshot next step",
|
|
79
|
+
fail: "Give up — the task cannot be completed",
|
|
80
|
+
new_tab: "Open a blank new tab"
|
|
81
|
+
};
|
|
82
|
+
if (opts?.allowWait !== false) criteria.wait = "Wait for the page to settle. Use at most once.";
|
|
83
|
+
if (opts?.allowDone !== false) criteria.done = "The task is finished — return the result titles you already found";
|
|
84
|
+
if (opts?.allowSearch !== false) criteria.search = "Open a NEW web search. Do not pick this if results are already on screen.";
|
|
85
|
+
if ((opts?.pagesBelow ?? 1) > 0) criteria.scroll_down = "Scroll down one viewport";
|
|
86
|
+
if ((opts?.pagesAbove ?? 1) > 0) criteria.scroll_up = "Scroll up one viewport";
|
|
87
|
+
const slice = elements.slice(0, max);
|
|
88
|
+
for (const e of slice) {
|
|
89
|
+
const label = e.label?.trim() || e.role || e.tag;
|
|
90
|
+
criteria[`click_${e.index}`] = `Click [${e.index}] ${e.role}: ${label}`;
|
|
91
|
+
if (e.isInput) criteria[`type_${e.index}`] = `Type into [${e.index}] ${label}`;
|
|
92
|
+
}
|
|
93
|
+
for (const tab of tabs) {
|
|
94
|
+
if (!tab.active) criteria[`switch_tab_${tab.id}`] = `Switch to tab ${tab.id} (${tab.url || "blank"})`;
|
|
95
|
+
if (tabs.length > 1) criteria[`close_tab_${tab.id}`] = `Close tab ${tab.id}`;
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
criteria,
|
|
99
|
+
toAction(label) {
|
|
100
|
+
return labelToAction(label);
|
|
101
|
+
}
|
|
102
|
+
};
|
|
112
103
|
}
|
|
113
104
|
function labelToAction(label, extras) {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
105
|
+
if (label === "back") return { action: "back" };
|
|
106
|
+
if (label === "scroll_down") return {
|
|
107
|
+
action: "scroll",
|
|
108
|
+
direction: "down"
|
|
109
|
+
};
|
|
110
|
+
if (label === "scroll_up") return {
|
|
111
|
+
action: "scroll",
|
|
112
|
+
direction: "up"
|
|
113
|
+
};
|
|
114
|
+
if (label === "wait") return {
|
|
115
|
+
action: "wait",
|
|
116
|
+
ms: 1500
|
|
117
|
+
};
|
|
118
|
+
if (label === "screenshot") return { action: "screenshot" };
|
|
119
|
+
if (label === "done") return {
|
|
120
|
+
action: "done",
|
|
121
|
+
result: extras?.doneResult ?? "Done"
|
|
122
|
+
};
|
|
123
|
+
if (label === "fail") return {
|
|
124
|
+
action: "fail",
|
|
125
|
+
reason: "Jev chose fail"
|
|
126
|
+
};
|
|
127
|
+
if (label === "search") return {
|
|
128
|
+
action: "search",
|
|
129
|
+
query: extras?.searchQuery ?? ""
|
|
130
|
+
};
|
|
131
|
+
if (label === "new_tab") return { action: "new_tab" };
|
|
132
|
+
if (label === "none") return void 0;
|
|
133
|
+
const click = /^click_(\d+)$/.exec(label);
|
|
134
|
+
if (click) return {
|
|
135
|
+
action: "click",
|
|
136
|
+
index: Number(click[1])
|
|
137
|
+
};
|
|
138
|
+
const type = /^type_(\d+)$/.exec(label);
|
|
139
|
+
if (type) return {
|
|
140
|
+
action: "type",
|
|
141
|
+
index: Number(type[1]),
|
|
142
|
+
text: extras?.typeText ?? ""
|
|
143
|
+
};
|
|
144
|
+
const sw = /^switch_tab_(.+)$/.exec(label);
|
|
145
|
+
if (sw) return {
|
|
146
|
+
action: "switch_tab",
|
|
147
|
+
tabId: sw[1]
|
|
148
|
+
};
|
|
149
|
+
const cl = /^close_tab_(.+)$/.exec(label);
|
|
150
|
+
if (cl) return {
|
|
151
|
+
action: "close_tab",
|
|
152
|
+
tabId: cl[1]
|
|
153
|
+
};
|
|
133
154
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
|
144
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
|
145
|
-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
146
|
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15"
|
|
155
|
+
//#endregion
|
|
156
|
+
//#region src/stealth.ts
|
|
157
|
+
const REALISTIC_USER_AGENTS = [
|
|
158
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
159
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
160
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
|
161
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
|
162
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
163
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15"
|
|
147
164
|
];
|
|
165
|
+
/** Pick a random realistic user-agent string. */
|
|
148
166
|
function pickUserAgent() {
|
|
149
|
-
|
|
167
|
+
return REALISTIC_USER_AGENTS[Math.floor(Math.random() * REALISTIC_USER_AGENTS.length)];
|
|
150
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* JavaScript that runs inside every page to patch common bot-detection vectors.
|
|
171
|
+
* Injected via Playwright's `context.addInitScript()`.
|
|
172
|
+
*/
|
|
151
173
|
function getStealthScript() {
|
|
152
|
-
|
|
153
|
-
//
|
|
174
|
+
return `
|
|
175
|
+
// ── navigator.webdriver ──────────────────────────────────────────
|
|
154
176
|
Object.defineProperty(navigator, 'webdriver', {
|
|
155
177
|
get: () => undefined,
|
|
156
178
|
configurable: true,
|
|
157
179
|
});
|
|
158
180
|
|
|
159
|
-
//
|
|
181
|
+
// ── navigator.plugins — appear non-empty ─────────────────────────
|
|
160
182
|
Object.defineProperty(navigator, 'plugins', {
|
|
161
183
|
get: () => {
|
|
162
184
|
const plugins = [
|
|
@@ -170,13 +192,13 @@ function getStealthScript() {
|
|
|
170
192
|
configurable: true,
|
|
171
193
|
});
|
|
172
194
|
|
|
173
|
-
//
|
|
195
|
+
// ── navigator.languages ──────────────────────────────────────────
|
|
174
196
|
Object.defineProperty(navigator, 'languages', {
|
|
175
197
|
get: () => ['en-US', 'en'],
|
|
176
198
|
configurable: true,
|
|
177
199
|
});
|
|
178
200
|
|
|
179
|
-
//
|
|
201
|
+
// ── navigator.permissions.query — hide "denied" for notifications ─
|
|
180
202
|
const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions);
|
|
181
203
|
window.navigator.permissions.query = (params) => {
|
|
182
204
|
if (params.name === 'notifications') {
|
|
@@ -185,7 +207,7 @@ function getStealthScript() {
|
|
|
185
207
|
return originalQuery(params);
|
|
186
208
|
};
|
|
187
209
|
|
|
188
|
-
//
|
|
210
|
+
// ── chrome runtime — make it look like a real Chrome ─────────────
|
|
189
211
|
if (!window.chrome) {
|
|
190
212
|
window.chrome = {};
|
|
191
213
|
}
|
|
@@ -196,7 +218,7 @@ function getStealthScript() {
|
|
|
196
218
|
};
|
|
197
219
|
}
|
|
198
220
|
|
|
199
|
-
//
|
|
221
|
+
// ── WebGL renderer — mask headless indicators ────────────────────
|
|
200
222
|
const getParameterOrig = WebGLRenderingContext.prototype.getParameter;
|
|
201
223
|
WebGLRenderingContext.prototype.getParameter = function(param) {
|
|
202
224
|
if (param === 37445) return 'Intel Inc.';
|
|
@@ -204,7 +226,7 @@ function getStealthScript() {
|
|
|
204
226
|
return getParameterOrig.call(this, param);
|
|
205
227
|
};
|
|
206
228
|
|
|
207
|
-
//
|
|
229
|
+
// ── WebGL2 renderer ──────────────────────────────────────────────
|
|
208
230
|
if (typeof WebGL2RenderingContext !== 'undefined') {
|
|
209
231
|
const getParam2Orig = WebGL2RenderingContext.prototype.getParameter;
|
|
210
232
|
WebGL2RenderingContext.prototype.getParameter = function(param) {
|
|
@@ -214,8 +236,8 @@ function getStealthScript() {
|
|
|
214
236
|
};
|
|
215
237
|
}
|
|
216
238
|
|
|
217
|
-
//
|
|
218
|
-
// Only watches childList additions, NOT every attribute mutation
|
|
239
|
+
// ── Remove "cdc_" Playwright/ChromeDriver markers from new elements ─
|
|
240
|
+
// Only watches childList additions, NOT every attribute mutation —
|
|
219
241
|
// attributes:true subtree:true is O(every DOM write) on heavy SPAs
|
|
220
242
|
// like FreightOS and noticeably slows the page.
|
|
221
243
|
try {
|
|
@@ -244,580 +266,582 @@ function getStealthScript() {
|
|
|
244
266
|
// An earlier version returned the parent window, which breaks legitimate
|
|
245
267
|
// page JS on any site that uses iframes (checkout widgets, embedded
|
|
246
268
|
// videos, A/B test scripts, analytics). The "anti-detection" value was
|
|
247
|
-
// minimal
|
|
269
|
+
// minimal — modern detectors don't rely on that property anyway — and
|
|
248
270
|
// the breakage caused pages to fail to boot, leaving font-test residue
|
|
249
271
|
// on screen.
|
|
250
272
|
`;
|
|
251
273
|
}
|
|
274
|
+
/**
|
|
275
|
+
* Build Playwright context options from a StealthConfig.
|
|
276
|
+
*/
|
|
252
277
|
function buildStealthContextOpts(config, viewport) {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
opts.geolocation = config.geolocation;
|
|
270
|
-
opts.permissions = ["geolocation"];
|
|
271
|
-
}
|
|
272
|
-
return opts;
|
|
278
|
+
const opts = {
|
|
279
|
+
viewport,
|
|
280
|
+
userAgent: config.userAgent ?? pickUserAgent(),
|
|
281
|
+
locale: config.locale ?? "en-US",
|
|
282
|
+
timezoneId: config.timezone ?? "America/New_York",
|
|
283
|
+
colorScheme: "light",
|
|
284
|
+
deviceScaleFactor: config.deviceScaleFactor ?? 1,
|
|
285
|
+
hasTouch: false,
|
|
286
|
+
javaScriptEnabled: true,
|
|
287
|
+
ignoreHTTPSErrors: config.ignoreHTTPSErrors ?? false
|
|
288
|
+
};
|
|
289
|
+
if (config.geolocation) {
|
|
290
|
+
opts.geolocation = config.geolocation;
|
|
291
|
+
opts.permissions = ["geolocation"];
|
|
292
|
+
}
|
|
293
|
+
return opts;
|
|
273
294
|
}
|
|
295
|
+
/**
|
|
296
|
+
* Build Playwright launch options for stealth.
|
|
297
|
+
*/
|
|
274
298
|
function buildStealthLaunchArgs(config) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
299
|
+
return {
|
|
300
|
+
args: [
|
|
301
|
+
"--disable-blink-features=AutomationControlled",
|
|
302
|
+
"--disable-infobars",
|
|
303
|
+
"--disable-dev-shm-usage",
|
|
304
|
+
"--no-first-run",
|
|
305
|
+
"--no-default-browser-check"
|
|
306
|
+
],
|
|
307
|
+
proxy: config.proxy ? {
|
|
308
|
+
server: config.proxy.server,
|
|
309
|
+
username: config.proxy.username,
|
|
310
|
+
password: config.proxy.password
|
|
311
|
+
} : void 0
|
|
312
|
+
};
|
|
284
313
|
}
|
|
285
|
-
|
|
286
|
-
|
|
314
|
+
//#endregion
|
|
315
|
+
//#region src/browser-provider.ts
|
|
316
|
+
/**
|
|
317
|
+
* Playwright wrapper with stealth anti-detection, human-like behavior,
|
|
318
|
+
* indexed DOM element resolution, and a rich action vocabulary.
|
|
319
|
+
*
|
|
320
|
+
* The `BrowserProvider` is intentionally LLM-agnostic — it exposes the
|
|
321
|
+
* primitives that `BrowserAgent` orchestrates via vision+DOM reasoning.
|
|
322
|
+
*/
|
|
287
323
|
var BrowserProvider = class {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
* This function intentionally runs entirely in the page context. It:
|
|
810
|
-
* - traverses the regular DOM + open shadow roots (deep)
|
|
811
|
-
* - applies a parent-chain visibility filter
|
|
812
|
-
* - applies a `cursor:pointer` second pass for custom widgets
|
|
813
|
-
* - hit-tests each candidate at its center to avoid overlay collisions
|
|
814
|
-
* - returns scroll context (pages above/below, hidden counts)
|
|
815
|
-
* - tags survivors with `data-bua-idx` for indexed actions
|
|
816
|
-
*/
|
|
817
|
-
extractorScriptSource() {
|
|
818
|
-
return (
|
|
819
|
-
/* js */
|
|
820
|
-
`
|
|
324
|
+
browser = null;
|
|
325
|
+
context = null;
|
|
326
|
+
page = null;
|
|
327
|
+
pages = /* @__PURE__ */ new Map();
|
|
328
|
+
activeTabId = "tab-0";
|
|
329
|
+
tabCounter = 0;
|
|
330
|
+
_viewport;
|
|
331
|
+
_videoDir;
|
|
332
|
+
_humanize;
|
|
333
|
+
/**
|
|
334
|
+
* Most recent DOM snapshot (one per `extractDOM` call). Indexed actions
|
|
335
|
+
* (`clickByIndex`, `inputByIndex`, …) resolve their `index` against this.
|
|
336
|
+
*/
|
|
337
|
+
_lastDom = [];
|
|
338
|
+
/** True if we connected over CDP (don't tear down the browser on close). */
|
|
339
|
+
_attached = false;
|
|
340
|
+
constructor() {
|
|
341
|
+
this._viewport = {
|
|
342
|
+
width: 1280,
|
|
343
|
+
height: 720
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
async launch(opts) {
|
|
347
|
+
const chromium = (await import("playwright")).chromium;
|
|
348
|
+
this._viewport = opts?.viewport ?? {
|
|
349
|
+
width: 1280,
|
|
350
|
+
height: 720
|
|
351
|
+
};
|
|
352
|
+
if (opts?.humanize) {
|
|
353
|
+
const h = typeof opts.humanize === "object" ? opts.humanize : {};
|
|
354
|
+
this._humanize = {
|
|
355
|
+
typingDelay: h.typingDelay ?? [40, 120],
|
|
356
|
+
clickJitter: h.clickJitter ?? 3,
|
|
357
|
+
actionDelay: h.actionDelay ?? [200, 800],
|
|
358
|
+
mouseMovement: h.mouseMovement ?? true
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
if (opts?.cdpUrl) {
|
|
362
|
+
this.browser = await chromium.connectOverCDP(opts.cdpUrl);
|
|
363
|
+
this._attached = true;
|
|
364
|
+
const contexts = this.browser.contexts();
|
|
365
|
+
this.context = contexts.length > 0 ? contexts[0] : await this.browser.newContext({ viewport: this._viewport });
|
|
366
|
+
const existingPages = this.context.pages();
|
|
367
|
+
this.page = existingPages.length > 0 ? existingPages[0] : await this.context.newPage();
|
|
368
|
+
this.tabCounter = 0;
|
|
369
|
+
this.activeTabId = "tab-0";
|
|
370
|
+
this.pages.set("tab-0", this.page);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const stealthEnabled = !!opts?.stealth;
|
|
374
|
+
const stealthCfg = typeof opts?.stealth === "object" ? opts.stealth : {};
|
|
375
|
+
const headless = opts?.headless ?? true;
|
|
376
|
+
const launchOpts = { headless };
|
|
377
|
+
const args = [];
|
|
378
|
+
if (stealthEnabled) {
|
|
379
|
+
const stealth = buildStealthLaunchArgs(stealthCfg);
|
|
380
|
+
args.push(...stealth.args);
|
|
381
|
+
if (stealth.proxy) launchOpts.proxy = stealth.proxy;
|
|
382
|
+
}
|
|
383
|
+
if (!headless) {
|
|
384
|
+
args.push(`--window-size=${this._viewport.width},${this._viewport.height + 140}`);
|
|
385
|
+
args.push("--window-position=0,0");
|
|
386
|
+
}
|
|
387
|
+
if (args.length) launchOpts.args = args;
|
|
388
|
+
this.browser = await chromium.launch(launchOpts);
|
|
389
|
+
let contextOpts;
|
|
390
|
+
if (stealthEnabled) contextOpts = buildStealthContextOpts(stealthCfg, this._viewport);
|
|
391
|
+
else contextOpts = {
|
|
392
|
+
viewport: this._viewport,
|
|
393
|
+
userAgent: pickUserAgent()
|
|
394
|
+
};
|
|
395
|
+
if (opts?.storageState) contextOpts.storageState = opts.storageState;
|
|
396
|
+
if (opts?.recordVideo) {
|
|
397
|
+
const dir = typeof opts.recordVideo === "object" ? opts.recordVideo.dir : "./browser-videos";
|
|
398
|
+
contextOpts.recordVideo = {
|
|
399
|
+
dir,
|
|
400
|
+
size: this._viewport
|
|
401
|
+
};
|
|
402
|
+
this._videoDir = dir;
|
|
403
|
+
}
|
|
404
|
+
this.context = await this.browser.newContext(contextOpts);
|
|
405
|
+
if (stealthEnabled && stealthCfg.patchFingerprint !== false) await this.context.addInitScript(getStealthScript());
|
|
406
|
+
this.page = await this.context.newPage();
|
|
407
|
+
this.tabCounter = 0;
|
|
408
|
+
this.activeTabId = "tab-0";
|
|
409
|
+
this.pages.set("tab-0", this.page);
|
|
410
|
+
}
|
|
411
|
+
async saveStorageState(path) {
|
|
412
|
+
this.ensureContext();
|
|
413
|
+
await this.context.storageState({ path });
|
|
414
|
+
}
|
|
415
|
+
async navigate(url) {
|
|
416
|
+
this.ensurePage();
|
|
417
|
+
if (!/^https?:\/\//i.test(url)) throw new Error(`Invalid URL scheme: only http:// and https:// are allowed`);
|
|
418
|
+
await this.page.goto(url, {
|
|
419
|
+
waitUntil: "domcontentloaded",
|
|
420
|
+
timeout: 3e4
|
|
421
|
+
});
|
|
422
|
+
await this.waitForStable(500);
|
|
423
|
+
}
|
|
424
|
+
async back() {
|
|
425
|
+
this.ensurePage();
|
|
426
|
+
await this.page.goBack({
|
|
427
|
+
waitUntil: "domcontentloaded",
|
|
428
|
+
timeout: 15e3
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
async screenshot() {
|
|
432
|
+
this.ensurePage();
|
|
433
|
+
return await this.page.screenshot({
|
|
434
|
+
type: "png",
|
|
435
|
+
fullPage: false,
|
|
436
|
+
scale: "css"
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
/** Viewport size in CSS pixels (matches screenshot dimensions). */
|
|
440
|
+
get viewport() {
|
|
441
|
+
return this._viewport;
|
|
442
|
+
}
|
|
443
|
+
/** Most recent DOM snapshot. Each entry has a stable `index`. */
|
|
444
|
+
get lastDom() {
|
|
445
|
+
return this._lastDom;
|
|
446
|
+
}
|
|
447
|
+
async click(x, y) {
|
|
448
|
+
this.ensurePage();
|
|
449
|
+
const [cx, cy] = this.clampToViewport(x, y);
|
|
450
|
+
const [fx, fy] = this.jitter(cx, cy);
|
|
451
|
+
if (this._humanize?.mouseMovement) await this.humanMouseMove(fx, fy);
|
|
452
|
+
await this.page.mouse.click(fx, fy);
|
|
453
|
+
await this.humanPause();
|
|
454
|
+
}
|
|
455
|
+
async type(text) {
|
|
456
|
+
this.ensurePage();
|
|
457
|
+
const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
|
|
458
|
+
await this.page.keyboard.type(text, { delay });
|
|
459
|
+
await this.humanPause();
|
|
460
|
+
}
|
|
461
|
+
async clickAndType(x, y, text) {
|
|
462
|
+
const [cx, cy] = this.clampToViewport(x, y);
|
|
463
|
+
await this.click(cx, cy);
|
|
464
|
+
await this.sleep(this._humanize ? this.randInt(150, 350) : 200);
|
|
465
|
+
const [fx, fy] = this.jitter(cx, cy);
|
|
466
|
+
await this.page.mouse.click(fx, fy, { clickCount: 3 });
|
|
467
|
+
await this.sleep(this._humanize ? this.randInt(80, 200) : 100);
|
|
468
|
+
await this.type(text);
|
|
469
|
+
}
|
|
470
|
+
async pressKey(key) {
|
|
471
|
+
this.ensurePage();
|
|
472
|
+
await this.page.keyboard.press(key);
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Send arbitrary keyboard keys / shortcuts. Accepts a single
|
|
476
|
+
* Playwright key spec (`"Enter"`, `"Control+l"`, `"Shift+ArrowDown"`)
|
|
477
|
+
* or a space-separated sequence (`"Tab Tab Enter"`).
|
|
478
|
+
*/
|
|
479
|
+
async sendKeys(keys) {
|
|
480
|
+
this.ensurePage();
|
|
481
|
+
const tokens = keys.trim().split(/\s+/).filter(Boolean);
|
|
482
|
+
for (const token of tokens) {
|
|
483
|
+
await this.page.keyboard.press(token);
|
|
484
|
+
await this.sleep(this._humanize ? this.randInt(40, 120) : 30);
|
|
485
|
+
}
|
|
486
|
+
await this.humanPause();
|
|
487
|
+
}
|
|
488
|
+
async scroll(direction, amount) {
|
|
489
|
+
this.ensurePage();
|
|
490
|
+
const base = amount ?? 400;
|
|
491
|
+
const jittered = this._humanize ? base + this.randInt(-40, 40) : base;
|
|
492
|
+
const scrollY = direction === "down" ? jittered : -jittered;
|
|
493
|
+
if (this._humanize) {
|
|
494
|
+
const steps = this.randInt(2, 4);
|
|
495
|
+
const perStep = scrollY / steps;
|
|
496
|
+
for (let i = 0; i < steps; i++) {
|
|
497
|
+
await this.page.mouse.wheel(0, perStep);
|
|
498
|
+
await this.sleep(this.randInt(30, 80));
|
|
499
|
+
}
|
|
500
|
+
} else await this.page.mouse.wheel(0, scrollY);
|
|
501
|
+
await this.humanPause();
|
|
502
|
+
}
|
|
503
|
+
/**
|
|
504
|
+
* Build a Playwright locator for a DOM-snapshot index. Each `extractDOM`
|
|
505
|
+
* call tags surviving elements with `data-bua-idx="<n>"`; we resolve by
|
|
506
|
+
* that attribute. Returns null if the index is unknown.
|
|
507
|
+
*/
|
|
508
|
+
locatorForIndex(index) {
|
|
509
|
+
if (!this.page) return null;
|
|
510
|
+
if (!this._lastDom.find((e) => e.index === index)) return null;
|
|
511
|
+
return this.page.locator(`[data-bua-idx="${index}"]`).first();
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Click an element by its DOM-snapshot index. The most reliable click
|
|
515
|
+
* path on dynamic pages — survives layout shifts and DPR oddities.
|
|
516
|
+
*/
|
|
517
|
+
async clickByIndex(index, opts) {
|
|
518
|
+
this.ensurePage();
|
|
519
|
+
const loc = this.locatorForIndex(index);
|
|
520
|
+
if (!loc) return false;
|
|
521
|
+
try {
|
|
522
|
+
await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {});
|
|
523
|
+
await loc.click({ timeout: opts?.timeout ?? 5e3 });
|
|
524
|
+
await this.humanPause();
|
|
525
|
+
return true;
|
|
526
|
+
} catch {
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Focus an indexed input, optionally clear it, and type. Returns false
|
|
532
|
+
* if the index couldn't be resolved or the input couldn't be focused.
|
|
533
|
+
*/
|
|
534
|
+
async inputByIndex(index, text, opts) {
|
|
535
|
+
this.ensurePage();
|
|
536
|
+
const loc = this.locatorForIndex(index);
|
|
537
|
+
if (!loc) return false;
|
|
538
|
+
try {
|
|
539
|
+
const timeout = opts?.timeout ?? 5e3;
|
|
540
|
+
await loc.scrollIntoViewIfNeeded({ timeout: 1500 }).catch(() => {});
|
|
541
|
+
if (opts?.clear !== false) await loc.fill("", { timeout });
|
|
542
|
+
const delay = this._humanize ? this.randInt(this._humanize.typingDelay[0], this._humanize.typingDelay[1]) : 30;
|
|
543
|
+
await loc.click({ timeout });
|
|
544
|
+
await loc.pressSequentially(text, { delay });
|
|
545
|
+
if (opts?.submit) await this.page.keyboard.press("Enter");
|
|
546
|
+
await this.humanPause();
|
|
547
|
+
return true;
|
|
548
|
+
} catch {
|
|
549
|
+
return false;
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
async uploadFileByIndex(index, path) {
|
|
553
|
+
this.ensurePage();
|
|
554
|
+
const loc = this.locatorForIndex(index);
|
|
555
|
+
if (!loc) return false;
|
|
556
|
+
try {
|
|
557
|
+
await loc.setInputFiles(path, { timeout: 5e3 });
|
|
558
|
+
await this.humanPause();
|
|
559
|
+
return true;
|
|
560
|
+
} catch {
|
|
561
|
+
return false;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
/** Scroll the indexed element into view (no click). */
|
|
565
|
+
async scrollIntoViewByIndex(index) {
|
|
566
|
+
this.ensurePage();
|
|
567
|
+
const loc = this.locatorForIndex(index);
|
|
568
|
+
if (!loc) return false;
|
|
569
|
+
try {
|
|
570
|
+
await loc.scrollIntoViewIfNeeded({ timeout: 3e3 });
|
|
571
|
+
return true;
|
|
572
|
+
} catch {
|
|
573
|
+
return false;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* Deterministic, DOM-based click using Playwright's text locator.
|
|
578
|
+
*
|
|
579
|
+
* Returns `true` if a matching, visible, clickable element was found and
|
|
580
|
+
* clicked within `timeout` ms; `false` otherwise (so the caller can fall
|
|
581
|
+
* back to coordinate clicking). Substring-matches by default — e.g.
|
|
582
|
+
* `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
|
|
583
|
+
*/
|
|
584
|
+
async clickByText(keyword, opts) {
|
|
585
|
+
this.ensurePage();
|
|
586
|
+
const timeout = opts?.timeout ?? 3e3;
|
|
587
|
+
try {
|
|
588
|
+
await this.page.locator(`text=${keyword}`).first().click({ timeout });
|
|
589
|
+
await this.humanPause();
|
|
590
|
+
return true;
|
|
591
|
+
} catch {
|
|
592
|
+
return false;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
/**
|
|
596
|
+
* Scroll the first occurrence of `text` into view. Returns false if no
|
|
597
|
+
* match was found within the timeout.
|
|
598
|
+
*/
|
|
599
|
+
async findText(text, opts) {
|
|
600
|
+
this.ensurePage();
|
|
601
|
+
const timeout = opts?.timeout ?? 3e3;
|
|
602
|
+
try {
|
|
603
|
+
await this.page.getByText(text).first().scrollIntoViewIfNeeded({ timeout });
|
|
604
|
+
return true;
|
|
605
|
+
} catch {
|
|
606
|
+
return false;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Read the options of a native `<select>` at the given DOM-snapshot
|
|
611
|
+
* index. Returns `[]` if the element is not a `<select>`.
|
|
612
|
+
*/
|
|
613
|
+
async dropdownOptions(index) {
|
|
614
|
+
this.ensurePage();
|
|
615
|
+
const loc = this.locatorForIndex(index);
|
|
616
|
+
if (!loc) return [];
|
|
617
|
+
try {
|
|
618
|
+
return await loc.evaluate((el) => {
|
|
619
|
+
if (el?.tagName !== "SELECT") return [];
|
|
620
|
+
return Array.from(el.options).map((opt) => ({
|
|
621
|
+
value: opt.value,
|
|
622
|
+
label: (opt.label || opt.textContent || "").trim(),
|
|
623
|
+
selected: !!opt.selected
|
|
624
|
+
}));
|
|
625
|
+
});
|
|
626
|
+
} catch {
|
|
627
|
+
return [];
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Select an option in a native `<select>` by its visible text or value.
|
|
632
|
+
* Returns false if the element isn't a `<select>` or no option matched.
|
|
633
|
+
*/
|
|
634
|
+
async selectDropdown(index, text) {
|
|
635
|
+
this.ensurePage();
|
|
636
|
+
const loc = this.locatorForIndex(index);
|
|
637
|
+
if (!loc) return false;
|
|
638
|
+
try {
|
|
639
|
+
await loc.selectOption({ label: text }, { timeout: 3e3 });
|
|
640
|
+
await this.humanPause();
|
|
641
|
+
return true;
|
|
642
|
+
} catch {
|
|
643
|
+
try {
|
|
644
|
+
await loc.selectOption({ value: text }, { timeout: 3e3 });
|
|
645
|
+
await this.humanPause();
|
|
646
|
+
return true;
|
|
647
|
+
} catch {
|
|
648
|
+
return false;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* Run arbitrary JS in the page context. The caller is responsible for
|
|
654
|
+
* gating this behind a config flag — the BrowserAgent only routes the
|
|
655
|
+
* `evaluate` action here when `allowEvaluate: true`.
|
|
656
|
+
*
|
|
657
|
+
* The code is wrapped in `(async () => { ... })()` and the return value
|
|
658
|
+
* is coerced to a string for the model.
|
|
659
|
+
*/
|
|
660
|
+
async evaluate(code) {
|
|
661
|
+
this.ensurePage();
|
|
662
|
+
try {
|
|
663
|
+
const result = await this.page.evaluate(new Function(`return (async () => { ${code} })()`));
|
|
664
|
+
if (result === void 0) return "undefined";
|
|
665
|
+
if (result === null) return "null";
|
|
666
|
+
if (typeof result === "string") return result;
|
|
667
|
+
try {
|
|
668
|
+
return JSON.stringify(result);
|
|
669
|
+
} catch {
|
|
670
|
+
return String(result);
|
|
671
|
+
}
|
|
672
|
+
} catch (e) {
|
|
673
|
+
throw new Error(`evaluate failed: ${e?.message ?? e}`);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Returns a clean text representation of the visible page body, with
|
|
678
|
+
* optional link extraction. Used by the BrowserAgent's `extract` action
|
|
679
|
+
* — the text is passed to a (usually cheap) LLM with the user's query.
|
|
680
|
+
*/
|
|
681
|
+
async pageText(opts) {
|
|
682
|
+
this.ensurePage();
|
|
683
|
+
const maxChars = opts?.maxChars ?? 2e4;
|
|
684
|
+
const extractLinks = !!opts?.extractLinks;
|
|
685
|
+
const collapsed = (await this.page.evaluate((withLinks) => {
|
|
686
|
+
const doc = globalThis.document;
|
|
687
|
+
const SKIP = /* @__PURE__ */ new Set([
|
|
688
|
+
"SCRIPT",
|
|
689
|
+
"STYLE",
|
|
690
|
+
"NOSCRIPT",
|
|
691
|
+
"TEMPLATE",
|
|
692
|
+
"SVG"
|
|
693
|
+
]);
|
|
694
|
+
const win = globalThis.window;
|
|
695
|
+
const lines = [];
|
|
696
|
+
function visit(node) {
|
|
697
|
+
if (!node) return;
|
|
698
|
+
if (node.nodeType === 3) {
|
|
699
|
+
const t = (node.nodeValue || "").trim();
|
|
700
|
+
if (t) lines.push(t);
|
|
701
|
+
return;
|
|
702
|
+
}
|
|
703
|
+
if (node.nodeType !== 1) return;
|
|
704
|
+
const tag = node.tagName.toUpperCase();
|
|
705
|
+
if (SKIP.has(tag)) return;
|
|
706
|
+
const style = win.getComputedStyle(node);
|
|
707
|
+
if (style && (style.visibility === "hidden" || style.display === "none")) return;
|
|
708
|
+
if (tag === "A" && withLinks) {
|
|
709
|
+
const href = node.getAttribute("href") || "";
|
|
710
|
+
const txt = (node.innerText || node.textContent || "").trim();
|
|
711
|
+
if (txt && href) {
|
|
712
|
+
lines.push(`[${txt}](${href})`);
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
for (const child of node.childNodes) visit(child);
|
|
717
|
+
}
|
|
718
|
+
visit(doc.body);
|
|
719
|
+
return lines.join("\n");
|
|
720
|
+
}, extractLinks)).replace(/\n{3,}/g, "\n\n").trim();
|
|
721
|
+
return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}\n…[truncated]` : collapsed;
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* Snapshot the interactive elements visible in the viewport, tag each
|
|
725
|
+
* with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
|
|
726
|
+
* return:
|
|
727
|
+
* - `text`: a human-readable string fed to the model
|
|
728
|
+
* - `elements`: the structured list with stable indices
|
|
729
|
+
* - `scroll`: spatial context (pages above/below, hidden interactive count)
|
|
730
|
+
*
|
|
731
|
+
* Five properties matter for accuracy:
|
|
732
|
+
* - **Hit-tested**: each listed coordinate / index actually reaches the
|
|
733
|
+
* labeled element (overlays / occlusion skip the entry).
|
|
734
|
+
* - **Visibility filtered (with parent chain)**: an element is dropped
|
|
735
|
+
* if itself OR any ancestor is `display:none`, `visibility:hidden`,
|
|
736
|
+
* `pointer-events:none`, or near-zero opacity.
|
|
737
|
+
* - **Shadow DOM piercing**: traverses open shadow roots so custom
|
|
738
|
+
* elements / web components are visible to the agent.
|
|
739
|
+
* - **Same-origin iframes**: walks into each accessible iframe and
|
|
740
|
+
* includes its interactive elements (offset by the iframe's screen
|
|
741
|
+
* position so the coordinates the model sees are still viewport-
|
|
742
|
+
* relative).
|
|
743
|
+
* - **`cursor: pointer` fallback pass**: catches custom React widgets
|
|
744
|
+
* that have no semantic role/href/onclick but are clickable.
|
|
745
|
+
*/
|
|
746
|
+
async extractDOM(opts) {
|
|
747
|
+
this.ensurePage();
|
|
748
|
+
const max = opts?.maxElements ?? 200;
|
|
749
|
+
const mainResult = await this.page.evaluate((limit) => {
|
|
750
|
+
return globalThis.__buaExtract(limit, 0, 0, "main");
|
|
751
|
+
}, max).catch(async () => {
|
|
752
|
+
await this.installExtractorScript();
|
|
753
|
+
return await this.page.evaluate((limit) => {
|
|
754
|
+
return globalThis.__buaExtract(limit, 0, 0, "main");
|
|
755
|
+
}, max);
|
|
756
|
+
});
|
|
757
|
+
const collected = mainResult.elements;
|
|
758
|
+
const scroll = mainResult.scroll;
|
|
759
|
+
try {
|
|
760
|
+
const frames = this.page.frames();
|
|
761
|
+
for (const frame of frames) {
|
|
762
|
+
if (frame === this.page.mainFrame()) continue;
|
|
763
|
+
if (collected.length >= max) break;
|
|
764
|
+
let bbox = null;
|
|
765
|
+
try {
|
|
766
|
+
const owner = await frame.frameElement();
|
|
767
|
+
if (owner) {
|
|
768
|
+
const rect = await owner.boundingBox();
|
|
769
|
+
if (rect) bbox = {
|
|
770
|
+
x: rect.x,
|
|
771
|
+
y: rect.y
|
|
772
|
+
};
|
|
773
|
+
}
|
|
774
|
+
} catch {
|
|
775
|
+
continue;
|
|
776
|
+
}
|
|
777
|
+
if (!bbox) continue;
|
|
778
|
+
let iframeRes = null;
|
|
779
|
+
try {
|
|
780
|
+
iframeRes = await frame.evaluate((args) => {
|
|
781
|
+
return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
|
|
782
|
+
}, {
|
|
783
|
+
limit: max - collected.length,
|
|
784
|
+
ox: bbox.x,
|
|
785
|
+
oy: bbox.y,
|
|
786
|
+
frame: frame.url() || "(iframe)"
|
|
787
|
+
});
|
|
788
|
+
} catch {
|
|
789
|
+
try {
|
|
790
|
+
await frame.evaluate(this.extractorScriptSource());
|
|
791
|
+
iframeRes = await frame.evaluate((args) => {
|
|
792
|
+
return globalThis.__buaExtract(args.limit, args.ox, args.oy, args.frame);
|
|
793
|
+
}, {
|
|
794
|
+
limit: max - collected.length,
|
|
795
|
+
ox: bbox.x,
|
|
796
|
+
oy: bbox.y,
|
|
797
|
+
frame: frame.url() || "(iframe)"
|
|
798
|
+
});
|
|
799
|
+
} catch {}
|
|
800
|
+
}
|
|
801
|
+
if (iframeRes?.elements?.length) {
|
|
802
|
+
const offset = collected.length;
|
|
803
|
+
for (let i = 0; i < iframeRes.elements.length; i++) {
|
|
804
|
+
const e = iframeRes.elements[i];
|
|
805
|
+
e.index = offset + i + 1;
|
|
806
|
+
collected.push(e);
|
|
807
|
+
if (collected.length >= max) break;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
} catch {}
|
|
812
|
+
this._lastDom = collected;
|
|
813
|
+
return {
|
|
814
|
+
text: collected.map((e) => {
|
|
815
|
+
const typeSuffix = e.type ? `(${e.type})` : "";
|
|
816
|
+
const frameSuffix = e.frame && e.frame !== "main" ? ` [frame]` : "";
|
|
817
|
+
return `[${e.index}] [${e.cx},${e.cy}] ${e.role}${typeSuffix}${frameSuffix}: "${e.label}"`;
|
|
818
|
+
}).join("\n"),
|
|
819
|
+
elements: collected,
|
|
820
|
+
scroll
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
/**
|
|
824
|
+
* Install the `__buaExtract` global on the main page. Idempotent —
|
|
825
|
+
* subsequent calls are no-ops.
|
|
826
|
+
*/
|
|
827
|
+
async installExtractorScript() {
|
|
828
|
+
if (!this.page) return;
|
|
829
|
+
await this.page.evaluate(this.extractorScriptSource());
|
|
830
|
+
}
|
|
831
|
+
/**
|
|
832
|
+
* The extractor source. Lives in its own method so we can also inject
|
|
833
|
+
* it into iframes that haven't yet had it loaded.
|
|
834
|
+
*
|
|
835
|
+
* This function intentionally runs entirely in the page context. It:
|
|
836
|
+
* - traverses the regular DOM + open shadow roots (deep)
|
|
837
|
+
* - applies a parent-chain visibility filter
|
|
838
|
+
* - applies a `cursor:pointer` second pass for custom widgets
|
|
839
|
+
* - hit-tests each candidate at its center to avoid overlay collisions
|
|
840
|
+
* - returns scroll context (pages above/below, hidden counts)
|
|
841
|
+
* - tags survivors with `data-bua-idx` for indexed actions
|
|
842
|
+
*/
|
|
843
|
+
extractorScriptSource() {
|
|
844
|
+
return `
|
|
821
845
|
(function () {
|
|
822
846
|
if (typeof window.__buaExtract === "function") return;
|
|
823
847
|
|
|
@@ -938,8 +962,10 @@ var BrowserProvider = class {
|
|
|
938
962
|
var title = el.getAttribute("title") || "";
|
|
939
963
|
var name = el.getAttribute("name") || "";
|
|
940
964
|
var href = el.getAttribute("href") || "";
|
|
941
|
-
|
|
942
|
-
|
|
965
|
+
// Field contents are never labels. Password, OTP and payment values
|
|
966
|
+
// must not enter model observations or recorded DOM snapshots.
|
|
967
|
+
var isField = tag === "input" || tag === "textarea" || el.isContentEditable === true;
|
|
968
|
+
var label = ariaLabel || (isField ? "" : text) || placeholder || title || name;
|
|
943
969
|
if (!label && href) label = href.slice(0, 60);
|
|
944
970
|
if (!label) label = "(" + tag + (type ? (" type=" + type) : "") + ")";
|
|
945
971
|
|
|
@@ -983,1761 +1009,1626 @@ var BrowserProvider = class {
|
|
|
983
1009
|
};
|
|
984
1010
|
};
|
|
985
1011
|
})();
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
}
|
|
1012
|
+
`;
|
|
1013
|
+
}
|
|
1014
|
+
async getPageInfo() {
|
|
1015
|
+
this.ensurePage();
|
|
1016
|
+
const url = this.page.url();
|
|
1017
|
+
let title = "";
|
|
1018
|
+
try {
|
|
1019
|
+
title = await this.page.title();
|
|
1020
|
+
} catch (err) {
|
|
1021
|
+
console.warn("[agentium/browser] Error getting page title:", err instanceof Error ? err.message : err);
|
|
1022
|
+
}
|
|
1023
|
+
return {
|
|
1024
|
+
url,
|
|
1025
|
+
title,
|
|
1026
|
+
viewportSize: this._viewport
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
async waitForStable(minWait = 300) {
|
|
1030
|
+
this.ensurePage();
|
|
1031
|
+
await this.sleep(minWait);
|
|
1032
|
+
try {
|
|
1033
|
+
await this.page.waitForLoadState("networkidle", { timeout: 5e3 });
|
|
1034
|
+
} catch (err) {
|
|
1035
|
+
console.warn("[agentium/browser] Error waiting for stable:", err instanceof Error ? err.message : err);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
async newTab(url) {
|
|
1039
|
+
this.ensureContext();
|
|
1040
|
+
const newPage = await this.context.newPage();
|
|
1041
|
+
this.tabCounter++;
|
|
1042
|
+
const tabId = `tab-${this.tabCounter}`;
|
|
1043
|
+
this.pages.set(tabId, newPage);
|
|
1044
|
+
if (url) await newPage.goto(url, {
|
|
1045
|
+
waitUntil: "domcontentloaded",
|
|
1046
|
+
timeout: 3e4
|
|
1047
|
+
});
|
|
1048
|
+
return tabId;
|
|
1049
|
+
}
|
|
1050
|
+
async switchTab(tabId) {
|
|
1051
|
+
const targetPage = this.pages.get(tabId);
|
|
1052
|
+
if (!targetPage) throw new Error(`Tab "${tabId}" not found`);
|
|
1053
|
+
this.page = targetPage;
|
|
1054
|
+
this.activeTabId = tabId;
|
|
1055
|
+
await this.page.bringToFront();
|
|
1056
|
+
}
|
|
1057
|
+
async closeTab(tabId) {
|
|
1058
|
+
if (this.pages.size <= 1) throw new Error("Cannot close the last tab");
|
|
1059
|
+
const targetPage = this.pages.get(tabId);
|
|
1060
|
+
if (!targetPage) throw new Error(`Tab "${tabId}" not found`);
|
|
1061
|
+
await targetPage.close();
|
|
1062
|
+
this.pages.delete(tabId);
|
|
1063
|
+
if (this.activeTabId === tabId) {
|
|
1064
|
+
const firstRemaining = this.pages.entries().next().value;
|
|
1065
|
+
if (firstRemaining) {
|
|
1066
|
+
this.activeTabId = firstRemaining[0];
|
|
1067
|
+
this.page = firstRemaining[1];
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
async visibleText(maxChars = 2e3) {
|
|
1072
|
+
this.ensurePage();
|
|
1073
|
+
return this.page.evaluate((n) => (globalThis.document?.body?.innerText ?? "").slice(0, n), maxChars);
|
|
1074
|
+
}
|
|
1075
|
+
/**
|
|
1076
|
+
* Grep visible page text. No LLM. Used by `search_page`.
|
|
1077
|
+
*/
|
|
1078
|
+
async searchPage(opts) {
|
|
1079
|
+
this.ensurePage();
|
|
1080
|
+
const pattern = opts.pattern;
|
|
1081
|
+
const useRegex = !!opts.regex;
|
|
1082
|
+
const caseSensitive = !!opts.caseSensitive;
|
|
1083
|
+
const maxResults = Math.min(Math.max(opts.maxResults ?? 20, 1), 50);
|
|
1084
|
+
const contextChars = opts.contextChars ?? 100;
|
|
1085
|
+
return this.page.evaluate(({ pattern, useRegex, caseSensitive, maxResults, contextChars }) => {
|
|
1086
|
+
const text = globalThis.document?.body?.innerText ?? "";
|
|
1087
|
+
const src = String(pattern);
|
|
1088
|
+
const flags = caseSensitive ? "g" : "gi";
|
|
1089
|
+
let re;
|
|
1090
|
+
try {
|
|
1091
|
+
re = useRegex ? new RegExp(src, flags) : new RegExp(src.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), flags);
|
|
1092
|
+
} catch {
|
|
1093
|
+
return [];
|
|
1094
|
+
}
|
|
1095
|
+
const matches = [];
|
|
1096
|
+
let m = re.exec(text);
|
|
1097
|
+
while (m && matches.length < Number(maxResults)) {
|
|
1098
|
+
const start = Math.max(0, m.index - Number(contextChars));
|
|
1099
|
+
const end = Math.min(text.length, m.index + m[0].length + Number(contextChars));
|
|
1100
|
+
matches.push({
|
|
1101
|
+
match: m[0],
|
|
1102
|
+
context: text.slice(start, end),
|
|
1103
|
+
index: m.index
|
|
1104
|
+
});
|
|
1105
|
+
if (!re.global) break;
|
|
1106
|
+
m = re.exec(text);
|
|
1107
|
+
}
|
|
1108
|
+
return matches;
|
|
1109
|
+
}, {
|
|
1110
|
+
pattern,
|
|
1111
|
+
useRegex,
|
|
1112
|
+
caseSensitive,
|
|
1113
|
+
maxResults,
|
|
1114
|
+
contextChars
|
|
1115
|
+
});
|
|
1116
|
+
}
|
|
1117
|
+
/**
|
|
1118
|
+
* querySelectorAll over the page. No LLM. Used by `find_elements`.
|
|
1119
|
+
*/
|
|
1120
|
+
async findElements(selector, opts) {
|
|
1121
|
+
this.ensurePage();
|
|
1122
|
+
const maxResults = Math.min(Math.max(opts?.maxResults ?? 50, 1), 80);
|
|
1123
|
+
return this.page.evaluate(({ selector, maxResults }) => {
|
|
1124
|
+
const doc = globalThis.document;
|
|
1125
|
+
let nodes;
|
|
1126
|
+
try {
|
|
1127
|
+
nodes = Array.from(doc.querySelectorAll(selector)).slice(0, maxResults);
|
|
1128
|
+
} catch {
|
|
1129
|
+
return [];
|
|
1130
|
+
}
|
|
1131
|
+
return nodes.map((el) => ({
|
|
1132
|
+
tag: String(el.tagName ?? "").toLowerCase(),
|
|
1133
|
+
text: String(el.textContent ?? "").trim().slice(0, 200),
|
|
1134
|
+
href: el.href ? String(el.href) : void 0
|
|
1135
|
+
}));
|
|
1136
|
+
}, {
|
|
1137
|
+
selector,
|
|
1138
|
+
maxResults
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
listTabs() {
|
|
1142
|
+
const tabs = [];
|
|
1143
|
+
for (const [id, pg] of this.pages) tabs.push({
|
|
1144
|
+
id,
|
|
1145
|
+
url: pg.url(),
|
|
1146
|
+
active: id === this.activeTabId
|
|
1147
|
+
});
|
|
1148
|
+
return tabs;
|
|
1149
|
+
}
|
|
1150
|
+
get currentTabId() {
|
|
1151
|
+
return this.activeTabId;
|
|
1152
|
+
}
|
|
1153
|
+
async getVideoPath(tabId) {
|
|
1154
|
+
const targetPage = tabId ? this.pages.get(tabId) : this.page;
|
|
1155
|
+
if (!targetPage) return null;
|
|
1156
|
+
try {
|
|
1157
|
+
const video = targetPage.video();
|
|
1158
|
+
if (!video) return null;
|
|
1159
|
+
return await video.path();
|
|
1160
|
+
} catch (err) {
|
|
1161
|
+
console.warn("[agentium/browser] Error getting video path:", err instanceof Error ? err.message : err);
|
|
1162
|
+
return null;
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
get videoDir() {
|
|
1166
|
+
return this._videoDir;
|
|
1167
|
+
}
|
|
1168
|
+
async close() {
|
|
1169
|
+
try {
|
|
1170
|
+
if (this.context && !this._attached) await this.context.close();
|
|
1171
|
+
} catch (err) {
|
|
1172
|
+
console.warn("[agentium/browser] Error closing context:", err instanceof Error ? err.message : err);
|
|
1173
|
+
}
|
|
1174
|
+
try {
|
|
1175
|
+
if (this.browser && !this._attached) await this.browser.close();
|
|
1176
|
+
else if (this.browser && this._attached) try {
|
|
1177
|
+
await this.browser.close();
|
|
1178
|
+
} catch {}
|
|
1179
|
+
} catch (err) {
|
|
1180
|
+
console.warn("[agentium/browser] Error closing browser:", err instanceof Error ? err.message : err);
|
|
1181
|
+
}
|
|
1182
|
+
this.page = null;
|
|
1183
|
+
this.context = null;
|
|
1184
|
+
this.browser = null;
|
|
1185
|
+
this.pages.clear();
|
|
1186
|
+
this._lastDom = [];
|
|
1187
|
+
}
|
|
1188
|
+
/** Add small random offset to coordinates to avoid pixel-perfect bot patterns. */
|
|
1189
|
+
jitter(x, y) {
|
|
1190
|
+
if (!this._humanize) return [x, y];
|
|
1191
|
+
const j = this._humanize.clickJitter;
|
|
1192
|
+
return [x + this.randInt(-j, j), y + this.randInt(-j, j)];
|
|
1193
|
+
}
|
|
1194
|
+
/**
|
|
1195
|
+
* Safety net: clamp coordinates returned by the vision model to the actual
|
|
1196
|
+
* viewport. If a model occasionally returns image-space coordinates from a
|
|
1197
|
+
* 2x screenshot (despite our `scale: "css"` fix), this prevents Playwright
|
|
1198
|
+
* from clicking at e.g. (2200, 1300) and either erroring or landing on a
|
|
1199
|
+
* random off-screen element.
|
|
1200
|
+
*/
|
|
1201
|
+
clampToViewport(x, y) {
|
|
1202
|
+
return [Math.max(0, Math.min(this._viewport.width - 1, Math.round(x))), Math.max(0, Math.min(this._viewport.height - 1, Math.round(y)))];
|
|
1203
|
+
}
|
|
1204
|
+
/**
|
|
1205
|
+
* Simulate human mouse movement using smoothstep interpolation.
|
|
1206
|
+
*/
|
|
1207
|
+
async humanMouseMove(targetX, targetY) {
|
|
1208
|
+
const steps = this.randInt(5, 12);
|
|
1209
|
+
const startX = this._viewport.width / 2;
|
|
1210
|
+
const startY = this._viewport.height / 2;
|
|
1211
|
+
for (let i = 1; i <= steps; i++) {
|
|
1212
|
+
const t = i / steps;
|
|
1213
|
+
const ease = t * t * (3 - 2 * t);
|
|
1214
|
+
const cx = startX + (targetX - startX) * ease + this.randInt(-2, 2);
|
|
1215
|
+
const cy = startY + (targetY - startY) * ease + this.randInt(-2, 2);
|
|
1216
|
+
await this.page.mouse.move(cx, cy);
|
|
1217
|
+
await this.sleep(this.randInt(5, 20));
|
|
1218
|
+
}
|
|
1219
|
+
await this.page.mouse.move(targetX, targetY);
|
|
1220
|
+
}
|
|
1221
|
+
/** Small random pause after an interaction. */
|
|
1222
|
+
async humanPause() {
|
|
1223
|
+
if (!this._humanize) return;
|
|
1224
|
+
const [min, max] = this._humanize.actionDelay;
|
|
1225
|
+
await this.sleep(this.randInt(min, max));
|
|
1226
|
+
}
|
|
1227
|
+
randInt(min, max) {
|
|
1228
|
+
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
1229
|
+
}
|
|
1230
|
+
ensurePage() {
|
|
1231
|
+
if (!this.page) throw new Error("Browser not launched. Call launch() first.");
|
|
1232
|
+
}
|
|
1233
|
+
ensureContext() {
|
|
1234
|
+
if (!this.context) throw new Error("Browser not launched. Call launch() first.");
|
|
1235
|
+
}
|
|
1236
|
+
sleep(ms) {
|
|
1237
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1238
|
+
}
|
|
1214
1239
|
};
|
|
1215
|
-
|
|
1216
|
-
|
|
1240
|
+
//#endregion
|
|
1241
|
+
//#region src/loop-detector.ts
|
|
1217
1242
|
var LoopDetector = class {
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
message: `Page hasn't changed in ${stagnantCount} steps \u2014 verify your action is taking effect.`
|
|
1303
|
-
};
|
|
1304
|
-
}
|
|
1305
|
-
return { severity: "none" };
|
|
1306
|
-
}
|
|
1307
|
-
/** Combine two advices, returning the more severe one. */
|
|
1308
|
-
static combine(a, b) {
|
|
1309
|
-
const rank = { none: 0, warn: 1, escalate: 2, abort: 3 };
|
|
1310
|
-
return rank[a.severity] >= rank[b.severity] ? a : b;
|
|
1311
|
-
}
|
|
1243
|
+
actionWindow = [];
|
|
1244
|
+
pageHistory = [];
|
|
1245
|
+
actionWindowSize;
|
|
1246
|
+
pageHistorySize;
|
|
1247
|
+
/** Thresholds: repeat counts at which each severity fires. */
|
|
1248
|
+
actionThresholds;
|
|
1249
|
+
/** Thresholds: consecutive unchanged-page counts at which each fires. */
|
|
1250
|
+
pageThresholds;
|
|
1251
|
+
constructor(opts) {
|
|
1252
|
+
this.actionWindowSize = opts?.actionWindowSize ?? 20;
|
|
1253
|
+
this.pageHistorySize = opts?.pageHistorySize ?? 10;
|
|
1254
|
+
this.actionThresholds = {
|
|
1255
|
+
warn: opts?.actionThresholds?.warn ?? 5,
|
|
1256
|
+
escalate: opts?.actionThresholds?.escalate ?? 8,
|
|
1257
|
+
abort: opts?.actionThresholds?.abort ?? 12
|
|
1258
|
+
};
|
|
1259
|
+
this.pageThresholds = {
|
|
1260
|
+
warn: opts?.pageThresholds?.warn ?? 5,
|
|
1261
|
+
escalate: opts?.pageThresholds?.escalate ?? 8,
|
|
1262
|
+
abort: opts?.pageThresholds?.abort ?? 12
|
|
1263
|
+
};
|
|
1264
|
+
}
|
|
1265
|
+
/**
|
|
1266
|
+
* Record an action and return advice for the runtime. `severity` of
|
|
1267
|
+
* `abort` means the loop should terminate now.
|
|
1268
|
+
*/
|
|
1269
|
+
recordAction(action) {
|
|
1270
|
+
const key = normalizeAction(action);
|
|
1271
|
+
this.actionWindow.push(key);
|
|
1272
|
+
if (this.actionWindow.length > this.actionWindowSize) this.actionWindow.shift();
|
|
1273
|
+
let count = 0;
|
|
1274
|
+
for (const k of this.actionWindow) if (k === key) count++;
|
|
1275
|
+
if (count >= this.actionThresholds.abort) return {
|
|
1276
|
+
severity: "abort",
|
|
1277
|
+
message: `Auto-stopped: the action "${key}" has been repeated ${count} times in the last ${this.actionWindow.length} steps with no apparent progress.`
|
|
1278
|
+
};
|
|
1279
|
+
if (count >= this.actionThresholds.escalate) return {
|
|
1280
|
+
severity: "escalate",
|
|
1281
|
+
message: `You have repeated essentially the same action ${count} times. The current approach is NOT working — pick a different strategy: scroll, navigate elsewhere, dismiss a popup, or use "fail" if the task truly cannot be done.`
|
|
1282
|
+
};
|
|
1283
|
+
if (count >= this.actionThresholds.warn) return {
|
|
1284
|
+
severity: "warn",
|
|
1285
|
+
message: `You've repeated this action ${count} times — if the page hasn't changed, try a different approach.`
|
|
1286
|
+
};
|
|
1287
|
+
return { severity: "none" };
|
|
1288
|
+
}
|
|
1289
|
+
/**
|
|
1290
|
+
* Record a page fingerprint and return advice if the page has been
|
|
1291
|
+
* stagnant. Counts consecutive identical fingerprints at the END of
|
|
1292
|
+
* the history (so a single navigation resets the counter).
|
|
1293
|
+
*/
|
|
1294
|
+
recordPage(fp) {
|
|
1295
|
+
this.pageHistory.push(fp);
|
|
1296
|
+
if (this.pageHistory.length > this.pageHistorySize) this.pageHistory.shift();
|
|
1297
|
+
let stagnantCount = 1;
|
|
1298
|
+
for (let i = this.pageHistory.length - 2; i >= 0; i--) {
|
|
1299
|
+
const prev = this.pageHistory[i];
|
|
1300
|
+
if (prev.url === fp.url && prev.interactiveCount === fp.interactiveCount && prev.textHash === fp.textHash) stagnantCount++;
|
|
1301
|
+
else break;
|
|
1302
|
+
}
|
|
1303
|
+
if (stagnantCount >= this.pageThresholds.abort) return {
|
|
1304
|
+
severity: "abort",
|
|
1305
|
+
message: `Auto-stopped: the page hasn't changed in ${stagnantCount} consecutive steps. Likely a blocking overlay or dead control.`
|
|
1306
|
+
};
|
|
1307
|
+
if (stagnantCount >= this.pageThresholds.escalate) return {
|
|
1308
|
+
severity: "escalate",
|
|
1309
|
+
message: `The page has not changed in ${stagnantCount} steps. Something is blocking progress — try dismissing popups, navigating elsewhere, or scrolling.`
|
|
1310
|
+
};
|
|
1311
|
+
if (stagnantCount >= this.pageThresholds.warn) return {
|
|
1312
|
+
severity: "warn",
|
|
1313
|
+
message: `Page hasn't changed in ${stagnantCount} steps — verify your action is taking effect.`
|
|
1314
|
+
};
|
|
1315
|
+
return { severity: "none" };
|
|
1316
|
+
}
|
|
1317
|
+
/** Combine two advices, returning the more severe one. */
|
|
1318
|
+
static combine(a, b) {
|
|
1319
|
+
const rank = {
|
|
1320
|
+
none: 0,
|
|
1321
|
+
warn: 1,
|
|
1322
|
+
escalate: 2,
|
|
1323
|
+
abort: 3
|
|
1324
|
+
};
|
|
1325
|
+
return rank[a.severity] >= rank[b.severity] ? a : b;
|
|
1326
|
+
}
|
|
1312
1327
|
};
|
|
1328
|
+
/**
|
|
1329
|
+
* Normalize an action for loop comparison. The goal is to canonicalize
|
|
1330
|
+
* tiny variations the LLM produces (e.g. different `description`s for
|
|
1331
|
+
* the same click) into a single key while keeping actions that are
|
|
1332
|
+
* fundamentally different (different `index`, `text`, `url`, …) distinct.
|
|
1333
|
+
*/
|
|
1313
1334
|
function normalizeAction(action) {
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
case "screenshot":
|
|
1345
|
-
return "screenshot";
|
|
1346
|
-
case "send_keys":
|
|
1347
|
-
return `send_keys:${action.keys}`;
|
|
1348
|
-
case "find_text":
|
|
1349
|
-
return `find_text:${truncate(action.text, 40)}`;
|
|
1350
|
-
case "evaluate":
|
|
1351
|
-
return `evaluate:${truncate(action.code, 80)}`;
|
|
1352
|
-
case "dropdown_options":
|
|
1353
|
-
return `dropdown_options:${action.index}`;
|
|
1354
|
-
case "select_dropdown":
|
|
1355
|
-
return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
|
|
1356
|
-
case "upload_file":
|
|
1357
|
-
return `upload_file:${action.index}:${action.path}`;
|
|
1358
|
-
case "extract":
|
|
1359
|
-
return `extract:${truncate(action.query, 80)}`;
|
|
1360
|
-
case "tool":
|
|
1361
|
-
return `tool:${action.name}`;
|
|
1362
|
-
case "done":
|
|
1363
|
-
case "fail":
|
|
1364
|
-
return action.action;
|
|
1365
|
-
}
|
|
1335
|
+
switch (action.action) {
|
|
1336
|
+
case "click":
|
|
1337
|
+
if (typeof action.index === "number") return `click:${action.index}`;
|
|
1338
|
+
if (typeof action.x === "number" && typeof action.y === "number") return `click:xy:${Math.round(action.x / 16)},${Math.round(action.y / 16)}`;
|
|
1339
|
+
return `click:?`;
|
|
1340
|
+
case "type":
|
|
1341
|
+
if (typeof action.index === "number") return `type:${action.index}:${truncate(action.text, 40)}`;
|
|
1342
|
+
return `type:?:${truncate(action.text, 40)}`;
|
|
1343
|
+
case "scroll": return `scroll:${action.direction}:${action.amount ?? "default"}${action.index != null ? `:${action.index}` : ""}`;
|
|
1344
|
+
case "navigate": return `navigate:${action.url}${action.newTab ? ":new" : ""}`;
|
|
1345
|
+
case "search": return `search:${truncate(action.query, 40)}:${action.engine ?? "default"}`;
|
|
1346
|
+
case "new_tab": return `new_tab:${action.url ?? ""}`;
|
|
1347
|
+
case "switch_tab": return `switch_tab:${action.tabId}`;
|
|
1348
|
+
case "close_tab": return `close_tab:${action.tabId}`;
|
|
1349
|
+
case "search_page": return `search_page:${truncate(action.pattern, 40)}`;
|
|
1350
|
+
case "find_elements": return `find_elements:${truncate(action.selector, 40)}`;
|
|
1351
|
+
case "back": return "back";
|
|
1352
|
+
case "wait": return `wait:${Math.round(action.ms / 1e3)}`;
|
|
1353
|
+
case "screenshot": return "screenshot";
|
|
1354
|
+
case "send_keys": return `send_keys:${action.keys}`;
|
|
1355
|
+
case "find_text": return `find_text:${truncate(action.text, 40)}`;
|
|
1356
|
+
case "evaluate": return `evaluate:${truncate(action.code, 80)}`;
|
|
1357
|
+
case "dropdown_options": return `dropdown_options:${action.index}`;
|
|
1358
|
+
case "select_dropdown": return `select_dropdown:${action.index}:${truncate(action.text, 40)}`;
|
|
1359
|
+
case "upload_file": return `upload_file:${action.index}:${action.path}`;
|
|
1360
|
+
case "extract": return `extract:${truncate(action.query, 80)}`;
|
|
1361
|
+
case "tool": return `tool:${action.name}`;
|
|
1362
|
+
case "done":
|
|
1363
|
+
case "fail": return action.action;
|
|
1364
|
+
}
|
|
1366
1365
|
}
|
|
1366
|
+
/**
|
|
1367
|
+
* Tiny non-cryptographic string hash, stable across runs. Used for the
|
|
1368
|
+
* DOM-text portion of `PageFingerprint`.
|
|
1369
|
+
*/
|
|
1367
1370
|
function fnvHash(s) {
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1371
|
+
let h = 2166136261;
|
|
1372
|
+
for (let i = 0; i < s.length; i++) {
|
|
1373
|
+
h ^= s.charCodeAt(i);
|
|
1374
|
+
h = h * 16777619 >>> 0;
|
|
1375
|
+
}
|
|
1376
|
+
return h;
|
|
1374
1377
|
}
|
|
1375
1378
|
function truncate(s, n) {
|
|
1376
|
-
|
|
1377
|
-
|
|
1379
|
+
if (!s) return "";
|
|
1380
|
+
return s.length <= n ? s : `${s.slice(0, n)}…`;
|
|
1378
1381
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1382
|
+
//#endregion
|
|
1383
|
+
//#region src/prompts.ts
|
|
1381
1384
|
function buildSystemPrompt(viewport, extraInstructions, credentialKeys, options) {
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
`- A short list of your previous actions (so you don't repeat yourself).`,
|
|
1410
|
-
``,
|
|
1411
|
-
`## Coordinate System`,
|
|
1412
|
-
`The browser viewport is ${viewport.width}\xD7${viewport.height} CSS pixels, top-left origin. Coordinates outside that range are rejected.`,
|
|
1413
|
-
``,
|
|
1414
|
-
`## Available Actions`,
|
|
1415
|
-
`Respond with a JSON object \u2014 or, when several independent actions can be safely batched (e.g. filling multiple form fields), an array of up to ${maxActions} JSON objects. The runtime executes them in order and stops early if the page navigates.`,
|
|
1416
|
-
``,
|
|
1417
|
-
`### click \u2014 by index (preferred)`,
|
|
1418
|
-
`\`{ "action": "click", "index": <n>, "description": "<what you are clicking, ideally with the visible text in quotes>" }\``,
|
|
1419
|
-
`If you cannot find a matching index (e.g. an element is partly off-screen and not in the snapshot), you may fall back to coordinates: \`{ "action": "click", "x": <n>, "y": <n>, "description": "Click on 'Cheapest' tab" }\`. The runtime will additionally try a Playwright text locator using the quoted phrase.`,
|
|
1420
|
-
``,
|
|
1421
|
-
`### type \u2014 by index (preferred)`,
|
|
1422
|
-
`Type into the element at \`index\`. The field is cleared first by default. Set \`"submit": true\` to press Enter after typing (e.g. submit a search).`,
|
|
1423
|
-
`\`{ "action": "type", "index": <n>, "text": "<text>", "clear": true, "submit": false }\``,
|
|
1424
|
-
`Fallback: \`{ "action": "type", "text": "<text>", "x": <n>, "y": <n> }\` clicks the coordinates first, then types.`,
|
|
1425
|
-
``,
|
|
1426
|
-
`### scroll`,
|
|
1427
|
-
`\`{ "action": "scroll", "direction": "down"|"up", "amount": 400 }\` or \`{ "action": "scroll", "index": <n> }\` to scroll a specific element into view.`,
|
|
1428
|
-
``,
|
|
1429
|
-
`### find_text`,
|
|
1430
|
-
`Scroll the first occurrence of a phrase into the viewport. Use this instead of repeated \`scroll\` actions when you know what you're hunting for.`,
|
|
1431
|
-
`\`{ "action": "find_text", "text": "Annual report 2024" }\``,
|
|
1432
|
-
``,
|
|
1433
|
-
`### send_keys`,
|
|
1434
|
-
`Send arbitrary keys / shortcuts. Single key, combo (with \`+\`), or space-separated sequence.`,
|
|
1435
|
-
`\`{ "action": "send_keys", "keys": "Tab Tab Enter" }\` \xB7 \`{ "action": "send_keys", "keys": "Control+l" }\` \xB7 \`{ "action": "send_keys", "keys": "Escape" }\``,
|
|
1436
|
-
``,
|
|
1437
|
-
`### dropdown_options / select_dropdown`,
|
|
1438
|
-
`Inspect or set a native \`<select>\` by index. **Do not click into native selects** \u2014 the OS overlay is not part of the DOM.`,
|
|
1439
|
-
`\`{ "action": "dropdown_options", "index": <n> }\` returns the option list in the next observation.`,
|
|
1440
|
-
`\`{ "action": "select_dropdown", "index": <n>, "text": "United States" }\` picks the matching option.`,
|
|
1441
|
-
``,
|
|
1442
|
-
`### upload_file`,
|
|
1443
|
-
`Set a file on an \`<input type="file">\` by index. \`path\` must be a path the runtime can read; the agent does NOT have a filesystem of its own.`,
|
|
1444
|
-
`\`{ "action": "upload_file", "index": <n>, "path": "/abs/path/to/file.pdf" }\``,
|
|
1445
|
-
``,
|
|
1446
|
-
`### navigate / back / search`,
|
|
1447
|
-
`\`{ "action": "navigate", "url": "<full URL>" }\` \xB7 \`{ "action": "navigate", "url": "<full URL>", "newTab": true }\` \xB7 \`{ "action": "back" }\``,
|
|
1448
|
-
`\`{ "action": "search", "query": "<query>", "engine": "duckduckgo"|"google"|"bing" }\` opens a search-engine results page (default engine: duckduckgo).`,
|
|
1449
|
-
``,
|
|
1450
|
-
`### tabs`,
|
|
1451
|
-
`Tab ids come from the **Open tabs** list in the observation.`,
|
|
1452
|
-
`\`{ "action": "new_tab", "url": "<optional URL>" }\` \xB7 \`{ "action": "switch_tab", "tabId": "tab-2" }\` \xB7 \`{ "action": "close_tab", "tabId": "tab-1" }\``,
|
|
1453
|
-
``,
|
|
1454
|
-
`### search_page / find_elements`,
|
|
1455
|
-
`Zero-LLM inspect. Use these instead of \`extract\` when you just need to grep visible text or run a CSS selector.`,
|
|
1456
|
-
`\`{ "action": "search_page", "pattern": "Total", "regex": false, "caseSensitive": false }\``,
|
|
1457
|
-
`\`{ "action": "find_elements", "selector": "a.storylink" }\``,
|
|
1458
|
-
``,
|
|
1459
|
-
`### wait / screenshot`,
|
|
1460
|
-
`\`{ "action": "wait", "ms": <ms \u2264 10000> }\` \xB7 \`{ "action": "screenshot" }\` (request a fresh image on the next step \u2014 only useful when vision mode is "auto").`,
|
|
1461
|
-
``,
|
|
1462
|
-
`### extract`,
|
|
1463
|
-
`Extract structured information from the current page using a secondary LLM. Cheaper than reasoning over multiple screenshots yourself when you just need facts/text. The extracted result is returned in the next observation as \`Last extract result:\`.`,
|
|
1464
|
-
`\`{ "action": "extract", "query": "List the top 5 result titles and their prices", "extractLinks": false }\``
|
|
1465
|
-
);
|
|
1466
|
-
if (allowEvaluate) {
|
|
1467
|
-
lines.push(
|
|
1468
|
-
``,
|
|
1469
|
-
`### evaluate (JS escape hatch)`,
|
|
1470
|
-
`Run arbitrary JavaScript in the page context. Use this ONLY when no other action fits \u2014 e.g. shadow DOM access, complex selectors, custom widget state. The return value is stringified and returned in the next observation.`,
|
|
1471
|
-
`\`{ "action": "evaluate", "code": "return document.title" }\``
|
|
1472
|
-
);
|
|
1473
|
-
}
|
|
1474
|
-
if (tools.length > 0) {
|
|
1475
|
-
lines.push(``, `### tool \u2014 invoke a custom tool`);
|
|
1476
|
-
lines.push(
|
|
1477
|
-
`In addition to browser actions, you have access to the following custom tools. Invoke them with \`{ "action": "tool", "name": "<tool>", "args": { ... } }\`. The result is returned in the next observation.`
|
|
1478
|
-
);
|
|
1479
|
-
for (const t of tools) {
|
|
1480
|
-
lines.push(`- **${t.name}** \u2014 ${t.description}`);
|
|
1481
|
-
}
|
|
1482
|
-
}
|
|
1483
|
-
lines.push(
|
|
1484
|
-
``,
|
|
1485
|
-
`### done / fail`,
|
|
1486
|
-
`\`{ "action": "done", "result": "<comprehensive summary of what was accomplished and any data the user asked for>" }\` \u2014 use when the task is complete.`,
|
|
1487
|
-
`\`{ "action": "fail", "reason": "<why>" }\` \u2014 use when the task cannot be completed even after several attempts.`,
|
|
1488
|
-
``,
|
|
1489
|
-
`## Rules`,
|
|
1490
|
-
`1. ALWAYS check the DOM snapshot first. If the target element is listed, use its \`index\`. Coordinates are a fallback.`,
|
|
1491
|
-
`2. For text-bearing targets, ALSO include the visible label in quotes in \`description\` \u2014 the runtime will use it as a third-tier fallback if the locator fails.`,
|
|
1492
|
-
`3. Dismiss cookie banners, consent dialogs, and modal popups FIRST \u2014 they intercept clicks on elements behind them.`,
|
|
1493
|
-
`4. If a previous action clearly hit the wrong thing, do NOT repeat it. Pick a different element or a different action.`,
|
|
1494
|
-
`5. Batch independent actions when safe (e.g. filling 3 form fields). Don't batch when an action navigates or substantially changes the DOM.`,
|
|
1495
|
-
`6. NEVER hallucinate. Only report data you can actually see in the snapshot, screenshot, or an extract result.`,
|
|
1496
|
-
`7. When the task is fully complete, return \`done\` IMMEDIATELY with a thorough result \u2014 don't add extra confirmation steps.`,
|
|
1497
|
-
`8. If after several attempts you cannot make progress, return \`fail\` with a clear reason.`
|
|
1498
|
-
);
|
|
1499
|
-
appendCredentials(lines, credentialKeys);
|
|
1500
|
-
if (extraInstructions) {
|
|
1501
|
-
lines.push(``, `## Additional Instructions`, extraInstructions);
|
|
1502
|
-
}
|
|
1503
|
-
lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
|
|
1504
|
-
return lines.join("\n");
|
|
1385
|
+
const maxActions = options?.maxActionsPerStep ?? 3;
|
|
1386
|
+
const allowEvaluate = !!options?.allowEvaluate;
|
|
1387
|
+
const tools = options?.tools ?? [];
|
|
1388
|
+
const useVision = options?.useVision ?? "auto";
|
|
1389
|
+
const useThinking = options?.useThinking ?? true;
|
|
1390
|
+
if (options?.overrideSystemMessage) {
|
|
1391
|
+
const lines = [options.overrideSystemMessage];
|
|
1392
|
+
appendCredentials(lines, credentialKeys);
|
|
1393
|
+
lines.push("", "## Response Format", responseFormatLines(maxActions, useThinking).join("\n"));
|
|
1394
|
+
return lines.join("\n");
|
|
1395
|
+
}
|
|
1396
|
+
const lines = [];
|
|
1397
|
+
lines.push(`You are a browser automation agent. Your job is to complete the user's task by interacting with a real web page.`, ``, `## What you receive each step`, `- The current URL and page title`, `- A numbered list of interactive elements visible in the viewport (the "DOM snapshot"). Each entry looks like \`[idx] [cx,cy] role(type): "label"\` where \`idx\` is a stable per-step index, \`cx,cy\` are the CSS-pixel center coordinates, and \`label\` is the visible text / aria-label / placeholder. **Use the index to act on elements** — it is the most reliable handle.`);
|
|
1398
|
+
if (useVision !== false) lines.push(`- A PNG screenshot of the same viewport at exactly ${viewport.width}×${viewport.height} pixels. Use it for visual context (layout, icons, charts) when the DOM snapshot is ambiguous.`);
|
|
1399
|
+
else lines.push(`- (Vision is disabled — no screenshot. Rely entirely on the DOM snapshot and page text.)`);
|
|
1400
|
+
lines.push(`- A short list of your previous actions (so you don't repeat yourself).`, ``, `## Coordinate System`, `The browser viewport is ${viewport.width}×${viewport.height} CSS pixels, top-left origin. Coordinates outside that range are rejected.`, ``, `## Available Actions`, `Respond with a JSON object — or, when several independent actions can be safely batched (e.g. filling multiple form fields), an array of up to ${maxActions} JSON objects. The runtime executes them in order and stops early if the page navigates.`, ``, `### click — by index (preferred)`, `\`{ "action": "click", "index": <n>, "description": "<what you are clicking, ideally with the visible text in quotes>" }\``, `If you cannot find a matching index (e.g. an element is partly off-screen and not in the snapshot), you may fall back to coordinates: \`{ "action": "click", "x": <n>, "y": <n>, "description": "Click on 'Cheapest' tab" }\`. The runtime will additionally try a Playwright text locator using the quoted phrase.`, ``, `### type — by index (preferred)`, `Type into the element at \`index\`. The field is cleared first by default. Set \`"submit": true\` to press Enter after typing (e.g. submit a search).`, `\`{ "action": "type", "index": <n>, "text": "<text>", "clear": true, "submit": false }\``, `Fallback: \`{ "action": "type", "text": "<text>", "x": <n>, "y": <n> }\` clicks the coordinates first, then types.`, ``, `### scroll`, `\`{ "action": "scroll", "direction": "down"|"up", "amount": 400 }\` or \`{ "action": "scroll", "index": <n> }\` to scroll a specific element into view.`, ``, `### find_text`, `Scroll the first occurrence of a phrase into the viewport. Use this instead of repeated \`scroll\` actions when you know what you're hunting for.`, `\`{ "action": "find_text", "text": "Annual report 2024" }\``, ``, `### send_keys`, `Send arbitrary keys / shortcuts. Single key, combo (with \`+\`), or space-separated sequence.`, `\`{ "action": "send_keys", "keys": "Tab Tab Enter" }\` · \`{ "action": "send_keys", "keys": "Control+l" }\` · \`{ "action": "send_keys", "keys": "Escape" }\``, ``, `### dropdown_options / select_dropdown`, `Inspect or set a native \`<select>\` by index. **Do not click into native selects** — the OS overlay is not part of the DOM.`, `\`{ "action": "dropdown_options", "index": <n> }\` returns the option list in the next observation.`, `\`{ "action": "select_dropdown", "index": <n>, "text": "United States" }\` picks the matching option.`, ``, `### upload_file`, `Set a file on an \`<input type="file">\` by index. \`path\` must be a path the runtime can read; the agent does NOT have a filesystem of its own.`, `\`{ "action": "upload_file", "index": <n>, "path": "/abs/path/to/file.pdf" }\``, ``, `### navigate / back / search`, `\`{ "action": "navigate", "url": "<full URL>" }\` · \`{ "action": "navigate", "url": "<full URL>", "newTab": true }\` · \`{ "action": "back" }\``, `\`{ "action": "search", "query": "<query>", "engine": "duckduckgo"|"google"|"bing" }\` opens a search-engine results page (default engine: duckduckgo).`, ``, `### tabs`, `Tab ids come from the **Open tabs** list in the observation.`, `\`{ "action": "new_tab", "url": "<optional URL>" }\` · \`{ "action": "switch_tab", "tabId": "tab-2" }\` · \`{ "action": "close_tab", "tabId": "tab-1" }\``, ``, `### search_page / find_elements`, `Zero-LLM inspect. Use these instead of \`extract\` when you just need to grep visible text or run a CSS selector.`, `\`{ "action": "search_page", "pattern": "Total", "regex": false, "caseSensitive": false }\``, `\`{ "action": "find_elements", "selector": "a.storylink" }\``, ``, `### wait / screenshot`, `\`{ "action": "wait", "ms": <ms ≤ 10000> }\` · \`{ "action": "screenshot" }\` (request a fresh image on the next step — only useful when vision mode is "auto").`, ``, `### extract`, `Extract structured information from the current page using a secondary LLM. Cheaper than reasoning over multiple screenshots yourself when you just need facts/text. The extracted result is returned in the next observation as \`Last extract result:\`.`, `\`{ "action": "extract", "query": "List the top 5 result titles and their prices", "extractLinks": false }\``);
|
|
1401
|
+
if (allowEvaluate) lines.push(``, `### evaluate (JS escape hatch)`, `Run arbitrary JavaScript in the page context. Use this ONLY when no other action fits — e.g. shadow DOM access, complex selectors, custom widget state. The return value is stringified and returned in the next observation.`, `\`{ "action": "evaluate", "code": "return document.title" }\``);
|
|
1402
|
+
if (tools.length > 0) {
|
|
1403
|
+
lines.push(``, `### tool — invoke a custom tool`);
|
|
1404
|
+
lines.push(`In addition to browser actions, you have access to the following custom tools. Invoke them with \`{ "action": "tool", "name": "<tool>", "args": { ... } }\`. The result is returned in the next observation.`);
|
|
1405
|
+
for (const t of tools) lines.push(`- **${t.name}** — ${t.description}`);
|
|
1406
|
+
}
|
|
1407
|
+
lines.push(``, `### done / fail`, `\`{ "action": "done", "result": "<comprehensive summary of what was accomplished and any data the user asked for>" }\` — use when the task is complete.`, `\`{ "action": "fail", "reason": "<why>" }\` — use when the task cannot be completed even after several attempts.`, ``, `## Rules`, `1. ALWAYS check the DOM snapshot first. If the target element is listed, use its \`index\`. Coordinates are a fallback.`, `2. For text-bearing targets, ALSO include the visible label in quotes in \`description\` — the runtime will use it as a third-tier fallback if the locator fails.`, `3. Dismiss cookie banners, consent dialogs, and modal popups FIRST — they intercept clicks on elements behind them.`, `4. If a previous action clearly hit the wrong thing, do NOT repeat it. Pick a different element or a different action.`, `5. Batch independent actions when safe (e.g. filling 3 form fields). Don't batch when an action navigates or substantially changes the DOM.`, `6. NEVER hallucinate. Only report data you can actually see in the snapshot, screenshot, or an extract result.`, `7. When the task is fully complete, return \`done\` IMMEDIATELY with a thorough result — don't add extra confirmation steps.`, `8. If after several attempts you cannot make progress, return \`fail\` with a clear reason.`);
|
|
1408
|
+
appendCredentials(lines, credentialKeys);
|
|
1409
|
+
if (extraInstructions) lines.push(``, `## Additional Instructions`, extraInstructions);
|
|
1410
|
+
lines.push(``, `## Response Format`, responseFormatLines(maxActions, useThinking).join("\n"));
|
|
1411
|
+
return lines.join("\n");
|
|
1505
1412
|
}
|
|
1506
1413
|
function appendCredentials(lines, credentialKeys) {
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
``,
|
|
1510
|
-
`## Secure Credentials`,
|
|
1511
|
-
`The following credential placeholders are available for use in "type" actions:`,
|
|
1512
|
-
...credentialKeys.map((k) => `- \`{{${k}}}\``),
|
|
1513
|
-
``,
|
|
1514
|
-
`When you need to fill in a login form or any field requiring these credentials,`,
|
|
1515
|
-
`use the EXACT placeholder (e.g. \`{{email}}\`) as the "text" value in a type action.`,
|
|
1516
|
-
`The system will securely replace them with real values at execution time.`,
|
|
1517
|
-
`NEVER guess, invent, or ask the user for the actual credential values.`,
|
|
1518
|
-
`NEVER include real credential values in "done" or "fail" results.`
|
|
1519
|
-
);
|
|
1414
|
+
if (!credentialKeys || credentialKeys.length === 0) return;
|
|
1415
|
+
lines.push(``, `## Secure Credentials`, `The following credential placeholders are available for use in "type" actions:`, ...credentialKeys.map((k) => `- \`{{${k}}}\``), ``, `When you need to fill in a login form or any field requiring these credentials,`, `use the EXACT placeholder (e.g. \`{{email}}\`) as the "text" value in a type action.`, `The system will securely replace them with real values at execution time.`, `NEVER guess, invent, or ask the user for the actual credential values.`, `NEVER include real credential values in "done" or "fail" results.`);
|
|
1520
1416
|
}
|
|
1521
1417
|
function responseFormatLines(maxActions, useThinking) {
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
``,
|
|
1542
|
-
`On the very first step, set "evaluation_previous_goal" to "n/a \u2014 first step".`,
|
|
1543
|
-
`Keep each text field to one short paragraph or a few lines. Be concrete.`,
|
|
1544
|
-
`Only the "action" field is executed; the others help you self-correct over multiple steps.`
|
|
1545
|
-
];
|
|
1418
|
+
if (!useThinking) return [`Respond with ONLY valid JSON. No markdown, no commentary.`, `Either a single action object, or an array of up to ${maxActions} action objects to execute in order.`];
|
|
1419
|
+
return [
|
|
1420
|
+
`Respond with ONLY a single valid JSON object. No markdown, no commentary.`,
|
|
1421
|
+
``,
|
|
1422
|
+
`Use this exact shape:`,
|
|
1423
|
+
"```json",
|
|
1424
|
+
`{`,
|
|
1425
|
+
` "thinking": "Short chain-of-thought: what do I see, what's my plan?",`,
|
|
1426
|
+
` "evaluation_previous_goal": "Did the previous action succeed? success/partial/failure + 1 line",`,
|
|
1427
|
+
` "memory": "Compact bullet list of facts to remember across steps (URLs, found data, …)",`,
|
|
1428
|
+
` "next_goal": "What I want to accomplish in THIS step",`,
|
|
1429
|
+
` "action": <single action object OR array of up to ${maxActions} action objects>`,
|
|
1430
|
+
`}`,
|
|
1431
|
+
"```",
|
|
1432
|
+
``,
|
|
1433
|
+
`On the very first step, set "evaluation_previous_goal" to "n/a — first step".`,
|
|
1434
|
+
`Keep each text field to one short paragraph or a few lines. Be concrete.`,
|
|
1435
|
+
`Only the "action" field is executed; the others help you self-correct over multiple steps.`
|
|
1436
|
+
];
|
|
1546
1437
|
}
|
|
1547
1438
|
function buildUserMessage(task, pageUrl, pageTitle, stepIndex, actionHistory, domSnapshot, lastExtract, scroll, nudge, stepBudget, tabs) {
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
if (nudge) {
|
|
1597
|
-
lines.push(``);
|
|
1598
|
-
lines.push(`**\u26A0 Runtime hint:** ${nudge}`);
|
|
1599
|
-
}
|
|
1600
|
-
lines.push(``);
|
|
1601
|
-
lines.push(`Decide the next action(s) to complete the task.`);
|
|
1602
|
-
return lines.join("\n");
|
|
1439
|
+
const lines = [];
|
|
1440
|
+
lines.push(`**Task:** ${task}`);
|
|
1441
|
+
lines.push(`**Current URL:** ${pageUrl}`);
|
|
1442
|
+
if (pageTitle) lines.push(`**Page Title:** ${pageTitle}`);
|
|
1443
|
+
if (stepBudget) lines.push(`**Step:** ${stepIndex + 1} of ${stepBudget.max} (${stepBudget.max - stepIndex - 1} remaining)`);
|
|
1444
|
+
else lines.push(`**Step:** ${stepIndex + 1}`);
|
|
1445
|
+
if (scroll) {
|
|
1446
|
+
const parts = [];
|
|
1447
|
+
parts.push(`${scroll.totalInteractive} interactive elements (${scroll.hiddenInteractive} hidden)`);
|
|
1448
|
+
if (scroll.pagesAbove > 0) parts.push(`${scroll.pagesAbove} page${scroll.pagesAbove === 1 ? "" : "s"} above`);
|
|
1449
|
+
if (scroll.pagesBelow > 0) parts.push(`${scroll.pagesBelow} page${scroll.pagesBelow === 1 ? "" : "s"} below`);
|
|
1450
|
+
if (scroll.pagesAbove === 0 && scroll.pagesBelow === 0) parts.push("fits in viewport");
|
|
1451
|
+
lines.push(`**Page stats:** ${parts.join(" · ")}`);
|
|
1452
|
+
}
|
|
1453
|
+
if (tabs && tabs.length > 0) {
|
|
1454
|
+
lines.push(`**Open tabs:**`);
|
|
1455
|
+
for (const tab of tabs) {
|
|
1456
|
+
const mark = tab.active ? " (active)" : "";
|
|
1457
|
+
lines.push(`- ${tab.id}${mark}: ${tab.url || "blank"}`);
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
if (domSnapshot !== void 0) {
|
|
1461
|
+
if (domSnapshot.trim().length === 0) {
|
|
1462
|
+
lines.push(``);
|
|
1463
|
+
lines.push(`**⚠ Empty page / no interactive elements detected.** The page may be blank, blocked by a captcha/anti-bot wall, mid-load, or rendered with shadow DOM in an unsupported way. Consider: \`wait\` + retry, \`navigate\` to a different URL, or \`fail\` if the site is blocking access.`);
|
|
1464
|
+
} else {
|
|
1465
|
+
lines.push(``);
|
|
1466
|
+
lines.push(`**Interactive elements (format: [idx] [cx,cy] role: "label"):**`);
|
|
1467
|
+
lines.push(domSnapshot);
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
if (lastExtract) {
|
|
1471
|
+
lines.push(``);
|
|
1472
|
+
lines.push(`**Last extract result:**`);
|
|
1473
|
+
lines.push(lastExtract);
|
|
1474
|
+
}
|
|
1475
|
+
if (actionHistory.length > 0) {
|
|
1476
|
+
lines.push(``);
|
|
1477
|
+
lines.push(`**Previous actions:**`);
|
|
1478
|
+
for (const entry of actionHistory.slice(-10)) lines.push(`- ${entry}`);
|
|
1479
|
+
}
|
|
1480
|
+
if (nudge) {
|
|
1481
|
+
lines.push(``);
|
|
1482
|
+
lines.push(`**⚠ Runtime hint:** ${nudge}`);
|
|
1483
|
+
}
|
|
1484
|
+
lines.push(``);
|
|
1485
|
+
lines.push(`Decide the next action(s) to complete the task.`);
|
|
1486
|
+
return lines.join("\n");
|
|
1603
1487
|
}
|
|
1604
1488
|
function summarizeAction(action) {
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
case "find_text":
|
|
1638
|
-
return `Scrolled to text: "${action.text}"`;
|
|
1639
|
-
case "evaluate":
|
|
1640
|
-
return `Evaluated JS`;
|
|
1641
|
-
case "dropdown_options":
|
|
1642
|
-
return `Read dropdown options at [${action.index}]`;
|
|
1643
|
-
case "select_dropdown":
|
|
1644
|
-
return `Selected "${action.text}" in dropdown [${action.index}]`;
|
|
1645
|
-
case "upload_file":
|
|
1646
|
-
return `Uploaded file to [${action.index}]: ${action.path}`;
|
|
1647
|
-
case "extract":
|
|
1648
|
-
return `Extracted: "${action.query}"`;
|
|
1649
|
-
case "tool":
|
|
1650
|
-
return `Called tool "${action.name}"`;
|
|
1651
|
-
case "done":
|
|
1652
|
-
return `Done: ${action.result}`;
|
|
1653
|
-
case "fail":
|
|
1654
|
-
return `Failed: ${action.reason}`;
|
|
1655
|
-
default:
|
|
1656
|
-
return JSON.stringify(action);
|
|
1657
|
-
}
|
|
1489
|
+
switch (action.action) {
|
|
1490
|
+
case "click":
|
|
1491
|
+
if (typeof action.index === "number") return `Clicked [${action.index}]: ${action.description ?? ""}`.trim();
|
|
1492
|
+
return `Clicked at (${action.x}, ${action.y}): ${action.description ?? ""}`.trim();
|
|
1493
|
+
case "type":
|
|
1494
|
+
if (typeof action.index === "number") return `Typed into [${action.index}]: "${action.text}"`;
|
|
1495
|
+
return action.x != null ? `Clicked (${action.x}, ${action.y}) and typed "${action.text}"` : `Typed "${action.text}"`;
|
|
1496
|
+
case "scroll":
|
|
1497
|
+
if (typeof action.index === "number") return `Scrolled [${action.index}] into view`;
|
|
1498
|
+
return `Scrolled ${action.direction}${action.amount ? ` ${action.amount}px` : ""}`;
|
|
1499
|
+
case "navigate": return action.newTab ? `Opened ${action.url} in a new tab` : `Navigated to ${action.url}`;
|
|
1500
|
+
case "search": return `Searched ${action.engine ?? "web"} for "${action.query}"`;
|
|
1501
|
+
case "new_tab": return action.url ? `Opened new tab: ${action.url}` : `Opened new tab`;
|
|
1502
|
+
case "switch_tab": return `Switched to ${action.tabId}`;
|
|
1503
|
+
case "close_tab": return `Closed ${action.tabId}`;
|
|
1504
|
+
case "search_page": return `Searched page for "${action.pattern}"`;
|
|
1505
|
+
case "find_elements": return `Found elements: ${action.selector}`;
|
|
1506
|
+
case "back": return `Went back to previous page`;
|
|
1507
|
+
case "wait": return `Waited ${action.ms}ms`;
|
|
1508
|
+
case "screenshot": return `Requested a fresh screenshot`;
|
|
1509
|
+
case "send_keys": return `Sent keys: ${action.keys}`;
|
|
1510
|
+
case "find_text": return `Scrolled to text: "${action.text}"`;
|
|
1511
|
+
case "evaluate": return `Evaluated JS`;
|
|
1512
|
+
case "dropdown_options": return `Read dropdown options at [${action.index}]`;
|
|
1513
|
+
case "select_dropdown": return `Selected "${action.text}" in dropdown [${action.index}]`;
|
|
1514
|
+
case "upload_file": return `Uploaded file to [${action.index}]: ${action.path}`;
|
|
1515
|
+
case "extract": return `Extracted: "${action.query}"`;
|
|
1516
|
+
case "tool": return `Called tool "${action.name}"`;
|
|
1517
|
+
case "done": return `Done: ${action.result}`;
|
|
1518
|
+
case "fail": return `Failed: ${action.reason}`;
|
|
1519
|
+
default: return JSON.stringify(action);
|
|
1520
|
+
}
|
|
1658
1521
|
}
|
|
1659
|
-
|
|
1660
|
-
|
|
1522
|
+
//#endregion
|
|
1523
|
+
//#region src/browser-agent.ts
|
|
1661
1524
|
var BrowserAgent = class {
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
${
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
${
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
1525
|
+
name;
|
|
1526
|
+
eventBus;
|
|
1527
|
+
model;
|
|
1528
|
+
pageExtractionLLM;
|
|
1529
|
+
fallbackModel;
|
|
1530
|
+
useThinking;
|
|
1531
|
+
historyWindow;
|
|
1532
|
+
instructions;
|
|
1533
|
+
extendSystemMessage;
|
|
1534
|
+
overrideSystemMessage;
|
|
1535
|
+
maxSteps;
|
|
1536
|
+
maxFailures;
|
|
1537
|
+
maxActionsPerStep;
|
|
1538
|
+
initialActions;
|
|
1539
|
+
useVision;
|
|
1540
|
+
directlyOpenUrl;
|
|
1541
|
+
headless;
|
|
1542
|
+
viewport;
|
|
1543
|
+
defaultStartUrl;
|
|
1544
|
+
waitAfterAction;
|
|
1545
|
+
maxRepeats;
|
|
1546
|
+
useDOM;
|
|
1547
|
+
allowEvaluate;
|
|
1548
|
+
allowedDomains;
|
|
1549
|
+
prohibitedDomains;
|
|
1550
|
+
storageState;
|
|
1551
|
+
cdpUrl;
|
|
1552
|
+
recordVideo;
|
|
1553
|
+
credentials;
|
|
1554
|
+
stealth;
|
|
1555
|
+
humanize;
|
|
1556
|
+
tools;
|
|
1557
|
+
approvalManager;
|
|
1558
|
+
executionPolicy;
|
|
1559
|
+
planner;
|
|
1560
|
+
jevModel;
|
|
1561
|
+
maxActionChoices;
|
|
1562
|
+
searchEngine;
|
|
1563
|
+
jevProvider = null;
|
|
1564
|
+
costTracker;
|
|
1565
|
+
memoryManager = null;
|
|
1566
|
+
logger;
|
|
1567
|
+
/** Access the MemoryManager (if memory is configured). */
|
|
1568
|
+
get memory() {
|
|
1569
|
+
return this.memoryManager;
|
|
1570
|
+
}
|
|
1571
|
+
constructor(config) {
|
|
1572
|
+
this.name = config.name;
|
|
1573
|
+
this.model = config.model;
|
|
1574
|
+
this.pageExtractionLLM = config.pageExtractionLLM ?? null;
|
|
1575
|
+
this.fallbackModel = config.fallbackModel ?? null;
|
|
1576
|
+
this.useThinking = config.useThinking ?? true;
|
|
1577
|
+
this.historyWindow = Math.max(0, config.historyWindow ?? 6);
|
|
1578
|
+
this.instructions = config.instructions;
|
|
1579
|
+
this.extendSystemMessage = config.extendSystemMessage;
|
|
1580
|
+
this.overrideSystemMessage = config.overrideSystemMessage;
|
|
1581
|
+
this.maxSteps = config.maxSteps ?? 30;
|
|
1582
|
+
this.maxFailures = config.maxFailures ?? 3;
|
|
1583
|
+
this.maxActionsPerStep = Math.max(1, config.maxActionsPerStep ?? 3);
|
|
1584
|
+
this.initialActions = config.initialActions ?? [];
|
|
1585
|
+
this.useVision = config.useVision ?? "auto";
|
|
1586
|
+
this.directlyOpenUrl = config.directlyOpenUrl ?? true;
|
|
1587
|
+
this.headless = config.headless ?? true;
|
|
1588
|
+
this.viewport = config.viewport ?? {
|
|
1589
|
+
width: 1280,
|
|
1590
|
+
height: 720
|
|
1591
|
+
};
|
|
1592
|
+
this.defaultStartUrl = config.startUrl;
|
|
1593
|
+
this.waitAfterAction = config.waitAfterAction ?? 1500;
|
|
1594
|
+
this.maxRepeats = config.maxRepeats ?? 3;
|
|
1595
|
+
this.useDOM = config.useDOM ?? true;
|
|
1596
|
+
this.allowEvaluate = config.allowEvaluate ?? false;
|
|
1597
|
+
this.allowedDomains = config.allowedDomains;
|
|
1598
|
+
this.prohibitedDomains = config.prohibitedDomains;
|
|
1599
|
+
this.storageState = config.storageState;
|
|
1600
|
+
this.cdpUrl = config.cdpUrl;
|
|
1601
|
+
this.recordVideo = config.recordVideo;
|
|
1602
|
+
this.credentials = config.credentials;
|
|
1603
|
+
this.stealth = config.stealth;
|
|
1604
|
+
this.humanize = config.humanize;
|
|
1605
|
+
this.eventBus = config.eventBus ?? new EventBus();
|
|
1606
|
+
this.tools = config.tools ?? [];
|
|
1607
|
+
this.executionPolicy = config.executionPolicy;
|
|
1608
|
+
this.approvalManager = config.approvalManager ?? (config.approval ? new ApprovalManager({
|
|
1609
|
+
...config.approval,
|
|
1610
|
+
eventBus: this.eventBus
|
|
1611
|
+
}) : null);
|
|
1612
|
+
this.planner = config.planner ?? "vision";
|
|
1613
|
+
this.jevModel = config.jevModel ?? "jev-latest";
|
|
1614
|
+
this.maxActionChoices = config.maxActionChoices ?? 40;
|
|
1615
|
+
this.searchEngine = config.searchEngine ?? "duckduckgo";
|
|
1616
|
+
this.costTracker = config.costTracker ?? null;
|
|
1617
|
+
this.logger = new Logger({
|
|
1618
|
+
prefix: `BrowserAgent:${config.name}`,
|
|
1619
|
+
level: config.logLevel ?? "silent"
|
|
1620
|
+
});
|
|
1621
|
+
if (config.memory) this.memoryManager = new MemoryManager(config.memory);
|
|
1622
|
+
}
|
|
1623
|
+
async run(task, opts) {
|
|
1624
|
+
const startTime = Date.now();
|
|
1625
|
+
const maxSteps = opts?.maxSteps ?? this.maxSteps;
|
|
1626
|
+
const sessionId = opts?.context?.sessionId ?? opts?.sessionId ?? `browser_${startTime}`;
|
|
1627
|
+
const userId = opts?.context?.userId ?? opts?.userId;
|
|
1628
|
+
const ctx = opts?.context ?? new RunContext({
|
|
1629
|
+
sessionId,
|
|
1630
|
+
userId,
|
|
1631
|
+
tenantId: opts?.tenantId,
|
|
1632
|
+
signal: opts?.signal,
|
|
1633
|
+
runMode: opts?.runMode,
|
|
1634
|
+
executionPolicy: this.executionPolicy,
|
|
1635
|
+
eventBus: this.eventBus
|
|
1636
|
+
});
|
|
1637
|
+
const executor = new ToolExecutor(this.tools, {
|
|
1638
|
+
approvalManager: this.approvalManager ?? void 0,
|
|
1639
|
+
executionPolicy: this.executionPolicy,
|
|
1640
|
+
agentName: this.name
|
|
1641
|
+
});
|
|
1642
|
+
const browser = new BrowserProvider();
|
|
1643
|
+
const steps = [];
|
|
1644
|
+
const actionHistory = [];
|
|
1645
|
+
const extractedContent = [];
|
|
1646
|
+
let lastExtractResult;
|
|
1647
|
+
let consecutiveFailures = 0;
|
|
1648
|
+
let lastActionWasScreenshot = false;
|
|
1649
|
+
const loop = new LoopDetector();
|
|
1650
|
+
/**
|
|
1651
|
+
* Rolling conversation history. Each entry stores the structured
|
|
1652
|
+
* agent envelope so we can replay model_thoughts to the model on
|
|
1653
|
+
* subsequent steps — this is what `historyWindow > 0` buys us.
|
|
1654
|
+
*/
|
|
1655
|
+
const historyTurns = [];
|
|
1656
|
+
let extraInstructions = [this.instructions, this.extendSystemMessage].filter(Boolean).join("\n\n");
|
|
1657
|
+
if (this.memoryManager) {
|
|
1658
|
+
await this.memoryManager.ensureReady();
|
|
1659
|
+
const memoryContext = await this.memoryManager.buildContext(sessionId, userId, task, this.name);
|
|
1660
|
+
if (memoryContext) extraInstructions = extraInstructions ? `${extraInstructions}\n\n${memoryContext}` : memoryContext;
|
|
1661
|
+
}
|
|
1662
|
+
const credentialKeys = this.credentials?.keys();
|
|
1663
|
+
const systemPrompt = buildSystemPrompt(this.viewport, extraInstructions || void 0, credentialKeys, {
|
|
1664
|
+
overrideSystemMessage: this.overrideSystemMessage,
|
|
1665
|
+
maxActionsPerStep: this.maxActionsPerStep,
|
|
1666
|
+
allowEvaluate: this.allowEvaluate,
|
|
1667
|
+
tools: this.tools,
|
|
1668
|
+
useVision: this.useVision,
|
|
1669
|
+
useDOM: this.useDOM,
|
|
1670
|
+
useThinking: this.useThinking
|
|
1671
|
+
});
|
|
1672
|
+
try {
|
|
1673
|
+
if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
|
|
1674
|
+
if (ctx.runMode === "plan") throw new Error("Native browser operations are not supported in plan mode");
|
|
1675
|
+
this.logger.info("Launching browser", {
|
|
1676
|
+
headless: this.headless,
|
|
1677
|
+
viewport: this.viewport,
|
|
1678
|
+
useDOM: this.useDOM,
|
|
1679
|
+
useVision: this.useVision,
|
|
1680
|
+
useThinking: this.useThinking,
|
|
1681
|
+
cdpUrl: this.cdpUrl ?? void 0,
|
|
1682
|
+
recordVideo: !!this.recordVideo,
|
|
1683
|
+
stealth: !!this.stealth,
|
|
1684
|
+
humanize: !!this.humanize
|
|
1685
|
+
});
|
|
1686
|
+
await browser.launch({
|
|
1687
|
+
headless: this.headless,
|
|
1688
|
+
viewport: this.viewport,
|
|
1689
|
+
storageState: this.storageState,
|
|
1690
|
+
recordVideo: this.recordVideo,
|
|
1691
|
+
stealth: this.stealth,
|
|
1692
|
+
humanize: this.humanize,
|
|
1693
|
+
cdpUrl: this.cdpUrl
|
|
1694
|
+
});
|
|
1695
|
+
const startUrl = opts?.startUrl ?? this.defaultStartUrl ?? this.detectUrlInTask(task);
|
|
1696
|
+
if (startUrl) {
|
|
1697
|
+
this.logger.info("Navigating to start URL", { url: startUrl });
|
|
1698
|
+
this.assertDomainAllowed(startUrl);
|
|
1699
|
+
await browser.navigate(startUrl);
|
|
1700
|
+
await this.navigationHealthCheck(browser, startUrl);
|
|
1701
|
+
}
|
|
1702
|
+
for (const ia of this.initialActions) {
|
|
1703
|
+
try {
|
|
1704
|
+
await this.executeAction(browser, ia, actionHistory, extractedContent, ctx, executor);
|
|
1705
|
+
} catch (e) {
|
|
1706
|
+
this.logger.warn("initialAction failed", {
|
|
1707
|
+
action: ia,
|
|
1708
|
+
error: e?.message
|
|
1709
|
+
});
|
|
1710
|
+
}
|
|
1711
|
+
await this.sleep(this.waitAfterAction);
|
|
1712
|
+
}
|
|
1713
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
1714
|
+
if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
|
|
1715
|
+
const pageInfo = await browser.getPageInfo();
|
|
1716
|
+
let domSnapshot;
|
|
1717
|
+
let scrollCtx;
|
|
1718
|
+
let elements = [];
|
|
1719
|
+
if (this.useDOM) {
|
|
1720
|
+
const dom = await browser.extractDOM();
|
|
1721
|
+
domSnapshot = dom.text;
|
|
1722
|
+
scrollCtx = dom.scroll;
|
|
1723
|
+
elements = dom.elements;
|
|
1724
|
+
}
|
|
1725
|
+
const tabs = browser.listTabs();
|
|
1726
|
+
const pageAdvice = loop.recordPage({
|
|
1727
|
+
url: pageInfo.url,
|
|
1728
|
+
interactiveCount: scrollCtx?.totalInteractive ?? 0,
|
|
1729
|
+
textHash: fnvHash(domSnapshot ?? "")
|
|
1730
|
+
});
|
|
1731
|
+
if (pageAdvice.severity === "abort") return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
1732
|
+
result: pageAdvice.message ?? "Auto-stopped: page is stagnant.",
|
|
1733
|
+
success: false
|
|
1734
|
+
});
|
|
1735
|
+
const wantVision = this.shouldCaptureVision(step, lastActionWasScreenshot);
|
|
1736
|
+
const screenshot = wantVision ? await browser.screenshot() : Buffer.alloc(0);
|
|
1737
|
+
if (wantVision) this.eventBus.emit("browser.screenshot", { data: screenshot });
|
|
1738
|
+
const isLastStep = step === maxSteps - 1;
|
|
1739
|
+
const nudgeParts = [];
|
|
1740
|
+
if (pageAdvice.severity !== "none" && pageAdvice.message) nudgeParts.push(pageAdvice.message);
|
|
1741
|
+
if (isLastStep) nudgeParts.push("This is your FINAL step. Return a `done` action right now summarizing whatever you have, even if partial.");
|
|
1742
|
+
const nudge = nudgeParts.length > 0 ? nudgeParts.join(" ") : void 0;
|
|
1743
|
+
const userText = buildUserMessage(task, pageInfo.url, pageInfo.title, step, actionHistory, domSnapshot, lastExtractResult, scrollCtx, nudge, {
|
|
1744
|
+
current: step,
|
|
1745
|
+
max: maxSteps
|
|
1746
|
+
}, tabs);
|
|
1747
|
+
this.logger.debug("Calling planner", {
|
|
1748
|
+
step,
|
|
1749
|
+
url: pageInfo.url,
|
|
1750
|
+
planner: this.planner,
|
|
1751
|
+
vision: wantVision
|
|
1752
|
+
});
|
|
1753
|
+
let envelope = null;
|
|
1754
|
+
let modelUsed = this.model;
|
|
1755
|
+
if (this.planner === "jev") {
|
|
1756
|
+
const planned = await this.planWithJev({
|
|
1757
|
+
task,
|
|
1758
|
+
url: pageInfo.url,
|
|
1759
|
+
title: pageInfo.title,
|
|
1760
|
+
elements,
|
|
1761
|
+
tabs,
|
|
1762
|
+
actionHistory,
|
|
1763
|
+
lastExtract: lastExtractResult,
|
|
1764
|
+
pagesBelow: scrollCtx?.pagesBelow,
|
|
1765
|
+
pagesAbove: scrollCtx?.pagesAbove,
|
|
1766
|
+
apiKey: opts?.apiKey,
|
|
1767
|
+
signal: ctx.signal
|
|
1768
|
+
});
|
|
1769
|
+
envelope = planned.envelope;
|
|
1770
|
+
modelUsed = planned.modelUsed;
|
|
1771
|
+
if (this.costTracker && planned.usage) this.costTracker.track({
|
|
1772
|
+
runId: sessionId,
|
|
1773
|
+
agentName: this.name,
|
|
1774
|
+
modelId: modelUsed.modelId,
|
|
1775
|
+
usage: planned.usage,
|
|
1776
|
+
sessionId,
|
|
1777
|
+
userId
|
|
1778
|
+
});
|
|
1779
|
+
} else {
|
|
1780
|
+
const messages = this.buildMessages(systemPrompt, historyTurns, userText, wantVision ? screenshot : null);
|
|
1781
|
+
const { response, modelUsed: used } = await this.callModelWithFallback(messages, opts?.apiKey, ctx.signal);
|
|
1782
|
+
modelUsed = used;
|
|
1783
|
+
if (!response) {
|
|
1784
|
+
consecutiveFailures++;
|
|
1785
|
+
actionHistory.push(`(model call failed — retrying, ${consecutiveFailures}/${this.maxFailures})`);
|
|
1786
|
+
if (consecutiveFailures > this.maxFailures) return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "model");
|
|
1787
|
+
continue;
|
|
1788
|
+
}
|
|
1789
|
+
if (this.costTracker && response.usage) this.costTracker.track({
|
|
1790
|
+
runId: sessionId,
|
|
1791
|
+
agentName: this.name,
|
|
1792
|
+
modelId: modelUsed.modelId,
|
|
1793
|
+
usage: response.usage,
|
|
1794
|
+
sessionId,
|
|
1795
|
+
userId
|
|
1796
|
+
});
|
|
1797
|
+
const raw = typeof response.message.content === "string" ? response.message.content : "";
|
|
1798
|
+
envelope = this.parseEnvelope(raw);
|
|
1799
|
+
}
|
|
1800
|
+
if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
|
|
1801
|
+
if (!envelope) {
|
|
1802
|
+
consecutiveFailures++;
|
|
1803
|
+
this.logger.warn("Failed to parse planner response", {
|
|
1804
|
+
planner: this.planner,
|
|
1805
|
+
consecutiveFailures
|
|
1806
|
+
});
|
|
1807
|
+
if (consecutiveFailures > this.maxFailures) return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "parse");
|
|
1808
|
+
actionHistory.push("(invalid JSON response — retrying)");
|
|
1809
|
+
continue;
|
|
1810
|
+
}
|
|
1811
|
+
consecutiveFailures = 0;
|
|
1812
|
+
historyTurns.push({
|
|
1813
|
+
userText,
|
|
1814
|
+
hasScreenshot: wantVision,
|
|
1815
|
+
envelope
|
|
1816
|
+
});
|
|
1817
|
+
if (this.historyWindow > 0 && historyTurns.length > this.historyWindow) historyTurns.splice(0, historyTurns.length - this.historyWindow);
|
|
1818
|
+
const actions = (Array.isArray(envelope.action) ? envelope.action : [envelope.action]).slice(0, this.maxActionsPerStep);
|
|
1819
|
+
let didTerminate = null;
|
|
1820
|
+
let didNavigate = false;
|
|
1821
|
+
lastActionWasScreenshot = false;
|
|
1822
|
+
for (let ai = 0; ai < actions.length; ai++) {
|
|
1823
|
+
if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
|
|
1824
|
+
const action = actions[ai];
|
|
1825
|
+
let summary = summarizeAction(action);
|
|
1826
|
+
if (this.credentials) summary = this.credentials.mask(summary);
|
|
1827
|
+
const advice = loop.recordAction(action);
|
|
1828
|
+
if (advice.severity === "abort" && action.action !== "done" && action.action !== "fail") {
|
|
1829
|
+
this.logger.warn("Loop detector aborting run", { advice });
|
|
1830
|
+
actionHistory.push(`⚠ ${advice.message ?? "loop detected — auto-stopping"}`);
|
|
1831
|
+
didTerminate = {
|
|
1832
|
+
result: advice.message ?? "Stuck in a loop — auto-stopped.",
|
|
1833
|
+
success: false
|
|
1834
|
+
};
|
|
1835
|
+
break;
|
|
1836
|
+
}
|
|
1837
|
+
actionHistory.push(summary);
|
|
1838
|
+
this.logger.info(`Step ${step + 1}.${ai + 1}: ${summary}`);
|
|
1839
|
+
this.eventBus.emit("browser.action", { action });
|
|
1840
|
+
if (action.action === "done") {
|
|
1841
|
+
didTerminate = {
|
|
1842
|
+
result: this.credentials ? this.credentials.mask(action.result) : action.result,
|
|
1843
|
+
success: true
|
|
1844
|
+
};
|
|
1845
|
+
break;
|
|
1846
|
+
}
|
|
1847
|
+
if (action.action === "fail") {
|
|
1848
|
+
didTerminate = {
|
|
1849
|
+
result: this.credentials ? this.credentials.mask(action.reason) : action.reason,
|
|
1850
|
+
success: false
|
|
1851
|
+
};
|
|
1852
|
+
break;
|
|
1853
|
+
}
|
|
1854
|
+
let stepOk = true;
|
|
1855
|
+
let stepOutput;
|
|
1856
|
+
try {
|
|
1857
|
+
const exec = await this.executeAction(browser, action, actionHistory, extractedContent, ctx, executor);
|
|
1858
|
+
stepOutput = exec?.output;
|
|
1859
|
+
if (exec?.didNavigate) {
|
|
1860
|
+
didNavigate = true;
|
|
1861
|
+
await this.navigationHealthCheck(browser, pageInfo.url);
|
|
1862
|
+
}
|
|
1863
|
+
if (exec?.output && (action.action === "extract" || action.action === "search" || action.action === "find_elements" || action.action === "search_page")) lastExtractResult = exec.output;
|
|
1864
|
+
if (action.action === "screenshot") lastActionWasScreenshot = true;
|
|
1865
|
+
} catch (e) {
|
|
1866
|
+
stepOk = false;
|
|
1867
|
+
consecutiveFailures++;
|
|
1868
|
+
this.logger.warn("Action failed", {
|
|
1869
|
+
action: action.action,
|
|
1870
|
+
error: e?.message,
|
|
1871
|
+
consecutiveFailures
|
|
1872
|
+
});
|
|
1873
|
+
actionHistory.push(`(error: ${e?.message ?? "unknown"})`);
|
|
1874
|
+
if (consecutiveFailures > this.maxFailures) {
|
|
1875
|
+
didTerminate = {
|
|
1876
|
+
result: `Action ${action.action} failed ${consecutiveFailures} times. Last error: ${e?.message}`,
|
|
1877
|
+
success: false
|
|
1878
|
+
};
|
|
1879
|
+
break;
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
steps.push({
|
|
1883
|
+
index: steps.length,
|
|
1884
|
+
action,
|
|
1885
|
+
screenshot,
|
|
1886
|
+
pageUrl: pageInfo.url,
|
|
1887
|
+
pageTitle: pageInfo.title,
|
|
1888
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
1889
|
+
dom: domSnapshot,
|
|
1890
|
+
output: stepOutput,
|
|
1891
|
+
ok: stepOk,
|
|
1892
|
+
thinking: envelope.thinking,
|
|
1893
|
+
evaluationPreviousGoal: envelope.evaluationPreviousGoal,
|
|
1894
|
+
memory: envelope.memory,
|
|
1895
|
+
nextGoal: envelope.nextGoal
|
|
1896
|
+
});
|
|
1897
|
+
this.eventBus.emit("browser.step", {
|
|
1898
|
+
index: steps.length - 1,
|
|
1899
|
+
action,
|
|
1900
|
+
pageUrl: pageInfo.url,
|
|
1901
|
+
screenshot
|
|
1902
|
+
});
|
|
1903
|
+
await this.sleep(this.waitAfterAction);
|
|
1904
|
+
if (didNavigate) break;
|
|
1905
|
+
}
|
|
1906
|
+
if (didTerminate) return await this.finalize(browser, steps, startTime, opts, extractedContent, didTerminate);
|
|
1907
|
+
}
|
|
1908
|
+
this.logger.warn("Max steps reached without explicit done", { maxSteps });
|
|
1909
|
+
return await this.forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, "max_steps");
|
|
1910
|
+
} catch (error) {
|
|
1911
|
+
this.logger.error("Browser agent error", { error: error.message });
|
|
1912
|
+
this.eventBus.emit("browser.error", { error });
|
|
1913
|
+
await browser.close();
|
|
1914
|
+
return {
|
|
1915
|
+
result: `Error: ${error.message}`,
|
|
1916
|
+
success: false,
|
|
1917
|
+
steps,
|
|
1918
|
+
finalUrl: "",
|
|
1919
|
+
finalScreenshot: Buffer.alloc(0),
|
|
1920
|
+
durationMs: Date.now() - startTime,
|
|
1921
|
+
extractedContent
|
|
1922
|
+
};
|
|
1923
|
+
} finally {
|
|
1924
|
+
if (!opts?.context) this.approvalManager?.cancelRun(ctx.runId);
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
/**
|
|
1928
|
+
* Returns a ToolDef that lets a regular Agent delegate browser tasks
|
|
1929
|
+
* to this BrowserAgent.
|
|
1930
|
+
*/
|
|
1931
|
+
asTool(config) {
|
|
1932
|
+
return {
|
|
1933
|
+
name: config?.name ?? "browse_web",
|
|
1934
|
+
description: config?.description ?? "Open a browser and autonomously complete a task on a website. Provide a clear task description and optionally a starting URL.",
|
|
1935
|
+
parameters: z.object({
|
|
1936
|
+
task: z.string().describe("What to do in the browser (e.g., 'Search for X and return the top 3 results')"),
|
|
1937
|
+
startUrl: z.string().optional().describe("URL to start at (e.g., 'https://www.google.com')")
|
|
1938
|
+
}),
|
|
1939
|
+
execute: async (args, ctx) => {
|
|
1940
|
+
return (await this.run(args.task, {
|
|
1941
|
+
startUrl: args.startUrl,
|
|
1942
|
+
context: ctx
|
|
1943
|
+
})).result;
|
|
1944
|
+
}
|
|
1945
|
+
};
|
|
1946
|
+
}
|
|
1947
|
+
shouldCaptureVision(step, lastActionWasScreenshot) {
|
|
1948
|
+
if (this.planner === "jev" && this.useVision !== true) return false;
|
|
1949
|
+
if (this.useVision === false) return false;
|
|
1950
|
+
if (this.useVision === true) return true;
|
|
1951
|
+
if (step === 0) return true;
|
|
1952
|
+
if (!this.useDOM) return true;
|
|
1953
|
+
return lastActionWasScreenshot;
|
|
1954
|
+
}
|
|
1955
|
+
detectUrlInTask(task) {
|
|
1956
|
+
if (!this.directlyOpenUrl) return void 0;
|
|
1957
|
+
const match = task.match(/https?:\/\/[^\s)<>"']+/);
|
|
1958
|
+
return match ? match[0] : void 0;
|
|
1959
|
+
}
|
|
1960
|
+
assertDomainAllowed(url) {
|
|
1961
|
+
if (!this.allowedDomains?.length && !this.prohibitedDomains?.length) return;
|
|
1962
|
+
let host;
|
|
1963
|
+
try {
|
|
1964
|
+
host = new URL(url).hostname.toLowerCase();
|
|
1965
|
+
} catch {
|
|
1966
|
+
throw new Error(`Cannot parse URL for domain check: ${url}`);
|
|
1967
|
+
}
|
|
1968
|
+
if (this.allowedDomains?.length && !this.allowedDomains.some((p) => matchDomain(host, p))) throw new Error(`Navigation blocked: ${host} is not in allowedDomains`);
|
|
1969
|
+
if (this.prohibitedDomains?.length && this.prohibitedDomains.some((p) => matchDomain(host, p))) throw new Error(`Navigation blocked: ${host} is in prohibitedDomains`);
|
|
1970
|
+
}
|
|
1971
|
+
/**
|
|
1972
|
+
* Build the message array sent to the model: system prompt + a compact
|
|
1973
|
+
* summary of older turns (if any) + the most recent `historyWindow`
|
|
1974
|
+
* turns verbatim + the current step's user message.
|
|
1975
|
+
*
|
|
1976
|
+
* Inspired by browser-use's history compaction. Keeps tokens bounded
|
|
1977
|
+
* while giving the model meaningful context about what it already
|
|
1978
|
+
* tried.
|
|
1979
|
+
*/
|
|
1980
|
+
buildMessages(systemPrompt, historyTurns, currentUserText, currentScreenshot) {
|
|
1981
|
+
const msgs = [{
|
|
1982
|
+
role: "system",
|
|
1983
|
+
content: systemPrompt
|
|
1984
|
+
}];
|
|
1985
|
+
if (this.historyWindow > 0 && historyTurns.length > 0) for (const turn of historyTurns) {
|
|
1986
|
+
const env = turn.envelope;
|
|
1987
|
+
const replay = [];
|
|
1988
|
+
if (env.thinking) replay.push(`thinking: ${env.thinking}`);
|
|
1989
|
+
if (env.evaluationPreviousGoal) replay.push(`evaluation: ${env.evaluationPreviousGoal}`);
|
|
1990
|
+
if (env.memory) replay.push(`memory: ${env.memory}`);
|
|
1991
|
+
if (env.nextGoal) replay.push(`next_goal: ${env.nextGoal}`);
|
|
1992
|
+
replay.push(`action: ${JSON.stringify(env.action)}`);
|
|
1993
|
+
msgs.push({
|
|
1994
|
+
role: "assistant",
|
|
1995
|
+
content: replay.join("\n")
|
|
1996
|
+
});
|
|
1997
|
+
}
|
|
1998
|
+
const content = [];
|
|
1999
|
+
if (currentScreenshot && currentScreenshot.length > 0) content.push({
|
|
2000
|
+
type: "image",
|
|
2001
|
+
data: currentScreenshot.toString("base64"),
|
|
2002
|
+
mimeType: "image/png"
|
|
2003
|
+
});
|
|
2004
|
+
content.push({
|
|
2005
|
+
type: "text",
|
|
2006
|
+
text: currentUserText
|
|
2007
|
+
});
|
|
2008
|
+
msgs.push({
|
|
2009
|
+
role: "user",
|
|
2010
|
+
content
|
|
2011
|
+
});
|
|
2012
|
+
return msgs;
|
|
2013
|
+
}
|
|
2014
|
+
/**
|
|
2015
|
+
* Call the primary model, retrying once with `fallbackModel` on
|
|
2016
|
+
* transient errors (5xx, 429, network). Returns the response or
|
|
2017
|
+
* `null` if both models failed.
|
|
2018
|
+
*/
|
|
2019
|
+
async callModelWithFallback(messages, apiKey, signal) {
|
|
2020
|
+
const reqOpts = {
|
|
2021
|
+
temperature: .1,
|
|
2022
|
+
maxTokens: 1024,
|
|
2023
|
+
apiKey,
|
|
2024
|
+
signal,
|
|
2025
|
+
responseFormat: "json"
|
|
2026
|
+
};
|
|
2027
|
+
try {
|
|
2028
|
+
signal?.throwIfAborted();
|
|
2029
|
+
const r = await this.model.generate(messages, reqOpts);
|
|
2030
|
+
signal?.throwIfAborted();
|
|
2031
|
+
return {
|
|
2032
|
+
response: r,
|
|
2033
|
+
modelUsed: this.model
|
|
2034
|
+
};
|
|
2035
|
+
} catch (e) {
|
|
2036
|
+
signal?.throwIfAborted();
|
|
2037
|
+
if (this.fallbackModel && this.isTransientError(e)) {
|
|
2038
|
+
this.logger.warn("Primary model failed; trying fallbackModel", {
|
|
2039
|
+
error: e?.message,
|
|
2040
|
+
primary: this.model.modelId,
|
|
2041
|
+
fallback: this.fallbackModel.modelId
|
|
2042
|
+
});
|
|
2043
|
+
try {
|
|
2044
|
+
const r = await this.fallbackModel.generate(messages, reqOpts);
|
|
2045
|
+
signal?.throwIfAborted();
|
|
2046
|
+
return {
|
|
2047
|
+
response: r,
|
|
2048
|
+
modelUsed: this.fallbackModel
|
|
2049
|
+
};
|
|
2050
|
+
} catch (e2) {
|
|
2051
|
+
signal?.throwIfAborted();
|
|
2052
|
+
this.logger.warn("Fallback model also failed", { error: e2?.message });
|
|
2053
|
+
return {
|
|
2054
|
+
response: null,
|
|
2055
|
+
modelUsed: this.model
|
|
2056
|
+
};
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
this.logger.warn("Model call failed (no fallback configured)", { error: e?.message });
|
|
2060
|
+
return {
|
|
2061
|
+
response: null,
|
|
2062
|
+
modelUsed: this.model
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
isTransientError(e) {
|
|
2067
|
+
const msg = String(e?.message ?? e).toLowerCase();
|
|
2068
|
+
if (msg.includes("rate limit") || msg.includes("429")) return true;
|
|
2069
|
+
if (msg.includes("timeout") || msg.includes("etimedout")) return true;
|
|
2070
|
+
if (msg.includes("econnreset") || msg.includes("network")) return true;
|
|
2071
|
+
if (/\b5\d\d\b/.test(msg)) return true;
|
|
2072
|
+
if (msg.includes("401") || msg.includes("402") || msg.includes("auth")) return true;
|
|
2073
|
+
return false;
|
|
2074
|
+
}
|
|
2075
|
+
getJev() {
|
|
2076
|
+
if (!this.jevProvider) this.jevProvider = jev(this.jevModel);
|
|
2077
|
+
return this.jevProvider;
|
|
2078
|
+
}
|
|
2079
|
+
/**
|
|
2080
|
+
* Ask Jev to pick one label from this frame's action space, then map it
|
|
2081
|
+
* to a BrowserAction. Type/search strings come from a text model or the task.
|
|
2082
|
+
*/
|
|
2083
|
+
async planWithJev(args) {
|
|
2084
|
+
const alreadySearched = args.actionHistory.some((h) => /^Searched /.test(h));
|
|
2085
|
+
const alreadyWaited = args.actionHistory.some((h) => /^Waited /.test(h));
|
|
2086
|
+
const foundTitles = args.lastExtract ?? titlesFromElements(args.elements);
|
|
2087
|
+
const space = buildActionSpace(args.elements, args.tabs, {
|
|
2088
|
+
max: this.maxActionChoices,
|
|
2089
|
+
pagesBelow: args.pagesBelow,
|
|
2090
|
+
pagesAbove: args.pagesAbove,
|
|
2091
|
+
allowSearch: !alreadySearched && !isSearchResultsUrl(args.url) && !isBlockedPageUrl(args.url),
|
|
2092
|
+
allowDone: looksLikeResultList(foundTitles),
|
|
2093
|
+
allowWait: !alreadyWaited
|
|
2094
|
+
});
|
|
2095
|
+
const provider = this.getJev();
|
|
2096
|
+
args.signal?.throwIfAborted();
|
|
2097
|
+
try {
|
|
2098
|
+
const response = await provider.generate([{
|
|
2099
|
+
role: "user",
|
|
2100
|
+
content: JSON.stringify({
|
|
2101
|
+
task: args.task,
|
|
2102
|
+
url: args.url,
|
|
2103
|
+
title: args.title,
|
|
2104
|
+
lastActions: args.actionHistory.slice(-8),
|
|
2105
|
+
lastExtract: args.lastExtract,
|
|
2106
|
+
visibleTitles: titlesFromElements(args.elements, 8),
|
|
2107
|
+
tabs: args.tabs,
|
|
2108
|
+
hint: looksLikeResultList(foundTitles) ? "Result titles are in lastExtract. Pick done." : alreadySearched ? "A search already ran. Wait or click a result title. Do not search again." : void 0
|
|
2109
|
+
})
|
|
2110
|
+
}], {
|
|
2111
|
+
questions: { action: choice("Which browser action next? Do not repeat lastActions. If search results are already visible, click a result or done.", space.criteria) },
|
|
2112
|
+
apiKey: args.apiKey,
|
|
2113
|
+
signal: args.signal
|
|
2114
|
+
});
|
|
2115
|
+
args.signal?.throwIfAborted();
|
|
2116
|
+
const raw = typeof response.message.content === "string" ? response.message.content : "";
|
|
2117
|
+
let pick;
|
|
2118
|
+
try {
|
|
2119
|
+
const answers = JSON.parse(raw);
|
|
2120
|
+
pick = typeof answers.action === "string" ? answers.action : answers.action?.choice;
|
|
2121
|
+
} catch {
|
|
2122
|
+
pick = void 0;
|
|
2123
|
+
}
|
|
2124
|
+
if (!pick) return {
|
|
2125
|
+
envelope: null,
|
|
2126
|
+
modelUsed: provider,
|
|
2127
|
+
usage: response.usage
|
|
2128
|
+
};
|
|
2129
|
+
const extras = {
|
|
2130
|
+
searchQuery: pick === "search" ? guessSearchQuery(args.task) : void 0,
|
|
2131
|
+
typeText: pick.startsWith("type_") ? await this.inferTypeText(args.task, pick, args.signal) : void 0,
|
|
2132
|
+
doneResult: pick === "done" ? looksLikeResultList(args.lastExtract ?? "") ? args.lastExtract : titlesFromElements(args.elements) || args.lastExtract || "Done" : void 0
|
|
2133
|
+
};
|
|
2134
|
+
const action = labelToAction(pick, extras);
|
|
2135
|
+
if (!action) return {
|
|
2136
|
+
envelope: null,
|
|
2137
|
+
modelUsed: provider,
|
|
2138
|
+
usage: response.usage
|
|
2139
|
+
};
|
|
2140
|
+
return {
|
|
2141
|
+
envelope: {
|
|
2142
|
+
action,
|
|
2143
|
+
nextGoal: pick
|
|
2144
|
+
},
|
|
2145
|
+
modelUsed: provider,
|
|
2146
|
+
usage: response.usage
|
|
2147
|
+
};
|
|
2148
|
+
} catch (e) {
|
|
2149
|
+
args.signal?.throwIfAborted();
|
|
2150
|
+
this.logger.warn("Jev planner failed", { error: e?.message });
|
|
2151
|
+
return {
|
|
2152
|
+
envelope: null,
|
|
2153
|
+
modelUsed: provider
|
|
2154
|
+
};
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
async pageLooksBlocked(browser) {
|
|
2158
|
+
if (isBlockedPageUrl((await browser.getPageInfo()).url)) return true;
|
|
2159
|
+
try {
|
|
2160
|
+
return isBotChallengeText(await browser.visibleText());
|
|
2161
|
+
} catch {
|
|
2162
|
+
return false;
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
async scrapeSerpTitles(browser) {
|
|
2166
|
+
for (const selector of SERP_TITLE_SELECTORS) {
|
|
2167
|
+
const found = await browser.findElements(selector, { maxResults: 8 });
|
|
2168
|
+
const labels = [];
|
|
2169
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2170
|
+
for (const f of found) {
|
|
2171
|
+
const text = f.text.replace(/\s+/g, " ").trim();
|
|
2172
|
+
if (!isResultTitle(text) || seen.has(text.toLowerCase())) continue;
|
|
2173
|
+
seen.add(text.toLowerCase());
|
|
2174
|
+
labels.push(text);
|
|
2175
|
+
}
|
|
2176
|
+
if (labels.length >= 2) return labels.slice(0, 5).map((t, i) => `${i + 1}. ${t}`).join("\n");
|
|
2177
|
+
}
|
|
2178
|
+
return "";
|
|
2179
|
+
}
|
|
2180
|
+
async inferTypeText(task, pick, signal) {
|
|
2181
|
+
signal?.throwIfAborted();
|
|
2182
|
+
const model = this.pageExtractionLLM ?? (this.model.providerId === "jev" ? null : this.model);
|
|
2183
|
+
if (!model) return guessSearchQuery(task);
|
|
2184
|
+
try {
|
|
2185
|
+
const response = await model.generate([{
|
|
2186
|
+
role: "user",
|
|
2187
|
+
content: `Task: ${task}\nThe next browser action is ${pick}. Reply with ONLY the text to type into that field. No quotes.`
|
|
2188
|
+
}], { signal });
|
|
2189
|
+
signal?.throwIfAborted();
|
|
2190
|
+
return (typeof response.message.content === "string" ? response.message.content.trim() : "").replace(/^["']|["']$/g, "").slice(0, 500);
|
|
2191
|
+
} catch {
|
|
2192
|
+
signal?.throwIfAborted();
|
|
2193
|
+
return guessSearchQuery(task);
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Parse the model's raw response into an `AgentOutput`. Tolerant to
|
|
2198
|
+
* three shapes:
|
|
2199
|
+
* - Full envelope: { thinking, evaluation_previous_goal, action, ... }
|
|
2200
|
+
* - Raw action object (legacy / `useThinking: false`)
|
|
2201
|
+
* - Raw action array
|
|
2202
|
+
*
|
|
2203
|
+
* Also strips ```json fences the model occasionally adds.
|
|
2204
|
+
*/
|
|
2205
|
+
parseEnvelope(raw) {
|
|
2206
|
+
if (!raw) return null;
|
|
2207
|
+
let s = raw.trim();
|
|
2208
|
+
const fence = s.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
|
|
2209
|
+
if (fence) s = fence[1].trim();
|
|
2210
|
+
let json;
|
|
2211
|
+
try {
|
|
2212
|
+
json = JSON.parse(s);
|
|
2213
|
+
} catch {
|
|
2214
|
+
const m = s.match(/[[{][\s\S]*[\]}]/);
|
|
2215
|
+
if (!m) return null;
|
|
2216
|
+
try {
|
|
2217
|
+
json = JSON.parse(m[0]);
|
|
2218
|
+
} catch {
|
|
2219
|
+
return null;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
if (Array.isArray(json)) return { action: json };
|
|
2223
|
+
if (json && typeof json === "object" && typeof json.action === "string") return { action: json };
|
|
2224
|
+
if (json && typeof json === "object" && "action" in json) {
|
|
2225
|
+
const out = { action: json.action };
|
|
2226
|
+
out.thinking = json.thinking ?? json.reasoning ?? void 0;
|
|
2227
|
+
out.evaluationPreviousGoal = json.evaluationPreviousGoal ?? json.evaluation_previous_goal ?? void 0;
|
|
2228
|
+
out.memory = json.memory ?? void 0;
|
|
2229
|
+
out.nextGoal = json.nextGoal ?? json.next_goal ?? void 0;
|
|
2230
|
+
return out;
|
|
2231
|
+
}
|
|
2232
|
+
return null;
|
|
2233
|
+
}
|
|
2234
|
+
/**
|
|
2235
|
+
* Quick post-navigation health check. If the page came back blank
|
|
2236
|
+
* (no body text and no interactive elements), reload once and wait
|
|
2237
|
+
* for stable. This catches the "FreightOS half-loaded font test" class
|
|
2238
|
+
* of failure before the LLM ever sees it.
|
|
2239
|
+
*/
|
|
2240
|
+
async navigationHealthCheck(browser, url) {
|
|
2241
|
+
try {
|
|
2242
|
+
const ok = await browser.pageText({ maxChars: 200 }).catch(() => "");
|
|
2243
|
+
if (ok && ok.trim().length > 5) return;
|
|
2244
|
+
this.logger.warn("Navigation produced an empty page; reloading once", { url });
|
|
2245
|
+
await this.sleep(800);
|
|
2246
|
+
try {
|
|
2247
|
+
await browser.navigate(url);
|
|
2248
|
+
} catch {}
|
|
2249
|
+
} catch {}
|
|
2250
|
+
}
|
|
2251
|
+
/**
|
|
2252
|
+
* Force-finalize the run with whatever partial data the agent has.
|
|
2253
|
+
* Called when:
|
|
2254
|
+
* - `maxSteps` is exhausted without an explicit `done`,
|
|
2255
|
+
* - `maxFailures` is exceeded,
|
|
2256
|
+
* - the model can't produce parseable JSON enough times to make
|
|
2257
|
+
* forward progress.
|
|
2258
|
+
* The result is composed from `extractedContent` + the last few
|
|
2259
|
+
* action summaries so the caller gets something useful instead of
|
|
2260
|
+
* just a one-line error.
|
|
2261
|
+
*/
|
|
2262
|
+
async forceDone(browser, steps, startTime, opts, extractedContent, actionHistory, reason) {
|
|
2263
|
+
const reasonLine = reason === "model" ? "Model call failed too many consecutive times." : reason === "parse" ? "Model produced invalid JSON too many times." : "Max step budget exhausted before an explicit `done`.";
|
|
2264
|
+
const tail = actionHistory.slice(-5).join(" → ");
|
|
2265
|
+
const extracts = extractedContent.length > 0 ? `\n\nExtracts so far:\n${extractedContent.join("\n---\n")}` : "";
|
|
2266
|
+
const result = `${reasonLine}\nLast actions: ${tail || "(none)"}${extracts}`;
|
|
2267
|
+
return await this.finalize(browser, steps, startTime, opts, extractedContent, {
|
|
2268
|
+
result,
|
|
2269
|
+
success: false
|
|
2270
|
+
});
|
|
2271
|
+
}
|
|
2272
|
+
async finalize(browser, steps, startTime, opts, extractedContent, outcome) {
|
|
2273
|
+
let finalScreenshot;
|
|
2274
|
+
let finalInfo;
|
|
2275
|
+
try {
|
|
2276
|
+
finalScreenshot = await browser.screenshot();
|
|
2277
|
+
finalInfo = await browser.getPageInfo();
|
|
2278
|
+
} catch {
|
|
2279
|
+
finalScreenshot = Buffer.alloc(0);
|
|
2280
|
+
finalInfo = {
|
|
2281
|
+
url: "",
|
|
2282
|
+
title: ""
|
|
2283
|
+
};
|
|
2284
|
+
}
|
|
2285
|
+
if (opts?.saveStorageState) try {
|
|
2286
|
+
await browser.saveStorageState(opts.saveStorageState);
|
|
2287
|
+
this.logger.info("Storage state saved", { path: opts.saveStorageState });
|
|
2288
|
+
} catch (e) {
|
|
2289
|
+
this.logger.warn("Failed to save storage state", { error: e.message });
|
|
2290
|
+
}
|
|
2291
|
+
let videoPath;
|
|
2292
|
+
if (this.recordVideo) videoPath = await browser.getVideoPath() ?? void 0;
|
|
2293
|
+
await browser.close();
|
|
2294
|
+
const output = {
|
|
2295
|
+
result: outcome.result,
|
|
2296
|
+
success: outcome.success,
|
|
2297
|
+
steps,
|
|
2298
|
+
finalUrl: finalInfo.url,
|
|
2299
|
+
finalScreenshot,
|
|
2300
|
+
durationMs: Date.now() - startTime,
|
|
2301
|
+
videoPath,
|
|
2302
|
+
extractedContent
|
|
2303
|
+
};
|
|
2304
|
+
if (this.memoryManager) {
|
|
2305
|
+
const sessionId = opts?.context?.sessionId ?? opts?.sessionId ?? `browser_${startTime}`;
|
|
2306
|
+
const userId = opts?.context?.userId ?? opts?.userId;
|
|
2307
|
+
const actionSummary = steps.map((s) => summarizeAction(s.action)).join("; ");
|
|
2308
|
+
const messages = [{
|
|
2309
|
+
role: "user",
|
|
2310
|
+
content: `Task: ${outcome.result}`
|
|
2311
|
+
}, {
|
|
2312
|
+
role: "assistant",
|
|
2313
|
+
content: `Actions: ${actionSummary}. Result: ${outcome.result}`
|
|
2314
|
+
}];
|
|
2315
|
+
this.memoryManager.appendMessages(sessionId, messages, this.model).catch((e) => this.logger.warn("Memory persist failed", { error: String(e) }));
|
|
2316
|
+
try {
|
|
2317
|
+
this.memoryManager.afterRun(sessionId, userId, messages, this.model, this.name);
|
|
2318
|
+
} catch (e) {
|
|
2319
|
+
this.logger.warn("Memory afterRun failed", { error: String(e) });
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
this.eventBus.emit("browser.done", {
|
|
2323
|
+
result: output.result,
|
|
2324
|
+
success: outcome.success,
|
|
2325
|
+
steps
|
|
2326
|
+
});
|
|
2327
|
+
return output;
|
|
2328
|
+
}
|
|
2329
|
+
/**
|
|
2330
|
+
* Execute a single action. Returns `{ output?, didNavigate? }` for the
|
|
2331
|
+
* caller's state tracking. May throw — the loop handles failure-budget
|
|
2332
|
+
* accounting in that case.
|
|
2333
|
+
*/
|
|
2334
|
+
async executeAction(browser, action, _actionHistory, extractedContent, ctx, executor) {
|
|
2335
|
+
if (ctx.signal?.aborted) throw new Error("Browser run cancelled");
|
|
2336
|
+
switch (action.action) {
|
|
2337
|
+
case "click": {
|
|
2338
|
+
if (typeof action.index === "number") {
|
|
2339
|
+
if (await browser.clickByIndex(action.index)) return {};
|
|
2340
|
+
}
|
|
2341
|
+
const keyword = this.extractClickKeyword(action.description);
|
|
2342
|
+
if (keyword && await browser.clickByText(keyword)) {
|
|
2343
|
+
this.logger.debug("Clicked by text", { keyword });
|
|
2344
|
+
return {};
|
|
2345
|
+
}
|
|
2346
|
+
if (typeof action.x === "number" && typeof action.y === "number") {
|
|
2347
|
+
await browser.click(action.x, action.y);
|
|
2348
|
+
return {};
|
|
2349
|
+
}
|
|
2350
|
+
throw new Error(`click action has no resolvable target (index/text/coordinates)`);
|
|
2351
|
+
}
|
|
2352
|
+
case "type": {
|
|
2353
|
+
const resolvedText = this.credentials ? this.credentials.resolve(action.text) : action.text;
|
|
2354
|
+
const submit = action.submit ?? resolvedText.includes("\n");
|
|
2355
|
+
const cleanText = submit ? resolvedText.replace(/\n+$/, "") : resolvedText;
|
|
2356
|
+
if (typeof action.index === "number") {
|
|
2357
|
+
if (await browser.inputByIndex(action.index, cleanText, {
|
|
2358
|
+
clear: action.clear,
|
|
2359
|
+
submit
|
|
2360
|
+
})) return {};
|
|
2361
|
+
}
|
|
2362
|
+
if (typeof action.x === "number" && typeof action.y === "number") {
|
|
2363
|
+
await browser.clickAndType(action.x, action.y, cleanText);
|
|
2364
|
+
if (submit) await browser.pressKey("Enter");
|
|
2365
|
+
return {};
|
|
2366
|
+
}
|
|
2367
|
+
await browser.type(cleanText);
|
|
2368
|
+
if (submit) await browser.pressKey("Enter");
|
|
2369
|
+
return {};
|
|
2370
|
+
}
|
|
2371
|
+
case "scroll":
|
|
2372
|
+
if (typeof action.index === "number") {
|
|
2373
|
+
await browser.scrollIntoViewByIndex(action.index);
|
|
2374
|
+
return {};
|
|
2375
|
+
}
|
|
2376
|
+
await browser.scroll(action.direction, action.amount);
|
|
2377
|
+
return {};
|
|
2378
|
+
case "navigate":
|
|
2379
|
+
this.assertDomainAllowed(action.url);
|
|
2380
|
+
if (action.newTab) {
|
|
2381
|
+
const tabId = await browser.newTab(action.url);
|
|
2382
|
+
await browser.switchTab(tabId);
|
|
2383
|
+
return {
|
|
2384
|
+
didNavigate: true,
|
|
2385
|
+
output: `Opened ${action.url} in ${tabId}`
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
await browser.navigate(action.url);
|
|
2389
|
+
return { didNavigate: true };
|
|
2390
|
+
case "search": {
|
|
2391
|
+
const engine = action.engine ?? this.searchEngine;
|
|
2392
|
+
const url = searchUrl(action.query, engine);
|
|
2393
|
+
this.assertDomainAllowed(url);
|
|
2394
|
+
await browser.navigate(url);
|
|
2395
|
+
let used = engine;
|
|
2396
|
+
if (await this.pageLooksBlocked(browser)) {
|
|
2397
|
+
if (engine === "duckduckgo") {
|
|
2398
|
+
const fallback = searchUrl(action.query, "bing");
|
|
2399
|
+
this.assertDomainAllowed(fallback);
|
|
2400
|
+
await browser.navigate(fallback);
|
|
2401
|
+
used = "bing";
|
|
2402
|
+
} else return {
|
|
2403
|
+
didNavigate: true,
|
|
2404
|
+
output: `Search blocked on ${engine}. Pick fail or navigate elsewhere.`
|
|
2405
|
+
};
|
|
2406
|
+
}
|
|
2407
|
+
await browser.waitForStable(400);
|
|
2408
|
+
return {
|
|
2409
|
+
didNavigate: true,
|
|
2410
|
+
output: await this.scrapeSerpTitles(browser) || `Searched ${used} for "${action.query}"`
|
|
2411
|
+
};
|
|
2412
|
+
}
|
|
2413
|
+
case "new_tab": {
|
|
2414
|
+
if (action.url) this.assertDomainAllowed(action.url);
|
|
2415
|
+
const tabId = await browser.newTab(action.url);
|
|
2416
|
+
await browser.switchTab(tabId);
|
|
2417
|
+
return {
|
|
2418
|
+
didNavigate: !!action.url,
|
|
2419
|
+
output: `Opened ${tabId}`
|
|
2420
|
+
};
|
|
2421
|
+
}
|
|
2422
|
+
case "switch_tab":
|
|
2423
|
+
await browser.switchTab(action.tabId);
|
|
2424
|
+
return {
|
|
2425
|
+
didNavigate: true,
|
|
2426
|
+
output: `Switched to ${action.tabId}`
|
|
2427
|
+
};
|
|
2428
|
+
case "close_tab":
|
|
2429
|
+
await browser.closeTab(action.tabId);
|
|
2430
|
+
return { output: `Closed ${action.tabId}` };
|
|
2431
|
+
case "search_page": {
|
|
2432
|
+
const hits = await browser.searchPage({
|
|
2433
|
+
pattern: action.pattern,
|
|
2434
|
+
regex: action.regex,
|
|
2435
|
+
caseSensitive: action.caseSensitive,
|
|
2436
|
+
maxResults: action.maxResults
|
|
2437
|
+
});
|
|
2438
|
+
const lines = hits.map((h, i) => `${i + 1}. "${h.match}" — …${h.context}…`);
|
|
2439
|
+
return { output: hits.length ? `search_page:\n${lines.join("\n")}` : "search_page: no matches" };
|
|
2440
|
+
}
|
|
2441
|
+
case "find_elements": {
|
|
2442
|
+
const found = await browser.findElements(action.selector, { maxResults: action.maxResults });
|
|
2443
|
+
const lines = found.map((el, i) => `${i + 1}. <${el.tag}> ${el.text}${el.href ? ` ${el.href}` : ""}`);
|
|
2444
|
+
return { output: found.length ? `find_elements:\n${lines.join("\n")}` : "find_elements: none" };
|
|
2445
|
+
}
|
|
2446
|
+
case "back":
|
|
2447
|
+
await browser.back();
|
|
2448
|
+
return { didNavigate: true };
|
|
2449
|
+
case "wait":
|
|
2450
|
+
await this.sleep(Math.min(action.ms, 1e4));
|
|
2451
|
+
return {};
|
|
2452
|
+
case "screenshot": return {};
|
|
2453
|
+
case "send_keys":
|
|
2454
|
+
await browser.sendKeys(action.keys);
|
|
2455
|
+
return {};
|
|
2456
|
+
case "find_text":
|
|
2457
|
+
await browser.findText(action.text);
|
|
2458
|
+
return {};
|
|
2459
|
+
case "evaluate":
|
|
2460
|
+
if (!this.allowEvaluate) throw new Error("evaluate is disabled. Set allowEvaluate: true on BrowserAgent to enable.");
|
|
2461
|
+
return { output: `evaluate → ${await browser.evaluate(action.code)}` };
|
|
2462
|
+
case "dropdown_options": {
|
|
2463
|
+
const formatted = (await browser.dropdownOptions(action.index)).map((o, i) => `${i + 1}. "${o.label}" (value="${o.value}")${o.selected ? " [selected]" : ""}`).join("\n");
|
|
2464
|
+
return { output: `Dropdown [${action.index}] options:\n${formatted || "(no options found)"}` };
|
|
2465
|
+
}
|
|
2466
|
+
case "select_dropdown":
|
|
2467
|
+
if (!await browser.selectDropdown(action.index, action.text)) throw new Error(`Could not select "${action.text}" on dropdown [${action.index}]`);
|
|
2468
|
+
return {};
|
|
2469
|
+
case "upload_file":
|
|
2470
|
+
if (!await browser.uploadFileByIndex(action.index, action.path)) throw new Error(`Could not upload "${action.path}" to [${action.index}]`);
|
|
2471
|
+
return {};
|
|
2472
|
+
case "extract": {
|
|
2473
|
+
const pageText = await browser.pageText({ extractLinks: action.extractLinks });
|
|
2474
|
+
const model = this.pageExtractionLLM ?? this.model;
|
|
2475
|
+
const messages = [{
|
|
2476
|
+
role: "system",
|
|
2477
|
+
content: "You extract information from web pages. Use ONLY the page content provided. If the requested information is not present, say so explicitly. Be concise and structured (lists/tables) when appropriate."
|
|
2478
|
+
}, {
|
|
2479
|
+
role: "user",
|
|
2480
|
+
content: `Query: ${action.query}\n\nPage content:\n${pageText}`
|
|
2481
|
+
}];
|
|
2482
|
+
const response = await model.generate(messages, {
|
|
2483
|
+
temperature: 0,
|
|
2484
|
+
maxTokens: 2048,
|
|
2485
|
+
signal: ctx.signal
|
|
2486
|
+
});
|
|
2487
|
+
ctx.signal?.throwIfAborted();
|
|
2488
|
+
const out = typeof response.message.content === "string" ? response.message.content : "";
|
|
2489
|
+
const masked = this.credentials ? this.credentials.mask(out) : out;
|
|
2490
|
+
extractedContent.push(masked);
|
|
2491
|
+
return { output: masked };
|
|
2492
|
+
}
|
|
2493
|
+
case "tool": {
|
|
2494
|
+
const tool = this.tools.find((t) => t.name === action.name);
|
|
2495
|
+
if (!tool) throw new Error(`Tool "${action.name}" is not registered on this BrowserAgent`);
|
|
2496
|
+
const [result] = await executor.executeAll([{
|
|
2497
|
+
id: `browser-tool-${ctx.runId}-${Date.now()}`,
|
|
2498
|
+
name: tool.name,
|
|
2499
|
+
arguments: action.args ?? {}
|
|
2500
|
+
}], ctx);
|
|
2501
|
+
if (result.error) throw new Error(result.error);
|
|
2502
|
+
const out = typeof result.result === "string" ? result.result : result.result.content;
|
|
2503
|
+
return { output: `${action.name} → ${out}` };
|
|
2504
|
+
}
|
|
2505
|
+
case "done":
|
|
2506
|
+
case "fail": return {};
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
sleep(ms) {
|
|
2510
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2511
|
+
}
|
|
2512
|
+
/**
|
|
2513
|
+
* Parse a quoted target keyword from a click action's `description`.
|
|
2514
|
+
* Returns `undefined` for generic / ambiguous labels (login buttons,
|
|
2515
|
+
* close, OK, etc.) where a substring text match could fire on the
|
|
2516
|
+
* wrong element.
|
|
2517
|
+
*/
|
|
2518
|
+
extractClickKeyword(description) {
|
|
2519
|
+
if (!description) return void 0;
|
|
2520
|
+
const match = description.match(/['"\u2018\u2019\u201C\u201D]([^'"\u2018\u2019\u201C\u201D]{1,80})['"\u2018\u2019\u201C\u201D]/);
|
|
2521
|
+
if (!match) return void 0;
|
|
2522
|
+
const keyword = match[1].trim();
|
|
2523
|
+
if (!keyword || keyword.length < 2) return void 0;
|
|
2524
|
+
if ((/* @__PURE__ */ new Set([
|
|
2525
|
+
"log in",
|
|
2526
|
+
"login",
|
|
2527
|
+
"sign in",
|
|
2528
|
+
"sign up",
|
|
2529
|
+
"submit",
|
|
2530
|
+
"close",
|
|
2531
|
+
"ok",
|
|
2532
|
+
"okay",
|
|
2533
|
+
"cancel",
|
|
2534
|
+
"yes",
|
|
2535
|
+
"no",
|
|
2536
|
+
"x",
|
|
2537
|
+
"continue",
|
|
2538
|
+
"next",
|
|
2539
|
+
"back",
|
|
2540
|
+
"accept",
|
|
2541
|
+
"dismiss",
|
|
2542
|
+
"got it",
|
|
2543
|
+
"agree",
|
|
2544
|
+
"i agree",
|
|
2545
|
+
"allow",
|
|
2546
|
+
"deny"
|
|
2547
|
+
])).has(keyword.toLowerCase())) return void 0;
|
|
2548
|
+
return keyword;
|
|
2549
|
+
}
|
|
2652
2550
|
};
|
|
2551
|
+
/**
|
|
2552
|
+
* Domain wildcard matcher. Supports:
|
|
2553
|
+
* "example.com" — exact match
|
|
2554
|
+
* "*.example.com" — example.com and any subdomain
|
|
2555
|
+
* "*" — anything
|
|
2556
|
+
* Case-insensitive.
|
|
2557
|
+
*/
|
|
2653
2558
|
function matchDomain(host, pattern) {
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2559
|
+
const h = host.toLowerCase();
|
|
2560
|
+
let p = pattern.toLowerCase().replace(/^https?\*?:\/\//, "").replace(/^\/+/, "");
|
|
2561
|
+
if (p.includes("/")) p = p.split("/")[0];
|
|
2562
|
+
if (p === "*") return true;
|
|
2563
|
+
if (p === h) return true;
|
|
2564
|
+
if (p.startsWith("*.")) {
|
|
2565
|
+
const base = p.slice(2);
|
|
2566
|
+
return h === base || h.endsWith(`.${base}`);
|
|
2567
|
+
}
|
|
2568
|
+
return false;
|
|
2664
2569
|
}
|
|
2665
|
-
|
|
2666
|
-
|
|
2570
|
+
//#endregion
|
|
2571
|
+
//#region src/credential-vault.ts
|
|
2572
|
+
/**
|
|
2573
|
+
* Secure credential store for BrowserAgent.
|
|
2574
|
+
*
|
|
2575
|
+
* Secrets are stored in memory and NEVER sent to the LLM.
|
|
2576
|
+
* The model works with placeholders (e.g. `{{email}}`, `{{password}}`),
|
|
2577
|
+
* and the agent resolves them to real values only at execution time.
|
|
2578
|
+
*/
|
|
2667
2579
|
var CredentialVault = class {
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
let masked = text;
|
|
2721
|
-
for (const [key, value] of this.secrets) {
|
|
2722
|
-
if (value && masked.includes(value)) {
|
|
2723
|
-
masked = masked.split(value).join(`{{${key}}}`);
|
|
2724
|
-
}
|
|
2725
|
-
}
|
|
2726
|
-
return masked;
|
|
2727
|
-
}
|
|
2728
|
-
};
|
|
2729
|
-
export {
|
|
2730
|
-
BrowserAgent,
|
|
2731
|
-
BrowserProvider,
|
|
2732
|
-
CredentialVault,
|
|
2733
|
-
buildActionSpace,
|
|
2734
|
-
guessSearchQuery,
|
|
2735
|
-
isBlockedPageUrl,
|
|
2736
|
-
isBotChallengeText,
|
|
2737
|
-
isResultTitle,
|
|
2738
|
-
isSearchResultsUrl,
|
|
2739
|
-
labelToAction,
|
|
2740
|
-
looksLikeResultList,
|
|
2741
|
-
searchUrl,
|
|
2742
|
-
titlesFromElements
|
|
2580
|
+
secrets = /* @__PURE__ */ new Map();
|
|
2581
|
+
constructor(initial) {
|
|
2582
|
+
if (initial) for (const [key, value] of Object.entries(initial)) this.set(key, value);
|
|
2583
|
+
}
|
|
2584
|
+
/** Store a credential. Key names become the placeholder: `{{key}}`. */
|
|
2585
|
+
set(key, value) {
|
|
2586
|
+
this.secrets.set(key.toLowerCase(), value);
|
|
2587
|
+
return this;
|
|
2588
|
+
}
|
|
2589
|
+
/** Retrieve a credential value. Returns undefined if not found. */
|
|
2590
|
+
get(key) {
|
|
2591
|
+
return this.secrets.get(key.toLowerCase());
|
|
2592
|
+
}
|
|
2593
|
+
has(key) {
|
|
2594
|
+
return this.secrets.has(key.toLowerCase());
|
|
2595
|
+
}
|
|
2596
|
+
/** List available placeholder names (never exposes values). */
|
|
2597
|
+
keys() {
|
|
2598
|
+
return [...this.secrets.keys()];
|
|
2599
|
+
}
|
|
2600
|
+
/**
|
|
2601
|
+
* Load credentials from environment variables.
|
|
2602
|
+
* Maps env var names to placeholder keys.
|
|
2603
|
+
*
|
|
2604
|
+
* @example
|
|
2605
|
+
* vault.fromEnv({ email: "LOGIN_EMAIL", password: "LOGIN_PASS" });
|
|
2606
|
+
*/
|
|
2607
|
+
fromEnv(mapping) {
|
|
2608
|
+
for (const [key, envVar] of Object.entries(mapping)) {
|
|
2609
|
+
const value = process.env[envVar];
|
|
2610
|
+
if (value) this.set(key, value);
|
|
2611
|
+
}
|
|
2612
|
+
return this;
|
|
2613
|
+
}
|
|
2614
|
+
/**
|
|
2615
|
+
* Replace `{{key}}` placeholders in text with actual credential values.
|
|
2616
|
+
* Used internally by BrowserAgent right before executing a type action.
|
|
2617
|
+
*/
|
|
2618
|
+
resolve(text) {
|
|
2619
|
+
return text.replace(/\{\{(\w+)\}\}/g, (_match, key) => {
|
|
2620
|
+
return this.get(key) ?? `{{${key}}}`;
|
|
2621
|
+
});
|
|
2622
|
+
}
|
|
2623
|
+
/**
|
|
2624
|
+
* Replace any occurrence of real credential values in text with
|
|
2625
|
+
* their `{{key}}` placeholder. Used to sanitize logs and action history.
|
|
2626
|
+
*/
|
|
2627
|
+
mask(text) {
|
|
2628
|
+
let masked = text;
|
|
2629
|
+
for (const [key, value] of this.secrets) if (value && masked.includes(value)) masked = masked.split(value).join(`{{${key}}}`);
|
|
2630
|
+
return masked;
|
|
2631
|
+
}
|
|
2743
2632
|
};
|
|
2633
|
+
//#endregion
|
|
2634
|
+
export { BrowserAgent, BrowserProvider, CredentialVault, buildActionSpace, guessSearchQuery, isBlockedPageUrl, isBotChallengeText, isResultTitle, isSearchResultsUrl, labelToAction, looksLikeResultList, searchUrl, titlesFromElements };
|