bwb-browser 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +43 -17
- package/README.md +88 -40
- package/lib/act.mjs +474 -215
- package/lib/browser.mjs +189 -32
- package/lib/config.mjs +180 -0
- package/lib/diagnose.mjs +13 -11
- package/lib/fetch.mjs +187 -75
- package/lib/fingerprint.mjs +73 -28
- package/lib/helpers.mjs +86 -39
- package/lib/session.mjs +34 -7
- package/lib/setup.mjs +94 -13
- package/lib/tabs.mjs +127 -15
- package/lib/urlpolicy.mjs +137 -0
- package/lib/vigil.mjs +44 -21
- package/package.json +15 -7
- package/server.mjs +433 -213
package/lib/fetch.mjs
CHANGED
|
@@ -6,8 +6,23 @@
|
|
|
6
6
|
* nothing until the first static fetch. Called by browser_goto before CDP.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { assertOutbound, UrlPolicyError } from "./urlpolicy.mjs";
|
|
10
|
+
|
|
9
11
|
const FETCH_MAX_BYTES = 2_000_000; // never buffer more than 2MB per page
|
|
10
12
|
const MIN_TEXT_CHARS = 500; // below this + JS markers => probably a JS shell
|
|
13
|
+
const MAX_REDIRECTS = 5;
|
|
14
|
+
const DEFAULT_MAX_CHARS = 20_000;
|
|
15
|
+
const MAX_LINKS = 40;
|
|
16
|
+
|
|
17
|
+
let USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 bwb-static";
|
|
18
|
+
try {
|
|
19
|
+
const { readFileSync } = await import("fs");
|
|
20
|
+
const { join, dirname } = await import("path");
|
|
21
|
+
const { fileURLToPath } = await import("url");
|
|
22
|
+
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
23
|
+
const { version } = JSON.parse(readFileSync(pkgPath, "utf8"));
|
|
24
|
+
if (version) USER_AGENT = `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 bwb-static/${version}`;
|
|
25
|
+
} catch { /* keep the fallback UA */ }
|
|
11
26
|
|
|
12
27
|
// Framework shells / client-rendered markers — presence + thin text = escalate
|
|
13
28
|
const JS_MARKERS = [
|
|
@@ -18,110 +33,207 @@ const JS_MARKERS = [
|
|
|
18
33
|
// Login / auth walls — static fetch can't pass, but the browser (sessions) might
|
|
19
34
|
const AUTH_MARKERS = ["password", "sign in", "log in", "login", "captcha"];
|
|
20
35
|
|
|
21
|
-
|
|
36
|
+
// Rendered through Readability
|
|
37
|
+
const HTML_TYPES = ["text/html", "application/xhtml+xml"];
|
|
38
|
+
// Served as-is: no article extraction, no "thin body" heuristics. A 31-byte
|
|
39
|
+
// robots.txt or a JSON file must NOT cost a Chromium spawn.
|
|
40
|
+
const VERBATIM_TYPES = [
|
|
41
|
+
"text/plain", "application/json", "application/ld+json", "text/xml",
|
|
42
|
+
"application/xml", "application/rss+xml", "application/atom+xml",
|
|
43
|
+
"text/markdown", "text/csv", "text/tab-separated-values",
|
|
44
|
+
];
|
|
45
|
+
const TEXT_TYPES = [...HTML_TYPES, ...VERBATIM_TYPES];
|
|
46
|
+
|
|
47
|
+
const STATIC_FETCH_DEFAULTS = {
|
|
48
|
+
timeout: 15000,
|
|
49
|
+
allowPrivate: false,
|
|
50
|
+
allowDomains: null,
|
|
51
|
+
raw: false,
|
|
52
|
+
maxChars: DEFAULT_MAX_CHARS,
|
|
53
|
+
};
|
|
22
54
|
|
|
23
55
|
/**
|
|
24
56
|
* Fetch + extract without Chromium.
|
|
25
57
|
* @returns {mode:'static',...} | {mode:'escalate', reason} | {mode:'error', error}
|
|
26
58
|
*/
|
|
27
|
-
export async function staticFetch(url,
|
|
59
|
+
export async function staticFetch(url, options = {}) {
|
|
60
|
+
const { timeout, allowPrivate, allowDomains, raw, maxChars } =
|
|
61
|
+
{ ...STATIC_FETCH_DEFAULTS, ...options };
|
|
62
|
+
|
|
63
|
+
// One abort timer for the whole exchange — headers AND body. Previously the
|
|
64
|
+
// timer was cleared in `finally` as soon as headers arrived, so a server that
|
|
65
|
+
// sent headers then trickled bytes hung forever.
|
|
66
|
+
const ctrl = new AbortController();
|
|
67
|
+
const timer = setTimeout(() => ctrl.abort(), timeout);
|
|
28
68
|
let res;
|
|
69
|
+
let target;
|
|
70
|
+
|
|
29
71
|
try {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
res = await fetch(
|
|
72
|
+
target = url;
|
|
73
|
+
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
74
|
+
const u = await assertOutbound(target, { allowPrivate, allowDomains });
|
|
75
|
+
res = await fetch(u.href, {
|
|
34
76
|
signal: ctrl.signal,
|
|
35
|
-
|
|
77
|
+
// Manual so every hop is re-validated: a public URL can redirect to
|
|
78
|
+
// 169.254.169.254 or file://.
|
|
79
|
+
redirect: "manual",
|
|
36
80
|
headers: {
|
|
37
|
-
"User-Agent":
|
|
38
|
-
Accept: "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.1",
|
|
81
|
+
"User-Agent": USER_AGENT,
|
|
82
|
+
Accept: "text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.9,*/*;q=0.1",
|
|
83
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
39
84
|
},
|
|
40
85
|
});
|
|
41
|
-
|
|
42
|
-
|
|
86
|
+
const location = res.headers.get("location");
|
|
87
|
+
if ([301, 302, 303, 307, 308].includes(res.status) && location) {
|
|
88
|
+
try { await res.body?.cancel(); } catch {}
|
|
89
|
+
target = new URL(location, u).href;
|
|
90
|
+
res = null;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
if (!res) {
|
|
96
|
+
return { mode: "error", error: `Too many redirects (>${MAX_REDIRECTS}) for ${url}` };
|
|
43
97
|
}
|
|
44
98
|
} catch (err) {
|
|
99
|
+
clearTimeout(timer);
|
|
100
|
+
if (err instanceof UrlPolicyError) return { mode: "error", error: err.message };
|
|
45
101
|
// Network-level failure — CDP shares the same network, but surfaces
|
|
46
102
|
// better diagnostics (diagnosePage). Escalate rather than hard-fail.
|
|
47
103
|
return { mode: "escalate", reason: `static fetch failed (${err.name}): ${err.message}` };
|
|
48
104
|
}
|
|
49
105
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
106
|
+
try {
|
|
107
|
+
// Auth walls: the browser (with sessions/cookies) may pass where fetch can't
|
|
108
|
+
if (res.status === 401 || res.status === 403 || res.status === 429) {
|
|
109
|
+
return { mode: "escalate", reason: `HTTP ${res.status} — possible auth wall / rate limit` };
|
|
110
|
+
}
|
|
111
|
+
if (!res.ok) {
|
|
112
|
+
// Dead URL — CDP won't resurrect it. Don't spawn Chromium for a 404.
|
|
113
|
+
return { mode: "error", error: `HTTP ${res.status} ${res.statusText} for ${res.url || url}` };
|
|
114
|
+
}
|
|
58
115
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
116
|
+
const rawType = (res.headers.get("content-type") || "").split(";")[0].trim().toLowerCase();
|
|
117
|
+
const contentType = rawType || "text/html";
|
|
118
|
+
if (!TEXT_TYPES.some((t) => contentType.startsWith(t))) {
|
|
119
|
+
return { mode: "escalate", reason: `non-text content (${rawType || "unknown"})` };
|
|
120
|
+
}
|
|
63
121
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
122
|
+
// Bounded body read. Chunks are concatenated as BYTES and decoded once —
|
|
123
|
+
// decoding per chunk mangles every multibyte character that straddles a
|
|
124
|
+
// network boundary (real servers split at arbitrary byte offsets).
|
|
125
|
+
let buf;
|
|
126
|
+
try {
|
|
127
|
+
const chunks = [];
|
|
128
|
+
let received = 0;
|
|
129
|
+
const reader = res.body.getReader();
|
|
130
|
+
for (;;) {
|
|
131
|
+
const { done, value } = await reader.read();
|
|
132
|
+
if (done) break;
|
|
133
|
+
received += value.length;
|
|
134
|
+
if (received > FETCH_MAX_BYTES) {
|
|
135
|
+
try { await reader.cancel(); } catch {}
|
|
136
|
+
return { mode: "escalate", reason: "page exceeds 2MB static cap — needs browser" };
|
|
137
|
+
}
|
|
138
|
+
chunks.push(Buffer.from(value));
|
|
76
139
|
}
|
|
77
|
-
|
|
140
|
+
buf = Buffer.concat(chunks, received);
|
|
141
|
+
} catch (err) {
|
|
142
|
+
return { mode: "escalate", reason: `body read failed: ${err.message}` };
|
|
78
143
|
}
|
|
79
|
-
} catch (err) {
|
|
80
|
-
return { mode: "escalate", reason: `body read failed: ${err.message}` };
|
|
81
|
-
}
|
|
82
144
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
145
|
+
const charset =
|
|
146
|
+
/charset=["']?([\w-]+)/i.exec(res.headers.get("content-type") || "")?.[1]
|
|
147
|
+
|| /<meta[^>]+charset=["']?([\w-]+)/i.exec(buf.subarray(0, 4096).toString("latin1"))?.[1];
|
|
148
|
+
let body;
|
|
149
|
+
try {
|
|
150
|
+
body = new TextDecoder(charset || "utf-8", { fatal: false }).decode(buf);
|
|
151
|
+
} catch {
|
|
152
|
+
body = new TextDecoder("utf-8", { fatal: false }).decode(buf);
|
|
87
153
|
}
|
|
88
|
-
return { mode: "static", title: "", text, finalUrl: res.url, bytes: html.length, confidence: "high" };
|
|
89
|
-
}
|
|
90
154
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
155
|
+
const finalUrl = res.url || target;
|
|
156
|
+
const cap = capText(body, maxChars);
|
|
157
|
+
|
|
158
|
+
if (VERBATIM_TYPES.some((t) => contentType.startsWith(t))) {
|
|
159
|
+
return {
|
|
160
|
+
mode: "static", title: "", text: cap.text, finalUrl, bytes: buf.length,
|
|
161
|
+
contentType, ...(cap.truncated ? { truncated: true, nextOffset: cap.nextOffset } : {}),
|
|
162
|
+
confidence: "high",
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (raw) {
|
|
167
|
+
return {
|
|
168
|
+
mode: "static", title: "", text: cap.text, finalUrl, bytes: buf.length,
|
|
169
|
+
contentType, raw: true,
|
|
170
|
+
...(cap.truncated ? { truncated: true, nextOffset: cap.nextOffset } : {}),
|
|
171
|
+
confidence: "high",
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// HTML → Readability (lazy deps: zero cost until first use)
|
|
176
|
+
let article, links = [];
|
|
177
|
+
try {
|
|
178
|
+
const { parseHTML } = await import("linkedom");
|
|
179
|
+
const { Readability } = await import("@mozilla/readability");
|
|
180
|
+
const { document } = parseHTML(body);
|
|
181
|
+
links = collectLinks(document);
|
|
182
|
+
article = new Readability(document).parse();
|
|
183
|
+
} catch (err) {
|
|
184
|
+
return { mode: "escalate", reason: `extraction crashed: ${err.message}` };
|
|
185
|
+
}
|
|
186
|
+
if (!article || !(article.textContent || "").trim()) {
|
|
187
|
+
return { mode: "escalate", reason: "no readable article found" };
|
|
188
|
+
}
|
|
104
189
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
190
|
+
const articleText = article.textContent.trim().replace(/\n{3,}/g, "\n\n");
|
|
191
|
+
const capped = capText(articleText, maxChars);
|
|
192
|
+
const lower = body.toLowerCase();
|
|
193
|
+
const hasJsShell = JS_MARKERS.some((m) => lower.includes(m.toLowerCase()));
|
|
194
|
+
const looksAuthed = AUTH_MARKERS.some((m) => lower.includes(m)) && articleText.length < MIN_TEXT_CHARS;
|
|
195
|
+
|
|
196
|
+
if (articleText.length < MIN_TEXT_CHARS && (hasJsShell || looksAuthed)) {
|
|
197
|
+
return {
|
|
198
|
+
mode: "escalate",
|
|
199
|
+
reason: looksAuthed && !hasJsShell
|
|
200
|
+
? "thin text behind possible login wall"
|
|
201
|
+
: "thin text + JS shell markers — needs rendering",
|
|
202
|
+
};
|
|
203
|
+
}
|
|
109
204
|
|
|
110
|
-
if (text.length < MIN_TEXT_CHARS && (hasJsShell || looksAuthed)) {
|
|
111
205
|
return {
|
|
112
|
-
mode: "
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
206
|
+
mode: "static",
|
|
207
|
+
title: article.title || "",
|
|
208
|
+
text: capped.text,
|
|
209
|
+
finalUrl,
|
|
210
|
+
bytes: buf.length,
|
|
211
|
+
links,
|
|
212
|
+
...(capped.truncated ? { truncated: true, nextOffset: capped.nextOffset } : {}),
|
|
213
|
+
confidence: articleText.length >= MIN_TEXT_CHARS ? "high" : "medium",
|
|
116
214
|
};
|
|
215
|
+
} finally {
|
|
216
|
+
clearTimeout(timer);
|
|
117
217
|
}
|
|
218
|
+
}
|
|
118
219
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
text,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
confidence: text.length >= MIN_TEXT_CHARS ? "high" : "medium",
|
|
126
|
-
};
|
|
220
|
+
/** Cap text at maxChars, reporting whether more remains. */
|
|
221
|
+
export function capText(text, maxChars = DEFAULT_MAX_CHARS) {
|
|
222
|
+
if (!maxChars || maxChars < 0 || text.length <= maxChars) {
|
|
223
|
+
return { text, truncated: false };
|
|
224
|
+
}
|
|
225
|
+
return { text: text.slice(0, maxChars), truncated: true, nextOffset: maxChars };
|
|
127
226
|
}
|
|
227
|
+
|
|
228
|
+
function collectLinks(document) {
|
|
229
|
+
const out = [];
|
|
230
|
+
try {
|
|
231
|
+
for (const a of document.querySelectorAll("a[href]")) {
|
|
232
|
+
if (out.length >= MAX_LINKS) break;
|
|
233
|
+
const href = a.getAttribute("href") || "";
|
|
234
|
+
if (!href || href.startsWith("#") || href.startsWith("javascript:")) continue;
|
|
235
|
+
out.push({ text: (a.textContent || "").trim().slice(0, 80), href });
|
|
236
|
+
}
|
|
237
|
+
} catch {}
|
|
238
|
+
return out;
|
|
239
|
+
}
|
package/lib/fingerprint.mjs
CHANGED
|
@@ -1,31 +1,67 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* bwb-browser —
|
|
2
|
+
* bwb-browser — Anti-automation fingerprint patches
|
|
3
3
|
*
|
|
4
|
-
* Applies standard browser
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* bot-detection systems in a realistic browser environment.
|
|
4
|
+
* Applies the standard browser-fingerprint countermeasures used by test rigs
|
|
5
|
+
* (Playwright's Stealth plugin, puppeteer-extra-plugin-stealth) so that
|
|
6
|
+
* bot-detection on a site you own does not fire purely because the browser is
|
|
7
|
+
* headless and automated. These patches are exactly what "stealth mode" means;
|
|
8
|
+
* the previous docs claimed they were "not stealth mode".
|
|
10
9
|
*
|
|
11
10
|
* How it works:
|
|
12
|
-
* -
|
|
13
|
-
* -
|
|
14
|
-
* -
|
|
15
|
-
* -
|
|
16
|
-
* -
|
|
11
|
+
* - navigator.webdriver → false
|
|
12
|
+
* - navigator.plugins — a plausible list
|
|
13
|
+
* - navigator.languages — ['en-US', 'en']
|
|
14
|
+
* - chrome.runtime — normalized
|
|
15
|
+
* - user agent — derived from the REAL Chromium via Browser.getVersion(),
|
|
16
|
+
* with only the HeadlessChrome token swapped for Chrome. Hardcoding
|
|
17
|
+
* "Chrome/126 … Linux x86_64" on a Termux ARM device with a Chromium 140
|
|
18
|
+
* binary is MORE detectable than saying nothing, and it disagreed with
|
|
19
|
+
* navigator.userAgentData and navigator.platform.
|
|
17
20
|
*
|
|
18
|
-
* NOTE:
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
+
* NOTE: scripts are injected via Page.addScriptToEvaluateOnNewDocument, so
|
|
22
|
+
* they only affect pages loaded AFTER applyRealisticProfile. Call before
|
|
23
|
+
* browser_goto.
|
|
21
24
|
*/
|
|
22
25
|
|
|
26
|
+
import { platform } from "os";
|
|
27
|
+
|
|
28
|
+
const PLATFORM_STRING = {
|
|
29
|
+
linux: "Linux x86_64",
|
|
30
|
+
darwin: "Macintosh; Intel Mac OS X 10_15_7",
|
|
31
|
+
win32: "Windows NT 10.0; Win64; x86",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
function uaPlatform() {
|
|
35
|
+
const p = platform();
|
|
36
|
+
if (PLATFORM_STRING[p]) return PLATFORM_STRING[p];
|
|
37
|
+
// Termux/Android reports linux; a phone is not x86_64.
|
|
38
|
+
if (process.env.TERMUX_VERSION) return "Linux; Android 13";
|
|
39
|
+
return PLATFORM_STRING.linux;
|
|
40
|
+
}
|
|
41
|
+
|
|
23
42
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
43
|
+
* Build a user agent that agrees with the actual browser build.
|
|
44
|
+
* Returns null when the version cannot be determined (keep Chromium's own).
|
|
26
45
|
*/
|
|
46
|
+
async function deriveUserAgent(protocol) {
|
|
47
|
+
const product = uaPlatform();
|
|
48
|
+
let version = null;
|
|
49
|
+
try {
|
|
50
|
+
const { Browser } = protocol;
|
|
51
|
+
const info = await Browser.getVersion();
|
|
52
|
+
version = (info?.product || "").match(/\d+\.\d+\.\d+\.\d+/)?.[0] || null;
|
|
53
|
+
} catch {}
|
|
54
|
+
if (!version) return null;
|
|
55
|
+
const headless = process.env.BWB_HEADLESS !== "false";
|
|
56
|
+
const engine = headless ? "HeadlessChrome" : "Chrome";
|
|
57
|
+
return (
|
|
58
|
+
`Mozilla/5.0 (${product}) AppleWebKit/537.36 (KHTML, like Gecko) ` +
|
|
59
|
+
`${engine}/${version} Safari/537.36`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
27
63
|
export async function applyRealisticProfile(protocol) {
|
|
28
|
-
const { Page, Network } = protocol;
|
|
64
|
+
const { Page, Network, Emulation } = protocol;
|
|
29
65
|
|
|
30
66
|
// Inject fingerprint-normalizing script for ALL new documents
|
|
31
67
|
await Page.addScriptToEvaluateOnNewDocument({
|
|
@@ -45,7 +81,7 @@ export async function applyRealisticProfile(protocol) {
|
|
|
45
81
|
|
|
46
82
|
// Realistic language preferences
|
|
47
83
|
Object.defineProperty(navigator, 'languages', {
|
|
48
|
-
get: () => ['en-US', 'en'
|
|
84
|
+
get: () => ['en-US', 'en'],
|
|
49
85
|
configurable: true,
|
|
50
86
|
});
|
|
51
87
|
|
|
@@ -82,24 +118,33 @@ export async function applyRealisticProfile(protocol) {
|
|
|
82
118
|
`,
|
|
83
119
|
});
|
|
84
120
|
|
|
85
|
-
//
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
});
|
|
91
|
-
|
|
121
|
+
// User agent derived from the real browser, with matching userAgentData so
|
|
122
|
+
// the override does not contradict itself.
|
|
123
|
+
let userAgent = null;
|
|
124
|
+
try { userAgent = await deriveUserAgent(protocol); } catch {}
|
|
125
|
+
if (userAgent) {
|
|
126
|
+
try { await Network.setUserAgentOverride({ userAgent }); } catch {}
|
|
127
|
+
try {
|
|
128
|
+
await Emulation?.setUserAgentOverride?.({
|
|
129
|
+
userAgent,
|
|
130
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
131
|
+
platform: uaPlatform(),
|
|
132
|
+
});
|
|
133
|
+
} catch {}
|
|
134
|
+
}
|
|
92
135
|
|
|
93
136
|
return {
|
|
94
137
|
status: "profile applied",
|
|
95
138
|
note: "Only affects pages loaded after this call. Navigate to a new page for the profile to take effect.",
|
|
139
|
+
userAgent: userAgent || "unchanged (real Chromium user agent kept)",
|
|
140
|
+
intendedUse: "Testing sites you own or have permission to test.",
|
|
96
141
|
techniques: [
|
|
97
142
|
"navigator.webdriver → false",
|
|
98
143
|
"navigator.plugins — realistic list",
|
|
99
144
|
"navigator.languages — configured",
|
|
100
145
|
"chrome.runtime — normalized",
|
|
101
146
|
"Permissions query — overridden",
|
|
102
|
-
"User-Agent —
|
|
147
|
+
"User-Agent — derived from the real Chromium build",
|
|
103
148
|
],
|
|
104
149
|
};
|
|
105
|
-
}
|
|
150
|
+
}
|
package/lib/helpers.mjs
CHANGED
|
@@ -2,46 +2,76 @@
|
|
|
2
2
|
* bwb-browser — Shared interaction helpers
|
|
3
3
|
*
|
|
4
4
|
* Functions for navigating, clicking, filling, and waiting for page elements
|
|
5
|
-
*
|
|
5
|
+
* over the CDP protocol.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import { assertNavigable } from "./urlpolicy.mjs";
|
|
9
|
+
|
|
8
10
|
// ─── Navigation Helper ────────────────────────────────────────────────────────
|
|
9
11
|
|
|
10
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Navigate and wait for load.
|
|
14
|
+
* @returns {{title: string, url: string, timedOut: boolean}}
|
|
15
|
+
* `url` is the URL the page ACTUALLY landed on (post-redirect), and
|
|
16
|
+
* `timedOut` says whether the load event simply never arrived. Reporting the
|
|
17
|
+
* *requested* URL as if it were loaded is how "went to X" quietly became
|
|
18
|
+
* "still on about:blank".
|
|
19
|
+
*/
|
|
20
|
+
export async function gotoUrl(page, runtime, url, timeoutMs, { allowDomains = null } = {}) {
|
|
21
|
+
assertNavigable(url, { allowDomains });
|
|
11
22
|
await page.enable();
|
|
12
23
|
|
|
13
|
-
|
|
14
|
-
const
|
|
24
|
+
let settled = false;
|
|
25
|
+
const loadPromise = page.loadEventFired().then(() => { settled = true; });
|
|
26
|
+
const domPromise = page.domContentEventFired().then(() => { settled = true; });
|
|
15
27
|
|
|
16
|
-
await page.navigate({ url });
|
|
28
|
+
const nav = await page.navigate({ url });
|
|
29
|
+
if (nav?.errorText) {
|
|
30
|
+
throw new Error(`Navigation failed: ${nav.errorText} (${url})`);
|
|
31
|
+
}
|
|
17
32
|
|
|
18
33
|
await Promise.race([
|
|
19
34
|
Promise.all([loadPromise, domPromise]),
|
|
20
|
-
new Promise(r => setTimeout(
|
|
35
|
+
new Promise((r) => setTimeout(r, timeoutMs)),
|
|
21
36
|
]);
|
|
22
37
|
|
|
23
38
|
// Small grace for JS framework rendering
|
|
24
|
-
await new Promise(r => setTimeout(r, 500));
|
|
39
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
25
40
|
|
|
41
|
+
let finalUrl = url;
|
|
42
|
+
try {
|
|
43
|
+
const { result } = await runtime.evaluate({ expression: "location.href" });
|
|
44
|
+
if (result?.value) finalUrl = result.value;
|
|
45
|
+
} catch {}
|
|
26
46
|
const { result } = await runtime.evaluate({ expression: "document.title" });
|
|
27
|
-
return { title: result?.value || "", url };
|
|
47
|
+
return { title: result?.value || "", url: finalUrl, timedOut: !settled };
|
|
28
48
|
}
|
|
29
49
|
|
|
30
50
|
// ─── Click Helper (uses CDP Input.dispatchMouseEvent) ─────────────────────────
|
|
31
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Scroll the element into view, hit-test the click point, then dispatch real
|
|
54
|
+
* mouse events. Without the scroll + hit-test an off-screen element silently
|
|
55
|
+
* "clicked" whatever happened to be at those coordinates.
|
|
56
|
+
*/
|
|
32
57
|
export async function clickElement(page, runtime, input, selector) {
|
|
33
58
|
const { result } = await runtime.evaluate({
|
|
34
59
|
expression: `(() => {
|
|
35
60
|
const el = document.querySelector(${JSON.stringify(selector)});
|
|
36
61
|
if (!el) return JSON.stringify({ error: 'NOT_FOUND' });
|
|
62
|
+
try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
|
|
37
63
|
const rect = el.getBoundingClientRect();
|
|
64
|
+
const x = Math.round(rect.x + rect.width / 2);
|
|
65
|
+
const y = Math.round(rect.y + rect.height / 2);
|
|
66
|
+
const at = document.elementFromPoint(x, y);
|
|
38
67
|
return JSON.stringify({
|
|
39
|
-
x
|
|
40
|
-
y: rect.y + rect.height / 2,
|
|
68
|
+
x, y,
|
|
41
69
|
width: rect.width,
|
|
42
70
|
height: rect.height,
|
|
43
71
|
tag: el.tagName,
|
|
44
|
-
text: (el.textContent || '').trim().slice(0,
|
|
72
|
+
text: (el.innerText || el.textContent || '').trim().slice(0, 80),
|
|
73
|
+
landedOn: at ? at.tagName.toLowerCase() + (at.className ? '.' + String(at.className).split(' ')[0] : '') : null,
|
|
74
|
+
hit: !!(at && (at === el || el.contains(at) || at.contains(el))),
|
|
45
75
|
});
|
|
46
76
|
})()`,
|
|
47
77
|
});
|
|
@@ -54,10 +84,13 @@ export async function clickElement(page, runtime, input, selector) {
|
|
|
54
84
|
if (info.error === "NOT_FOUND") {
|
|
55
85
|
throw new Error(`Element not found: ${selector}`);
|
|
56
86
|
}
|
|
87
|
+
if (!info.width || !info.height) {
|
|
88
|
+
throw new Error(`Element is not visible (zero size): ${selector}`);
|
|
89
|
+
}
|
|
57
90
|
|
|
58
91
|
// Dispatch real mouse events via CDP Input domain (ONLY — no native JS click)
|
|
59
|
-
const x =
|
|
60
|
-
|
|
92
|
+
const { x, y } = info;
|
|
93
|
+
await input.dispatchMouseEvent({ type: "mouseMoved", x, y, button: "none" });
|
|
61
94
|
await input.dispatchMouseEvent({ type: "mousePressed", x, y, button: "left", clickCount: 1 });
|
|
62
95
|
await input.dispatchMouseEvent({ type: "mouseReleased", x, y, button: "left", clickCount: 1 });
|
|
63
96
|
|
|
@@ -70,28 +103,36 @@ export async function fillElement(page, runtime, input, selector, text) {
|
|
|
70
103
|
const { result } = await runtime.evaluate({
|
|
71
104
|
expression: `(() => {
|
|
72
105
|
const el = document.querySelector(${JSON.stringify(selector)});
|
|
73
|
-
if (!el) return 'NOT_FOUND';
|
|
106
|
+
if (!el) return JSON.stringify({ error: 'NOT_FOUND' });
|
|
74
107
|
el.focus();
|
|
75
|
-
|
|
76
|
-
|
|
108
|
+
// Select existing content so insertText REPLACES it. focus() alone
|
|
109
|
+
// appends, and a bare Ctrl+A without the modifier bit does nothing.
|
|
110
|
+
try {
|
|
111
|
+
if (el.isContentEditable) { document.execCommand('selectAll'); }
|
|
112
|
+
else if (typeof el.select === 'function') { el.select(); }
|
|
113
|
+
else if ('value' in el) { el.value = ''; }
|
|
114
|
+
} catch {}
|
|
115
|
+
return JSON.stringify({
|
|
116
|
+
focused: true,
|
|
117
|
+
tag: el.tagName ? el.tagName.toLowerCase() : '',
|
|
118
|
+
type: (el.type || '').toLowerCase(),
|
|
119
|
+
contenteditable: String(el.isContentEditable) === 'true',
|
|
120
|
+
});
|
|
77
121
|
})()`,
|
|
78
122
|
returnByValue: true,
|
|
79
123
|
});
|
|
80
124
|
|
|
81
|
-
|
|
125
|
+
let info;
|
|
126
|
+
try { info = JSON.parse(result.value); } catch { info = { error: "NOT_FOUND" }; }
|
|
127
|
+
|
|
128
|
+
if (info.error === "NOT_FOUND") {
|
|
82
129
|
throw new Error(`Element not found: ${selector}`);
|
|
83
130
|
}
|
|
84
131
|
|
|
85
|
-
//
|
|
86
|
-
|
|
87
|
-
await input.dispatchKeyEvent({ type: "keyDown", key: "a" });
|
|
88
|
-
await input.dispatchKeyEvent({ type: "keyUp", key: "a" });
|
|
89
|
-
await input.dispatchKeyEvent({ type: "keyUp", key: "Control" });
|
|
90
|
-
await input.dispatchKeyEvent({ type: "keyDown", key: "Delete" });
|
|
91
|
-
await input.dispatchKeyEvent({ type: "keyUp", key: "Delete" });
|
|
92
|
-
|
|
93
|
-
// Insert text via CDP Input domain
|
|
132
|
+
// Insert text via CDP Input domain (goes through the page's own input
|
|
133
|
+
// handlers, so React/Vue bindings see it).
|
|
94
134
|
await input.insertText({ text });
|
|
135
|
+
return info;
|
|
95
136
|
}
|
|
96
137
|
|
|
97
138
|
// ─── waitForSelector Helper ──────────────────────────────────────────────────
|
|
@@ -102,18 +143,24 @@ export async function waitForSelector(runtime, selector, opts = {}) {
|
|
|
102
143
|
const start = Date.now();
|
|
103
144
|
|
|
104
145
|
while (Date.now() - start < timeout) {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
146
|
+
let info = {};
|
|
147
|
+
try {
|
|
148
|
+
const { result } = await runtime.evaluate({
|
|
149
|
+
expression: `(() => {
|
|
150
|
+
const el = document.querySelector(${JSON.stringify(selector)});
|
|
151
|
+
if (!el) return JSON.stringify({ status: "NOT_FOUND" });
|
|
152
|
+
const rect = el.getBoundingClientRect();
|
|
153
|
+
const hidden = rect.width === 0 || rect.height === 0;
|
|
154
|
+
const text = (el.innerText || el.textContent || "").trim().slice(0, 200);
|
|
155
|
+
return JSON.stringify({ status: "FOUND", tag: el.tagName, text, hidden });
|
|
156
|
+
})()`,
|
|
157
|
+
});
|
|
158
|
+
info = JSON.parse(result?.value || "{}");
|
|
159
|
+
} catch {
|
|
160
|
+
// "Execution context was destroyed" fires on every navigation; keep
|
|
161
|
+
// polling instead of aborting the whole wait.
|
|
162
|
+
info = {};
|
|
163
|
+
}
|
|
117
164
|
|
|
118
165
|
if (disappear && info.status === "NOT_FOUND") return true;
|
|
119
166
|
if (!disappear && info.status === "FOUND" && (!opts.visible || !info.hidden)) return true;
|
|
@@ -122,4 +169,4 @@ export async function waitForSelector(runtime, selector, opts = {}) {
|
|
|
122
169
|
}
|
|
123
170
|
|
|
124
171
|
throw new Error(`browser_waitForSelector: "${selector}" not ${disappear ? "disappeared" : "found"} within ${timeout}ms`);
|
|
125
|
-
}
|
|
172
|
+
}
|