barebrowse 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/README.md +13 -1
- package/barebrowse.context.md +30 -5
- package/cli.js +39 -3
- package/commands/barebrowse/SKILL.md +6 -0
- package/commands/barebrowse.md +1 -0
- package/mcp-server.js +108 -9
- package/package.json +1 -1
- package/src/auth.js +25 -7
- package/src/ax-snapshot.js +24 -5
- package/src/bidi.js +3 -2
- package/src/blocklist-firefox.js +12 -4
- package/src/daemon.js +27 -2
- package/src/data.js +2 -2
- package/src/firefox-page.js +126 -25
- package/src/index.js +152 -37
- package/src/navigation.js +12 -0
- package/src/prune.js +3 -2
- package/src/site-lock.js +184 -0
- package/src/zoom.js +250 -0
- package/types/auth.d.ts +26 -0
- package/types/blocklist-firefox.d.ts +2 -1
- package/types/data.d.ts +2 -1
- package/types/firefox-page.d.ts +14 -2
- package/types/index.d.ts +16 -3
- package/types/navigation.d.ts +12 -0
- package/types/prune.d.ts +2 -0
- package/types/site-lock.d.ts +70 -0
- package/types/zoom.d.ts +51 -0
package/package.json
CHANGED
package/src/auth.js
CHANGED
|
@@ -56,6 +56,16 @@ export function findFirefoxCookieDb() {
|
|
|
56
56
|
|
|
57
57
|
// --- Chromium cookie decryption (Linux) ---
|
|
58
58
|
|
|
59
|
+
/**
|
|
60
|
+
* kwallet-query output -> Chromium keyring password. Chromium feeds the stored
|
|
61
|
+
* string to PBKDF2 as-is (it is not base64 to decode).
|
|
62
|
+
* @param {string} stdout
|
|
63
|
+
* @returns {string}
|
|
64
|
+
*/
|
|
65
|
+
export function kwalletPassword(stdout) {
|
|
66
|
+
return stdout.trim();
|
|
67
|
+
}
|
|
68
|
+
|
|
59
69
|
/**
|
|
60
70
|
* Get Chromium encryption password from OS keyring.
|
|
61
71
|
* Tries KWallet (KDE) first, then GNOME keyring, then fallback.
|
|
@@ -63,11 +73,11 @@ export function findFirefoxCookieDb() {
|
|
|
63
73
|
function getChromiumPassword() {
|
|
64
74
|
// KDE / KWallet
|
|
65
75
|
try {
|
|
66
|
-
const
|
|
76
|
+
const pw = kwalletPassword(execSync(
|
|
67
77
|
'kwallet-query -r "Chromium Safe Storage" -f "Chromium Keys" kdewallet',
|
|
68
78
|
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }
|
|
69
|
-
)
|
|
70
|
-
if (
|
|
79
|
+
));
|
|
80
|
+
if (pw) return pw;
|
|
71
81
|
} catch { /* not KDE or no entry */ }
|
|
72
82
|
|
|
73
83
|
// GNOME keyring / libsecret
|
|
@@ -87,7 +97,7 @@ function getChromiumPassword() {
|
|
|
87
97
|
* Derive AES key from Chromium keyring password.
|
|
88
98
|
* Chrome Linux: PBKDF2-SHA1, salt='saltysalt', 1 iteration, 16-byte key.
|
|
89
99
|
*/
|
|
90
|
-
function deriveKey(password) {
|
|
100
|
+
export function deriveKey(password) {
|
|
91
101
|
return pbkdf2Sync(password, 'saltysalt', 1, 16, 'sha1');
|
|
92
102
|
}
|
|
93
103
|
|
|
@@ -97,7 +107,7 @@ function deriveKey(password) {
|
|
|
97
107
|
* @param {Buffer} aesKey - Derived AES key
|
|
98
108
|
* @returns {string} Decrypted cookie value
|
|
99
109
|
*/
|
|
100
|
-
function decryptCookie(encrypted, aesKey) {
|
|
110
|
+
export function decryptCookie(encrypted, aesKey) {
|
|
101
111
|
const buf = Buffer.from(encrypted);
|
|
102
112
|
if (buf.length === 0) return '';
|
|
103
113
|
|
|
@@ -121,7 +131,7 @@ function decryptCookie(encrypted, aesKey) {
|
|
|
121
131
|
* @param {string} [domain] - Filter by domain (e.g. '.github.com')
|
|
122
132
|
* @returns {Array<object>} Cookies in CDP Network.setCookie format
|
|
123
133
|
*/
|
|
124
|
-
function extractChromiumCookies(dbPath, domain) {
|
|
134
|
+
export function extractChromiumCookies(dbPath, domain) {
|
|
125
135
|
const password = getChromiumPassword();
|
|
126
136
|
const aesKey = deriveKey(password);
|
|
127
137
|
|
|
@@ -132,10 +142,18 @@ function extractChromiumCookies(dbPath, domain) {
|
|
|
132
142
|
CAST(expires_utc AS TEXT) AS expires_utc, is_secure, is_httponly, samesite
|
|
133
143
|
FROM cookies`;
|
|
134
144
|
const params = [];
|
|
145
|
+
// Skip partitioned (CHIPS) rows: a non-empty top_frame_site_key marks a cookie
|
|
146
|
+
// scoped to a top-level site, and it can share name+domain+path with the real
|
|
147
|
+
// one. Older Chromium schemas lack the column, so only filter when it exists.
|
|
148
|
+
const where = [];
|
|
149
|
+
if (db.prepare('PRAGMA table_info(cookies)').all().some((c) => c.name === 'top_frame_site_key')) {
|
|
150
|
+
where.push(`top_frame_site_key = ''`);
|
|
151
|
+
}
|
|
135
152
|
if (domain) {
|
|
136
|
-
|
|
153
|
+
where.push(`host_key LIKE ?`);
|
|
137
154
|
params.push(`%${domain}%`);
|
|
138
155
|
}
|
|
156
|
+
if (where.length) sql += ` WHERE ${where.join(' AND ')}`;
|
|
139
157
|
|
|
140
158
|
const stmt = db.prepare(sql);
|
|
141
159
|
const rows = params.length ? stmt.all(...params) : stmt.all();
|
package/src/ax-snapshot.js
CHANGED
|
@@ -49,6 +49,10 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
|
|
|
49
49
|
'tab', 'menuitem', 'option', 'treeitem', 'switch', 'checkbox', 'radio',
|
|
50
50
|
]);
|
|
51
51
|
|
|
52
|
+
// Table cells name from content like the roles above, but CDP also keeps
|
|
53
|
+
// their text as StaticText children; without them act/read prune the cell away.
|
|
54
|
+
const KEEP_TEXT_CHILDREN = new Set(['cell', 'columnheader', 'rowheader']);
|
|
55
|
+
|
|
52
56
|
// HTML tag → implicit ARIA role (the common, high-value subset).
|
|
53
57
|
const TAG_ROLE = {
|
|
54
58
|
A: (el) => (el.hasAttribute('href') ? 'link' : 'generic'),
|
|
@@ -159,6 +163,23 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
|
|
|
159
163
|
return '';
|
|
160
164
|
}
|
|
161
165
|
|
|
166
|
+
// Descendant text for name-from-content, with an <img>'s alt standing in for
|
|
167
|
+
// the image (accname step 2F), so an image-only link is named like in CDP.
|
|
168
|
+
function contentName(el) {
|
|
169
|
+
let out = '';
|
|
170
|
+
const add = (n) => {
|
|
171
|
+
for (const c of n.childNodes) {
|
|
172
|
+
if (c.nodeType === 3) out += c.textContent;
|
|
173
|
+
else if (c.nodeType === 1) {
|
|
174
|
+
if (c.tagName === 'IMG') out += ' ' + (c.getAttribute('alt') || '') + ' ';
|
|
175
|
+
else add(c);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
add(el);
|
|
180
|
+
return out.replace(/\s+/g, ' ').trim();
|
|
181
|
+
}
|
|
182
|
+
|
|
162
183
|
function props(el, role) {
|
|
163
184
|
const p = {};
|
|
164
185
|
if (role === 'checkbox' || role === 'radio' || role === 'switch') p.checked = !!el.checked;
|
|
@@ -190,9 +211,7 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
|
|
|
190
211
|
// text for roles that name from content (button/link/heading/…), matching
|
|
191
212
|
// CDP — otherwise those nodes come back nameless.
|
|
192
213
|
let name = accName(el);
|
|
193
|
-
if (!name && NAME_FROM_CONTENT.has(role))
|
|
194
|
-
name = el.textContent.replace(/\s+/g, ' ').trim();
|
|
195
|
-
}
|
|
214
|
+
if (!name && NAME_FROM_CONTENT.has(role)) name = contentName(el);
|
|
196
215
|
const node = makeNode(role, name, props(el, role));
|
|
197
216
|
el.setAttribute(REF_ATTR, node.nodeId);
|
|
198
217
|
|
|
@@ -201,7 +220,7 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
|
|
|
201
220
|
// would otherwise flood every snapshot). A multi/size listbox keeps them.
|
|
202
221
|
if (el.tagName === 'SELECT' && !el.multiple) return node;
|
|
203
222
|
|
|
204
|
-
if (!NAME_FROM_CONTENT.has(role)) {
|
|
223
|
+
if (!NAME_FROM_CONTENT.has(role) || KEEP_TEXT_CHILDREN.has(role)) {
|
|
205
224
|
for (const child of kidsOf(el)) {
|
|
206
225
|
if (child.nodeType === 3) {
|
|
207
226
|
const t = child.textContent.replace(/\s+/g, ' ').trim();
|
|
@@ -213,7 +232,7 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
|
|
|
213
232
|
}
|
|
214
233
|
} else {
|
|
215
234
|
// name-from-content: still recurse into element children for nested
|
|
216
|
-
// interactives (e.g. a link inside a heading), but drop bare text.
|
|
235
|
+
// interactives (e.g. a link inside a heading), but drop bare text (cells excepted, above).
|
|
217
236
|
for (const child of kidsOf(el)) {
|
|
218
237
|
if (child.nodeType !== 1) continue;
|
|
219
238
|
const c = walk(child);
|
package/src/bidi.js
CHANGED
|
@@ -98,11 +98,12 @@ export async function createBiDi(wsUrl) {
|
|
|
98
98
|
* @param {string} context - browsing-context id
|
|
99
99
|
* @param {string} expression - JS source to evaluate
|
|
100
100
|
* @param {boolean} [awaitPromise=true]
|
|
101
|
+
* @param {string} [sandbox] - run in this isolated realm (page script cannot see or alter it)
|
|
101
102
|
* @returns {Promise<*>} the deserialized primitive value (result.value)
|
|
102
103
|
*/
|
|
103
|
-
async evaluate(context, expression, awaitPromise = true) {
|
|
104
|
+
async evaluate(context, expression, awaitPromise = true, sandbox) {
|
|
104
105
|
const res = await client.send('script.evaluate', {
|
|
105
|
-
expression, target: { context }, awaitPromise,
|
|
106
|
+
expression, target: sandbox ? { context, sandbox } : { context }, awaitPromise,
|
|
106
107
|
});
|
|
107
108
|
if (res.type === 'exception') {
|
|
108
109
|
const d = res.exceptionDetails;
|
package/src/blocklist-firefox.js
CHANGED
|
@@ -13,9 +13,15 @@
|
|
|
13
13
|
* request pauses for one continue/fail round-trip to Node. On a typical page
|
|
14
14
|
* (tens–low-hundreds of requests over a local socket) that's negligible, and
|
|
15
15
|
* it's the only route that preserves glob parity without a second list.
|
|
16
|
+
*
|
|
17
|
+
* The site lock (`allowHosts`, site-lock.js) plugs into this same handler: a
|
|
18
|
+
* second catch-all handler would race this one on the same paused request, so
|
|
19
|
+
* there is exactly one continue/fail decision per request, and the intercept
|
|
20
|
+
* exists whenever either the blocklist or the lock is active.
|
|
16
21
|
*/
|
|
17
22
|
|
|
18
23
|
import { resolveBlocklistPatterns, makeBlockMatcher } from './blocklist.js';
|
|
24
|
+
import { createFirefoxLockCheck } from './site-lock.js';
|
|
19
25
|
|
|
20
26
|
/**
|
|
21
27
|
* Install ad/tracker blocking on a BiDi session. Idempotent per session is the
|
|
@@ -26,13 +32,15 @@ import { resolveBlocklistPatterns, makeBlockMatcher } from './blocklist.js';
|
|
|
26
32
|
* @param {boolean} [pageOpts.blockAds] - false disables the default list.
|
|
27
33
|
* @param {string[]} [pageOpts.blockUrls] - extra CDP-format globs; extend the
|
|
28
34
|
* default unless blockAds is false, in which case they're the whole list.
|
|
35
|
+
* @param {import('./site-lock.js').SiteLock|null} [lock] - Site lock to enforce in the same handler.
|
|
29
36
|
* @returns {Promise<void>}
|
|
30
37
|
*/
|
|
31
|
-
export async function applyFirefoxBlocklist(bidi, pageOpts = {}) {
|
|
38
|
+
export async function applyFirefoxBlocklist(bidi, pageOpts = {}, lock = null) {
|
|
32
39
|
const patterns = resolveBlocklistPatterns(pageOpts);
|
|
33
|
-
if (!patterns.length) return;
|
|
40
|
+
if (!patterns.length && !lock) return;
|
|
34
41
|
|
|
35
|
-
const isBlocked = makeBlockMatcher(patterns);
|
|
42
|
+
const isBlocked = patterns.length ? makeBlockMatcher(patterns) : () => false;
|
|
43
|
+
const lockBlocks = lock ? await createFirefoxLockCheck(bidi, lock) : null;
|
|
36
44
|
|
|
37
45
|
await bidi.send('network.addIntercept', {
|
|
38
46
|
phases: ['beforeRequestSent'],
|
|
@@ -47,7 +55,7 @@ export async function applyFirefoxBlocklist(bidi, pageOpts = {}) {
|
|
|
47
55
|
const id = e.request?.request;
|
|
48
56
|
if (!id) return;
|
|
49
57
|
try {
|
|
50
|
-
if (isBlocked(e.request.url)) {
|
|
58
|
+
if (isBlocked(e.request.url) || (lockBlocks && await lockBlocks(e))) {
|
|
51
59
|
await bidi.send('network.failRequest', { request: id });
|
|
52
60
|
} else {
|
|
53
61
|
await bidi.send('network.continueRequest', { request: id });
|
package/src/daemon.js
CHANGED
|
@@ -11,6 +11,7 @@ import { writeFileSync, mkdirSync, existsSync, readFileSync, unlinkSync } from '
|
|
|
11
11
|
import { randomBytes, timingSafeEqual } from 'node:crypto';
|
|
12
12
|
import { join, resolve } from 'node:path';
|
|
13
13
|
import { connect } from './index.js';
|
|
14
|
+
import { overNote } from './zoom.js';
|
|
14
15
|
import { formatReadable } from './readable.js';
|
|
15
16
|
import { cleanOld } from './output-clean.js';
|
|
16
17
|
import { isHttpError } from './navigation.js';
|
|
@@ -66,6 +67,7 @@ export function buildDaemonArgs(opts, absDir, initialUrl, cliPath) {
|
|
|
66
67
|
}
|
|
67
68
|
if (opts.blockPrivateNetwork) args.push('--block-private-network');
|
|
68
69
|
if (opts.uploadDir) args.push('--upload-dir', opts.uploadDir);
|
|
70
|
+
if (opts.allowHosts) args.push('--allow-hosts', opts.allowHosts.join(','));
|
|
69
71
|
return args;
|
|
70
72
|
}
|
|
71
73
|
|
|
@@ -235,6 +237,7 @@ export async function runDaemon(opts, outputDir, initialUrl) {
|
|
|
235
237
|
blockUrls: opts.blockUrls,
|
|
236
238
|
blockPrivateNetwork: opts.blockPrivateNetwork,
|
|
237
239
|
uploadDir: opts.uploadDir,
|
|
240
|
+
allowHosts: opts.allowHosts,
|
|
238
241
|
// incognito neuters injectCookies() session-wide; cookies:false (below)
|
|
239
242
|
// is the legacy per-open equivalent. Either yields a logged-out session.
|
|
240
243
|
incognito: opts.incognito || opts.cookies === false,
|
|
@@ -300,12 +303,23 @@ export async function runDaemon(opts, outputDir, initialUrl) {
|
|
|
300
303
|
}
|
|
301
304
|
await attachCapture(page);
|
|
302
305
|
|
|
306
|
+
// The first snapshot after a goto waits once for the network to go quiet (idle 500ms,
|
|
307
|
+
// capped 3s): a page's load event is not its last request. Library timing is unchanged.
|
|
308
|
+
let settlePending = false;
|
|
309
|
+
async function settleOnce(settle) {
|
|
310
|
+
if (!settlePending) return;
|
|
311
|
+
settlePending = false;
|
|
312
|
+
if (settle === false) return;
|
|
313
|
+
try { await page.waitForNetworkIdle({ idle: 500, timeout: 3000 }); } catch { /* best-effort */ }
|
|
314
|
+
}
|
|
315
|
+
|
|
303
316
|
// Navigate to initial URL if provided
|
|
304
317
|
if (initialUrl) {
|
|
305
318
|
if (opts.cookies !== false) {
|
|
306
319
|
try { await page.injectCookies(initialUrl, { browser: opts.browser }); } catch { /* no cookies */ }
|
|
307
320
|
}
|
|
308
321
|
await page.goto(initialUrl, opts.timeout ? Number(opts.timeout) : 30000);
|
|
322
|
+
settlePending = true;
|
|
309
323
|
}
|
|
310
324
|
|
|
311
325
|
// Default prune mode
|
|
@@ -315,15 +329,26 @@ export async function runDaemon(opts, outputDir, initialUrl) {
|
|
|
315
329
|
const handlers = {
|
|
316
330
|
async goto({ url, timeout }) {
|
|
317
331
|
const { status } = await page.goto(url, timeout || 30000);
|
|
332
|
+
settlePending = true;
|
|
318
333
|
return isHttpError(status) ? { ok: true, status } : { ok: true };
|
|
319
334
|
},
|
|
320
335
|
|
|
321
|
-
async snapshot({ mode, links }) {
|
|
336
|
+
async snapshot({ mode, links, within, map, full, maxChars, settle }) {
|
|
337
|
+
await settleOnce(settle);
|
|
322
338
|
const pruneMode = mode || defaultPruneMode;
|
|
323
|
-
const text = await page.snapshot({ mode: pruneMode, links });
|
|
339
|
+
const text = await page.snapshot({ mode: pruneMode, links, within, map });
|
|
324
340
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
325
341
|
const file = join(absDir, `page-${ts}.yml`);
|
|
326
342
|
writeFilePrivate(file, text);
|
|
343
|
+
// Over the cap with no zoom asked for: the file stays complete, and the
|
|
344
|
+
// map (a few % of the page) comes back too, to zoom from. Same as MCP.
|
|
345
|
+
if (within == null && !map && !full && text.length > (maxChars ?? 30000)) {
|
|
346
|
+
const outline = await page.snapshot({ links, map: true });
|
|
347
|
+
if (outline.length <= (maxChars ?? 30000)) return { ok: true, file, map: outline };
|
|
348
|
+
}
|
|
349
|
+
if ((within != null || map) && !full && text.length > (maxChars ?? 30000)) {
|
|
350
|
+
return { ok: true, file, note: overNote(within != null ? 'zoom' : 'map', text.length, maxChars ?? 30000, true) };
|
|
351
|
+
}
|
|
327
352
|
return { ok: true, file };
|
|
328
353
|
},
|
|
329
354
|
|
package/src/data.js
CHANGED
|
@@ -88,7 +88,7 @@ export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(s.indexOf('\n'
|
|
|
88
88
|
* Render a raw ARIA tree in data mode.
|
|
89
89
|
*
|
|
90
90
|
* @param {object} root - Root node from buildTree() (CDP vocabulary)
|
|
91
|
-
* @param {{ links?: boolean }} [opts] - links: the tree's link nodes carry `urls` (annotateLinks); adds the legend note
|
|
91
|
+
* @param {{ links?: boolean, whole?: boolean }} [opts] - links: the tree's link nodes carry `urls` (annotateLinks); adds the legend note. whole: root is a zoom box (snapshot `within`), so no main-only scope
|
|
92
92
|
* @returns {string} LEGEND line, then one line per record, YAML-ish like formatTree(); '' when there are no records
|
|
93
93
|
*/
|
|
94
94
|
export function dataSnapshot(root, opts = {}) {
|
|
@@ -283,7 +283,7 @@ export function dataSnapshot(root, opts = {}) {
|
|
|
283
283
|
if (tops.length === 1 && (tops[0].role === 'RootWebArea' || tops[0].role === 'WebArea')) tops = tops[0].children;
|
|
284
284
|
const find = (ns, role) => { for (const n of ns) { if (n.role === role) return n; const f = find(n.children, role); if (f) return f; } return null; };
|
|
285
285
|
const main = find(tops, 'main');
|
|
286
|
-
if (main && main.children.length) tops = [main];
|
|
286
|
+
if (!opts.whole && main && main.children.length) tops = [main];
|
|
287
287
|
const res = xform({ role: 'root', name: '', children: tops, props: {} }, []);
|
|
288
288
|
|
|
289
289
|
const lines = [];
|
package/src/firefox-page.js
CHANGED
|
@@ -20,6 +20,7 @@ import { prune as pruneTree } from './prune.js';
|
|
|
20
20
|
import { dataSnapshot, stripLegend } from './data.js';
|
|
21
21
|
import { annotateLinks } from './links.js';
|
|
22
22
|
import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
|
|
23
|
+
import { parseWithin, zoomSnapshot } from './zoom.js';
|
|
23
24
|
import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
|
|
24
25
|
import { scopedCookiesForUrl } from './auth.js';
|
|
25
26
|
import { assertNavigable, assertUploadAllowed } from './url-guard.js';
|
|
@@ -28,7 +29,10 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
|
|
|
28
29
|
import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
29
30
|
import { isChallengePage, countNodes } from './challenge.js';
|
|
30
31
|
import { withTimeout } from './with-timeout.js';
|
|
31
|
-
import { navFailed, httpWarning } from './navigation.js';
|
|
32
|
+
import { navFailed, navBlocked, httpWarning } from './navigation.js';
|
|
33
|
+
|
|
34
|
+
// Isolated BiDi realm holding the per-document stale-ref token (page script cannot reach it).
|
|
35
|
+
const DOC_SANDBOX = 'barebrowse';
|
|
32
36
|
|
|
33
37
|
/** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
|
|
34
38
|
const BIDI_KEYS = {
|
|
@@ -46,6 +50,7 @@ const BIDI_KEYS = {
|
|
|
46
50
|
* @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
|
|
47
51
|
* @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
|
|
48
52
|
* @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
|
|
53
|
+
* @param {?import('./site-lock.js').SiteLock} [opts.siteLock] - Site lock (allowHosts) state; the request-level enforcement is installed by applyFirefoxBlocklist, this side maps it to NAV_BLOCKED and snapshot notes.
|
|
49
54
|
* @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
|
|
50
55
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs after goto().
|
|
51
56
|
* @param {boolean} [opts.hybrid=false] - Relaunch headed on a bot-challenge page and retry.
|
|
@@ -56,6 +61,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
56
61
|
const defaultPruneMode = opts.pruneMode || 'act';
|
|
57
62
|
const urlGuard = opts.urlGuard || {};
|
|
58
63
|
const uploadDir = opts.uploadDir || null;
|
|
64
|
+
const lock = opts.siteLock || null;
|
|
59
65
|
const incognito = !!opts.incognito;
|
|
60
66
|
const consent = opts.consent !== false;
|
|
61
67
|
// True when the consent pass after the latest goto() found nothing: the first
|
|
@@ -89,6 +95,10 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
89
95
|
// ref (string int) → owning browsing-context id, rebuilt every snapshot so a
|
|
90
96
|
// click after a snapshot routes to the frame the element actually lives in.
|
|
91
97
|
let refContexts = new Map();
|
|
98
|
+
// context id → identity token stamped on that context's document at snapshot time.
|
|
99
|
+
// A document restored from the back/forward cache keeps its old data-bb-ref marks;
|
|
100
|
+
// its token differs from the one remembered here, so its refs are rejected.
|
|
101
|
+
let docTokens = new Map();
|
|
92
102
|
|
|
93
103
|
/**
|
|
94
104
|
* Depth-first list of the ACTIVE tab's contexts (main frame + descendant
|
|
@@ -117,6 +127,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
117
127
|
async function buildTree() {
|
|
118
128
|
const contextIds = await allContexts();
|
|
119
129
|
refContexts = new Map();
|
|
130
|
+
docTokens = new Map();
|
|
120
131
|
|
|
121
132
|
// Snapshot each context, assigning refs from a shared running counter so
|
|
122
133
|
// they're globally unique (matching CDP's flat integer refs).
|
|
@@ -125,7 +136,11 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
125
136
|
for (const ctx of contextIds) {
|
|
126
137
|
let raw;
|
|
127
138
|
try {
|
|
139
|
+
const token = Math.random().toString(36).slice(2);
|
|
128
140
|
raw = await bidi.evaluate(ctx, axSnapshotExpression(base), false);
|
|
141
|
+
// The token lives in a sandbox realm: page script can neither read, delete nor break it.
|
|
142
|
+
await bidi.evaluate(ctx, `globalThis.__bbDoc = ${JSON.stringify(token)}`, false, DOC_SANDBOX);
|
|
143
|
+
docTokens.set(ctx, token);
|
|
129
144
|
} catch { continue; } // frame navigated mid-snapshot — skip it
|
|
130
145
|
const { tree, count } = JSON.parse(raw);
|
|
131
146
|
for (let r = base + 1; r <= base + count; r++) refContexts.set(String(r), ctx);
|
|
@@ -159,17 +174,54 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
159
174
|
const context = refContexts.get(String(ref));
|
|
160
175
|
if (!context) throw new Error(`Unknown ref: ${ref} (snapshot first, or it's stale)`);
|
|
161
176
|
const res = await bidi.send('script.evaluate', {
|
|
162
|
-
expression: `document.querySelector('[${REF_ATTR}="${ref}"]')`,
|
|
163
|
-
target: { context }, awaitPromise: false, resultOwnership: 'root',
|
|
177
|
+
expression: `globalThis.__bbDoc === ${JSON.stringify(docTokens.get(context))} ? document.querySelector('[${REF_ATTR}="${ref}"]') : 'stale'`,
|
|
178
|
+
target: { context, sandbox: DOC_SANDBOX }, awaitPromise: false, resultOwnership: 'root',
|
|
164
179
|
});
|
|
180
|
+
if (res.type === 'success' && res.result.type === 'string') {
|
|
181
|
+
throw new Error(`No element found for ref "${ref}" — the page navigated since the snapshot that gave it; take a fresh snapshot`);
|
|
182
|
+
}
|
|
165
183
|
if (res.type === 'exception' || !res.result || res.result.type !== 'node') {
|
|
166
184
|
throw new Error(`Could not resolve element for ref ${ref}`);
|
|
167
185
|
}
|
|
168
186
|
return { context, sharedId: res.result.sharedId };
|
|
169
187
|
}
|
|
170
188
|
|
|
189
|
+
/** BiDi pointer moves do not scroll (Chromium's click does): bring the element on-screen first. */
|
|
190
|
+
async function scrollIntoView({ context, sharedId }) {
|
|
191
|
+
await bidi.send('script.callFunction', {
|
|
192
|
+
functionDeclaration: `function(){ this.scrollIntoView({ block: 'center', inline: 'center' }); }`,
|
|
193
|
+
this: { sharedId }, target: { context }, awaitPromise: false,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Mark the elements behind `within` ref selectors (in their frame) before a rebuild renumbers the refs. */
|
|
198
|
+
async function markRefs(selectors) {
|
|
199
|
+
const marks = new Map(); // old ref → context
|
|
200
|
+
for (const { ref } of selectors) {
|
|
201
|
+
if (!ref) continue;
|
|
202
|
+
const context = refContexts.get(ref);
|
|
203
|
+
if (!context) throw new Error(`within: ref ${ref} is not in the last snapshot (refs renumber on every snapshot and reset on navigation) — take a fresh ref from snapshot({ map: true })`);
|
|
204
|
+
const ok = await bidi.evaluate(context, `(() => { if (globalThis.__bbDoc !== ${JSON.stringify(docTokens.get(context))}) return false; const e = document.querySelector('[${REF_ATTR}="${ref}"]'); if (e) e.setAttribute('data-bb-zoom', '${ref}'); return !!e; })()`, false, DOC_SANDBOX).catch(() => false);
|
|
205
|
+
if (!ok) throw new Error(`within: the element behind ref ${ref} is no longer on the page — take a fresh ref from snapshot({ map: true })`);
|
|
206
|
+
marks.set(ref, context);
|
|
207
|
+
}
|
|
208
|
+
return marks;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Read (and clear) the marks after a rebuild: old ref → its new ref, or null when the element is gone. */
|
|
212
|
+
async function readMarks(marks) {
|
|
213
|
+
const refNodes = new Map();
|
|
214
|
+
for (const [ref, context] of marks) {
|
|
215
|
+
const now = await bidi.evaluate(context, `(() => { const e = document.querySelector('[data-bb-zoom="${ref}"]'); if (!e) return null; e.removeAttribute('data-bb-zoom'); return e.getAttribute('${REF_ATTR}'); })()`, false).catch(() => null);
|
|
216
|
+
refNodes.set(ref, now);
|
|
217
|
+
}
|
|
218
|
+
return refNodes;
|
|
219
|
+
}
|
|
220
|
+
|
|
171
221
|
async function pointerClick(ref) {
|
|
172
|
-
const
|
|
222
|
+
const target = await resolveRef(ref);
|
|
223
|
+
await scrollIntoView(target);
|
|
224
|
+
const { context, sharedId } = target;
|
|
173
225
|
await bidi.send('input.performActions', {
|
|
174
226
|
context,
|
|
175
227
|
actions: [{
|
|
@@ -282,6 +334,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
282
334
|
const offStarted = bidi.on('network.responseStarted', (e) => {
|
|
283
335
|
if (e.navigation && e.context === topContext) started.push(e);
|
|
284
336
|
});
|
|
337
|
+
lock?.startGoto(topContext);
|
|
285
338
|
try {
|
|
286
339
|
const res = await withTimeout(
|
|
287
340
|
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
@@ -289,6 +342,10 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
289
342
|
const last = events.filter((e) => e.navigation === res.navigation).pop();
|
|
290
343
|
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
291
344
|
} catch (err) {
|
|
345
|
+
// A lock-refused document (the URL or a redirect hop) surfaces as
|
|
346
|
+
// NS_ERROR_ABORT; report it as NAV_BLOCKED, not NAV_FAILED.
|
|
347
|
+
const blocked = lock?.takeGotoBlock(topContext);
|
|
348
|
+
if (blocked) throw navBlocked(blocked.url, blocked.host);
|
|
292
349
|
const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
|
|
293
350
|
if (!m) throw err;
|
|
294
351
|
if (m[1] === 'NS_BINDING_ABORTED') {
|
|
@@ -302,6 +359,25 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
302
359
|
}
|
|
303
360
|
}
|
|
304
361
|
|
|
362
|
+
// Settle after the accept click like consent.js does (1s): a consent form
|
|
363
|
+
// submits and navigates, and the click returns before the next page lands, so
|
|
364
|
+
// the snapshot would still show the interstitial.
|
|
365
|
+
// That landed document is the page goto() now reports (not the interstitial's).
|
|
366
|
+
async function consentClick(ref) {
|
|
367
|
+
const docs = [];
|
|
368
|
+
const off = bidi.on('network.responseCompleted', (e) => {
|
|
369
|
+
if (e.navigation && e.context === topContext) docs.push(e);
|
|
370
|
+
});
|
|
371
|
+
try {
|
|
372
|
+
await pointerClick(ref);
|
|
373
|
+
await new Promise((r) => setTimeout(r, 1000));
|
|
374
|
+
} finally {
|
|
375
|
+
off();
|
|
376
|
+
const last = docs.pop();
|
|
377
|
+
if (last) lastNavigation = { url: last.response.url, status: last.response.status };
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
305
381
|
/**
|
|
306
382
|
* Post-navigation routine shared by goto() and the hybrid retry: settle for
|
|
307
383
|
* dynamic content, auto-dismiss consent (best-effort), then report whether
|
|
@@ -317,9 +393,13 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
317
393
|
try { root = await buildTree(); } catch { /* null → treated as challenge */ }
|
|
318
394
|
consentRecheck = null;
|
|
319
395
|
if (consent && root) {
|
|
320
|
-
try { if (!(await dismissConsentFirefox(root,
|
|
396
|
+
try { if (!(await dismissConsentFirefox(root, consentClick))) consentRecheck = await docKey(); }
|
|
321
397
|
catch { /* consent dismissal is best-effort */ }
|
|
322
398
|
}
|
|
399
|
+
// The tree above was built for these internal passes, not shown to the caller: refs
|
|
400
|
+
// from before this navigation must stay dead (Chromium's goto never hands out refs
|
|
401
|
+
// either), or an old ref number resolves to a different element of the new page.
|
|
402
|
+
refContexts = new Map();
|
|
323
403
|
return isChallengePage(root, countNodes(root));
|
|
324
404
|
}
|
|
325
405
|
|
|
@@ -359,6 +439,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
359
439
|
// Same navigation guard the CDP path enforces — block file:/chrome:/
|
|
360
440
|
// view-source: and (optionally) private-network hosts before navigating.
|
|
361
441
|
assertNavigable(url, urlGuard);
|
|
442
|
+
lock?.assertAllowed(url);
|
|
362
443
|
refContexts = new Map(); // refs from the previous page are now stale
|
|
363
444
|
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
364
445
|
lastNavigation = null;
|
|
@@ -396,24 +477,38 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
396
477
|
|
|
397
478
|
/** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
|
|
398
479
|
async snapshot(pruneOpts = {}) {
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
480
|
+
// A `within` ref names an element of the previous snapshot. Refs renumber
|
|
481
|
+
// on every build, so mark the elements now and read their new refs after it.
|
|
482
|
+
const within = (pruneOpts || undefined)?.within;
|
|
483
|
+
const selectors = within != null ? parseWithin(within) : undefined;
|
|
484
|
+
const marks = await markRefs(selectors || []);
|
|
485
|
+
let root, refNodes;
|
|
486
|
+
try {
|
|
487
|
+
root = await buildTree();
|
|
488
|
+
if (root && consentRecheck) {
|
|
489
|
+
const key = consentRecheck;
|
|
490
|
+
consentRecheck = null; // once per navigation
|
|
491
|
+
try {
|
|
492
|
+
if (key === await docKey() && await dismissConsentFirefox(root, consentClick)) root = await buildTree();
|
|
493
|
+
} catch { /* best-effort */ }
|
|
494
|
+
}
|
|
495
|
+
} finally { refNodes = await readMarks(marks); }
|
|
496
|
+
for (const [ref, now] of refNodes) {
|
|
497
|
+
if (now == null) throw new Error(`within: the element behind ref ${ref} is no longer on the page — take a fresh ref from snapshot({ map: true })`);
|
|
406
498
|
}
|
|
407
499
|
if (!root) return '';
|
|
408
500
|
|
|
409
501
|
const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
|
|
410
502
|
const raw = formatTree(root);
|
|
411
|
-
const warn = httpWarning(lastNavigation?.status);
|
|
503
|
+
const warn = httpWarning(lastNavigation?.status) + (lock?.takeNotes(topContext) ?? '');
|
|
412
504
|
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
413
505
|
const tree = links ? annotateLinks(root, pageUrl) : root;
|
|
414
506
|
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
415
507
|
|
|
416
508
|
const mode = pruneOpts?.mode || defaultPruneMode;
|
|
509
|
+
if (pruneOpts?.map || selectors) {
|
|
510
|
+
return zoomSnapshot(tree, { head: `url: ${pageUrl}\n` + warn, map: true, selectors, refNodes, mode, links });
|
|
511
|
+
}
|
|
417
512
|
let out;
|
|
418
513
|
if (mode === 'data') out = dataSnapshot(tree, { links });
|
|
419
514
|
else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
|
|
@@ -430,7 +525,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
430
525
|
|
|
431
526
|
async click(ref) { await pointerClick(ref); },
|
|
432
527
|
|
|
433
|
-
/** @param {string} ref @param {string} text @param {import('./index.js').TypeOptions} [typeOpts] */
|
|
528
|
+
/** @param {string|number} ref @param {string} text @param {import('./index.js').TypeOptions} [typeOpts] */
|
|
434
529
|
async type(ref, text, typeOpts = {}) {
|
|
435
530
|
const { context, sharedId } = await resolveRef(ref);
|
|
436
531
|
// Focus the field (and optionally clear it) in-page, then send real key
|
|
@@ -458,7 +553,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
458
553
|
},
|
|
459
554
|
|
|
460
555
|
async hover(ref) {
|
|
461
|
-
const
|
|
556
|
+
const target = await resolveRef(ref);
|
|
557
|
+
await scrollIntoView(target);
|
|
558
|
+
const { context, sharedId } = target;
|
|
462
559
|
await bidi.send('input.performActions', {
|
|
463
560
|
context,
|
|
464
561
|
actions: [{
|
|
@@ -529,18 +626,22 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
529
626
|
if (from.context !== to.context) {
|
|
530
627
|
throw new Error('drag() between elements in different frames is not supported');
|
|
531
628
|
}
|
|
532
|
-
|
|
629
|
+
// Two calls so each target is scrolled into view before its move; the
|
|
630
|
+
// pressed-pointer state carries over between performActions calls.
|
|
631
|
+
const drag = (actions) => bidi.send('input.performActions', {
|
|
533
632
|
context: from.context,
|
|
534
|
-
actions: [{
|
|
535
|
-
type: 'pointer', id: 'mouse', parameters: { pointerType: 'mouse' },
|
|
536
|
-
actions: [
|
|
537
|
-
{ type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: from.sharedId } } },
|
|
538
|
-
{ type: 'pointerDown', button: 0 },
|
|
539
|
-
{ type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: to.sharedId } } },
|
|
540
|
-
{ type: 'pointerUp', button: 0 },
|
|
541
|
-
],
|
|
542
|
-
}],
|
|
633
|
+
actions: [{ type: 'pointer', id: 'mouse', parameters: { pointerType: 'mouse' }, actions }],
|
|
543
634
|
});
|
|
635
|
+
await scrollIntoView(from);
|
|
636
|
+
await drag([
|
|
637
|
+
{ type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: from.sharedId } } },
|
|
638
|
+
{ type: 'pointerDown', button: 0 },
|
|
639
|
+
]);
|
|
640
|
+
await scrollIntoView(to);
|
|
641
|
+
await drag([
|
|
642
|
+
{ type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: to.sharedId } } },
|
|
643
|
+
{ type: 'pointerUp', button: 0 },
|
|
644
|
+
]);
|
|
544
645
|
},
|
|
545
646
|
|
|
546
647
|
async upload(ref, files) {
|