barebrowse 0.24.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +24 -3
- package/barebrowse.context.md +11 -7
- package/cli.js +9 -7
- package/commands/barebrowse/SKILL.md +141 -0
- package/commands/barebrowse.md +140 -0
- package/mcp-server.js +10 -7
- package/package.json +2 -1
- package/src/aria.js +5 -0
- package/src/ax-snapshot.js +1 -0
- package/src/bareagent.js +7 -5
- package/src/consent-firefox.js +2 -2
- package/src/consent-patterns.js +10 -0
- package/src/consent.js +2 -2
- package/src/daemon.js +5 -4
- package/src/data.js +315 -0
- package/src/firefox-page.js +90 -16
- package/src/index.js +128 -27
- package/src/links.js +46 -0
- package/src/navigation.js +42 -0
- package/src/prune.js +11 -3
- package/types/aria.d.ts +0 -6
- package/types/consent-patterns.d.ts +2 -0
- package/types/data.d.ts +15 -0
- package/types/firefox-page.d.ts +7 -3
- package/types/index.d.ts +21 -6
- package/types/links.d.ts +28 -0
- package/types/navigation.d.ts +44 -0
- package/types/prune.d.ts +2 -0
package/src/firefox-page.js
CHANGED
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
|
|
18
18
|
import { formatTree } from './aria.js';
|
|
19
19
|
import { prune as pruneTree } from './prune.js';
|
|
20
|
+
import { dataSnapshot, stripLegend } from './data.js';
|
|
21
|
+
import { annotateLinks } from './links.js';
|
|
20
22
|
import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
|
|
21
23
|
import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
|
|
22
24
|
import { scopedCookiesForUrl } from './auth.js';
|
|
@@ -26,6 +28,7 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
|
|
|
26
28
|
import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
27
29
|
import { isChallengePage, countNodes } from './challenge.js';
|
|
28
30
|
import { withTimeout } from './with-timeout.js';
|
|
31
|
+
import { navFailed, httpWarning } from './navigation.js';
|
|
29
32
|
|
|
30
33
|
/** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
|
|
31
34
|
const BIDI_KEYS = {
|
|
@@ -39,7 +42,8 @@ const BIDI_KEYS = {
|
|
|
39
42
|
* Build a Firefox/BiDi-backed page object.
|
|
40
43
|
* @param {object} bidi - BiDi client from createBiDi()
|
|
41
44
|
* @param {object} [opts]
|
|
42
|
-
* @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Default prune mode.
|
|
45
|
+
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Default prune mode.
|
|
46
|
+
* @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
|
|
43
47
|
* @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
|
|
44
48
|
* @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
|
|
45
49
|
* @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
|
|
@@ -54,6 +58,15 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
54
58
|
const uploadDir = opts.uploadDir || null;
|
|
55
59
|
const incognito = !!opts.incognito;
|
|
56
60
|
const consent = opts.consent !== false;
|
|
61
|
+
// True when the consent pass after the latest goto() found nothing: the first
|
|
62
|
+
// snapshot() re-checks once (late-rendered CMP dialogs).
|
|
63
|
+
// Document identity (tab + performance.timeOrigin) the latest goto() landed on
|
|
64
|
+
// when its consent pass found nothing; null when no re-check is pending.
|
|
65
|
+
let consentRecheck = null;
|
|
66
|
+
async function docKey() {
|
|
67
|
+
try { return `${topContext}:${await bidi.evaluate(topContext, 'performance.timeOrigin', false)}`; }
|
|
68
|
+
catch { return null; }
|
|
69
|
+
}
|
|
57
70
|
// Hybrid mode: on a bot-challenge page, relaunch headed and retry. The
|
|
58
71
|
// relaunch tears down this Firefox+BiDi and returns a fresh one, so the
|
|
59
72
|
// browser lifecycle is owned by connectFirefox, which supplies relaunchHeaded.
|
|
@@ -61,6 +74,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
61
74
|
const relaunchHeaded = opts.relaunchHeaded || null;
|
|
62
75
|
let currentlyHeaded = !!opts.headed; // already headed → no fallback needed
|
|
63
76
|
let botBlocked = false;
|
|
77
|
+
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
78
|
+
/** @type {import('./navigation.js').NavigationOutcome|null} */
|
|
79
|
+
let lastNavigation = null;
|
|
64
80
|
// Last injectCookies(url, opts) args, so a hybrid relaunch (a brand-new
|
|
65
81
|
// Firefox profile) can re-inject them — otherwise the headed retry loads
|
|
66
82
|
// unauthenticated, defeating hybrid on an auth-gated challenge page.
|
|
@@ -244,7 +260,46 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
244
260
|
async function setupSubscriptions() {
|
|
245
261
|
await setupDialogs();
|
|
246
262
|
await setupDownloads();
|
|
247
|
-
await bidi.subscribe(['browsingContext.load']);
|
|
263
|
+
await bidi.subscribe(['browsingContext.load', 'network.responseCompleted', 'network.responseStarted']);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* browsingContext.navigate(wait:'complete') plus the navigation's outcome:
|
|
268
|
+
* the LAST network.responseCompleted for its navigation id in the top
|
|
269
|
+
* context (one fires per redirect hop; the final hop is where we landed).
|
|
270
|
+
* A network failure is rewrapped as a typed NAV_FAILED error. NS_BINDING_ABORTED
|
|
271
|
+
* (a 204, or a download replacing nothing) is not a failure, as with CDP's
|
|
272
|
+
* ERR_ABORTED: no responseCompleted fires for it, so the status comes from
|
|
273
|
+
* the last network.responseStarted (a 204 reports 204, as on Chromium).
|
|
274
|
+
* @returns {Promise<import('./navigation.js').NavigationOutcome>}
|
|
275
|
+
*/
|
|
276
|
+
async function navigateTo(url, timeout) {
|
|
277
|
+
const events = [];
|
|
278
|
+
const started = [];
|
|
279
|
+
const off = bidi.on('network.responseCompleted', (e) => {
|
|
280
|
+
if (e.navigation && e.context === topContext) events.push(e);
|
|
281
|
+
});
|
|
282
|
+
const offStarted = bidi.on('network.responseStarted', (e) => {
|
|
283
|
+
if (e.navigation && e.context === topContext) started.push(e);
|
|
284
|
+
});
|
|
285
|
+
try {
|
|
286
|
+
const res = await withTimeout(
|
|
287
|
+
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
288
|
+
timeout, `goto(${url})`);
|
|
289
|
+
const last = events.filter((e) => e.navigation === res.navigation).pop();
|
|
290
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
291
|
+
} catch (err) {
|
|
292
|
+
const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
|
|
293
|
+
if (!m) throw err;
|
|
294
|
+
if (m[1] === 'NS_BINDING_ABORTED') {
|
|
295
|
+
const last = started.pop();
|
|
296
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
297
|
+
}
|
|
298
|
+
throw navFailed(url, m[1]);
|
|
299
|
+
} finally {
|
|
300
|
+
off();
|
|
301
|
+
offStarted();
|
|
302
|
+
}
|
|
248
303
|
}
|
|
249
304
|
|
|
250
305
|
/**
|
|
@@ -260,8 +315,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
260
315
|
// the pre-dismiss tree avoids a second full AX reconstruction per goto.
|
|
261
316
|
let root = null;
|
|
262
317
|
try { root = await buildTree(); } catch { /* null → treated as challenge */ }
|
|
318
|
+
consentRecheck = null;
|
|
263
319
|
if (consent && root) {
|
|
264
|
-
try { await dismissConsentFirefox(root, (ref) => pointerClick(ref)); }
|
|
320
|
+
try { if (!(await dismissConsentFirefox(root, (ref) => pointerClick(ref)))) consentRecheck = await docKey(); }
|
|
265
321
|
catch { /* consent dismissal is best-effort */ }
|
|
266
322
|
}
|
|
267
323
|
return isChallengePage(root, countNodes(root));
|
|
@@ -274,6 +330,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
274
330
|
get context() { return topContext; },
|
|
275
331
|
/** Whether the last goto() landed on a bot-challenge page (parity w/ CDP). */
|
|
276
332
|
get botBlocked() { return botBlocked; },
|
|
333
|
+
/** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
|
|
334
|
+
get lastNavigation() { return lastNavigation; },
|
|
277
335
|
|
|
278
336
|
/** @type {'firefox'} Engine driving this session (see also `capabilities`). */
|
|
279
337
|
engine: 'firefox',
|
|
@@ -302,9 +360,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
302
360
|
// view-source: and (optionally) private-network hosts before navigating.
|
|
303
361
|
assertNavigable(url, urlGuard);
|
|
304
362
|
refContexts = new Map(); // refs from the previous page are now stale
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
363
|
+
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
364
|
+
lastNavigation = null;
|
|
365
|
+
lastNavigation = await navigateTo(url, timeout);
|
|
308
366
|
// Settle + consent + challenge detection (mirrors the CDP path).
|
|
309
367
|
botBlocked = await afterNavigate();
|
|
310
368
|
|
|
@@ -326,32 +384,43 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
326
384
|
if (lastInject) {
|
|
327
385
|
try { await page.injectCookies(lastInject.url, lastInject.cookieOpts); } catch { /* best-effort */ }
|
|
328
386
|
}
|
|
329
|
-
await
|
|
330
|
-
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
331
|
-
timeout, `goto(${url})`);
|
|
387
|
+
lastNavigation = await navigateTo(url, timeout);
|
|
332
388
|
botBlocked = await afterNavigate();
|
|
333
389
|
} catch {
|
|
334
390
|
// Headed relaunch failed (no display?) — keep the headless result;
|
|
335
391
|
// botBlocked stays true so the caller can see it didn't clear.
|
|
336
392
|
}
|
|
337
393
|
}
|
|
394
|
+
return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
|
|
338
395
|
},
|
|
339
396
|
|
|
340
397
|
/** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
|
|
341
398
|
async snapshot(pruneOpts = {}) {
|
|
342
|
-
|
|
399
|
+
let root = await buildTree();
|
|
400
|
+
if (root && consentRecheck) {
|
|
401
|
+
const key = consentRecheck;
|
|
402
|
+
consentRecheck = null; // once per navigation
|
|
403
|
+
try {
|
|
404
|
+
if (key === await docKey() && await dismissConsentFirefox(root, (ref) => pointerClick(ref))) root = await buildTree();
|
|
405
|
+
} catch { /* best-effort */ }
|
|
406
|
+
}
|
|
343
407
|
if (!root) return '';
|
|
344
408
|
|
|
345
409
|
const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
|
|
346
410
|
const raw = formatTree(root);
|
|
347
|
-
|
|
411
|
+
const warn = httpWarning(lastNavigation?.status);
|
|
412
|
+
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
413
|
+
const tree = links ? annotateLinks(root, pageUrl) : root;
|
|
414
|
+
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
348
415
|
|
|
349
416
|
const mode = pruneOpts?.mode || defaultPruneMode;
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
const
|
|
353
|
-
|
|
354
|
-
|
|
417
|
+
let out;
|
|
418
|
+
if (mode === 'data') out = dataSnapshot(tree, { links });
|
|
419
|
+
else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
|
|
420
|
+
const sized = stripLegend(out); // the legend is not content
|
|
421
|
+
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars`
|
|
422
|
+
+ ` (${raw.length ? Math.round((1 - sized.length / raw.length) * 100) : 0}% pruned)`;
|
|
423
|
+
return stats + '\n' + warn + out;
|
|
355
424
|
},
|
|
356
425
|
|
|
357
426
|
async readable() {
|
|
@@ -492,6 +561,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
492
561
|
bidi.send('browsingContext.reload', { context: topContext, wait: 'complete' }),
|
|
493
562
|
30000, 'reload');
|
|
494
563
|
refContexts = new Map();
|
|
564
|
+
consentRecheck = null;
|
|
565
|
+
lastNavigation = null; // the reload's status is not read; don't keep the old one
|
|
495
566
|
await new Promise((r) => setTimeout(r, 300));
|
|
496
567
|
},
|
|
497
568
|
|
|
@@ -528,6 +599,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
528
599
|
if (index < 0 || index >= tree.length) throw new Error(`Tab index ${index} out of range (0-${tree.length - 1})`);
|
|
529
600
|
topContext = tree[index].context;
|
|
530
601
|
refContexts = new Map(); // refs from the previous tab are invalid
|
|
602
|
+
lastNavigation = null; // the outcome belonged to the tab we left
|
|
531
603
|
await bidi.send('browsingContext.activate', { context: topContext }).catch(() => {});
|
|
532
604
|
},
|
|
533
605
|
|
|
@@ -644,6 +716,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
644
716
|
bidi.send('browsingContext.traverseHistory', { context: topContext, delta }),
|
|
645
717
|
30000, 'history navigation');
|
|
646
718
|
refContexts = new Map();
|
|
719
|
+
consentRecheck = null;
|
|
720
|
+
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
647
721
|
await new Promise((r) => setTimeout(r, 500));
|
|
648
722
|
}
|
|
649
723
|
|
package/src/index.js
CHANGED
|
@@ -16,6 +16,8 @@ import { createFirefoxPage } from './firefox-page.js';
|
|
|
16
16
|
import { formatTree } from './aria.js';
|
|
17
17
|
import { authenticate } from './auth.js';
|
|
18
18
|
import { prune as pruneTree } from './prune.js';
|
|
19
|
+
import { dataSnapshot, stripLegend } from './data.js';
|
|
20
|
+
import { annotateLinks } from './links.js';
|
|
19
21
|
import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
|
|
20
22
|
import { dismissConsent } from './consent.js';
|
|
21
23
|
import { applyStealth } from './stealth.js';
|
|
@@ -27,6 +29,7 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
|
27
29
|
import { isChallengePage } from './challenge.js';
|
|
28
30
|
import { withTimeout } from './with-timeout.js';
|
|
29
31
|
import { readable as extractReadable } from './readable.js';
|
|
32
|
+
import { navFailed, httpWarning } from './navigation.js';
|
|
30
33
|
import { assertNavigable, assertUploadAllowed } from './url-guard.js';
|
|
31
34
|
import { join as pathJoin } from 'node:path';
|
|
32
35
|
import { chmodSync } from 'node:fs';
|
|
@@ -58,7 +61,8 @@ import { chmodSync } from 'node:fs';
|
|
|
58
61
|
* @param {{width: number, height: number}} [opts.viewport] - Viewport dimensions.
|
|
59
62
|
* @param {string} [opts.browser] - Source browser for cookie extraction.
|
|
60
63
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
|
|
61
|
-
* @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Pruning mode.
|
|
64
|
+
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
65
|
+
* @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
|
|
62
66
|
* @returns {Promise<string>} ARIA snapshot text
|
|
63
67
|
*/
|
|
64
68
|
export async function browse(url, opts = {}) {
|
|
@@ -105,7 +109,7 @@ export async function browse(url, opts = {}) {
|
|
|
105
109
|
}
|
|
106
110
|
|
|
107
111
|
// Step 4: Navigate and wait for load
|
|
108
|
-
await navigate(page, url, timeout);
|
|
112
|
+
let outcome = await navigate(page, url, timeout);
|
|
109
113
|
|
|
110
114
|
// Step 4.5: Auto-dismiss cookie consent dialogs
|
|
111
115
|
if (opts.consent !== false) {
|
|
@@ -129,7 +133,7 @@ export async function browse(url, opts = {}) {
|
|
|
129
133
|
if (!noAuth) {
|
|
130
134
|
try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
|
|
131
135
|
}
|
|
132
|
-
await navigate(page, url, timeout);
|
|
136
|
+
outcome = await navigate(page, url, timeout);
|
|
133
137
|
if (opts.consent !== false) await dismissConsent(page.session);
|
|
134
138
|
({ tree } = await ariaTree(page));
|
|
135
139
|
} catch {
|
|
@@ -139,19 +143,27 @@ export async function browse(url, opts = {}) {
|
|
|
139
143
|
|
|
140
144
|
// Step 6: Prune for agent consumption
|
|
141
145
|
const raw = formatTree(tree);
|
|
146
|
+
const links = !!opts.links;
|
|
147
|
+
let linked = tree;
|
|
148
|
+
if (links) {
|
|
149
|
+
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
150
|
+
linked = annotateLinks(tree, entries[currentIndex]?.url || url);
|
|
151
|
+
}
|
|
142
152
|
let snapshot;
|
|
143
153
|
if (opts.prune !== false) {
|
|
144
|
-
|
|
145
|
-
|
|
154
|
+
snapshot = opts.pruneMode === 'data'
|
|
155
|
+
? dataSnapshot(linked, { links })
|
|
156
|
+
: formatTree(pruneTree(linked, { mode: opts.pruneMode || 'act', links }));
|
|
146
157
|
} else {
|
|
147
|
-
snapshot = raw;
|
|
158
|
+
snapshot = links ? formatTree(linked) : raw;
|
|
148
159
|
}
|
|
149
|
-
const
|
|
160
|
+
const sized = stripLegend(snapshot); // the legend is not content
|
|
161
|
+
const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
150
162
|
const actMode = !opts.pruneMode || opts.pruneMode === 'act';
|
|
151
163
|
const hint = (actMode && raw.length > 5000 && snapshot.length < 500 && snapshot.length < raw.length * 0.05)
|
|
152
164
|
? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
|
|
153
165
|
: '';
|
|
154
|
-
snapshot = stats + '\n' + hint + snapshot;
|
|
166
|
+
snapshot = stats + '\n' + hint + httpWarning(outcome.status) + snapshot;
|
|
155
167
|
|
|
156
168
|
// Step 7: Clean up
|
|
157
169
|
await cdp.send('Target.closeTarget', { targetId: page.targetId });
|
|
@@ -201,10 +213,20 @@ export async function browse(url, opts = {}) {
|
|
|
201
213
|
* @property {boolean} cookieInjection - Cookie extraction/injection is active (off in incognito)
|
|
202
214
|
*/
|
|
203
215
|
|
|
216
|
+
/**
|
|
217
|
+
* What `goto()` resolves to: the final URL (after redirects) and the main
|
|
218
|
+
* document's HTTP status (0 when no HTTP response was observed, e.g. an aborted navigation).
|
|
219
|
+
* Network failures reject with an Error whose `code` is 'NAV_FAILED' and
|
|
220
|
+
* `reason` the browser's raw error string.
|
|
221
|
+
* @typedef {import('./navigation.js').NavigationOutcome} NavigationOutcome
|
|
222
|
+
*/
|
|
223
|
+
|
|
204
224
|
/**
|
|
205
225
|
* Options for `page.snapshot()`. Pass `false` instead of an object to skip
|
|
206
226
|
* pruning and return the raw ARIA tree.
|
|
207
|
-
*
|
|
227
|
+
* `links: true` adds each link's absolute URL after its name (default off,
|
|
228
|
+
* or the `connect({ links })` default).
|
|
229
|
+
* @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
|
|
208
230
|
*/
|
|
209
231
|
|
|
210
232
|
/**
|
|
@@ -257,7 +279,8 @@ export async function browse(url, opts = {}) {
|
|
|
257
279
|
* @param {boolean} [opts.incognito=false] - Clean, unauthenticated session:
|
|
258
280
|
* skip storageState loading and make injectCookies() a no-op, so no auth
|
|
259
281
|
* ever enters the session even if a caller injects unconditionally.
|
|
260
|
-
* @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Pruning mode.
|
|
282
|
+
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
283
|
+
* @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
|
|
261
284
|
* @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
|
|
262
285
|
* 'firefox' drives over WebDriver BiDi (a separate transport / page object);
|
|
263
286
|
* 'chromium' (default) uses CDP.
|
|
@@ -364,6 +387,24 @@ export async function connect(opts = {}) {
|
|
|
364
387
|
let page = await createPage(cdp, !currentlyHeaded, pageOpts);
|
|
365
388
|
let refMap = new Map();
|
|
366
389
|
let botBlocked = false;
|
|
390
|
+
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
391
|
+
/** @type {NavigationOutcome|null} */
|
|
392
|
+
let lastNavigation = null;
|
|
393
|
+
// Set when the consent pass after the latest goto() found nothing: the first
|
|
394
|
+
// snapshot() re-checks once, since some CMPs (Google Funding Choices) render
|
|
395
|
+
// their dialog 1-3s after load. Holds the document identity (tab + main-frame
|
|
396
|
+
// loaderId) the goto landed on, so a snapshot of any other document (click,
|
|
397
|
+
// redirect, switchTab) never re-checks. loaderId changes on every
|
|
398
|
+
// cross-document navigation but not on pushState/replaceState/hash change.
|
|
399
|
+
// Null when no re-check is pending.
|
|
400
|
+
// Also set in attach mode (goto() runs consent there too).
|
|
401
|
+
let consentRecheck = null;
|
|
402
|
+
async function navKey() {
|
|
403
|
+
try {
|
|
404
|
+
const { frameTree } = await page.session.send('Page.getFrameTree');
|
|
405
|
+
return `${page.targetId}:${frameTree.frame.loaderId}`;
|
|
406
|
+
} catch { return null; }
|
|
407
|
+
}
|
|
367
408
|
|
|
368
409
|
// Suppress permission prompts. Skipped in attach mode — Browser.setPermission
|
|
369
410
|
// is browser-wide (no origin scope here), so flipping permissions to denied
|
|
@@ -470,6 +511,8 @@ export async function connect(opts = {}) {
|
|
|
470
511
|
// silently resolving to whatever backendNodeId happens to still be in
|
|
471
512
|
// the map.
|
|
472
513
|
refMap = new Map();
|
|
514
|
+
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
515
|
+
lastNavigation = null;
|
|
473
516
|
// Switch back to headless if we fell back to headed previously.
|
|
474
517
|
// Not in attach mode — we never own the browser there, so there's
|
|
475
518
|
// nothing to rewind.
|
|
@@ -486,9 +529,10 @@ export async function connect(opts = {}) {
|
|
|
486
529
|
currentlyHeaded = false;
|
|
487
530
|
}
|
|
488
531
|
|
|
489
|
-
await navigate(page, url, timeout);
|
|
532
|
+
lastNavigation = await navigate(page, url, timeout);
|
|
490
533
|
if (opts.consent !== false) {
|
|
491
|
-
await dismissConsent(page.session);
|
|
534
|
+
const dismissed = await dismissConsent(page.session);
|
|
535
|
+
if (!dismissed) consentRecheck = await navKey();
|
|
492
536
|
}
|
|
493
537
|
|
|
494
538
|
// Check for bot challenge
|
|
@@ -509,8 +553,11 @@ export async function connect(opts = {}) {
|
|
|
509
553
|
page = await createPage(cdp, false, pageOpts);
|
|
510
554
|
setupDialogHandler(page.session);
|
|
511
555
|
await suppressPermissions(cdp);
|
|
512
|
-
await navigate(page, url, timeout);
|
|
513
|
-
|
|
556
|
+
lastNavigation = await navigate(page, url, timeout);
|
|
557
|
+
consentRecheck = null;
|
|
558
|
+
if (opts.consent !== false && !(await dismissConsent(page.session))) {
|
|
559
|
+
consentRecheck = await navKey();
|
|
560
|
+
}
|
|
514
561
|
|
|
515
562
|
// Re-check after headed fallback
|
|
516
563
|
const after = await ariaTree(page);
|
|
@@ -520,6 +567,7 @@ export async function connect(opts = {}) {
|
|
|
520
567
|
// Headed launch failed (no display?) — keep headless result, botBlocked stays true
|
|
521
568
|
}
|
|
522
569
|
}
|
|
570
|
+
return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
|
|
523
571
|
},
|
|
524
572
|
|
|
525
573
|
async goBack() {
|
|
@@ -529,6 +577,8 @@ export async function connect(opts = {}) {
|
|
|
529
577
|
await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex - 1].id });
|
|
530
578
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
531
579
|
refMap = new Map(); // refs from the previous page are now invalid
|
|
580
|
+
consentRecheck = null;
|
|
581
|
+
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
532
582
|
},
|
|
533
583
|
|
|
534
584
|
async goForward() {
|
|
@@ -538,6 +588,8 @@ export async function connect(opts = {}) {
|
|
|
538
588
|
await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex + 1].id });
|
|
539
589
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
540
590
|
refMap = new Map();
|
|
591
|
+
consentRecheck = null;
|
|
592
|
+
lastNavigation = null;
|
|
541
593
|
},
|
|
542
594
|
|
|
543
595
|
async reload(reloadOpts = {}) {
|
|
@@ -548,6 +600,8 @@ export async function connect(opts = {}) {
|
|
|
548
600
|
});
|
|
549
601
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
550
602
|
refMap = new Map(); // refs from the pre-reload page are invalid
|
|
603
|
+
consentRecheck = null;
|
|
604
|
+
lastNavigation = null; // the reload's status is not read; don't keep the old one
|
|
551
605
|
},
|
|
552
606
|
|
|
553
607
|
/** @param {string} url @param {CookieOptions} [cookieOpts] */
|
|
@@ -560,17 +614,28 @@ export async function connect(opts = {}) {
|
|
|
560
614
|
|
|
561
615
|
/** @param {SnapshotOptions|false} [pruneOpts] */
|
|
562
616
|
async snapshot(pruneOpts = {}) {
|
|
617
|
+
if (consentRecheck) {
|
|
618
|
+
const key = consentRecheck;
|
|
619
|
+
consentRecheck = null; // once per navigation
|
|
620
|
+
// Only if we are still on the document the goto landed on.
|
|
621
|
+
if (key === await navKey()) {
|
|
622
|
+
try { await dismissConsent(page.session); } catch { /* best-effort */ }
|
|
623
|
+
}
|
|
624
|
+
}
|
|
563
625
|
const result = await ariaTree(page);
|
|
564
626
|
refMap = result.refMap;
|
|
565
627
|
const raw = formatTree(result.tree);
|
|
566
628
|
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
567
629
|
const pageUrl = entries[currentIndex]?.url || '';
|
|
568
|
-
const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : ''
|
|
569
|
-
|
|
570
|
-
const
|
|
571
|
-
const
|
|
572
|
-
|
|
573
|
-
const
|
|
630
|
+
const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
|
|
631
|
+
+ httpWarning(lastNavigation?.status);
|
|
632
|
+
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
633
|
+
const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
|
|
634
|
+
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
635
|
+
const mode = pruneOpts?.mode || opts.pruneMode || 'act';
|
|
636
|
+
const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
|
|
637
|
+
const sized = stripLegend(out); // the legend is not content
|
|
638
|
+
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
574
639
|
const hint = (mode === 'act' && raw.length > 5000 && out.length < 500 && out.length < raw.length * 0.05)
|
|
575
640
|
? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
|
|
576
641
|
: '';
|
|
@@ -668,6 +733,7 @@ export async function connect(opts = {}) {
|
|
|
668
733
|
const oldSessionId = page.sessionId;
|
|
669
734
|
page = await attachToExistingTarget(cdp, target.targetId, pageOpts);
|
|
670
735
|
refMap = new Map(); // refs from the previous tab are no longer valid
|
|
736
|
+
lastNavigation = null; // the outcome belonged to the tab we left
|
|
671
737
|
setupDialogHandler(page.session);
|
|
672
738
|
try { await cdp.send('Target.detachFromTarget', { sessionId: oldSessionId }); } catch {}
|
|
673
739
|
},
|
|
@@ -712,6 +778,8 @@ export async function connect(opts = {}) {
|
|
|
712
778
|
},
|
|
713
779
|
|
|
714
780
|
get botBlocked() { return botBlocked; },
|
|
781
|
+
/** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
|
|
782
|
+
get lastNavigation() { return lastNavigation; },
|
|
715
783
|
|
|
716
784
|
/** @type {'chromium'} Engine driving this session (see also `capabilities`) */
|
|
717
785
|
engine: 'chromium',
|
|
@@ -786,12 +854,13 @@ export async function connect(opts = {}) {
|
|
|
786
854
|
return {
|
|
787
855
|
async goto(url, timeout = 30000) {
|
|
788
856
|
assertNavigable(url, urlGuard);
|
|
789
|
-
await navigate(tab, url, timeout);
|
|
857
|
+
const outcome = await navigate(tab, url, timeout);
|
|
790
858
|
if (opts.consent !== false) {
|
|
791
859
|
await dismissConsent(tab.session);
|
|
792
860
|
}
|
|
793
861
|
const { tree, nodeCount } = await ariaTree(tab);
|
|
794
862
|
tabBotBlocked = isChallengePage(tree, nodeCount);
|
|
863
|
+
return outcome;
|
|
795
864
|
},
|
|
796
865
|
get botBlocked() { return tabBotBlocked; },
|
|
797
866
|
/** @param {string} url @param {CookieOptions} [cookieOpts] */
|
|
@@ -925,6 +994,7 @@ async function connectFirefox(opts) {
|
|
|
925
994
|
|
|
926
995
|
const page = await createFirefoxPage(bidi, {
|
|
927
996
|
pruneMode: opts.pruneMode,
|
|
997
|
+
links: opts.links,
|
|
928
998
|
urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
|
|
929
999
|
uploadDir: opts.uploadDir || null,
|
|
930
1000
|
incognito: !!opts.incognito,
|
|
@@ -1092,14 +1162,45 @@ export function _resetBlocklistWarning() {
|
|
|
1092
1162
|
}
|
|
1093
1163
|
|
|
1094
1164
|
/**
|
|
1095
|
-
* Navigate to a URL and wait for the page to load.
|
|
1165
|
+
* Navigate to a URL and wait for the page to load. Resolves to the outcome of
|
|
1166
|
+
* the main-document navigation (final URL after redirects, HTTP status);
|
|
1167
|
+
* rejects with a NAV_FAILED error when the browser could not reach a document.
|
|
1168
|
+
* @returns {Promise<NavigationOutcome>}
|
|
1096
1169
|
*/
|
|
1097
1170
|
async function navigate(page, url, timeout = 30000) {
|
|
1098
|
-
const
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1171
|
+
const { session } = page;
|
|
1172
|
+
// Main-document responses (every redirect hop fires one; the last wins).
|
|
1173
|
+
const docs = [];
|
|
1174
|
+
const offDoc = session.on('Network.responseReceived', (p) => { if (p.type === 'Document') docs.push(p); });
|
|
1175
|
+
// Load waiter, hand-rolled (not session.once) so it can be cancelled when
|
|
1176
|
+
// Page.navigate fails or aborts and no load event will ever come.
|
|
1177
|
+
let cancel = () => {};
|
|
1178
|
+
const loaded = /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
|
|
1179
|
+
const timer = setTimeout(() => { off(); reject(new Error('Timeout waiting for CDP event: Page.loadEventFired')); }, timeout);
|
|
1180
|
+
const off = session.on('Page.loadEventFired', () => { cancel(); resolve(undefined); });
|
|
1181
|
+
cancel = () => { clearTimeout(timer); off(); };
|
|
1182
|
+
}));
|
|
1183
|
+
loaded.catch(() => {}); // rejection is only read if we await it below
|
|
1184
|
+
try {
|
|
1185
|
+
const nav = await session.send('Page.navigate', { url });
|
|
1186
|
+
// ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
|
|
1187
|
+
// no document and fires no load event, so don't wait for one.
|
|
1188
|
+
const aborted = nav.errorText === 'net::ERR_ABORTED';
|
|
1189
|
+
// ERR_HTTP_RESPONSE_CODE_FAILURE is a 4xx/5xx with an empty body: the
|
|
1190
|
+
// document did load (Chromium's error page fires load, the Document
|
|
1191
|
+
// response carries the status), so report it like any other HTTP error.
|
|
1192
|
+
const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
|
|
1193
|
+
if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
|
|
1194
|
+
if (!aborted) await loaded;
|
|
1195
|
+
// Brief settle time for dynamic content
|
|
1196
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1197
|
+
// loaderId is shared by every redirect hop of one navigation.
|
|
1198
|
+
const last = docs.filter((d) => d.loaderId === nav.loaderId).pop();
|
|
1199
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
1200
|
+
} finally {
|
|
1201
|
+
cancel();
|
|
1202
|
+
offDoc();
|
|
1203
|
+
}
|
|
1103
1204
|
}
|
|
1104
1205
|
|
|
1105
1206
|
/**
|
package/src/links.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* links.js — opt-in link URLs for snapshots (`links: true`).
|
|
3
|
+
*
|
|
4
|
+
* `annotateLinks()` stamps `urls: [url]` on every link node that has a usable
|
|
5
|
+
* browser-serialized absolute URL. aria.js, prune.js and data.js print/carry
|
|
6
|
+
* `urls` — they never see the option, so with links off nothing changes.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Normalise a link's URL for a snapshot line. Returns '' when it should not be
|
|
11
|
+
* shown: no URL, `javascript:`, or a same-document link (the page URL, only
|
|
12
|
+
* the #fragment differs). A raw space, `<` or `>` (browsers encode them, but
|
|
13
|
+
* not always in the fragment) is percent-encoded so a line splits cleanly on
|
|
14
|
+
* ` <` and `> `.
|
|
15
|
+
*
|
|
16
|
+
* @param {string|undefined} url - Absolute URL as serialized by the browser
|
|
17
|
+
* @param {string} [pageUrl] - URL of the page the link is on
|
|
18
|
+
* @returns {string}
|
|
19
|
+
*/
|
|
20
|
+
export function linkUrl(url, pageUrl = '') {
|
|
21
|
+
if (typeof url !== 'string') return '';
|
|
22
|
+
const u = url.trim();
|
|
23
|
+
if (!u || /^javascript:/i.test(u)) return '';
|
|
24
|
+
if (pageUrl && u.split('#')[0] === pageUrl.split('#')[0]) return '';
|
|
25
|
+
return u.replace(/ /g, '%20').replace(/</g, '%3C').replace(/>/g, '%3E');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Copy a tree with `urls` set on its link nodes (input is not mutated).
|
|
30
|
+
*
|
|
31
|
+
* @param {object|null} node - Root from buildTree()
|
|
32
|
+
* @param {string} pageUrl
|
|
33
|
+
* @returns {object|null}
|
|
34
|
+
*/
|
|
35
|
+
export function annotateLinks(node, pageUrl) {
|
|
36
|
+
if (!node) return node;
|
|
37
|
+
const out = { ...node, children: node.children.map((c) => annotateLinks(c, pageUrl)) };
|
|
38
|
+
if (node.role === 'link') {
|
|
39
|
+
const u = linkUrl(node.properties?.url, pageUrl);
|
|
40
|
+
if (u) out.urls = [u];
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** ` <u1> <u2>` for a node's urls ('' when none). */
|
|
46
|
+
export const urlsText = (urls) => (urls || []).map((u) => ` <${u}>`).join('');
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* navigation.js — Shared pieces of the goto() outcome contract, so both
|
|
3
|
+
* engines (CDP in index.js, BiDi in firefox-page.js) report it identically.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What goto() resolves to: where the page ended up and how the server answered.
|
|
8
|
+
* `status` is 0 when no HTTP response was observed for the navigation (e.g. an
|
|
9
|
+
* aborted navigation that never got one).
|
|
10
|
+
* @typedef {{ url: string, status: number }} NavigationOutcome
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Typed error for a navigation that never reached a document (DNS failure,
|
|
15
|
+
* connection refused, TLS error, ...). `code` is 'NAV_FAILED'; `reason` is the
|
|
16
|
+
* browser's raw error string (`net::ERR_NAME_NOT_RESOLVED`,
|
|
17
|
+
* `NS_ERROR_UNKNOWN_HOST`, ...).
|
|
18
|
+
* @param {string} url
|
|
19
|
+
* @param {string} reason
|
|
20
|
+
* @returns {Error & {code: 'NAV_FAILED', reason: string}}
|
|
21
|
+
*/
|
|
22
|
+
export function navFailed(url, reason) {
|
|
23
|
+
return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* True for a final HTTP status that is not 2xx (0 = unknown is not an error).
|
|
28
|
+
* @param {number|undefined} status
|
|
29
|
+
*/
|
|
30
|
+
export function isHttpError(status) {
|
|
31
|
+
return !!status && (status < 200 || status >= 300);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Warning line for a non-2xx final status ('' for 2xx and for unknown 0), in
|
|
36
|
+
* the style of the bot-challenge line, newline-terminated for prepending.
|
|
37
|
+
* @param {number|undefined} status
|
|
38
|
+
* @returns {string}
|
|
39
|
+
*/
|
|
40
|
+
export function httpWarning(status) {
|
|
41
|
+
return isHttpError(status) ? `[HTTP ${status} — the page may be an error page]\n` : '';
|
|
42
|
+
}
|