barebrowse 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/README.md +32 -1
- package/barebrowse.context.md +10 -5
- package/cli.js +24 -6
- package/commands/barebrowse/SKILL.md +142 -0
- package/commands/barebrowse.md +141 -0
- package/mcp-server.js +78 -9
- package/package.json +2 -1
- package/src/aria.js +5 -0
- package/src/ax-snapshot.js +1 -0
- package/src/bareagent.js +5 -3
- package/src/blocklist-firefox.js +12 -4
- package/src/daemon.js +7 -4
- package/src/data.js +24 -9
- package/src/firefox-page.js +71 -12
- package/src/index.js +127 -24
- package/src/links.js +46 -0
- package/src/navigation.js +54 -0
- package/src/prune.js +11 -3
- package/src/site-lock.js +184 -0
- package/types/aria.d.ts +0 -6
- package/types/blocklist-firefox.d.ts +2 -1
- package/types/data.d.ts +6 -1
- package/types/firefox-page.d.ts +17 -1
- package/types/index.d.ts +22 -3
- package/types/links.d.ts +28 -0
- package/types/navigation.d.ts +56 -0
- package/types/prune.d.ts +2 -0
- package/types/site-lock.d.ts +70 -0
package/src/index.js
CHANGED
|
@@ -17,6 +17,7 @@ import { formatTree } from './aria.js';
|
|
|
17
17
|
import { authenticate } from './auth.js';
|
|
18
18
|
import { prune as pruneTree } from './prune.js';
|
|
19
19
|
import { dataSnapshot, stripLegend } from './data.js';
|
|
20
|
+
import { annotateLinks } from './links.js';
|
|
20
21
|
import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
|
|
21
22
|
import { dismissConsent } from './consent.js';
|
|
22
23
|
import { applyStealth } from './stealth.js';
|
|
@@ -28,6 +29,8 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
|
28
29
|
import { isChallengePage } from './challenge.js';
|
|
29
30
|
import { withTimeout } from './with-timeout.js';
|
|
30
31
|
import { readable as extractReadable } from './readable.js';
|
|
32
|
+
import { navFailed, navBlocked, httpWarning } from './navigation.js';
|
|
33
|
+
import { createSiteLock, applySiteLockCDP } from './site-lock.js';
|
|
31
34
|
import { assertNavigable, assertUploadAllowed } from './url-guard.js';
|
|
32
35
|
import { join as pathJoin } from 'node:path';
|
|
33
36
|
import { chmodSync } from 'node:fs';
|
|
@@ -60,6 +63,10 @@ import { chmodSync } from 'node:fs';
|
|
|
60
63
|
* @param {string} [opts.browser] - Source browser for cookie extraction.
|
|
61
64
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
|
|
62
65
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
66
|
+
* @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
|
|
67
|
+
* @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (the URL, its
|
|
68
|
+
* redirects) to hosts outside this list reject with NAV_BLOCKED. Entries are `host`
|
|
69
|
+
* or `*.host` (strict subdomains). Subresources and iframes are not locked.
|
|
63
70
|
* @returns {Promise<string>} ARIA snapshot text
|
|
64
71
|
*/
|
|
65
72
|
export async function browse(url, opts = {}) {
|
|
@@ -71,6 +78,8 @@ export async function browse(url, opts = {}) {
|
|
|
71
78
|
// Reject local-resource schemes (and optionally private hosts) before we
|
|
72
79
|
// spend a browser launch on a URL we won't navigate to.
|
|
73
80
|
assertNavigable(url, { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork });
|
|
81
|
+
const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
|
|
82
|
+
lock?.assertAllowed(url);
|
|
74
83
|
|
|
75
84
|
let browser = null;
|
|
76
85
|
let cdp = null;
|
|
@@ -88,6 +97,7 @@ export async function browse(url, opts = {}) {
|
|
|
88
97
|
browser = await launch(launchOpts);
|
|
89
98
|
cdp = await createCDP(browser.wsUrl);
|
|
90
99
|
}
|
|
100
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
91
101
|
|
|
92
102
|
// Step 2: Create a new page target and attach
|
|
93
103
|
const pageOpts = { viewport: opts.viewport, blockAds: opts.blockAds, blockUrls: opts.blockUrls };
|
|
@@ -106,7 +116,7 @@ export async function browse(url, opts = {}) {
|
|
|
106
116
|
}
|
|
107
117
|
|
|
108
118
|
// Step 4: Navigate and wait for load
|
|
109
|
-
await navigate(page, url, timeout);
|
|
119
|
+
let outcome = await navigate(page, url, timeout, lock);
|
|
110
120
|
|
|
111
121
|
// Step 4.5: Auto-dismiss cookie consent dialogs
|
|
112
122
|
if (opts.consent !== false) {
|
|
@@ -125,12 +135,13 @@ export async function browse(url, opts = {}) {
|
|
|
125
135
|
try {
|
|
126
136
|
browser = await launch({ ...launchOpts, headed: true });
|
|
127
137
|
cdp = await createCDP(browser.wsUrl);
|
|
138
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
128
139
|
page = await createPage(cdp, false, pageOpts);
|
|
129
140
|
await suppressPermissions(cdp);
|
|
130
141
|
if (!noAuth) {
|
|
131
142
|
try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
|
|
132
143
|
}
|
|
133
|
-
await navigate(page, url, timeout);
|
|
144
|
+
outcome = await navigate(page, url, timeout, lock);
|
|
134
145
|
if (opts.consent !== false) await dismissConsent(page.session);
|
|
135
146
|
({ tree } = await ariaTree(page));
|
|
136
147
|
} catch {
|
|
@@ -140,21 +151,27 @@ export async function browse(url, opts = {}) {
|
|
|
140
151
|
|
|
141
152
|
// Step 6: Prune for agent consumption
|
|
142
153
|
const raw = formatTree(tree);
|
|
154
|
+
const links = !!opts.links;
|
|
155
|
+
let linked = tree;
|
|
156
|
+
if (links) {
|
|
157
|
+
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
158
|
+
linked = annotateLinks(tree, entries[currentIndex]?.url || url);
|
|
159
|
+
}
|
|
143
160
|
let snapshot;
|
|
144
161
|
if (opts.prune !== false) {
|
|
145
162
|
snapshot = opts.pruneMode === 'data'
|
|
146
|
-
? dataSnapshot(
|
|
147
|
-
: formatTree(pruneTree(
|
|
163
|
+
? dataSnapshot(linked, { links })
|
|
164
|
+
: formatTree(pruneTree(linked, { mode: opts.pruneMode || 'act', links }));
|
|
148
165
|
} else {
|
|
149
|
-
snapshot = raw;
|
|
166
|
+
snapshot = links ? formatTree(linked) : raw;
|
|
150
167
|
}
|
|
151
|
-
const sized =
|
|
168
|
+
const sized = stripLegend(snapshot); // the legend is not content
|
|
152
169
|
const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
153
170
|
const actMode = !opts.pruneMode || opts.pruneMode === 'act';
|
|
154
171
|
const hint = (actMode && raw.length > 5000 && snapshot.length < 500 && snapshot.length < raw.length * 0.05)
|
|
155
172
|
? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
|
|
156
173
|
: '';
|
|
157
|
-
snapshot = stats + '\n' + hint + snapshot;
|
|
174
|
+
snapshot = stats + '\n' + hint + httpWarning(outcome.status) + snapshot;
|
|
158
175
|
|
|
159
176
|
// Step 7: Clean up
|
|
160
177
|
await cdp.send('Target.closeTarget', { targetId: page.targetId });
|
|
@@ -204,10 +221,20 @@ export async function browse(url, opts = {}) {
|
|
|
204
221
|
* @property {boolean} cookieInjection - Cookie extraction/injection is active (off in incognito)
|
|
205
222
|
*/
|
|
206
223
|
|
|
224
|
+
/**
|
|
225
|
+
* What `goto()` resolves to: the final URL (after redirects) and the main
|
|
226
|
+
* document's HTTP status (0 when no HTTP response was observed, e.g. an aborted navigation).
|
|
227
|
+
* Network failures reject with an Error whose `code` is 'NAV_FAILED' and
|
|
228
|
+
* `reason` the browser's raw error string.
|
|
229
|
+
* @typedef {import('./navigation.js').NavigationOutcome} NavigationOutcome
|
|
230
|
+
*/
|
|
231
|
+
|
|
207
232
|
/**
|
|
208
233
|
* Options for `page.snapshot()`. Pass `false` instead of an object to skip
|
|
209
234
|
* pruning and return the raw ARIA tree.
|
|
210
|
-
*
|
|
235
|
+
* `links: true` adds each link's absolute URL after its name (default off,
|
|
236
|
+
* or the `connect({ links })` default).
|
|
237
|
+
* @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
|
|
211
238
|
*/
|
|
212
239
|
|
|
213
240
|
/**
|
|
@@ -261,6 +288,12 @@ export async function browse(url, opts = {}) {
|
|
|
261
288
|
* skip storageState loading and make injectCookies() a no-op, so no auth
|
|
262
289
|
* ever enters the session even if a caller injects unconditionally.
|
|
263
290
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
291
|
+
* @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
|
|
292
|
+
* @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (goto, redirects,
|
|
293
|
+
* clicked links, JS jumps, popups) to hosts outside this list are refused and the page
|
|
294
|
+
* stays put. Entries are `host` or `*.host` (strict subdomains). goto() rejects with
|
|
295
|
+
* NAV_BLOCKED; in-page blocks add a `[blocked navigation to <host>]` line to the next
|
|
296
|
+
* snapshot. Subresources, iframes and WebSocket are not locked. Not allowed with `port`.
|
|
264
297
|
* @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
|
|
265
298
|
* 'firefox' drives over WebDriver BiDi (a separate transport / page object);
|
|
266
299
|
* 'chromium' (default) uses CDP.
|
|
@@ -319,6 +352,12 @@ export async function connect(opts = {}) {
|
|
|
319
352
|
|
|
320
353
|
const mode = opts.mode || 'headless';
|
|
321
354
|
const attachMode = !!opts.port;
|
|
355
|
+
// Validate before launching anything. Attach mode is refused: the lock is a
|
|
356
|
+
// browser-level intercept that would reach the user's own tabs.
|
|
357
|
+
if (opts.allowHosts && attachMode) {
|
|
358
|
+
throw new Error('allowHosts is not supported in attach mode (port): the site lock is browser-wide and would affect your own tabs');
|
|
359
|
+
}
|
|
360
|
+
const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
|
|
322
361
|
// Incognito: run a clean, unauthenticated session. Skips storageState loading
|
|
323
362
|
// and neuters injectCookies() so no auth ever enters the session — even when
|
|
324
363
|
// a caller (MCP goto, daemon) calls injectCookies() unconditionally. The
|
|
@@ -350,6 +389,7 @@ export async function connect(opts = {}) {
|
|
|
350
389
|
browser = await launch(launchOpts);
|
|
351
390
|
cdp = await createCDP(browser.wsUrl);
|
|
352
391
|
}
|
|
392
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
353
393
|
|
|
354
394
|
// In attach mode we don't know (and shouldn't assume) the user's headed/
|
|
355
395
|
// headless state — treat it as headed so stealth patches are skipped
|
|
@@ -367,6 +407,9 @@ export async function connect(opts = {}) {
|
|
|
367
407
|
let page = await createPage(cdp, !currentlyHeaded, pageOpts);
|
|
368
408
|
let refMap = new Map();
|
|
369
409
|
let botBlocked = false;
|
|
410
|
+
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
411
|
+
/** @type {NavigationOutcome|null} */
|
|
412
|
+
let lastNavigation = null;
|
|
370
413
|
// Set when the consent pass after the latest goto() found nothing: the first
|
|
371
414
|
// snapshot() re-checks once, since some CMPs (Google Funding Choices) render
|
|
372
415
|
// their dialog 1-3s after load. Holds the document identity (tab + main-frame
|
|
@@ -483,12 +526,14 @@ export async function connect(opts = {}) {
|
|
|
483
526
|
return {
|
|
484
527
|
async goto(url, timeout = 30000) {
|
|
485
528
|
assertNavigable(url, urlGuard);
|
|
529
|
+
lock?.assertAllowed(url);
|
|
486
530
|
// Refs from the previous page are about to become invalid — clear
|
|
487
531
|
// before navigating so a stale click(ref) errors clearly instead of
|
|
488
532
|
// silently resolving to whatever backendNodeId happens to still be in
|
|
489
533
|
// the map.
|
|
490
534
|
refMap = new Map();
|
|
491
535
|
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
536
|
+
lastNavigation = null;
|
|
492
537
|
// Switch back to headless if we fell back to headed previously.
|
|
493
538
|
// Not in attach mode — we never own the browser there, so there's
|
|
494
539
|
// nothing to rewind.
|
|
@@ -499,13 +544,14 @@ export async function connect(opts = {}) {
|
|
|
499
544
|
|
|
500
545
|
browser = await launch(launchOpts);
|
|
501
546
|
cdp = await createCDP(browser.wsUrl);
|
|
547
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
502
548
|
page = await createPage(cdp, true, pageOpts);
|
|
503
549
|
setupDialogHandler(page.session);
|
|
504
550
|
await suppressPermissions(cdp);
|
|
505
551
|
currentlyHeaded = false;
|
|
506
552
|
}
|
|
507
553
|
|
|
508
|
-
await navigate(page, url, timeout);
|
|
554
|
+
lastNavigation = await navigate(page, url, timeout, lock);
|
|
509
555
|
if (opts.consent !== false) {
|
|
510
556
|
const dismissed = await dismissConsent(page.session);
|
|
511
557
|
if (!dismissed) consentRecheck = await navKey();
|
|
@@ -526,10 +572,11 @@ export async function connect(opts = {}) {
|
|
|
526
572
|
try {
|
|
527
573
|
browser = await launch({ ...launchOpts, headed: true });
|
|
528
574
|
cdp = await createCDP(browser.wsUrl);
|
|
575
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
529
576
|
page = await createPage(cdp, false, pageOpts);
|
|
530
577
|
setupDialogHandler(page.session);
|
|
531
578
|
await suppressPermissions(cdp);
|
|
532
|
-
await navigate(page, url, timeout);
|
|
579
|
+
lastNavigation = await navigate(page, url, timeout, lock);
|
|
533
580
|
consentRecheck = null;
|
|
534
581
|
if (opts.consent !== false && !(await dismissConsent(page.session))) {
|
|
535
582
|
consentRecheck = await navKey();
|
|
@@ -543,6 +590,7 @@ export async function connect(opts = {}) {
|
|
|
543
590
|
// Headed launch failed (no display?) — keep headless result, botBlocked stays true
|
|
544
591
|
}
|
|
545
592
|
}
|
|
593
|
+
return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
|
|
546
594
|
},
|
|
547
595
|
|
|
548
596
|
async goBack() {
|
|
@@ -553,6 +601,7 @@ export async function connect(opts = {}) {
|
|
|
553
601
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
554
602
|
refMap = new Map(); // refs from the previous page are now invalid
|
|
555
603
|
consentRecheck = null;
|
|
604
|
+
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
556
605
|
},
|
|
557
606
|
|
|
558
607
|
async goForward() {
|
|
@@ -563,6 +612,7 @@ export async function connect(opts = {}) {
|
|
|
563
612
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
564
613
|
refMap = new Map();
|
|
565
614
|
consentRecheck = null;
|
|
615
|
+
lastNavigation = null;
|
|
566
616
|
},
|
|
567
617
|
|
|
568
618
|
async reload(reloadOpts = {}) {
|
|
@@ -574,6 +624,7 @@ export async function connect(opts = {}) {
|
|
|
574
624
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
575
625
|
refMap = new Map(); // refs from the pre-reload page are invalid
|
|
576
626
|
consentRecheck = null;
|
|
627
|
+
lastNavigation = null; // the reload's status is not read; don't keep the old one
|
|
577
628
|
},
|
|
578
629
|
|
|
579
630
|
/** @param {string} url @param {CookieOptions} [cookieOpts] */
|
|
@@ -599,11 +650,14 @@ export async function connect(opts = {}) {
|
|
|
599
650
|
const raw = formatTree(result.tree);
|
|
600
651
|
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
601
652
|
const pageUrl = entries[currentIndex]?.url || '';
|
|
602
|
-
const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : ''
|
|
603
|
-
|
|
653
|
+
const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
|
|
654
|
+
+ httpWarning(lastNavigation?.status) + (lock?.takeNotes(page.targetId) ?? '');
|
|
655
|
+
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
656
|
+
const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
|
|
657
|
+
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
604
658
|
const mode = pruneOpts?.mode || opts.pruneMode || 'act';
|
|
605
|
-
const out = mode === 'data' ? dataSnapshot(
|
|
606
|
-
const sized =
|
|
659
|
+
const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
|
|
660
|
+
const sized = stripLegend(out); // the legend is not content
|
|
607
661
|
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
608
662
|
const hint = (mode === 'act' && raw.length > 5000 && out.length < 500 && out.length < raw.length * 0.05)
|
|
609
663
|
? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
|
|
@@ -702,6 +756,7 @@ export async function connect(opts = {}) {
|
|
|
702
756
|
const oldSessionId = page.sessionId;
|
|
703
757
|
page = await attachToExistingTarget(cdp, target.targetId, pageOpts);
|
|
704
758
|
refMap = new Map(); // refs from the previous tab are no longer valid
|
|
759
|
+
lastNavigation = null; // the outcome belonged to the tab we left
|
|
705
760
|
setupDialogHandler(page.session);
|
|
706
761
|
try { await cdp.send('Target.detachFromTarget', { sessionId: oldSessionId }); } catch {}
|
|
707
762
|
},
|
|
@@ -746,6 +801,8 @@ export async function connect(opts = {}) {
|
|
|
746
801
|
},
|
|
747
802
|
|
|
748
803
|
get botBlocked() { return botBlocked; },
|
|
804
|
+
/** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
|
|
805
|
+
get lastNavigation() { return lastNavigation; },
|
|
749
806
|
|
|
750
807
|
/** @type {'chromium'} Engine driving this session (see also `capabilities`) */
|
|
751
808
|
engine: 'chromium',
|
|
@@ -820,12 +877,14 @@ export async function connect(opts = {}) {
|
|
|
820
877
|
return {
|
|
821
878
|
async goto(url, timeout = 30000) {
|
|
822
879
|
assertNavigable(url, urlGuard);
|
|
823
|
-
|
|
880
|
+
lock?.assertAllowed(url);
|
|
881
|
+
const outcome = await navigate(tab, url, timeout, lock);
|
|
824
882
|
if (opts.consent !== false) {
|
|
825
883
|
await dismissConsent(tab.session);
|
|
826
884
|
}
|
|
827
885
|
const { tree, nodeCount } = await ariaTree(tab);
|
|
828
886
|
tabBotBlocked = isChallengePage(tree, nodeCount);
|
|
887
|
+
return outcome;
|
|
829
888
|
},
|
|
830
889
|
get botBlocked() { return tabBotBlocked; },
|
|
831
890
|
/** @param {string} url @param {CookieOptions} [cookieOpts] */
|
|
@@ -905,10 +964,11 @@ async function suppressPermissions(cdp) {
|
|
|
905
964
|
* v0.16.0. Still Firefox-limited: `reload({ignoreCache})` (upstream BiDi gap).
|
|
906
965
|
* The navigation guard (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir`
|
|
907
966
|
* sandbox, `incognito`, `proxy`, `viewport`, and `pruneMode` all apply.
|
|
908
|
-
* @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls })
|
|
967
|
+
* @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls, allowHosts })
|
|
909
968
|
*/
|
|
910
969
|
async function connectFirefox(opts) {
|
|
911
970
|
const hybrid = opts.mode === 'hybrid';
|
|
971
|
+
const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null; // validates before any launch
|
|
912
972
|
const blockOpts = {
|
|
913
973
|
blockAds: opts.blockAds !== undefined ? opts.blockAds : true,
|
|
914
974
|
blockUrls: opts.blockUrls,
|
|
@@ -937,7 +997,7 @@ async function connectFirefox(opts) {
|
|
|
937
997
|
});
|
|
938
998
|
const bidi = await createBiDi(browser.wsUrl);
|
|
939
999
|
if (!headed) await applyFirefoxStealth(bidi);
|
|
940
|
-
await applyFirefoxBlocklist(bidi, blockOpts);
|
|
1000
|
+
await applyFirefoxBlocklist(bidi, blockOpts, lock);
|
|
941
1001
|
const { contexts } = await bidi.send('browsingContext.getTree', {});
|
|
942
1002
|
return { browser, bidi, topContext: contexts[0].context };
|
|
943
1003
|
}
|
|
@@ -959,8 +1019,10 @@ async function connectFirefox(opts) {
|
|
|
959
1019
|
|
|
960
1020
|
const page = await createFirefoxPage(bidi, {
|
|
961
1021
|
pruneMode: opts.pruneMode,
|
|
1022
|
+
links: opts.links,
|
|
962
1023
|
urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
|
|
963
1024
|
uploadDir: opts.uploadDir || null,
|
|
1025
|
+
siteLock: lock,
|
|
964
1026
|
incognito: !!opts.incognito,
|
|
965
1027
|
consent: opts.consent,
|
|
966
1028
|
hybrid,
|
|
@@ -1126,14 +1188,55 @@ export function _resetBlocklistWarning() {
|
|
|
1126
1188
|
}
|
|
1127
1189
|
|
|
1128
1190
|
/**
|
|
1129
|
-
* Navigate to a URL and wait for the page to load.
|
|
1191
|
+
* Navigate to a URL and wait for the page to load. Resolves to the outcome of
|
|
1192
|
+
* the main-document navigation (final URL after redirects, HTTP status);
|
|
1193
|
+
* rejects with a NAV_FAILED error when the browser could not reach a document,
|
|
1194
|
+
* or NAV_BLOCKED when the site lock refused the URL or a redirect hop.
|
|
1195
|
+
* @param {*} page
|
|
1196
|
+
* @param {string} url
|
|
1197
|
+
* @param {number} [timeout]
|
|
1198
|
+
* @param {import('./site-lock.js').SiteLock|null} [lock]
|
|
1199
|
+
* @returns {Promise<NavigationOutcome>}
|
|
1130
1200
|
*/
|
|
1131
|
-
async function navigate(page, url, timeout = 30000) {
|
|
1132
|
-
const
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1201
|
+
async function navigate(page, url, timeout = 30000, lock = null) {
|
|
1202
|
+
const { session } = page;
|
|
1203
|
+
// Main-document responses (every redirect hop fires one; the last wins).
|
|
1204
|
+
const docs = [];
|
|
1205
|
+
const offDoc = session.on('Network.responseReceived', (p) => { if (p.type === 'Document') docs.push(p); });
|
|
1206
|
+
// Load waiter, hand-rolled (not session.once) so it can be cancelled when
|
|
1207
|
+
// Page.navigate fails or aborts and no load event will ever come.
|
|
1208
|
+
let cancel = () => {};
|
|
1209
|
+
const loaded = /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
|
|
1210
|
+
const timer = setTimeout(() => { off(); reject(new Error('Timeout waiting for CDP event: Page.loadEventFired')); }, timeout);
|
|
1211
|
+
const off = session.on('Page.loadEventFired', () => { cancel(); resolve(undefined); });
|
|
1212
|
+
cancel = () => { clearTimeout(timer); off(); };
|
|
1213
|
+
}));
|
|
1214
|
+
loaded.catch(() => {}); // rejection is only read if we await it below
|
|
1215
|
+
try {
|
|
1216
|
+
lock?.startGoto(page.targetId);
|
|
1217
|
+
const nav = await session.send('Page.navigate', { url });
|
|
1218
|
+
// A lock-refused document (the URL or a redirect hop) also surfaces as
|
|
1219
|
+
// ERR_ABORTED, so this must come before the aborted branch below.
|
|
1220
|
+
const blocked = lock?.takeGotoBlock(page.targetId);
|
|
1221
|
+
if (blocked) throw navBlocked(blocked.url, blocked.host);
|
|
1222
|
+
// ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
|
|
1223
|
+
// no document and fires no load event, so don't wait for one.
|
|
1224
|
+
const aborted = nav.errorText === 'net::ERR_ABORTED';
|
|
1225
|
+
// ERR_HTTP_RESPONSE_CODE_FAILURE is a 4xx/5xx with an empty body: the
|
|
1226
|
+
// document did load (Chromium's error page fires load, the Document
|
|
1227
|
+
// response carries the status), so report it like any other HTTP error.
|
|
1228
|
+
const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
|
|
1229
|
+
if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
|
|
1230
|
+
if (!aborted) await loaded;
|
|
1231
|
+
// Brief settle time for dynamic content
|
|
1232
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1233
|
+
// loaderId is shared by every redirect hop of one navigation.
|
|
1234
|
+
const last = docs.filter((d) => d.loaderId === nav.loaderId).pop();
|
|
1235
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
1236
|
+
} finally {
|
|
1237
|
+
cancel();
|
|
1238
|
+
offDoc();
|
|
1239
|
+
}
|
|
1137
1240
|
}
|
|
1138
1241
|
|
|
1139
1242
|
/**
|
package/src/links.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* links.js — opt-in link URLs for snapshots (`links: true`).
|
|
3
|
+
*
|
|
4
|
+
* `annotateLinks()` stamps `urls: [url]` on every link node that has a usable
|
|
5
|
+
* browser-serialized absolute URL. aria.js, prune.js and data.js print/carry
|
|
6
|
+
* `urls` — they never see the option, so with links off nothing changes.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Normalise a link's URL for a snapshot line. Returns '' when it should not be
|
|
11
|
+
* shown: no URL, `javascript:`, or a same-document link (the page URL, only
|
|
12
|
+
* the #fragment differs). A raw space, `<` or `>` (browsers encode them, but
|
|
13
|
+
* not always in the fragment) is percent-encoded so a line splits cleanly on
|
|
14
|
+
* ` <` and `> `.
|
|
15
|
+
*
|
|
16
|
+
* @param {string|undefined} url - Absolute URL as serialized by the browser
|
|
17
|
+
* @param {string} [pageUrl] - URL of the page the link is on
|
|
18
|
+
* @returns {string}
|
|
19
|
+
*/
|
|
20
|
+
export function linkUrl(url, pageUrl = '') {
|
|
21
|
+
if (typeof url !== 'string') return '';
|
|
22
|
+
const u = url.trim();
|
|
23
|
+
if (!u || /^javascript:/i.test(u)) return '';
|
|
24
|
+
if (pageUrl && u.split('#')[0] === pageUrl.split('#')[0]) return '';
|
|
25
|
+
return u.replace(/ /g, '%20').replace(/</g, '%3C').replace(/>/g, '%3E');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Copy a tree with `urls` set on its link nodes (input is not mutated).
|
|
30
|
+
*
|
|
31
|
+
* @param {object|null} node - Root from buildTree()
|
|
32
|
+
* @param {string} pageUrl
|
|
33
|
+
* @returns {object|null}
|
|
34
|
+
*/
|
|
35
|
+
export function annotateLinks(node, pageUrl) {
|
|
36
|
+
if (!node) return node;
|
|
37
|
+
const out = { ...node, children: node.children.map((c) => annotateLinks(c, pageUrl)) };
|
|
38
|
+
if (node.role === 'link') {
|
|
39
|
+
const u = linkUrl(node.properties?.url, pageUrl);
|
|
40
|
+
if (u) out.urls = [u];
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** ` <u1> <u2>` for a node's urls ('' when none). */
|
|
46
|
+
export const urlsText = (urls) => (urls || []).map((u) => ` <${u}>`).join('');
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* navigation.js — Shared pieces of the goto() outcome contract, so both
|
|
3
|
+
* engines (CDP in index.js, BiDi in firefox-page.js) report it identically.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What goto() resolves to: where the page ended up and how the server answered.
|
|
8
|
+
* `status` is 0 when no HTTP response was observed for the navigation (e.g. an
|
|
9
|
+
* aborted navigation that never got one).
|
|
10
|
+
* @typedef {{ url: string, status: number }} NavigationOutcome
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Typed error for a navigation that never reached a document (DNS failure,
|
|
15
|
+
* connection refused, TLS error, ...). `code` is 'NAV_FAILED'; `reason` is the
|
|
16
|
+
* browser's raw error string (`net::ERR_NAME_NOT_RESOLVED`,
|
|
17
|
+
* `NS_ERROR_UNKNOWN_HOST`, ...).
|
|
18
|
+
* @param {string} url
|
|
19
|
+
* @param {string} reason
|
|
20
|
+
* @returns {Error & {code: 'NAV_FAILED', reason: string}}
|
|
21
|
+
*/
|
|
22
|
+
export function navFailed(url, reason) {
|
|
23
|
+
return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Typed error for a top-level navigation the site lock (`allowHosts`) refused:
|
|
28
|
+
* the target, or a redirect hop of it, is off the allowed list. `code` is
|
|
29
|
+
* 'NAV_BLOCKED'; `host` is the refused hostname. The page stays where it was.
|
|
30
|
+
* @param {string} url - The refused URL (the redirect target, if it was a hop).
|
|
31
|
+
* @param {string} host
|
|
32
|
+
* @returns {Error & {code: 'NAV_BLOCKED', host: string}}
|
|
33
|
+
*/
|
|
34
|
+
export function navBlocked(url, host) {
|
|
35
|
+
return Object.assign(new Error(`Navigation to ${url} blocked: ${host} is not in allowHosts`), { code: /** @type {const} */ ('NAV_BLOCKED'), host });
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* True for a final HTTP status that is not 2xx (0 = unknown is not an error).
|
|
40
|
+
* @param {number|undefined} status
|
|
41
|
+
*/
|
|
42
|
+
export function isHttpError(status) {
|
|
43
|
+
return !!status && (status < 200 || status >= 300);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Warning line for a non-2xx final status ('' for 2xx and for unknown 0), in
|
|
48
|
+
* the style of the bot-challenge line, newline-terminated for prepending.
|
|
49
|
+
* @param {number|undefined} status
|
|
50
|
+
* @returns {string}
|
|
51
|
+
*/
|
|
52
|
+
export function httpWarning(status) {
|
|
53
|
+
return isHttpError(status) ? `[HTTP ${status} — the page may be an error page]\n` : '';
|
|
54
|
+
}
|
package/src/prune.js
CHANGED
|
@@ -62,6 +62,7 @@ const SKIP_ROLES = new Set([
|
|
|
62
62
|
* @param {object} [options]
|
|
63
63
|
* @param {'act'|'browse'|'navigate'|'full'|'read'} [options.mode='act'] - Pruning mode ('read' is an alias for 'browse')
|
|
64
64
|
* @param {string} [options.context=''] - Search context for relevance filtering
|
|
65
|
+
* @param {boolean} [options.links=false] - Links carry `urls` (see links.js): nodes that drop their children keep the URLs of links beneath them
|
|
65
66
|
* @returns {object|null} Pruned tree
|
|
66
67
|
*/
|
|
67
68
|
export function prune(tree, options = {}) {
|
|
@@ -80,7 +81,7 @@ export function prune(tree, options = {}) {
|
|
|
80
81
|
nodes = extractRegions(nodes, allowedRegions);
|
|
81
82
|
|
|
82
83
|
// Step 2: Prune nodes
|
|
83
|
-
const ctx = { mode, parentRole: null, keywords };
|
|
84
|
+
const ctx = { mode, parentRole: null, keywords, links: !!options.links };
|
|
84
85
|
nodes = nodes.map((n) => pruneNode(n, ctx)).filter(Boolean);
|
|
85
86
|
|
|
86
87
|
// Step 3: Collapse structural wrappers
|
|
@@ -204,7 +205,7 @@ function pruneNode(node, ctx) {
|
|
|
204
205
|
return { ...node, children: pruneChildren(node.children, ctx) };
|
|
205
206
|
}
|
|
206
207
|
if (node.role === 'group' && node.name) {
|
|
207
|
-
if (!isBrowse && /kleuren|colors?|couleurs?|farben/i.test(node.name)) {
|
|
208
|
+
if (!isBrowse && !ctx.links && /kleuren|colors?|couleurs?|farben/i.test(node.name)) {
|
|
208
209
|
return collapseColors(node);
|
|
209
210
|
}
|
|
210
211
|
return { ...node, children: pruneChildren(node.children, ctx) };
|
|
@@ -217,7 +218,7 @@ function pruneNode(node, ctx) {
|
|
|
217
218
|
return null;
|
|
218
219
|
}
|
|
219
220
|
}
|
|
220
|
-
return { ...node, children: [] };
|
|
221
|
+
return { ...node, children: [], ...carryUrls(node, ctx) };
|
|
221
222
|
}
|
|
222
223
|
|
|
223
224
|
// StaticText — CDP equivalent of mcprune's "text" nodes
|
|
@@ -448,6 +449,13 @@ function flatten(nodes) {
|
|
|
448
449
|
return result;
|
|
449
450
|
}
|
|
450
451
|
|
|
452
|
+
// Links on: a node that drops its children keeps the URLs of the links beneath it.
|
|
453
|
+
function carryUrls(node, ctx) {
|
|
454
|
+
if (!ctx.links) return {};
|
|
455
|
+
const urls = [...new Set(flatten([node]).flatMap((n) => n.urls || []))];
|
|
456
|
+
return urls.length ? { urls } : {};
|
|
457
|
+
}
|
|
458
|
+
|
|
451
459
|
function condenseCard(node) {
|
|
452
460
|
const all = flatten([node]);
|
|
453
461
|
const link = all.find((n) => n.role === 'link' && n.name);
|