barebrowse 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/README.md +13 -1
- package/barebrowse.context.md +30 -5
- package/cli.js +39 -3
- package/commands/barebrowse/SKILL.md +6 -0
- package/commands/barebrowse.md +1 -0
- package/mcp-server.js +108 -9
- package/package.json +1 -1
- package/src/auth.js +25 -7
- package/src/ax-snapshot.js +24 -5
- package/src/bidi.js +3 -2
- package/src/blocklist-firefox.js +12 -4
- package/src/daemon.js +27 -2
- package/src/data.js +2 -2
- package/src/firefox-page.js +126 -25
- package/src/index.js +152 -37
- package/src/navigation.js +12 -0
- package/src/prune.js +3 -2
- package/src/site-lock.js +184 -0
- package/src/zoom.js +250 -0
- package/types/auth.d.ts +26 -0
- package/types/blocklist-firefox.d.ts +2 -1
- package/types/data.d.ts +2 -1
- package/types/firefox-page.d.ts +14 -2
- package/types/index.d.ts +16 -3
- package/types/navigation.d.ts +12 -0
- package/types/prune.d.ts +2 -0
- package/types/site-lock.d.ts +70 -0
- package/types/zoom.d.ts +51 -0
package/src/index.js
CHANGED
|
@@ -18,6 +18,7 @@ import { authenticate } from './auth.js';
|
|
|
18
18
|
import { prune as pruneTree } from './prune.js';
|
|
19
19
|
import { dataSnapshot, stripLegend } from './data.js';
|
|
20
20
|
import { annotateLinks } from './links.js';
|
|
21
|
+
import { parseWithin, zoomSnapshot } from './zoom.js';
|
|
21
22
|
import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
|
|
22
23
|
import { dismissConsent } from './consent.js';
|
|
23
24
|
import { applyStealth } from './stealth.js';
|
|
@@ -29,7 +30,8 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
|
29
30
|
import { isChallengePage } from './challenge.js';
|
|
30
31
|
import { withTimeout } from './with-timeout.js';
|
|
31
32
|
import { readable as extractReadable } from './readable.js';
|
|
32
|
-
import { navFailed, httpWarning } from './navigation.js';
|
|
33
|
+
import { navFailed, navBlocked, httpWarning } from './navigation.js';
|
|
34
|
+
import { createSiteLock, applySiteLockCDP } from './site-lock.js';
|
|
33
35
|
import { assertNavigable, assertUploadAllowed } from './url-guard.js';
|
|
34
36
|
import { join as pathJoin } from 'node:path';
|
|
35
37
|
import { chmodSync } from 'node:fs';
|
|
@@ -63,6 +65,9 @@ import { chmodSync } from 'node:fs';
|
|
|
63
65
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
|
|
64
66
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
65
67
|
* @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
|
|
68
|
+
* @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (the URL, its
|
|
69
|
+
* redirects) to hosts outside this list reject with NAV_BLOCKED. Entries are `host`
|
|
70
|
+
* or `*.host` (strict subdomains). Subresources and iframes are not locked.
|
|
66
71
|
* @returns {Promise<string>} ARIA snapshot text
|
|
67
72
|
*/
|
|
68
73
|
export async function browse(url, opts = {}) {
|
|
@@ -74,6 +79,8 @@ export async function browse(url, opts = {}) {
|
|
|
74
79
|
// Reject local-resource schemes (and optionally private hosts) before we
|
|
75
80
|
// spend a browser launch on a URL we won't navigate to.
|
|
76
81
|
assertNavigable(url, { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork });
|
|
82
|
+
const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
|
|
83
|
+
lock?.assertAllowed(url);
|
|
77
84
|
|
|
78
85
|
let browser = null;
|
|
79
86
|
let cdp = null;
|
|
@@ -91,6 +98,7 @@ export async function browse(url, opts = {}) {
|
|
|
91
98
|
browser = await launch(launchOpts);
|
|
92
99
|
cdp = await createCDP(browser.wsUrl);
|
|
93
100
|
}
|
|
101
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
94
102
|
|
|
95
103
|
// Step 2: Create a new page target and attach
|
|
96
104
|
const pageOpts = { viewport: opts.viewport, blockAds: opts.blockAds, blockUrls: opts.blockUrls };
|
|
@@ -109,7 +117,7 @@ export async function browse(url, opts = {}) {
|
|
|
109
117
|
}
|
|
110
118
|
|
|
111
119
|
// Step 4: Navigate and wait for load
|
|
112
|
-
let outcome = await navigate(page, url, timeout);
|
|
120
|
+
let outcome = await navigate(page, url, timeout, lock);
|
|
113
121
|
|
|
114
122
|
// Step 4.5: Auto-dismiss cookie consent dialogs
|
|
115
123
|
if (opts.consent !== false) {
|
|
@@ -128,12 +136,13 @@ export async function browse(url, opts = {}) {
|
|
|
128
136
|
try {
|
|
129
137
|
browser = await launch({ ...launchOpts, headed: true });
|
|
130
138
|
cdp = await createCDP(browser.wsUrl);
|
|
139
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
131
140
|
page = await createPage(cdp, false, pageOpts);
|
|
132
141
|
await suppressPermissions(cdp);
|
|
133
142
|
if (!noAuth) {
|
|
134
143
|
try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
|
|
135
144
|
}
|
|
136
|
-
outcome = await navigate(page, url, timeout);
|
|
145
|
+
outcome = await navigate(page, url, timeout, lock);
|
|
137
146
|
if (opts.consent !== false) await dismissConsent(page.session);
|
|
138
147
|
({ tree } = await ariaTree(page));
|
|
139
148
|
} catch {
|
|
@@ -226,7 +235,14 @@ export async function browse(url, opts = {}) {
|
|
|
226
235
|
* pruning and return the raw ARIA tree.
|
|
227
236
|
* `links: true` adds each link's absolute URL after its name (default off,
|
|
228
237
|
* or the `connect({ links })` default).
|
|
229
|
-
*
|
|
238
|
+
* `map: true` returns an outline of the page's big boxes (ref + size) instead of
|
|
239
|
+
* content. `within` zooms the snapshot to one box, or several (an array, output
|
|
240
|
+
* in page order): a ref from the map, a role (`'table'`), role + accessible name
|
|
241
|
+
* (`'list "Issues"'`) or an unnamed box by the heading before it
|
|
242
|
+
* (`'list after "Other flights"'`). Any `mode` and `links` apply inside a zoom,
|
|
243
|
+
* where the box is the whole page. Two or more matches return the map with the
|
|
244
|
+
* matches marked; none, or a ref that is gone, throws. `within` wins over `map`.
|
|
245
|
+
* @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean, map?: boolean, within?: string|number|Array<string|number> }} SnapshotOptions
|
|
230
246
|
*/
|
|
231
247
|
|
|
232
248
|
/**
|
|
@@ -281,6 +297,11 @@ export async function browse(url, opts = {}) {
|
|
|
281
297
|
* ever enters the session even if a caller injects unconditionally.
|
|
282
298
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
283
299
|
* @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
|
|
300
|
+
* @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (goto, redirects,
|
|
301
|
+
* clicked links, JS jumps, popups) to hosts outside this list are refused and the page
|
|
302
|
+
* stays put. Entries are `host` or `*.host` (strict subdomains). goto() rejects with
|
|
303
|
+
* NAV_BLOCKED; in-page blocks add a `[blocked navigation to <host>]` line to the next
|
|
304
|
+
* snapshot. Subresources, iframes and WebSocket are not locked. Not allowed with `port`.
|
|
284
305
|
* @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
|
|
285
306
|
* 'firefox' drives over WebDriver BiDi (a separate transport / page object);
|
|
286
307
|
* 'chromium' (default) uses CDP.
|
|
@@ -339,6 +360,12 @@ export async function connect(opts = {}) {
|
|
|
339
360
|
|
|
340
361
|
const mode = opts.mode || 'headless';
|
|
341
362
|
const attachMode = !!opts.port;
|
|
363
|
+
// Validate before launching anything. Attach mode is refused: the lock is a
|
|
364
|
+
// browser-level intercept that would reach the user's own tabs.
|
|
365
|
+
if (opts.allowHosts && attachMode) {
|
|
366
|
+
throw new Error('allowHosts is not supported in attach mode (port): the site lock is browser-wide and would affect your own tabs');
|
|
367
|
+
}
|
|
368
|
+
const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
|
|
342
369
|
// Incognito: run a clean, unauthenticated session. Skips storageState loading
|
|
343
370
|
// and neuters injectCookies() so no auth ever enters the session — even when
|
|
344
371
|
// a caller (MCP goto, daemon) calls injectCookies() unconditionally. The
|
|
@@ -370,6 +397,7 @@ export async function connect(opts = {}) {
|
|
|
370
397
|
browser = await launch(launchOpts);
|
|
371
398
|
cdp = await createCDP(browser.wsUrl);
|
|
372
399
|
}
|
|
400
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
373
401
|
|
|
374
402
|
// In attach mode we don't know (and shouldn't assume) the user's headed/
|
|
375
403
|
// headless state — treat it as headed so stealth patches are skipped
|
|
@@ -386,6 +414,9 @@ export async function connect(opts = {}) {
|
|
|
386
414
|
};
|
|
387
415
|
let page = await createPage(cdp, !currentlyHeaded, pageOpts);
|
|
388
416
|
let refMap = new Map();
|
|
417
|
+
// Main-document identity (navKey) when the current refs were handed out; read again
|
|
418
|
+
// fresh at every ref use, so a click/press that navigated to a new document kills them.
|
|
419
|
+
let refDoc = null;
|
|
389
420
|
let botBlocked = false;
|
|
390
421
|
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
391
422
|
/** @type {NavigationOutcome|null} */
|
|
@@ -406,6 +437,38 @@ export async function connect(opts = {}) {
|
|
|
406
437
|
} catch { return null; }
|
|
407
438
|
}
|
|
408
439
|
|
|
440
|
+
/** Resolve a ref from the last snapshot; throws when the document it belongs to is gone. */
|
|
441
|
+
async function refEntry(ref) {
|
|
442
|
+
const entry = refMap.get(String(ref));
|
|
443
|
+
if (!entry) throw new Error(`No element found for ref "${ref}"`);
|
|
444
|
+
if (refDoc) {
|
|
445
|
+
const now = await navKey();
|
|
446
|
+
if (now && now !== refDoc) {
|
|
447
|
+
refMap = new Map(); refDoc = null;
|
|
448
|
+
throw new Error(`No element found for ref "${ref}" — the page navigated since the snapshot that gave it; take a fresh snapshot`);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
return entry;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// A consent accept that navigates (a form submit, Google's consent.google.com) lands on
|
|
455
|
+
// the real page: report that document in lastNavigation, not the interstitial's.
|
|
456
|
+
async function dismissConsentTracked() {
|
|
457
|
+
const docs = [];
|
|
458
|
+
let off = () => {};
|
|
459
|
+
try {
|
|
460
|
+
const { frameTree } = await page.session.send('Page.getFrameTree');
|
|
461
|
+
off = page.session.on('Network.responseReceived', (p) => {
|
|
462
|
+
if (p.type === 'Document' && p.frameId === frameTree.frame.id) docs.push(p);
|
|
463
|
+
});
|
|
464
|
+
} catch { /* no tracking; dismissal itself still runs */ }
|
|
465
|
+
try { return await dismissConsent(page.session); } finally {
|
|
466
|
+
off();
|
|
467
|
+
const last = docs.pop();
|
|
468
|
+
if (last) lastNavigation = { url: last.response.url, status: last.response.status };
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
409
472
|
// Suppress permission prompts. Skipped in attach mode — Browser.setPermission
|
|
410
473
|
// is browser-wide (no origin scope here), so flipping permissions to denied
|
|
411
474
|
// would leak into the user's other tabs.
|
|
@@ -506,6 +569,7 @@ export async function connect(opts = {}) {
|
|
|
506
569
|
return {
|
|
507
570
|
async goto(url, timeout = 30000) {
|
|
508
571
|
assertNavigable(url, urlGuard);
|
|
572
|
+
lock?.assertAllowed(url);
|
|
509
573
|
// Refs from the previous page are about to become invalid — clear
|
|
510
574
|
// before navigating so a stale click(ref) errors clearly instead of
|
|
511
575
|
// silently resolving to whatever backendNodeId happens to still be in
|
|
@@ -523,15 +587,16 @@ export async function connect(opts = {}) {
|
|
|
523
587
|
|
|
524
588
|
browser = await launch(launchOpts);
|
|
525
589
|
cdp = await createCDP(browser.wsUrl);
|
|
590
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
526
591
|
page = await createPage(cdp, true, pageOpts);
|
|
527
592
|
setupDialogHandler(page.session);
|
|
528
593
|
await suppressPermissions(cdp);
|
|
529
594
|
currentlyHeaded = false;
|
|
530
595
|
}
|
|
531
596
|
|
|
532
|
-
lastNavigation = await navigate(page, url, timeout);
|
|
597
|
+
lastNavigation = await navigate(page, url, timeout, lock);
|
|
533
598
|
if (opts.consent !== false) {
|
|
534
|
-
const dismissed = await
|
|
599
|
+
const dismissed = await dismissConsentTracked();
|
|
535
600
|
if (!dismissed) consentRecheck = await navKey();
|
|
536
601
|
}
|
|
537
602
|
|
|
@@ -550,12 +615,13 @@ export async function connect(opts = {}) {
|
|
|
550
615
|
try {
|
|
551
616
|
browser = await launch({ ...launchOpts, headed: true });
|
|
552
617
|
cdp = await createCDP(browser.wsUrl);
|
|
618
|
+
if (lock) await applySiteLockCDP(cdp, lock);
|
|
553
619
|
page = await createPage(cdp, false, pageOpts);
|
|
554
620
|
setupDialogHandler(page.session);
|
|
555
621
|
await suppressPermissions(cdp);
|
|
556
|
-
lastNavigation = await navigate(page, url, timeout);
|
|
622
|
+
lastNavigation = await navigate(page, url, timeout, lock);
|
|
557
623
|
consentRecheck = null;
|
|
558
|
-
if (opts.consent !== false && !(await
|
|
624
|
+
if (opts.consent !== false && !(await dismissConsentTracked())) {
|
|
559
625
|
consentRecheck = await navKey();
|
|
560
626
|
}
|
|
561
627
|
|
|
@@ -573,9 +639,7 @@ export async function connect(opts = {}) {
|
|
|
573
639
|
async goBack() {
|
|
574
640
|
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
575
641
|
if (currentIndex <= 0) throw new Error('No previous page in history');
|
|
576
|
-
|
|
577
|
-
await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex - 1].id });
|
|
578
|
-
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
642
|
+
await traverse(page, entries[currentIndex - 1].id);
|
|
579
643
|
refMap = new Map(); // refs from the previous page are now invalid
|
|
580
644
|
consentRecheck = null;
|
|
581
645
|
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
@@ -584,9 +648,7 @@ export async function connect(opts = {}) {
|
|
|
584
648
|
async goForward() {
|
|
585
649
|
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
586
650
|
if (currentIndex >= entries.length - 1) throw new Error('No next page in history');
|
|
587
|
-
|
|
588
|
-
await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex + 1].id });
|
|
589
|
-
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
651
|
+
await traverse(page, entries[currentIndex + 1].id);
|
|
590
652
|
refMap = new Map();
|
|
591
653
|
consentRecheck = null;
|
|
592
654
|
lastNavigation = null;
|
|
@@ -619,20 +681,38 @@ export async function connect(opts = {}) {
|
|
|
619
681
|
consentRecheck = null; // once per navigation
|
|
620
682
|
// Only if we are still on the document the goto landed on.
|
|
621
683
|
if (key === await navKey()) {
|
|
622
|
-
try { await
|
|
684
|
+
try { await dismissConsentTracked(); } catch { /* best-effort */ }
|
|
623
685
|
}
|
|
624
686
|
}
|
|
687
|
+
// a navigation since the last snapshot leaves nothing for a `within` ref to name
|
|
688
|
+
const prevRefMap = refDoc && (await navKey() ?? refDoc) !== refDoc ? new Map() : refMap;
|
|
689
|
+
const docBefore = await navKey(); // before the tree: a navigation mid-build leaves the refs stale, not live
|
|
625
690
|
const result = await ariaTree(page);
|
|
626
691
|
refMap = result.refMap;
|
|
692
|
+
refDoc = docBefore;
|
|
627
693
|
const raw = formatTree(result.tree);
|
|
628
694
|
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
629
695
|
const pageUrl = entries[currentIndex]?.url || '';
|
|
630
696
|
const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
|
|
631
|
-
+ httpWarning(lastNavigation?.status);
|
|
697
|
+
+ httpWarning(lastNavigation?.status) + (lock?.takeNotes(page.targetId) ?? '');
|
|
632
698
|
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
633
699
|
const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
|
|
634
700
|
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
635
701
|
const mode = pruneOpts?.mode || opts.pruneMode || 'act';
|
|
702
|
+
if (pruneOpts?.map || pruneOpts?.within != null) {
|
|
703
|
+
// A ref names a node of the previous snapshot: find it again by its backend node.
|
|
704
|
+
const selectors = pruneOpts.within != null ? parseWithin(pruneOpts.within) : undefined;
|
|
705
|
+
const refNodes = new Map();
|
|
706
|
+
for (const s of selectors || []) {
|
|
707
|
+
if (!s.ref) continue;
|
|
708
|
+
const old = prevRefMap.get(s.ref);
|
|
709
|
+
if (!old) throw new Error(`within: ref ${s.ref} is not in the last snapshot (refs renumber on every snapshot and reset on navigation) — take a fresh ref from snapshot({ map: true })`);
|
|
710
|
+
const now = [...result.refMap].find(([, e]) => e.session === old.session && e.backendNodeId === old.backendNodeId);
|
|
711
|
+
if (!now) throw new Error(`within: the element behind ref ${s.ref} is no longer on the page — take a fresh ref from snapshot({ map: true })`);
|
|
712
|
+
refNodes.set(s.ref, now[0]);
|
|
713
|
+
}
|
|
714
|
+
return zoomSnapshot(tree, { head: `url: ${pageUrl}\n` + warn, map: true, selectors, refNodes, mode, links });
|
|
715
|
+
}
|
|
636
716
|
const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
|
|
637
717
|
const sized = stripLegend(out); // the legend is not content
|
|
638
718
|
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
@@ -650,15 +730,13 @@ export async function connect(opts = {}) {
|
|
|
650
730
|
},
|
|
651
731
|
|
|
652
732
|
async click(ref) {
|
|
653
|
-
const entry =
|
|
654
|
-
if (!entry) throw new Error(`No element found for ref "${ref}"`);
|
|
733
|
+
const entry = await refEntry(ref);
|
|
655
734
|
await cdpClick(entry.session, entry.backendNodeId);
|
|
656
735
|
},
|
|
657
736
|
|
|
658
|
-
/** @param {string} ref @param {string} text @param {TypeOptions} [typeOpts] */
|
|
737
|
+
/** @param {string|number} ref @param {string} text @param {TypeOptions} [typeOpts] */
|
|
659
738
|
async type(ref, text, typeOpts = {}) {
|
|
660
|
-
const entry =
|
|
661
|
-
if (!entry) throw new Error(`No element found for ref "${ref}"`);
|
|
739
|
+
const entry = await refEntry(ref);
|
|
662
740
|
await cdpType(entry.session, entry.backendNodeId, text, typeOpts);
|
|
663
741
|
},
|
|
664
742
|
|
|
@@ -671,22 +749,18 @@ export async function connect(opts = {}) {
|
|
|
671
749
|
},
|
|
672
750
|
|
|
673
751
|
async hover(ref) {
|
|
674
|
-
const entry =
|
|
675
|
-
if (!entry) throw new Error(`No element found for ref "${ref}"`);
|
|
752
|
+
const entry = await refEntry(ref);
|
|
676
753
|
await cdpHover(entry.session, entry.backendNodeId);
|
|
677
754
|
},
|
|
678
755
|
|
|
679
756
|
async select(ref, value) {
|
|
680
|
-
const entry =
|
|
681
|
-
if (!entry) throw new Error(`No element found for ref "${ref}"`);
|
|
757
|
+
const entry = await refEntry(ref);
|
|
682
758
|
await cdpSelect(entry.session, entry.backendNodeId, value);
|
|
683
759
|
},
|
|
684
760
|
|
|
685
761
|
async drag(fromRef, toRef) {
|
|
686
|
-
const from =
|
|
687
|
-
const to =
|
|
688
|
-
if (!from) throw new Error(`No element found for ref "${fromRef}"`);
|
|
689
|
-
if (!to) throw new Error(`No element found for ref "${toRef}"`);
|
|
762
|
+
const from = await refEntry(fromRef);
|
|
763
|
+
const to = await refEntry(toRef);
|
|
690
764
|
// Drag across different frames isn't physically meaningful — bail
|
|
691
765
|
// rather than mix sessions and produce nonsense coordinates.
|
|
692
766
|
if (from.session !== to.session) {
|
|
@@ -696,8 +770,7 @@ export async function connect(opts = {}) {
|
|
|
696
770
|
},
|
|
697
771
|
|
|
698
772
|
async upload(ref, files) {
|
|
699
|
-
const entry =
|
|
700
|
-
if (!entry) throw new Error(`No element found for ref "${ref}"`);
|
|
773
|
+
const entry = await refEntry(ref);
|
|
701
774
|
// Upload sandbox: when uploadDir is set, every path must resolve
|
|
702
775
|
// (symlinks included, via realpath) inside it. Stops a prompt-injected
|
|
703
776
|
// agent from attaching ~/.ssh/id_rsa or other arbitrary local files.
|
|
@@ -854,7 +927,8 @@ export async function connect(opts = {}) {
|
|
|
854
927
|
return {
|
|
855
928
|
async goto(url, timeout = 30000) {
|
|
856
929
|
assertNavigable(url, urlGuard);
|
|
857
|
-
|
|
930
|
+
lock?.assertAllowed(url);
|
|
931
|
+
const outcome = await navigate(tab, url, timeout, lock);
|
|
858
932
|
if (opts.consent !== false) {
|
|
859
933
|
await dismissConsent(tab.session);
|
|
860
934
|
}
|
|
@@ -940,10 +1014,11 @@ async function suppressPermissions(cdp) {
|
|
|
940
1014
|
* v0.16.0. Still Firefox-limited: `reload({ignoreCache})` (upstream BiDi gap).
|
|
941
1015
|
* The navigation guard (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir`
|
|
942
1016
|
* sandbox, `incognito`, `proxy`, `viewport`, and `pruneMode` all apply.
|
|
943
|
-
* @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls })
|
|
1017
|
+
* @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls, allowHosts })
|
|
944
1018
|
*/
|
|
945
1019
|
async function connectFirefox(opts) {
|
|
946
1020
|
const hybrid = opts.mode === 'hybrid';
|
|
1021
|
+
const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null; // validates before any launch
|
|
947
1022
|
const blockOpts = {
|
|
948
1023
|
blockAds: opts.blockAds !== undefined ? opts.blockAds : true,
|
|
949
1024
|
blockUrls: opts.blockUrls,
|
|
@@ -972,7 +1047,7 @@ async function connectFirefox(opts) {
|
|
|
972
1047
|
});
|
|
973
1048
|
const bidi = await createBiDi(browser.wsUrl);
|
|
974
1049
|
if (!headed) await applyFirefoxStealth(bidi);
|
|
975
|
-
await applyFirefoxBlocklist(bidi, blockOpts);
|
|
1050
|
+
await applyFirefoxBlocklist(bidi, blockOpts, lock);
|
|
976
1051
|
const { contexts } = await bidi.send('browsingContext.getTree', {});
|
|
977
1052
|
return { browser, bidi, topContext: contexts[0].context };
|
|
978
1053
|
}
|
|
@@ -997,6 +1072,7 @@ async function connectFirefox(opts) {
|
|
|
997
1072
|
links: opts.links,
|
|
998
1073
|
urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
|
|
999
1074
|
uploadDir: opts.uploadDir || null,
|
|
1075
|
+
siteLock: lock,
|
|
1000
1076
|
incognito: !!opts.incognito,
|
|
1001
1077
|
consent: opts.consent,
|
|
1002
1078
|
hybrid,
|
|
@@ -1161,13 +1237,45 @@ export function _resetBlocklistWarning() {
|
|
|
1161
1237
|
blocklistWarned = false;
|
|
1162
1238
|
}
|
|
1163
1239
|
|
|
1240
|
+
/**
|
|
1241
|
+
* Go to a history entry and wait until the traversal has landed. A bfcache
|
|
1242
|
+
* restore or a same-document entry never fires Page.loadEventFired, so wait
|
|
1243
|
+
* for the main frame's Page.frameStoppedLoading instead: it follows the load
|
|
1244
|
+
* event on a normal load and fires on both of those paths. Subscribed before
|
|
1245
|
+
* the navigate call so it cannot be missed; times out at 30s like before.
|
|
1246
|
+
* @param {*} page
|
|
1247
|
+
* @param {string} entryId history entry id (from Page.getNavigationHistory)
|
|
1248
|
+
*/
|
|
1249
|
+
async function traverse(page, entryId) {
|
|
1250
|
+
const { session } = page;
|
|
1251
|
+
const { frameTree } = await session.send('Page.getFrameTree');
|
|
1252
|
+
const mainId = frameTree.frame.id;
|
|
1253
|
+
let off = () => {};
|
|
1254
|
+
const landed = new Promise((resolve, reject) => {
|
|
1255
|
+
const timer = setTimeout(() => { off(); reject(new Error('timeout')); }, 30000);
|
|
1256
|
+
off = session.on('Page.frameStoppedLoading', (p) => {
|
|
1257
|
+
if (p.frameId === mainId) { clearTimeout(timer); off(); resolve(undefined); }
|
|
1258
|
+
});
|
|
1259
|
+
});
|
|
1260
|
+
landed.catch(() => {});
|
|
1261
|
+
try {
|
|
1262
|
+
await session.send('Page.navigateToHistoryEntry', { entryId });
|
|
1263
|
+
} catch (e) { off(); throw e; }
|
|
1264
|
+
try { await landed; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1164
1267
|
/**
|
|
1165
1268
|
* Navigate to a URL and wait for the page to load. Resolves to the outcome of
|
|
1166
1269
|
* the main-document navigation (final URL after redirects, HTTP status);
|
|
1167
|
-
* rejects with a NAV_FAILED error when the browser could not reach a document
|
|
1270
|
+
* rejects with a NAV_FAILED error when the browser could not reach a document,
|
|
1271
|
+
* or NAV_BLOCKED when the site lock refused the URL or a redirect hop.
|
|
1272
|
+
* @param {*} page
|
|
1273
|
+
* @param {string} url
|
|
1274
|
+
* @param {number} [timeout]
|
|
1275
|
+
* @param {import('./site-lock.js').SiteLock|null} [lock]
|
|
1168
1276
|
* @returns {Promise<NavigationOutcome>}
|
|
1169
1277
|
*/
|
|
1170
|
-
async function navigate(page, url, timeout = 30000) {
|
|
1278
|
+
async function navigate(page, url, timeout = 30000, lock = null) {
|
|
1171
1279
|
const { session } = page;
|
|
1172
1280
|
// Main-document responses (every redirect hop fires one; the last wins).
|
|
1173
1281
|
const docs = [];
|
|
@@ -1182,7 +1290,12 @@ async function navigate(page, url, timeout = 30000) {
|
|
|
1182
1290
|
}));
|
|
1183
1291
|
loaded.catch(() => {}); // rejection is only read if we await it below
|
|
1184
1292
|
try {
|
|
1293
|
+
lock?.startGoto(page.targetId);
|
|
1185
1294
|
const nav = await session.send('Page.navigate', { url });
|
|
1295
|
+
// A lock-refused document (the URL or a redirect hop) also surfaces as
|
|
1296
|
+
// ERR_ABORTED, so this must come before the aborted branch below.
|
|
1297
|
+
const blocked = lock?.takeGotoBlock(page.targetId);
|
|
1298
|
+
if (blocked) throw navBlocked(blocked.url, blocked.host);
|
|
1186
1299
|
// ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
|
|
1187
1300
|
// no document and fires no load event, so don't wait for one.
|
|
1188
1301
|
const aborted = nav.errorText === 'net::ERR_ABORTED';
|
|
@@ -1191,7 +1304,9 @@ async function navigate(page, url, timeout = 30000) {
|
|
|
1191
1304
|
// response carries the status), so report it like any other HTTP error.
|
|
1192
1305
|
const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
|
|
1193
1306
|
if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
|
|
1194
|
-
|
|
1307
|
+
// A same-document navigation (fragment-only change) commits no new document: Page.navigate
|
|
1308
|
+
// returns no loaderId and no load event follows, so don't wait for one.
|
|
1309
|
+
if (!aborted && nav.loaderId) await loaded;
|
|
1195
1310
|
// Brief settle time for dynamic content
|
|
1196
1311
|
await new Promise((r) => setTimeout(r, 500));
|
|
1197
1312
|
// loaderId is shared by every redirect hop of one navigation.
|
package/src/navigation.js
CHANGED
|
@@ -23,6 +23,18 @@ export function navFailed(url, reason) {
|
|
|
23
23
|
return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Typed error for a top-level navigation the site lock (`allowHosts`) refused:
|
|
28
|
+
* the target, or a redirect hop of it, is off the allowed list. `code` is
|
|
29
|
+
* 'NAV_BLOCKED'; `host` is the refused hostname. The page stays where it was.
|
|
30
|
+
* @param {string} url - The refused URL (the redirect target, if it was a hop).
|
|
31
|
+
* @param {string} host
|
|
32
|
+
* @returns {Error & {code: 'NAV_BLOCKED', host: string}}
|
|
33
|
+
*/
|
|
34
|
+
export function navBlocked(url, host) {
|
|
35
|
+
return Object.assign(new Error(`Navigation to ${url} blocked: ${host} is not in allowHosts`), { code: /** @type {const} */ ('NAV_BLOCKED'), host });
|
|
36
|
+
}
|
|
37
|
+
|
|
26
38
|
/**
|
|
27
39
|
* True for a final HTTP status that is not 2xx (0 = unknown is not an error).
|
|
28
40
|
* @param {number|undefined} status
|
package/src/prune.js
CHANGED
|
@@ -62,6 +62,7 @@ const SKIP_ROLES = new Set([
|
|
|
62
62
|
* @param {object} [options]
|
|
63
63
|
* @param {'act'|'browse'|'navigate'|'full'|'read'} [options.mode='act'] - Pruning mode ('read' is an alias for 'browse')
|
|
64
64
|
* @param {string} [options.context=''] - Search context for relevance filtering
|
|
65
|
+
* @param {boolean} [options.whole=false] - `tree` is a zoom box (snapshot `within`): it is the whole page, so the landmark step (1) is skipped
|
|
65
66
|
* @param {boolean} [options.links=false] - Links carry `urls` (see links.js): nodes that drop their children keep the URLs of links beneath them
|
|
66
67
|
* @returns {object|null} Pruned tree
|
|
67
68
|
*/
|
|
@@ -77,8 +78,8 @@ export function prune(tree, options = {}) {
|
|
|
77
78
|
// Wrap as array for pipeline
|
|
78
79
|
let nodes = tree ? [tree] : [];
|
|
79
80
|
|
|
80
|
-
// Step 1: Extract landmark regions
|
|
81
|
-
nodes = extractRegions(nodes, allowedRegions);
|
|
81
|
+
// Step 1: Extract landmark regions (a zoom box is the whole page: keep it)
|
|
82
|
+
if (!options.whole) nodes = extractRegions(nodes, allowedRegions);
|
|
82
83
|
|
|
83
84
|
// Step 2: Prune nodes
|
|
84
85
|
const ctx = { mode, parentRole: null, keywords, links: !!options.links };
|
package/src/site-lock.js
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* site-lock.js — Optional top-frame site lock (`allowHosts`).
|
|
3
|
+
*
|
|
4
|
+
* Top-level navigations (goto, redirect hops, clicked links, JS jumps, popups)
|
|
5
|
+
* to a host outside the allow list are failed in the browser before any request
|
|
6
|
+
* leaves, so the page stays put. Deliberately NOT locked: subresources
|
|
7
|
+
* (img/script/css/XHR/fetch), iframes, WebSocket — the lock stops an agent
|
|
8
|
+
* being steered to another site, not the page from loading its own assets.
|
|
9
|
+
* The engine wiring lives next to each transport (applySiteLockCDP here,
|
|
10
|
+
* blocklist-firefox.js for BiDi); this file is the shared policy + state.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { navBlocked } from './navigation.js';
|
|
14
|
+
|
|
15
|
+
const HOST_RE = /^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$/;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Validate and normalize an allow list. Entry = `host` or `*.host`; anything
|
|
19
|
+
* else (scheme, port, path, spaces, empty, bare `*`) throws — a typo must never
|
|
20
|
+
* silently widen or drop the lock.
|
|
21
|
+
* @param {string[]} list
|
|
22
|
+
* @returns {string[]} lowercase entries
|
|
23
|
+
*/
|
|
24
|
+
export function parseAllowHosts(list) {
|
|
25
|
+
if (!Array.isArray(list) || !list.length) throw new Error('allowHosts must be a non-empty array of hostnames');
|
|
26
|
+
return list.map((raw) => {
|
|
27
|
+
const e = typeof raw === 'string' ? raw.trim().toLowerCase() : raw;
|
|
28
|
+
const host = typeof e === 'string' && e.startsWith('*.') ? e.slice(2) : e;
|
|
29
|
+
if (typeof host !== 'string' || !HOST_RE.test(host) || host.includes('..')) {
|
|
30
|
+
throw new Error(`allowHosts: invalid entry ${JSON.stringify(raw)} (expected "host" or "*.host", no scheme/port/path)`);
|
|
31
|
+
}
|
|
32
|
+
return e;
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Build a URL matcher from an allow list. Non-http(s) URLs (about:, data:,
|
|
38
|
+
* blob:) are allowed; `host` matches that host exactly, `*.host` matches its
|
|
39
|
+
* strict subdomains only (not `host` itself). An unparseable URL is refused.
|
|
40
|
+
* @param {string[]} list
|
|
41
|
+
* @returns {(url: string) => boolean}
|
|
42
|
+
*/
|
|
43
|
+
export function makeHostMatcher(list) {
|
|
44
|
+
const entries = parseAllowHosts(list);
|
|
45
|
+
const exact = new Set(entries.filter((e) => !e.startsWith('*.')));
|
|
46
|
+
const suffixes = entries.filter((e) => e.startsWith('*.')).map((e) => e.slice(1)); // ".host"
|
|
47
|
+
return (url) => {
|
|
48
|
+
let u;
|
|
49
|
+
try { u = new URL(url); } catch { return false; }
|
|
50
|
+
if (u.protocol !== 'http:' && u.protocol !== 'https:') return true;
|
|
51
|
+
const host = u.hostname.replace(/\.$/, '');
|
|
52
|
+
return exact.has(host) || suffixes.some((s) => host.endsWith(s));
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The lock's shared state, one per connect()/browse() call, surviving
|
|
58
|
+
* relaunches and new tabs (each transport just keeps calling record()).
|
|
59
|
+
* @param {string[]} list
|
|
60
|
+
*/
|
|
61
|
+
export function createSiteLock(list) {
|
|
62
|
+
const allowed = makeHostMatcher(list);
|
|
63
|
+
const hostOf = (url) => { try { return new URL(url).hostname; } catch { return url; } };
|
|
64
|
+
/** @type {{url: string, host: string, ctx: string}[]} */
|
|
65
|
+
let gotoBlocks = [];
|
|
66
|
+
// Snapshot notes belong to the tab/context where the block happened, so one
|
|
67
|
+
// tab's (or a popup's, shown if the agent switches to it) never leak into another's.
|
|
68
|
+
/** @type {Map<string, Set<string>>} */
|
|
69
|
+
const noted = new Map();
|
|
70
|
+
return {
|
|
71
|
+
allowed,
|
|
72
|
+
/** @param {string} url @throws NAV_BLOCKED when url is off the list */
|
|
73
|
+
assertAllowed(url) {
|
|
74
|
+
if (!allowed(url)) throw navBlocked(url, hostOf(url));
|
|
75
|
+
},
|
|
76
|
+
/** A top-level document request was refused. ctx = tab/context it belongs to. */
|
|
77
|
+
record(url, ctx) {
|
|
78
|
+
const host = hostOf(url);
|
|
79
|
+
gotoBlocks.push({ url, host, ctx });
|
|
80
|
+
const hosts = noted.get(ctx) ?? new Set();
|
|
81
|
+
noted.set(ctx, hosts.add(host));
|
|
82
|
+
},
|
|
83
|
+
/** Call right before a goto navigates; drops that tab's pending notes too. */
|
|
84
|
+
startGoto(ctx) { gotoBlocks = []; noted.delete(ctx); },
|
|
85
|
+
/**
|
|
86
|
+
* The block that hit this goto's own tab (a popup's block is not the
|
|
87
|
+
* goto's), consumed; it is reported as NAV_BLOCKED, so no snapshot note.
|
|
88
|
+
*/
|
|
89
|
+
takeGotoBlock(ctx) {
|
|
90
|
+
const b = gotoBlocks.find((x) => x.ctx === ctx);
|
|
91
|
+
if (!b) return null;
|
|
92
|
+
gotoBlocks = gotoBlocks.filter((x) => x !== b);
|
|
93
|
+
noted.get(ctx)?.delete(b.host);
|
|
94
|
+
return { url: b.url, host: b.host };
|
|
95
|
+
},
|
|
96
|
+
/** Snapshot lines for this tab's in-page blocked jumps since the last call. */
|
|
97
|
+
takeNotes(ctx) {
|
|
98
|
+
const out = [...(noted.get(ctx) ?? [])].map((h) => `[blocked navigation to ${h}]\n`).join('');
|
|
99
|
+
noted.delete(ctx);
|
|
100
|
+
return out;
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** @typedef {ReturnType<typeof createSiteLock>} SiteLock */
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Chromium: lock every page target of this browser. A BROWSER-level
|
|
109
|
+
* Fetch.enable (root client, no sessionId) pauses document requests of every
|
|
110
|
+
* target, popups included, so one handler covers tabs we create later and
|
|
111
|
+
* windows the page opens itself. A request is top-level when its frameId is a
|
|
112
|
+
* page target id (OOPIF iframes are 'iframe' targets, subframes share their
|
|
113
|
+
* parent's id space). 'Aborted' (not BlockedByClient) fails the request but
|
|
114
|
+
* keeps the old page instead of swapping in a chrome-error:// page.
|
|
115
|
+
* Call once per CDP connection, before the first navigation.
|
|
116
|
+
* @param {object} cdp - root CDP client
|
|
117
|
+
* @param {SiteLock} lock
|
|
118
|
+
*/
|
|
119
|
+
export async function applySiteLockCDP(cdp, lock) {
|
|
120
|
+
const pageTargets = new Set();
|
|
121
|
+
const subframes = new Set(); // ids a refresh proved are not page targets
|
|
122
|
+
cdp.on('Target.targetCreated', ({ targetInfo }) => {
|
|
123
|
+
if (targetInfo.type === 'page') pageTargets.add(targetInfo.targetId);
|
|
124
|
+
});
|
|
125
|
+
cdp.on('Target.targetDestroyed', ({ targetId }) => pageTargets.delete(targetId));
|
|
126
|
+
await cdp.send('Target.setDiscoverTargets', { discover: true });
|
|
127
|
+
|
|
128
|
+
cdp.on('Fetch.requestPaused', async (p, sessionId) => {
|
|
129
|
+
// A popup's create event can lag its first request: ask the browser
|
|
130
|
+
// before treating an unknown frame as a subframe (and if that fails,
|
|
131
|
+
// fail closed: treat it as top-level).
|
|
132
|
+
if (!pageTargets.has(p.frameId) && !subframes.has(p.frameId)) {
|
|
133
|
+
try {
|
|
134
|
+
const { targetInfos } = await cdp.send('Target.getTargets');
|
|
135
|
+
for (const t of targetInfos) if (t.type === 'page') pageTargets.add(t.targetId);
|
|
136
|
+
if (!pageTargets.has(p.frameId)) subframes.add(p.frameId);
|
|
137
|
+
} catch { pageTargets.add(p.frameId); }
|
|
138
|
+
}
|
|
139
|
+
const ok = !pageTargets.has(p.frameId) || lock.allowed(p.request.url);
|
|
140
|
+
if (!ok) lock.record(p.request.url, p.frameId);
|
|
141
|
+
try {
|
|
142
|
+
if (ok) await cdp.send('Fetch.continueRequest', { requestId: p.requestId }, sessionId);
|
|
143
|
+
else await cdp.send('Fetch.failRequest', { requestId: p.requestId, errorReason: 'Aborted' }, sessionId);
|
|
144
|
+
} catch { /* target closed between the pause and our reply */ }
|
|
145
|
+
});
|
|
146
|
+
await cdp.send('Fetch.enable', { patterns: [{ resourceType: 'Document', requestStage: 'Request' }] });
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Firefox/BiDi: build the "is this paused request a locked top-level load?"
|
|
151
|
+
* check that blocklist-firefox.js's single intercept handler calls (two
|
|
152
|
+
* catch-all handlers on one paused request would race — one continues before
|
|
153
|
+
* the other fails). A request is a top-level document load when it carries a
|
|
154
|
+
* navigation id and its context has no parent. Context ids are tracked from
|
|
155
|
+
* contextCreated and refreshed from getTree when an unknown one shows up, so a
|
|
156
|
+
* popup whose create event lags its first request can't slip through; one that
|
|
157
|
+
* is still unknown after the refresh is treated as top-level (fail closed).
|
|
158
|
+
* Redirect hops arrive as their own beforeRequestSent, so each is checked.
|
|
159
|
+
* @param {object} bidi - BiDi client
|
|
160
|
+
* @param {SiteLock} lock
|
|
161
|
+
* @returns {Promise<(e: object) => Promise<boolean>>} true = fail this request
|
|
162
|
+
*/
|
|
163
|
+
export async function createFirefoxLockCheck(bidi, lock) {
|
|
164
|
+
const tops = new Set();
|
|
165
|
+
const known = new Set();
|
|
166
|
+
async function refresh() {
|
|
167
|
+
const { contexts } = await bidi.send('browsingContext.getTree', {});
|
|
168
|
+
(function walk(nodes, top) {
|
|
169
|
+
for (const n of nodes) { known.add(n.context); if (top) tops.add(n.context); walk(n.children || [], false); }
|
|
170
|
+
})(contexts, true);
|
|
171
|
+
}
|
|
172
|
+
bidi.on('browsingContext.contextCreated', (c) => { known.add(c.context); if (!c.parent) tops.add(c.context); });
|
|
173
|
+
bidi.on('browsingContext.contextDestroyed', (c) => { known.delete(c.context); tops.delete(c.context); });
|
|
174
|
+
await bidi.subscribe(['browsingContext.contextCreated', 'browsingContext.contextDestroyed']);
|
|
175
|
+
await refresh();
|
|
176
|
+
return async (e) => {
|
|
177
|
+
if (e.navigation == null) return false; // subresource
|
|
178
|
+
if (!known.has(e.context)) { try { await refresh(); } catch { tops.add(e.context); } }
|
|
179
|
+
if (known.has(e.context) && !tops.has(e.context)) return false; // iframe
|
|
180
|
+
if (lock.allowed(e.request.url)) return false;
|
|
181
|
+
lock.record(e.request.url, e.context);
|
|
182
|
+
return true;
|
|
183
|
+
};
|
|
184
|
+
}
|