barebrowse 0.26.0 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js CHANGED
@@ -18,6 +18,7 @@ import { authenticate } from './auth.js';
18
18
  import { prune as pruneTree } from './prune.js';
19
19
  import { dataSnapshot, stripLegend } from './data.js';
20
20
  import { annotateLinks } from './links.js';
21
+ import { parseWithin, zoomSnapshot } from './zoom.js';
21
22
  import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
22
23
  import { dismissConsent } from './consent.js';
23
24
  import { applyStealth } from './stealth.js';
@@ -29,7 +30,8 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
29
30
  import { isChallengePage } from './challenge.js';
30
31
  import { withTimeout } from './with-timeout.js';
31
32
  import { readable as extractReadable } from './readable.js';
32
- import { navFailed, httpWarning } from './navigation.js';
33
+ import { navFailed, navBlocked, httpWarning } from './navigation.js';
34
+ import { createSiteLock, applySiteLockCDP } from './site-lock.js';
33
35
  import { assertNavigable, assertUploadAllowed } from './url-guard.js';
34
36
  import { join as pathJoin } from 'node:path';
35
37
  import { chmodSync } from 'node:fs';
@@ -63,6 +65,9 @@ import { chmodSync } from 'node:fs';
63
65
  * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
64
66
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
65
67
  * @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
68
+ * @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (the URL, its
69
+ * redirects) to hosts outside this list reject with NAV_BLOCKED. Entries are `host`
70
+ * or `*.host` (strict subdomains). Subresources and iframes are not locked.
66
71
  * @returns {Promise<string>} ARIA snapshot text
67
72
  */
68
73
  export async function browse(url, opts = {}) {
@@ -74,6 +79,8 @@ export async function browse(url, opts = {}) {
74
79
  // Reject local-resource schemes (and optionally private hosts) before we
75
80
  // spend a browser launch on a URL we won't navigate to.
76
81
  assertNavigable(url, { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork });
82
+ const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
83
+ lock?.assertAllowed(url);
77
84
 
78
85
  let browser = null;
79
86
  let cdp = null;
@@ -91,6 +98,7 @@ export async function browse(url, opts = {}) {
91
98
  browser = await launch(launchOpts);
92
99
  cdp = await createCDP(browser.wsUrl);
93
100
  }
101
+ if (lock) await applySiteLockCDP(cdp, lock);
94
102
 
95
103
  // Step 2: Create a new page target and attach
96
104
  const pageOpts = { viewport: opts.viewport, blockAds: opts.blockAds, blockUrls: opts.blockUrls };
@@ -109,7 +117,7 @@ export async function browse(url, opts = {}) {
109
117
  }
110
118
 
111
119
  // Step 4: Navigate and wait for load
112
- let outcome = await navigate(page, url, timeout);
120
+ let outcome = await navigate(page, url, timeout, lock);
113
121
 
114
122
  // Step 4.5: Auto-dismiss cookie consent dialogs
115
123
  if (opts.consent !== false) {
@@ -128,12 +136,13 @@ export async function browse(url, opts = {}) {
128
136
  try {
129
137
  browser = await launch({ ...launchOpts, headed: true });
130
138
  cdp = await createCDP(browser.wsUrl);
139
+ if (lock) await applySiteLockCDP(cdp, lock);
131
140
  page = await createPage(cdp, false, pageOpts);
132
141
  await suppressPermissions(cdp);
133
142
  if (!noAuth) {
134
143
  try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
135
144
  }
136
- outcome = await navigate(page, url, timeout);
145
+ outcome = await navigate(page, url, timeout, lock);
137
146
  if (opts.consent !== false) await dismissConsent(page.session);
138
147
  ({ tree } = await ariaTree(page));
139
148
  } catch {
@@ -226,7 +235,14 @@ export async function browse(url, opts = {}) {
226
235
  * pruning and return the raw ARIA tree.
227
236
  * `links: true` adds each link's absolute URL after its name (default off,
228
237
  * or the `connect({ links })` default).
229
- * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
238
+ * `map: true` returns an outline of the page's big boxes (ref + size) instead of
239
+ * content. `within` zooms the snapshot to one box, or several (an array, output
240
+ * in page order): a ref from the map, a role (`'table'`), role + accessible name
241
+ * (`'list "Issues"'`) or an unnamed box by the heading before it
242
+ * (`'list after "Other flights"'`). Any `mode` and `links` apply inside a zoom,
243
+ * where the box is the whole page. Two or more matches return the map with the
244
+ * matches marked; none, or a ref that is gone, throws. `within` wins over `map`.
245
+ * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean, map?: boolean, within?: string|number|Array<string|number> }} SnapshotOptions
230
246
  */
231
247
 
232
248
  /**
@@ -281,6 +297,11 @@ export async function browse(url, opts = {}) {
281
297
  * ever enters the session even if a caller injects unconditionally.
282
298
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
283
299
  * @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
300
+ * @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (goto, redirects,
301
+ * clicked links, JS jumps, popups) to hosts outside this list are refused and the page
302
+ * stays put. Entries are `host` or `*.host` (strict subdomains). goto() rejects with
303
+ * NAV_BLOCKED; in-page blocks add a `[blocked navigation to <host>]` line to the next
304
+ * snapshot. Subresources, iframes and WebSocket are not locked. Not allowed with `port`.
284
305
  * @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
285
306
  * 'firefox' drives over WebDriver BiDi (a separate transport / page object);
286
307
  * 'chromium' (default) uses CDP.
@@ -339,6 +360,12 @@ export async function connect(opts = {}) {
339
360
 
340
361
  const mode = opts.mode || 'headless';
341
362
  const attachMode = !!opts.port;
363
+ // Validate before launching anything. Attach mode is refused: the lock is a
364
+ // browser-level intercept that would reach the user's own tabs.
365
+ if (opts.allowHosts && attachMode) {
366
+ throw new Error('allowHosts is not supported in attach mode (port): the site lock is browser-wide and would affect your own tabs');
367
+ }
368
+ const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
342
369
  // Incognito: run a clean, unauthenticated session. Skips storageState loading
343
370
  // and neuters injectCookies() so no auth ever enters the session — even when
344
371
  // a caller (MCP goto, daemon) calls injectCookies() unconditionally. The
@@ -370,6 +397,7 @@ export async function connect(opts = {}) {
370
397
  browser = await launch(launchOpts);
371
398
  cdp = await createCDP(browser.wsUrl);
372
399
  }
400
+ if (lock) await applySiteLockCDP(cdp, lock);
373
401
 
374
402
  // In attach mode we don't know (and shouldn't assume) the user's headed/
375
403
  // headless state — treat it as headed so stealth patches are skipped
@@ -386,6 +414,9 @@ export async function connect(opts = {}) {
386
414
  };
387
415
  let page = await createPage(cdp, !currentlyHeaded, pageOpts);
388
416
  let refMap = new Map();
417
+ // Main-document identity (navKey) when the current refs were handed out; read again
418
+ // fresh at every ref use, so a click/press that navigated to a new document kills them.
419
+ let refDoc = null;
389
420
  let botBlocked = false;
390
421
  // Outcome of the latest goto() ({url, status}); null before any goto.
391
422
  /** @type {NavigationOutcome|null} */
@@ -406,6 +437,38 @@ export async function connect(opts = {}) {
406
437
  } catch { return null; }
407
438
  }
408
439
 
440
+ /** Resolve a ref from the last snapshot; throws when the document it belongs to is gone. */
441
+ async function refEntry(ref) {
442
+ const entry = refMap.get(String(ref));
443
+ if (!entry) throw new Error(`No element found for ref "${ref}"`);
444
+ if (refDoc) {
445
+ const now = await navKey();
446
+ if (now && now !== refDoc) {
447
+ refMap = new Map(); refDoc = null;
448
+ throw new Error(`No element found for ref "${ref}" — the page navigated since the snapshot that gave it; take a fresh snapshot`);
449
+ }
450
+ }
451
+ return entry;
452
+ }
453
+
454
+ // A consent accept that navigates (a form submit, Google's consent.google.com) lands on
455
+ // the real page: report that document in lastNavigation, not the interstitial's.
456
+ async function dismissConsentTracked() {
457
+ const docs = [];
458
+ let off = () => {};
459
+ try {
460
+ const { frameTree } = await page.session.send('Page.getFrameTree');
461
+ off = page.session.on('Network.responseReceived', (p) => {
462
+ if (p.type === 'Document' && p.frameId === frameTree.frame.id) docs.push(p);
463
+ });
464
+ } catch { /* no tracking; dismissal itself still runs */ }
465
+ try { return await dismissConsent(page.session); } finally {
466
+ off();
467
+ const last = docs.pop();
468
+ if (last) lastNavigation = { url: last.response.url, status: last.response.status };
469
+ }
470
+ }
471
+
409
472
  // Suppress permission prompts. Skipped in attach mode — Browser.setPermission
410
473
  // is browser-wide (no origin scope here), so flipping permissions to denied
411
474
  // would leak into the user's other tabs.
@@ -506,6 +569,7 @@ export async function connect(opts = {}) {
506
569
  return {
507
570
  async goto(url, timeout = 30000) {
508
571
  assertNavigable(url, urlGuard);
572
+ lock?.assertAllowed(url);
509
573
  // Refs from the previous page are about to become invalid — clear
510
574
  // before navigating so a stale click(ref) errors clearly instead of
511
575
  // silently resolving to whatever backendNodeId happens to still be in
@@ -523,15 +587,16 @@ export async function connect(opts = {}) {
523
587
 
524
588
  browser = await launch(launchOpts);
525
589
  cdp = await createCDP(browser.wsUrl);
590
+ if (lock) await applySiteLockCDP(cdp, lock);
526
591
  page = await createPage(cdp, true, pageOpts);
527
592
  setupDialogHandler(page.session);
528
593
  await suppressPermissions(cdp);
529
594
  currentlyHeaded = false;
530
595
  }
531
596
 
532
- lastNavigation = await navigate(page, url, timeout);
597
+ lastNavigation = await navigate(page, url, timeout, lock);
533
598
  if (opts.consent !== false) {
534
- const dismissed = await dismissConsent(page.session);
599
+ const dismissed = await dismissConsentTracked();
535
600
  if (!dismissed) consentRecheck = await navKey();
536
601
  }
537
602
 
@@ -550,12 +615,13 @@ export async function connect(opts = {}) {
550
615
  try {
551
616
  browser = await launch({ ...launchOpts, headed: true });
552
617
  cdp = await createCDP(browser.wsUrl);
618
+ if (lock) await applySiteLockCDP(cdp, lock);
553
619
  page = await createPage(cdp, false, pageOpts);
554
620
  setupDialogHandler(page.session);
555
621
  await suppressPermissions(cdp);
556
- lastNavigation = await navigate(page, url, timeout);
622
+ lastNavigation = await navigate(page, url, timeout, lock);
557
623
  consentRecheck = null;
558
- if (opts.consent !== false && !(await dismissConsent(page.session))) {
624
+ if (opts.consent !== false && !(await dismissConsentTracked())) {
559
625
  consentRecheck = await navKey();
560
626
  }
561
627
 
@@ -573,9 +639,7 @@ export async function connect(opts = {}) {
573
639
  async goBack() {
574
640
  const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
575
641
  if (currentIndex <= 0) throw new Error('No previous page in history');
576
- const loadPromise = page.session.once('Page.loadEventFired', 30000);
577
- await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex - 1].id });
578
- try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
642
+ await traverse(page, entries[currentIndex - 1].id);
579
643
  refMap = new Map(); // refs from the previous page are now invalid
580
644
  consentRecheck = null;
581
645
  lastNavigation = null; // no outcome is known for the history entry we landed on
@@ -584,9 +648,7 @@ export async function connect(opts = {}) {
584
648
  async goForward() {
585
649
  const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
586
650
  if (currentIndex >= entries.length - 1) throw new Error('No next page in history');
587
- const loadPromise = page.session.once('Page.loadEventFired', 30000);
588
- await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex + 1].id });
589
- try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
651
+ await traverse(page, entries[currentIndex + 1].id);
590
652
  refMap = new Map();
591
653
  consentRecheck = null;
592
654
  lastNavigation = null;
@@ -619,20 +681,38 @@ export async function connect(opts = {}) {
619
681
  consentRecheck = null; // once per navigation
620
682
  // Only if we are still on the document the goto landed on.
621
683
  if (key === await navKey()) {
622
- try { await dismissConsent(page.session); } catch { /* best-effort */ }
684
+ try { await dismissConsentTracked(); } catch { /* best-effort */ }
623
685
  }
624
686
  }
687
+ // a navigation since the last snapshot leaves nothing for a `within` ref to name
688
+ const prevRefMap = refDoc && (await navKey() ?? refDoc) !== refDoc ? new Map() : refMap;
689
+ const docBefore = await navKey(); // before the tree: a navigation mid-build leaves the refs stale, not live
625
690
  const result = await ariaTree(page);
626
691
  refMap = result.refMap;
692
+ refDoc = docBefore;
627
693
  const raw = formatTree(result.tree);
628
694
  const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
629
695
  const pageUrl = entries[currentIndex]?.url || '';
630
696
  const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
631
- + httpWarning(lastNavigation?.status);
697
+ + httpWarning(lastNavigation?.status) + (lock?.takeNotes(page.targetId) ?? '');
632
698
  const links = !!((pruneOpts || undefined)?.links ?? opts.links);
633
699
  const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
634
700
  if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
635
701
  const mode = pruneOpts?.mode || opts.pruneMode || 'act';
702
+ if (pruneOpts?.map || pruneOpts?.within != null) {
703
+ // A ref names a node of the previous snapshot: find it again by its backend node.
704
+ const selectors = pruneOpts.within != null ? parseWithin(pruneOpts.within) : undefined;
705
+ const refNodes = new Map();
706
+ for (const s of selectors || []) {
707
+ if (!s.ref) continue;
708
+ const old = prevRefMap.get(s.ref);
709
+ if (!old) throw new Error(`within: ref ${s.ref} is not in the last snapshot (refs renumber on every snapshot and reset on navigation) — take a fresh ref from snapshot({ map: true })`);
710
+ const now = [...result.refMap].find(([, e]) => e.session === old.session && e.backendNodeId === old.backendNodeId);
711
+ if (!now) throw new Error(`within: the element behind ref ${s.ref} is no longer on the page — take a fresh ref from snapshot({ map: true })`);
712
+ refNodes.set(s.ref, now[0]);
713
+ }
714
+ return zoomSnapshot(tree, { head: `url: ${pageUrl}\n` + warn, map: true, selectors, refNodes, mode, links });
715
+ }
636
716
  const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
637
717
  const sized = stripLegend(out); // the legend is not content
638
718
  const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
@@ -650,15 +730,13 @@ export async function connect(opts = {}) {
650
730
  },
651
731
 
652
732
  async click(ref) {
653
- const entry = refMap.get(ref);
654
- if (!entry) throw new Error(`No element found for ref "${ref}"`);
733
+ const entry = await refEntry(ref);
655
734
  await cdpClick(entry.session, entry.backendNodeId);
656
735
  },
657
736
 
658
- /** @param {string} ref @param {string} text @param {TypeOptions} [typeOpts] */
737
+ /** @param {string|number} ref @param {string} text @param {TypeOptions} [typeOpts] */
659
738
  async type(ref, text, typeOpts = {}) {
660
- const entry = refMap.get(ref);
661
- if (!entry) throw new Error(`No element found for ref "${ref}"`);
739
+ const entry = await refEntry(ref);
662
740
  await cdpType(entry.session, entry.backendNodeId, text, typeOpts);
663
741
  },
664
742
 
@@ -671,22 +749,18 @@ export async function connect(opts = {}) {
671
749
  },
672
750
 
673
751
  async hover(ref) {
674
- const entry = refMap.get(ref);
675
- if (!entry) throw new Error(`No element found for ref "${ref}"`);
752
+ const entry = await refEntry(ref);
676
753
  await cdpHover(entry.session, entry.backendNodeId);
677
754
  },
678
755
 
679
756
  async select(ref, value) {
680
- const entry = refMap.get(ref);
681
- if (!entry) throw new Error(`No element found for ref "${ref}"`);
757
+ const entry = await refEntry(ref);
682
758
  await cdpSelect(entry.session, entry.backendNodeId, value);
683
759
  },
684
760
 
685
761
  async drag(fromRef, toRef) {
686
- const from = refMap.get(fromRef);
687
- const to = refMap.get(toRef);
688
- if (!from) throw new Error(`No element found for ref "${fromRef}"`);
689
- if (!to) throw new Error(`No element found for ref "${toRef}"`);
762
+ const from = await refEntry(fromRef);
763
+ const to = await refEntry(toRef);
690
764
  // Drag across different frames isn't physically meaningful — bail
691
765
  // rather than mix sessions and produce nonsense coordinates.
692
766
  if (from.session !== to.session) {
@@ -696,8 +770,7 @@ export async function connect(opts = {}) {
696
770
  },
697
771
 
698
772
  async upload(ref, files) {
699
- const entry = refMap.get(ref);
700
- if (!entry) throw new Error(`No element found for ref "${ref}"`);
773
+ const entry = await refEntry(ref);
701
774
  // Upload sandbox: when uploadDir is set, every path must resolve
702
775
  // (symlinks included, via realpath) inside it. Stops a prompt-injected
703
776
  // agent from attaching ~/.ssh/id_rsa or other arbitrary local files.
@@ -854,7 +927,8 @@ export async function connect(opts = {}) {
854
927
  return {
855
928
  async goto(url, timeout = 30000) {
856
929
  assertNavigable(url, urlGuard);
857
- const outcome = await navigate(tab, url, timeout);
930
+ lock?.assertAllowed(url);
931
+ const outcome = await navigate(tab, url, timeout, lock);
858
932
  if (opts.consent !== false) {
859
933
  await dismissConsent(tab.session);
860
934
  }
@@ -940,10 +1014,11 @@ async function suppressPermissions(cdp) {
940
1014
  * v0.16.0. Still Firefox-limited: `reload({ignoreCache})` (upstream BiDi gap).
941
1015
  * The navigation guard (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir`
942
1016
  * sandbox, `incognito`, `proxy`, `viewport`, and `pruneMode` all apply.
943
- * @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls })
1017
+ * @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls, allowHosts })
944
1018
  */
945
1019
  async function connectFirefox(opts) {
946
1020
  const hybrid = opts.mode === 'hybrid';
1021
+ const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null; // validates before any launch
947
1022
  const blockOpts = {
948
1023
  blockAds: opts.blockAds !== undefined ? opts.blockAds : true,
949
1024
  blockUrls: opts.blockUrls,
@@ -972,7 +1047,7 @@ async function connectFirefox(opts) {
972
1047
  });
973
1048
  const bidi = await createBiDi(browser.wsUrl);
974
1049
  if (!headed) await applyFirefoxStealth(bidi);
975
- await applyFirefoxBlocklist(bidi, blockOpts);
1050
+ await applyFirefoxBlocklist(bidi, blockOpts, lock);
976
1051
  const { contexts } = await bidi.send('browsingContext.getTree', {});
977
1052
  return { browser, bidi, topContext: contexts[0].context };
978
1053
  }
@@ -997,6 +1072,7 @@ async function connectFirefox(opts) {
997
1072
  links: opts.links,
998
1073
  urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
999
1074
  uploadDir: opts.uploadDir || null,
1075
+ siteLock: lock,
1000
1076
  incognito: !!opts.incognito,
1001
1077
  consent: opts.consent,
1002
1078
  hybrid,
@@ -1161,13 +1237,45 @@ export function _resetBlocklistWarning() {
1161
1237
  blocklistWarned = false;
1162
1238
  }
1163
1239
 
1240
+ /**
1241
+ * Go to a history entry and wait until the traversal has landed. A bfcache
1242
+ * restore or a same-document entry never fires Page.loadEventFired, so wait
1243
+ * for the main frame's Page.frameStoppedLoading instead: it follows the load
1244
+ * event on a normal load and fires on both of those paths. Subscribed before
1245
+ * the navigate call so it cannot be missed; times out at 30s like before.
1246
+ * @param {*} page
1247
+ * @param {string} entryId history entry id (from Page.getNavigationHistory)
1248
+ */
1249
+ async function traverse(page, entryId) {
1250
+ const { session } = page;
1251
+ const { frameTree } = await session.send('Page.getFrameTree');
1252
+ const mainId = frameTree.frame.id;
1253
+ let off = () => {};
1254
+ const landed = new Promise((resolve, reject) => {
1255
+ const timer = setTimeout(() => { off(); reject(new Error('timeout')); }, 30000);
1256
+ off = session.on('Page.frameStoppedLoading', (p) => {
1257
+ if (p.frameId === mainId) { clearTimeout(timer); off(); resolve(undefined); }
1258
+ });
1259
+ });
1260
+ landed.catch(() => {});
1261
+ try {
1262
+ await session.send('Page.navigateToHistoryEntry', { entryId });
1263
+ } catch (e) { off(); throw e; }
1264
+ try { await landed; } catch { await new Promise((r) => setTimeout(r, 500)); }
1265
+ }
1266
+
1164
1267
  /**
1165
1268
  * Navigate to a URL and wait for the page to load. Resolves to the outcome of
1166
1269
  * the main-document navigation (final URL after redirects, HTTP status);
1167
- * rejects with a NAV_FAILED error when the browser could not reach a document.
1270
+ * rejects with a NAV_FAILED error when the browser could not reach a document,
1271
+ * or NAV_BLOCKED when the site lock refused the URL or a redirect hop.
1272
+ * @param {*} page
1273
+ * @param {string} url
1274
+ * @param {number} [timeout]
1275
+ * @param {import('./site-lock.js').SiteLock|null} [lock]
1168
1276
  * @returns {Promise<NavigationOutcome>}
1169
1277
  */
1170
- async function navigate(page, url, timeout = 30000) {
1278
+ async function navigate(page, url, timeout = 30000, lock = null) {
1171
1279
  const { session } = page;
1172
1280
  // Main-document responses (every redirect hop fires one; the last wins).
1173
1281
  const docs = [];
@@ -1182,7 +1290,12 @@ async function navigate(page, url, timeout = 30000) {
1182
1290
  }));
1183
1291
  loaded.catch(() => {}); // rejection is only read if we await it below
1184
1292
  try {
1293
+ lock?.startGoto(page.targetId);
1185
1294
  const nav = await session.send('Page.navigate', { url });
1295
+ // A lock-refused document (the URL or a redirect hop) also surfaces as
1296
+ // ERR_ABORTED, so this must come before the aborted branch below.
1297
+ const blocked = lock?.takeGotoBlock(page.targetId);
1298
+ if (blocked) throw navBlocked(blocked.url, blocked.host);
1186
1299
  // ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
1187
1300
  // no document and fires no load event, so don't wait for one.
1188
1301
  const aborted = nav.errorText === 'net::ERR_ABORTED';
@@ -1191,7 +1304,9 @@ async function navigate(page, url, timeout = 30000) {
1191
1304
  // response carries the status), so report it like any other HTTP error.
1192
1305
  const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
1193
1306
  if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
1194
- if (!aborted) await loaded;
1307
+ // A same-document navigation (fragment-only change) commits no new document: Page.navigate
1308
+ // returns no loaderId and no load event follows, so don't wait for one.
1309
+ if (!aborted && nav.loaderId) await loaded;
1195
1310
  // Brief settle time for dynamic content
1196
1311
  await new Promise((r) => setTimeout(r, 500));
1197
1312
  // loaderId is shared by every redirect hop of one navigation.
package/src/navigation.js CHANGED
@@ -23,6 +23,18 @@ export function navFailed(url, reason) {
23
23
  return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
24
24
  }
25
25
 
26
+ /**
27
+ * Typed error for a top-level navigation the site lock (`allowHosts`) refused:
28
+ * the target, or a redirect hop of it, is off the allowed list. `code` is
29
+ * 'NAV_BLOCKED'; `host` is the refused hostname. The page stays where it was.
30
+ * @param {string} url - The refused URL (the redirect target, if it was a hop).
31
+ * @param {string} host
32
+ * @returns {Error & {code: 'NAV_BLOCKED', host: string}}
33
+ */
34
+ export function navBlocked(url, host) {
35
+ return Object.assign(new Error(`Navigation to ${url} blocked: ${host} is not in allowHosts`), { code: /** @type {const} */ ('NAV_BLOCKED'), host });
36
+ }
37
+
26
38
  /**
27
39
  * True for a final HTTP status that is not 2xx (0 = unknown is not an error).
28
40
  * @param {number|undefined} status
package/src/prune.js CHANGED
@@ -62,6 +62,7 @@ const SKIP_ROLES = new Set([
62
62
  * @param {object} [options]
63
63
  * @param {'act'|'browse'|'navigate'|'full'|'read'} [options.mode='act'] - Pruning mode ('read' is an alias for 'browse')
64
64
  * @param {string} [options.context=''] - Search context for relevance filtering
65
+ * @param {boolean} [options.whole=false] - `tree` is a zoom box (snapshot `within`): it is the whole page, so the landmark step (1) is skipped
65
66
  * @param {boolean} [options.links=false] - Links carry `urls` (see links.js): nodes that drop their children keep the URLs of links beneath them
66
67
  * @returns {object|null} Pruned tree
67
68
  */
@@ -77,8 +78,8 @@ export function prune(tree, options = {}) {
77
78
  // Wrap as array for pipeline
78
79
  let nodes = tree ? [tree] : [];
79
80
 
80
- // Step 1: Extract landmark regions
81
- nodes = extractRegions(nodes, allowedRegions);
81
+ // Step 1: Extract landmark regions (a zoom box is the whole page: keep it)
82
+ if (!options.whole) nodes = extractRegions(nodes, allowedRegions);
82
83
 
83
84
  // Step 2: Prune nodes
84
85
  const ctx = { mode, parentRole: null, keywords, links: !!options.links };
@@ -0,0 +1,184 @@
1
+ /**
2
+ * site-lock.js — Optional top-frame site lock (`allowHosts`).
3
+ *
4
+ * Top-level navigations (goto, redirect hops, clicked links, JS jumps, popups)
5
+ * to a host outside the allow list are failed in the browser before any request
6
+ * leaves, so the page stays put. Deliberately NOT locked: subresources
7
+ * (img/script/css/XHR/fetch), iframes, WebSocket — the lock stops an agent
8
+ * being steered to another site, not the page from loading its own assets.
9
+ * The engine wiring lives next to each transport (applySiteLockCDP here,
10
+ * blocklist-firefox.js for BiDi); this file is the shared policy + state.
11
+ */
12
+
13
+ import { navBlocked } from './navigation.js';
14
+
15
+ const HOST_RE = /^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$/;
16
+
17
+ /**
18
+ * Validate and normalize an allow list. Entry = `host` or `*.host`; anything
19
+ * else (scheme, port, path, spaces, empty, bare `*`) throws — a typo must never
20
+ * silently widen or drop the lock.
21
+ * @param {string[]} list
22
+ * @returns {string[]} lowercase entries
23
+ */
24
+ export function parseAllowHosts(list) {
25
+ if (!Array.isArray(list) || !list.length) throw new Error('allowHosts must be a non-empty array of hostnames');
26
+ return list.map((raw) => {
27
+ const e = typeof raw === 'string' ? raw.trim().toLowerCase() : raw;
28
+ const host = typeof e === 'string' && e.startsWith('*.') ? e.slice(2) : e;
29
+ if (typeof host !== 'string' || !HOST_RE.test(host) || host.includes('..')) {
30
+ throw new Error(`allowHosts: invalid entry ${JSON.stringify(raw)} (expected "host" or "*.host", no scheme/port/path)`);
31
+ }
32
+ return e;
33
+ });
34
+ }
35
+
36
+ /**
37
+ * Build a URL matcher from an allow list. Non-http(s) URLs (about:, data:,
38
+ * blob:) are allowed; `host` matches that host exactly, `*.host` matches its
39
+ * strict subdomains only (not `host` itself). An unparseable URL is refused.
40
+ * @param {string[]} list
41
+ * @returns {(url: string) => boolean}
42
+ */
43
+ export function makeHostMatcher(list) {
44
+ const entries = parseAllowHosts(list);
45
+ const exact = new Set(entries.filter((e) => !e.startsWith('*.')));
46
+ const suffixes = entries.filter((e) => e.startsWith('*.')).map((e) => e.slice(1)); // ".host"
47
+ return (url) => {
48
+ let u;
49
+ try { u = new URL(url); } catch { return false; }
50
+ if (u.protocol !== 'http:' && u.protocol !== 'https:') return true;
51
+ const host = u.hostname.replace(/\.$/, '');
52
+ return exact.has(host) || suffixes.some((s) => host.endsWith(s));
53
+ };
54
+ }
55
+
56
+ /**
57
+ * The lock's shared state, one per connect()/browse() call, surviving
58
+ * relaunches and new tabs (each transport just keeps calling record()).
59
+ * @param {string[]} list
60
+ */
61
+ export function createSiteLock(list) {
62
+ const allowed = makeHostMatcher(list);
63
+ const hostOf = (url) => { try { return new URL(url).hostname; } catch { return url; } };
64
+ /** @type {{url: string, host: string, ctx: string}[]} */
65
+ let gotoBlocks = [];
66
+ // Snapshot notes belong to the tab/context where the block happened, so one
67
+ // tab's (or a popup's, shown if the agent switches to it) never leak into another's.
68
+ /** @type {Map<string, Set<string>>} */
69
+ const noted = new Map();
70
+ return {
71
+ allowed,
72
+ /** @param {string} url @throws NAV_BLOCKED when url is off the list */
73
+ assertAllowed(url) {
74
+ if (!allowed(url)) throw navBlocked(url, hostOf(url));
75
+ },
76
+ /** A top-level document request was refused. ctx = tab/context it belongs to. */
77
+ record(url, ctx) {
78
+ const host = hostOf(url);
79
+ gotoBlocks.push({ url, host, ctx });
80
+ const hosts = noted.get(ctx) ?? new Set();
81
+ noted.set(ctx, hosts.add(host));
82
+ },
83
+ /** Call right before a goto navigates; drops that tab's pending notes too. */
84
+ startGoto(ctx) { gotoBlocks = []; noted.delete(ctx); },
85
+ /**
86
+ * The block that hit this goto's own tab (a popup's block is not the
87
+ * goto's), consumed; it is reported as NAV_BLOCKED, so no snapshot note.
88
+ */
89
+ takeGotoBlock(ctx) {
90
+ const b = gotoBlocks.find((x) => x.ctx === ctx);
91
+ if (!b) return null;
92
+ gotoBlocks = gotoBlocks.filter((x) => x !== b);
93
+ noted.get(ctx)?.delete(b.host);
94
+ return { url: b.url, host: b.host };
95
+ },
96
+ /** Snapshot lines for this tab's in-page blocked jumps since the last call. */
97
+ takeNotes(ctx) {
98
+ const out = [...(noted.get(ctx) ?? [])].map((h) => `[blocked navigation to ${h}]\n`).join('');
99
+ noted.delete(ctx);
100
+ return out;
101
+ },
102
+ };
103
+ }
104
+
105
+ /** @typedef {ReturnType<typeof createSiteLock>} SiteLock */
106
+
107
+ /**
108
+ * Chromium: lock every page target of this browser. A BROWSER-level
109
+ * Fetch.enable (root client, no sessionId) pauses document requests of every
110
+ * target, popups included, so one handler covers tabs we create later and
111
+ * windows the page opens itself. A request is top-level when its frameId is a
112
+ * page target id (OOPIF iframes are 'iframe' targets, subframes share their
113
+ * parent's id space). 'Aborted' (not BlockedByClient) fails the request but
114
+ * keeps the old page instead of swapping in a chrome-error:// page.
115
+ * Call once per CDP connection, before the first navigation.
116
+ * @param {object} cdp - root CDP client
117
+ * @param {SiteLock} lock
118
+ */
119
+ export async function applySiteLockCDP(cdp, lock) {
120
+ const pageTargets = new Set();
121
+ const subframes = new Set(); // ids a refresh proved are not page targets
122
+ cdp.on('Target.targetCreated', ({ targetInfo }) => {
123
+ if (targetInfo.type === 'page') pageTargets.add(targetInfo.targetId);
124
+ });
125
+ cdp.on('Target.targetDestroyed', ({ targetId }) => pageTargets.delete(targetId));
126
+ await cdp.send('Target.setDiscoverTargets', { discover: true });
127
+
128
+ cdp.on('Fetch.requestPaused', async (p, sessionId) => {
129
+ // A popup's create event can lag its first request: ask the browser
130
+ // before treating an unknown frame as a subframe (and if that fails,
131
+ // fail closed: treat it as top-level).
132
+ if (!pageTargets.has(p.frameId) && !subframes.has(p.frameId)) {
133
+ try {
134
+ const { targetInfos } = await cdp.send('Target.getTargets');
135
+ for (const t of targetInfos) if (t.type === 'page') pageTargets.add(t.targetId);
136
+ if (!pageTargets.has(p.frameId)) subframes.add(p.frameId);
137
+ } catch { pageTargets.add(p.frameId); }
138
+ }
139
+ const ok = !pageTargets.has(p.frameId) || lock.allowed(p.request.url);
140
+ if (!ok) lock.record(p.request.url, p.frameId);
141
+ try {
142
+ if (ok) await cdp.send('Fetch.continueRequest', { requestId: p.requestId }, sessionId);
143
+ else await cdp.send('Fetch.failRequest', { requestId: p.requestId, errorReason: 'Aborted' }, sessionId);
144
+ } catch { /* target closed between the pause and our reply */ }
145
+ });
146
+ await cdp.send('Fetch.enable', { patterns: [{ resourceType: 'Document', requestStage: 'Request' }] });
147
+ }
148
+
149
+ /**
150
+ * Firefox/BiDi: build the "is this paused request a locked top-level load?"
151
+ * check that blocklist-firefox.js's single intercept handler calls (two
152
+ * catch-all handlers on one paused request would race — one continues before
153
+ * the other fails). A request is a top-level document load when it carries a
154
+ * navigation id and its context has no parent. Context ids are tracked from
155
+ * contextCreated and refreshed from getTree when an unknown one shows up, so a
156
+ * popup whose create event lags its first request can't slip through; one that
157
+ * is still unknown after the refresh is treated as top-level (fail closed).
158
+ * Redirect hops arrive as their own beforeRequestSent, so each is checked.
159
+ * @param {object} bidi - BiDi client
160
+ * @param {SiteLock} lock
161
+ * @returns {Promise<(e: object) => Promise<boolean>>} true = fail this request
162
+ */
163
+ export async function createFirefoxLockCheck(bidi, lock) {
164
+ const tops = new Set();
165
+ const known = new Set();
166
+ async function refresh() {
167
+ const { contexts } = await bidi.send('browsingContext.getTree', {});
168
+ (function walk(nodes, top) {
169
+ for (const n of nodes) { known.add(n.context); if (top) tops.add(n.context); walk(n.children || [], false); }
170
+ })(contexts, true);
171
+ }
172
+ bidi.on('browsingContext.contextCreated', (c) => { known.add(c.context); if (!c.parent) tops.add(c.context); });
173
+ bidi.on('browsingContext.contextDestroyed', (c) => { known.delete(c.context); tops.delete(c.context); });
174
+ await bidi.subscribe(['browsingContext.contextCreated', 'browsingContext.contextDestroyed']);
175
+ await refresh();
176
+ return async (e) => {
177
+ if (e.navigation == null) return false; // subresource
178
+ if (!known.has(e.context)) { try { await refresh(); } catch { tops.add(e.context); } }
179
+ if (known.has(e.context) && !tops.has(e.context)) return false; // iframe
180
+ if (lock.allowed(e.request.url)) return false;
181
+ lock.record(e.request.url, e.context);
182
+ return true;
183
+ };
184
+ }