barebrowse 0.25.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js CHANGED
@@ -17,6 +17,7 @@ import { formatTree } from './aria.js';
17
17
  import { authenticate } from './auth.js';
18
18
  import { prune as pruneTree } from './prune.js';
19
19
  import { dataSnapshot, stripLegend } from './data.js';
20
+ import { annotateLinks } from './links.js';
20
21
  import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
21
22
  import { dismissConsent } from './consent.js';
22
23
  import { applyStealth } from './stealth.js';
@@ -28,6 +29,8 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
28
29
  import { isChallengePage } from './challenge.js';
29
30
  import { withTimeout } from './with-timeout.js';
30
31
  import { readable as extractReadable } from './readable.js';
32
+ import { navFailed, navBlocked, httpWarning } from './navigation.js';
33
+ import { createSiteLock, applySiteLockCDP } from './site-lock.js';
31
34
  import { assertNavigable, assertUploadAllowed } from './url-guard.js';
32
35
  import { join as pathJoin } from 'node:path';
33
36
  import { chmodSync } from 'node:fs';
@@ -60,6 +63,10 @@ import { chmodSync } from 'node:fs';
60
63
  * @param {string} [opts.browser] - Source browser for cookie extraction.
61
64
  * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
62
65
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
66
+ * @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
67
+ * @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (the URL, its
68
+ * redirects) to hosts outside this list reject with NAV_BLOCKED. Entries are `host`
69
+ * or `*.host` (strict subdomains). Subresources and iframes are not locked.
63
70
  * @returns {Promise<string>} ARIA snapshot text
64
71
  */
65
72
  export async function browse(url, opts = {}) {
@@ -71,6 +78,8 @@ export async function browse(url, opts = {}) {
71
78
  // Reject local-resource schemes (and optionally private hosts) before we
72
79
  // spend a browser launch on a URL we won't navigate to.
73
80
  assertNavigable(url, { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork });
81
+ const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
82
+ lock?.assertAllowed(url);
74
83
 
75
84
  let browser = null;
76
85
  let cdp = null;
@@ -88,6 +97,7 @@ export async function browse(url, opts = {}) {
88
97
  browser = await launch(launchOpts);
89
98
  cdp = await createCDP(browser.wsUrl);
90
99
  }
100
+ if (lock) await applySiteLockCDP(cdp, lock);
91
101
 
92
102
  // Step 2: Create a new page target and attach
93
103
  const pageOpts = { viewport: opts.viewport, blockAds: opts.blockAds, blockUrls: opts.blockUrls };
@@ -106,7 +116,7 @@ export async function browse(url, opts = {}) {
106
116
  }
107
117
 
108
118
  // Step 4: Navigate and wait for load
109
- await navigate(page, url, timeout);
119
+ let outcome = await navigate(page, url, timeout, lock);
110
120
 
111
121
  // Step 4.5: Auto-dismiss cookie consent dialogs
112
122
  if (opts.consent !== false) {
@@ -125,12 +135,13 @@ export async function browse(url, opts = {}) {
125
135
  try {
126
136
  browser = await launch({ ...launchOpts, headed: true });
127
137
  cdp = await createCDP(browser.wsUrl);
138
+ if (lock) await applySiteLockCDP(cdp, lock);
128
139
  page = await createPage(cdp, false, pageOpts);
129
140
  await suppressPermissions(cdp);
130
141
  if (!noAuth) {
131
142
  try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
132
143
  }
133
- await navigate(page, url, timeout);
144
+ outcome = await navigate(page, url, timeout, lock);
134
145
  if (opts.consent !== false) await dismissConsent(page.session);
135
146
  ({ tree } = await ariaTree(page));
136
147
  } catch {
@@ -140,21 +151,27 @@ export async function browse(url, opts = {}) {
140
151
 
141
152
  // Step 6: Prune for agent consumption
142
153
  const raw = formatTree(tree);
154
+ const links = !!opts.links;
155
+ let linked = tree;
156
+ if (links) {
157
+ const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
158
+ linked = annotateLinks(tree, entries[currentIndex]?.url || url);
159
+ }
143
160
  let snapshot;
144
161
  if (opts.prune !== false) {
145
162
  snapshot = opts.pruneMode === 'data'
146
- ? dataSnapshot(tree)
147
- : formatTree(pruneTree(tree, { mode: opts.pruneMode || 'act' }));
163
+ ? dataSnapshot(linked, { links })
164
+ : formatTree(pruneTree(linked, { mode: opts.pruneMode || 'act', links }));
148
165
  } else {
149
- snapshot = raw;
166
+ snapshot = links ? formatTree(linked) : raw;
150
167
  }
151
- const sized = opts.pruneMode === 'data' ? stripLegend(snapshot) : snapshot; // the legend is not content
168
+ const sized = stripLegend(snapshot); // the legend is not content
152
169
  const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
153
170
  const actMode = !opts.pruneMode || opts.pruneMode === 'act';
154
171
  const hint = (actMode && raw.length > 5000 && snapshot.length < 500 && snapshot.length < raw.length * 0.05)
155
172
  ? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
156
173
  : '';
157
- snapshot = stats + '\n' + hint + snapshot;
174
+ snapshot = stats + '\n' + hint + httpWarning(outcome.status) + snapshot;
158
175
 
159
176
  // Step 7: Clean up
160
177
  await cdp.send('Target.closeTarget', { targetId: page.targetId });
@@ -204,10 +221,20 @@ export async function browse(url, opts = {}) {
204
221
  * @property {boolean} cookieInjection - Cookie extraction/injection is active (off in incognito)
205
222
  */
206
223
 
224
+ /**
225
+ * What `goto()` resolves to: the final URL (after redirects) and the main
226
+ * document's HTTP status (0 when no HTTP response was observed, e.g. an aborted navigation).
227
+ * Network failures reject with an Error whose `code` is 'NAV_FAILED' and
228
+ * `reason` the browser's raw error string.
229
+ * @typedef {import('./navigation.js').NavigationOutcome} NavigationOutcome
230
+ */
231
+
207
232
  /**
208
233
  * Options for `page.snapshot()`. Pass `false` instead of an object to skip
209
234
  * pruning and return the raw ARIA tree.
210
- * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data' }} SnapshotOptions
235
+ * `links: true` adds each link's absolute URL after its name (default off,
236
+ * or the `connect({ links })` default).
237
+ * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
211
238
  */
212
239
 
213
240
  /**
@@ -261,6 +288,12 @@ export async function browse(url, opts = {}) {
261
288
  * skip storageState loading and make injectCookies() a no-op, so no auth
262
289
  * ever enters the session even if a caller injects unconditionally.
263
290
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
291
+ * @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
292
+ * @param {string[]} [opts.allowHosts] - Site lock: top-level navigations (goto, redirects,
293
+ * clicked links, JS jumps, popups) to hosts outside this list are refused and the page
294
+ * stays put. Entries are `host` or `*.host` (strict subdomains). goto() rejects with
295
+ * NAV_BLOCKED; in-page blocks add a `[blocked navigation to <host>]` line to the next
296
+ * snapshot. Subresources, iframes and WebSocket are not locked. Not allowed with `port`.
264
297
  * @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
265
298
  * 'firefox' drives over WebDriver BiDi (a separate transport / page object);
266
299
  * 'chromium' (default) uses CDP.
@@ -319,6 +352,12 @@ export async function connect(opts = {}) {
319
352
 
320
353
  const mode = opts.mode || 'headless';
321
354
  const attachMode = !!opts.port;
355
+ // Validate before launching anything. Attach mode is refused: the lock is a
356
+ // browser-level intercept that would reach the user's own tabs.
357
+ if (opts.allowHosts && attachMode) {
358
+ throw new Error('allowHosts is not supported in attach mode (port): the site lock is browser-wide and would affect your own tabs');
359
+ }
360
+ const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null;
322
361
  // Incognito: run a clean, unauthenticated session. Skips storageState loading
323
362
  // and neuters injectCookies() so no auth ever enters the session — even when
324
363
  // a caller (MCP goto, daemon) calls injectCookies() unconditionally. The
@@ -350,6 +389,7 @@ export async function connect(opts = {}) {
350
389
  browser = await launch(launchOpts);
351
390
  cdp = await createCDP(browser.wsUrl);
352
391
  }
392
+ if (lock) await applySiteLockCDP(cdp, lock);
353
393
 
354
394
  // In attach mode we don't know (and shouldn't assume) the user's headed/
355
395
  // headless state — treat it as headed so stealth patches are skipped
@@ -367,6 +407,9 @@ export async function connect(opts = {}) {
367
407
  let page = await createPage(cdp, !currentlyHeaded, pageOpts);
368
408
  let refMap = new Map();
369
409
  let botBlocked = false;
410
+ // Outcome of the latest goto() ({url, status}); null before any goto.
411
+ /** @type {NavigationOutcome|null} */
412
+ let lastNavigation = null;
370
413
  // Set when the consent pass after the latest goto() found nothing: the first
371
414
  // snapshot() re-checks once, since some CMPs (Google Funding Choices) render
372
415
  // their dialog 1-3s after load. Holds the document identity (tab + main-frame
@@ -483,12 +526,14 @@ export async function connect(opts = {}) {
483
526
  return {
484
527
  async goto(url, timeout = 30000) {
485
528
  assertNavigable(url, urlGuard);
529
+ lock?.assertAllowed(url);
486
530
  // Refs from the previous page are about to become invalid — clear
487
531
  // before navigating so a stale click(ref) errors clearly instead of
488
532
  // silently resolving to whatever backendNodeId happens to still be in
489
533
  // the map.
490
534
  refMap = new Map();
491
535
  consentRecheck = null; // a throwing navigate() must not leave the old page's flag
536
+ lastNavigation = null;
492
537
  // Switch back to headless if we fell back to headed previously.
493
538
  // Not in attach mode — we never own the browser there, so there's
494
539
  // nothing to rewind.
@@ -499,13 +544,14 @@ export async function connect(opts = {}) {
499
544
 
500
545
  browser = await launch(launchOpts);
501
546
  cdp = await createCDP(browser.wsUrl);
547
+ if (lock) await applySiteLockCDP(cdp, lock);
502
548
  page = await createPage(cdp, true, pageOpts);
503
549
  setupDialogHandler(page.session);
504
550
  await suppressPermissions(cdp);
505
551
  currentlyHeaded = false;
506
552
  }
507
553
 
508
- await navigate(page, url, timeout);
554
+ lastNavigation = await navigate(page, url, timeout, lock);
509
555
  if (opts.consent !== false) {
510
556
  const dismissed = await dismissConsent(page.session);
511
557
  if (!dismissed) consentRecheck = await navKey();
@@ -526,10 +572,11 @@ export async function connect(opts = {}) {
526
572
  try {
527
573
  browser = await launch({ ...launchOpts, headed: true });
528
574
  cdp = await createCDP(browser.wsUrl);
575
+ if (lock) await applySiteLockCDP(cdp, lock);
529
576
  page = await createPage(cdp, false, pageOpts);
530
577
  setupDialogHandler(page.session);
531
578
  await suppressPermissions(cdp);
532
- await navigate(page, url, timeout);
579
+ lastNavigation = await navigate(page, url, timeout, lock);
533
580
  consentRecheck = null;
534
581
  if (opts.consent !== false && !(await dismissConsent(page.session))) {
535
582
  consentRecheck = await navKey();
@@ -543,6 +590,7 @@ export async function connect(opts = {}) {
543
590
  // Headed launch failed (no display?) — keep headless result, botBlocked stays true
544
591
  }
545
592
  }
593
+ return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
546
594
  },
547
595
 
548
596
  async goBack() {
@@ -553,6 +601,7 @@ export async function connect(opts = {}) {
553
601
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
554
602
  refMap = new Map(); // refs from the previous page are now invalid
555
603
  consentRecheck = null;
604
+ lastNavigation = null; // no outcome is known for the history entry we landed on
556
605
  },
557
606
 
558
607
  async goForward() {
@@ -563,6 +612,7 @@ export async function connect(opts = {}) {
563
612
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
564
613
  refMap = new Map();
565
614
  consentRecheck = null;
615
+ lastNavigation = null;
566
616
  },
567
617
 
568
618
  async reload(reloadOpts = {}) {
@@ -574,6 +624,7 @@ export async function connect(opts = {}) {
574
624
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
575
625
  refMap = new Map(); // refs from the pre-reload page are invalid
576
626
  consentRecheck = null;
627
+ lastNavigation = null; // the reload's status is not read; don't keep the old one
577
628
  },
578
629
 
579
630
  /** @param {string} url @param {CookieOptions} [cookieOpts] */
@@ -599,11 +650,14 @@ export async function connect(opts = {}) {
599
650
  const raw = formatTree(result.tree);
600
651
  const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
601
652
  const pageUrl = entries[currentIndex]?.url || '';
602
- const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '';
603
- if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + raw;
653
+ const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
654
+ + httpWarning(lastNavigation?.status) + (lock?.takeNotes(page.targetId) ?? '');
655
+ const links = !!((pruneOpts || undefined)?.links ?? opts.links);
656
+ const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
657
+ if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
604
658
  const mode = pruneOpts?.mode || opts.pruneMode || 'act';
605
- const out = mode === 'data' ? dataSnapshot(result.tree) : formatTree(pruneTree(result.tree, { mode }));
606
- const sized = mode === 'data' ? stripLegend(out) : out; // the legend is not content
659
+ const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
660
+ const sized = stripLegend(out); // the legend is not content
607
661
  const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
608
662
  const hint = (mode === 'act' && raw.length > 5000 && out.length < 500 && out.length < raw.length * 0.05)
609
663
  ? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
@@ -702,6 +756,7 @@ export async function connect(opts = {}) {
702
756
  const oldSessionId = page.sessionId;
703
757
  page = await attachToExistingTarget(cdp, target.targetId, pageOpts);
704
758
  refMap = new Map(); // refs from the previous tab are no longer valid
759
+ lastNavigation = null; // the outcome belonged to the tab we left
705
760
  setupDialogHandler(page.session);
706
761
  try { await cdp.send('Target.detachFromTarget', { sessionId: oldSessionId }); } catch {}
707
762
  },
@@ -746,6 +801,8 @@ export async function connect(opts = {}) {
746
801
  },
747
802
 
748
803
  get botBlocked() { return botBlocked; },
804
+ /** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
805
+ get lastNavigation() { return lastNavigation; },
749
806
 
750
807
  /** @type {'chromium'} Engine driving this session (see also `capabilities`) */
751
808
  engine: 'chromium',
@@ -820,12 +877,14 @@ export async function connect(opts = {}) {
820
877
  return {
821
878
  async goto(url, timeout = 30000) {
822
879
  assertNavigable(url, urlGuard);
823
- await navigate(tab, url, timeout);
880
+ lock?.assertAllowed(url);
881
+ const outcome = await navigate(tab, url, timeout, lock);
824
882
  if (opts.consent !== false) {
825
883
  await dismissConsent(tab.session);
826
884
  }
827
885
  const { tree, nodeCount } = await ariaTree(tab);
828
886
  tabBotBlocked = isChallengePage(tree, nodeCount);
887
+ return outcome;
829
888
  },
830
889
  get botBlocked() { return tabBotBlocked; },
831
890
  /** @param {string} url @param {CookieOptions} [cookieOpts] */
@@ -905,10 +964,11 @@ async function suppressPermissions(cdp) {
905
964
  * v0.16.0. Still Firefox-limited: `reload({ignoreCache})` (upstream BiDi gap).
906
965
  * The navigation guard (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir`
907
966
  * sandbox, `incognito`, `proxy`, `viewport`, and `pruneMode` all apply.
908
- * @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls })
967
+ * @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito, downloadPath, blockAds, blockUrls, allowHosts })
909
968
  */
910
969
  async function connectFirefox(opts) {
911
970
  const hybrid = opts.mode === 'hybrid';
971
+ const lock = opts.allowHosts ? createSiteLock(opts.allowHosts) : null; // validates before any launch
912
972
  const blockOpts = {
913
973
  blockAds: opts.blockAds !== undefined ? opts.blockAds : true,
914
974
  blockUrls: opts.blockUrls,
@@ -937,7 +997,7 @@ async function connectFirefox(opts) {
937
997
  });
938
998
  const bidi = await createBiDi(browser.wsUrl);
939
999
  if (!headed) await applyFirefoxStealth(bidi);
940
- await applyFirefoxBlocklist(bidi, blockOpts);
1000
+ await applyFirefoxBlocklist(bidi, blockOpts, lock);
941
1001
  const { contexts } = await bidi.send('browsingContext.getTree', {});
942
1002
  return { browser, bidi, topContext: contexts[0].context };
943
1003
  }
@@ -959,8 +1019,10 @@ async function connectFirefox(opts) {
959
1019
 
960
1020
  const page = await createFirefoxPage(bidi, {
961
1021
  pruneMode: opts.pruneMode,
1022
+ links: opts.links,
962
1023
  urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
963
1024
  uploadDir: opts.uploadDir || null,
1025
+ siteLock: lock,
964
1026
  incognito: !!opts.incognito,
965
1027
  consent: opts.consent,
966
1028
  hybrid,
@@ -1126,14 +1188,55 @@ export function _resetBlocklistWarning() {
1126
1188
  }
1127
1189
 
1128
1190
  /**
1129
- * Navigate to a URL and wait for the page to load.
1191
+ * Navigate to a URL and wait for the page to load. Resolves to the outcome of
1192
+ * the main-document navigation (final URL after redirects, HTTP status);
1193
+ * rejects with a NAV_FAILED error when the browser could not reach a document,
1194
+ * or NAV_BLOCKED when the site lock refused the URL or a redirect hop.
1195
+ * @param {*} page
1196
+ * @param {string} url
1197
+ * @param {number} [timeout]
1198
+ * @param {import('./site-lock.js').SiteLock|null} [lock]
1199
+ * @returns {Promise<NavigationOutcome>}
1130
1200
  */
1131
- async function navigate(page, url, timeout = 30000) {
1132
- const loadPromise = page.session.once('Page.loadEventFired', timeout);
1133
- await page.session.send('Page.navigate', { url });
1134
- await loadPromise;
1135
- // Brief settle time for dynamic content
1136
- await new Promise((r) => setTimeout(r, 500));
1201
+ async function navigate(page, url, timeout = 30000, lock = null) {
1202
+ const { session } = page;
1203
+ // Main-document responses (every redirect hop fires one; the last wins).
1204
+ const docs = [];
1205
+ const offDoc = session.on('Network.responseReceived', (p) => { if (p.type === 'Document') docs.push(p); });
1206
+ // Load waiter, hand-rolled (not session.once) so it can be cancelled when
1207
+ // Page.navigate fails or aborts and no load event will ever come.
1208
+ let cancel = () => {};
1209
+ const loaded = /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
1210
+ const timer = setTimeout(() => { off(); reject(new Error('Timeout waiting for CDP event: Page.loadEventFired')); }, timeout);
1211
+ const off = session.on('Page.loadEventFired', () => { cancel(); resolve(undefined); });
1212
+ cancel = () => { clearTimeout(timer); off(); };
1213
+ }));
1214
+ loaded.catch(() => {}); // rejection is only read if we await it below
1215
+ try {
1216
+ lock?.startGoto(page.targetId);
1217
+ const nav = await session.send('Page.navigate', { url });
1218
+ // A lock-refused document (the URL or a redirect hop) also surfaces as
1219
+ // ERR_ABORTED, so this must come before the aborted branch below.
1220
+ const blocked = lock?.takeGotoBlock(page.targetId);
1221
+ if (blocked) throw navBlocked(blocked.url, blocked.host);
1222
+ // ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
1223
+ // no document and fires no load event, so don't wait for one.
1224
+ const aborted = nav.errorText === 'net::ERR_ABORTED';
1225
+ // ERR_HTTP_RESPONSE_CODE_FAILURE is a 4xx/5xx with an empty body: the
1226
+ // document did load (Chromium's error page fires load, the Document
1227
+ // response carries the status), so report it like any other HTTP error.
1228
+ const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
1229
+ if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
1230
+ if (!aborted) await loaded;
1231
+ // Brief settle time for dynamic content
1232
+ await new Promise((r) => setTimeout(r, 500));
1233
+ // loaderId is shared by every redirect hop of one navigation.
1234
+ const last = docs.filter((d) => d.loaderId === nav.loaderId).pop();
1235
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
1236
+ } finally {
1237
+ cancel();
1238
+ offDoc();
1239
+ }
1137
1240
  }
1138
1241
 
1139
1242
  /**
package/src/links.js ADDED
@@ -0,0 +1,46 @@
1
+ /**
2
+ * links.js — opt-in link URLs for snapshots (`links: true`).
3
+ *
4
+ * `annotateLinks()` stamps `urls: [url]` on every link node that has a usable
5
+ * browser-serialized absolute URL. aria.js, prune.js and data.js print/carry
6
+ * `urls` — they never see the option, so with links off nothing changes.
7
+ */
8
+
9
+ /**
10
+ * Normalise a link's URL for a snapshot line. Returns '' when it should not be
11
+ * shown: no URL, `javascript:`, or a same-document link (the page URL, only
12
+ * the #fragment differs). A raw space, `<` or `>` (browsers encode them, but
13
+ * not always in the fragment) is percent-encoded so a line splits cleanly on
14
+ * ` <` and `> `.
15
+ *
16
+ * @param {string|undefined} url - Absolute URL as serialized by the browser
17
+ * @param {string} [pageUrl] - URL of the page the link is on
18
+ * @returns {string}
19
+ */
20
+ export function linkUrl(url, pageUrl = '') {
21
+ if (typeof url !== 'string') return '';
22
+ const u = url.trim();
23
+ if (!u || /^javascript:/i.test(u)) return '';
24
+ if (pageUrl && u.split('#')[0] === pageUrl.split('#')[0]) return '';
25
+ return u.replace(/ /g, '%20').replace(/</g, '%3C').replace(/>/g, '%3E');
26
+ }
27
+
28
+ /**
29
+ * Copy a tree with `urls` set on its link nodes (input is not mutated).
30
+ *
31
+ * @param {object|null} node - Root from buildTree()
32
+ * @param {string} pageUrl
33
+ * @returns {object|null}
34
+ */
35
+ export function annotateLinks(node, pageUrl) {
36
+ if (!node) return node;
37
+ const out = { ...node, children: node.children.map((c) => annotateLinks(c, pageUrl)) };
38
+ if (node.role === 'link') {
39
+ const u = linkUrl(node.properties?.url, pageUrl);
40
+ if (u) out.urls = [u];
41
+ }
42
+ return out;
43
+ }
44
+
45
+ /** ` <u1> <u2>` for a node's urls ('' when none). */
46
+ export const urlsText = (urls) => (urls || []).map((u) => ` <${u}>`).join('');
@@ -0,0 +1,54 @@
1
+ /**
2
+ * navigation.js — Shared pieces of the goto() outcome contract, so both
3
+ * engines (CDP in index.js, BiDi in firefox-page.js) report it identically.
4
+ */
5
+
6
+ /**
7
+ * What goto() resolves to: where the page ended up and how the server answered.
8
+ * `status` is 0 when no HTTP response was observed for the navigation (e.g. an
9
+ * aborted navigation that never got one).
10
+ * @typedef {{ url: string, status: number }} NavigationOutcome
11
+ */
12
+
13
+ /**
14
+ * Typed error for a navigation that never reached a document (DNS failure,
15
+ * connection refused, TLS error, ...). `code` is 'NAV_FAILED'; `reason` is the
16
+ * browser's raw error string (`net::ERR_NAME_NOT_RESOLVED`,
17
+ * `NS_ERROR_UNKNOWN_HOST`, ...).
18
+ * @param {string} url
19
+ * @param {string} reason
20
+ * @returns {Error & {code: 'NAV_FAILED', reason: string}}
21
+ */
22
+ export function navFailed(url, reason) {
23
+ return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
24
+ }
25
+
26
+ /**
27
+ * Typed error for a top-level navigation the site lock (`allowHosts`) refused:
28
+ * the target, or a redirect hop of it, is off the allowed list. `code` is
29
+ * 'NAV_BLOCKED'; `host` is the refused hostname. The page stays where it was.
30
+ * @param {string} url - The refused URL (the redirect target, if it was a hop).
31
+ * @param {string} host
32
+ * @returns {Error & {code: 'NAV_BLOCKED', host: string}}
33
+ */
34
+ export function navBlocked(url, host) {
35
+ return Object.assign(new Error(`Navigation to ${url} blocked: ${host} is not in allowHosts`), { code: /** @type {const} */ ('NAV_BLOCKED'), host });
36
+ }
37
+
38
+ /**
39
+ * True for a final HTTP status that is not 2xx (0 = unknown is not an error).
40
+ * @param {number|undefined} status
41
+ */
42
+ export function isHttpError(status) {
43
+ return !!status && (status < 200 || status >= 300);
44
+ }
45
+
46
+ /**
47
+ * Warning line for a non-2xx final status ('' for 2xx and for unknown 0), in
48
+ * the style of the bot-challenge line, newline-terminated for prepending.
49
+ * @param {number|undefined} status
50
+ * @returns {string}
51
+ */
52
+ export function httpWarning(status) {
53
+ return isHttpError(status) ? `[HTTP ${status} — the page may be an error page]\n` : '';
54
+ }
package/src/prune.js CHANGED
@@ -62,6 +62,7 @@ const SKIP_ROLES = new Set([
62
62
  * @param {object} [options]
63
63
  * @param {'act'|'browse'|'navigate'|'full'|'read'} [options.mode='act'] - Pruning mode ('read' is an alias for 'browse')
64
64
  * @param {string} [options.context=''] - Search context for relevance filtering
65
+ * @param {boolean} [options.links=false] - Links carry `urls` (see links.js): nodes that drop their children keep the URLs of links beneath them
65
66
  * @returns {object|null} Pruned tree
66
67
  */
67
68
  export function prune(tree, options = {}) {
@@ -80,7 +81,7 @@ export function prune(tree, options = {}) {
80
81
  nodes = extractRegions(nodes, allowedRegions);
81
82
 
82
83
  // Step 2: Prune nodes
83
- const ctx = { mode, parentRole: null, keywords };
84
+ const ctx = { mode, parentRole: null, keywords, links: !!options.links };
84
85
  nodes = nodes.map((n) => pruneNode(n, ctx)).filter(Boolean);
85
86
 
86
87
  // Step 3: Collapse structural wrappers
@@ -204,7 +205,7 @@ function pruneNode(node, ctx) {
204
205
  return { ...node, children: pruneChildren(node.children, ctx) };
205
206
  }
206
207
  if (node.role === 'group' && node.name) {
207
- if (!isBrowse && /kleuren|colors?|couleurs?|farben/i.test(node.name)) {
208
+ if (!isBrowse && !ctx.links && /kleuren|colors?|couleurs?|farben/i.test(node.name)) {
208
209
  return collapseColors(node);
209
210
  }
210
211
  return { ...node, children: pruneChildren(node.children, ctx) };
@@ -217,7 +218,7 @@ function pruneNode(node, ctx) {
217
218
  return null;
218
219
  }
219
220
  }
220
- return { ...node, children: [] };
221
+ return { ...node, children: [], ...carryUrls(node, ctx) };
221
222
  }
222
223
 
223
224
  // StaticText — CDP equivalent of mcprune's "text" nodes
@@ -448,6 +449,13 @@ function flatten(nodes) {
448
449
  return result;
449
450
  }
450
451
 
452
+ // Links on: a node that drops its children keeps the URLs of the links beneath it.
453
+ function carryUrls(node, ctx) {
454
+ if (!ctx.links) return {};
455
+ const urls = [...new Set(flatten([node]).flatMap((n) => n.urls || []))];
456
+ return urls.length ? { urls } : {};
457
+ }
458
+
451
459
  function condenseCard(node) {
452
460
  const all = flatten([node]);
453
461
  const link = all.find((n) => n.role === 'link' && n.name);