barebrowse 0.24.1 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,6 +17,8 @@
17
17
 
18
18
  import { formatTree } from './aria.js';
19
19
  import { prune as pruneTree } from './prune.js';
20
+ import { dataSnapshot, stripLegend } from './data.js';
21
+ import { annotateLinks } from './links.js';
20
22
  import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
21
23
  import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
22
24
  import { scopedCookiesForUrl } from './auth.js';
@@ -26,6 +28,7 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
26
28
  import { decideDialog, dialogLogEntry } from './dialog.js';
27
29
  import { isChallengePage, countNodes } from './challenge.js';
28
30
  import { withTimeout } from './with-timeout.js';
31
+ import { navFailed, httpWarning } from './navigation.js';
29
32
 
30
33
  /** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
31
34
  const BIDI_KEYS = {
@@ -39,7 +42,8 @@ const BIDI_KEYS = {
39
42
  * Build a Firefox/BiDi-backed page object.
40
43
  * @param {object} bidi - BiDi client from createBiDi()
41
44
  * @param {object} [opts]
42
- * @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Default prune mode.
45
+ * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Default prune mode.
46
+ * @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
43
47
  * @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
44
48
  * @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
45
49
  * @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
@@ -54,6 +58,15 @@ export async function createFirefoxPage(bidi, opts = {}) {
54
58
  const uploadDir = opts.uploadDir || null;
55
59
  const incognito = !!opts.incognito;
56
60
  const consent = opts.consent !== false;
61
+ // True when the consent pass after the latest goto() found nothing: the first
62
+ // snapshot() re-checks once (late-rendered CMP dialogs).
63
+ // Document identity (tab + performance.timeOrigin) the latest goto() landed on
64
+ // when its consent pass found nothing; null when no re-check is pending.
65
+ let consentRecheck = null;
66
+ async function docKey() {
67
+ try { return `${topContext}:${await bidi.evaluate(topContext, 'performance.timeOrigin', false)}`; }
68
+ catch { return null; }
69
+ }
57
70
  // Hybrid mode: on a bot-challenge page, relaunch headed and retry. The
58
71
  // relaunch tears down this Firefox+BiDi and returns a fresh one, so the
59
72
  // browser lifecycle is owned by connectFirefox, which supplies relaunchHeaded.
@@ -61,6 +74,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
61
74
  const relaunchHeaded = opts.relaunchHeaded || null;
62
75
  let currentlyHeaded = !!opts.headed; // already headed → no fallback needed
63
76
  let botBlocked = false;
77
+ // Outcome of the latest goto() ({url, status}); null before any goto.
78
+ /** @type {import('./navigation.js').NavigationOutcome|null} */
79
+ let lastNavigation = null;
64
80
  // Last injectCookies(url, opts) args, so a hybrid relaunch (a brand-new
65
81
  // Firefox profile) can re-inject them — otherwise the headed retry loads
66
82
  // unauthenticated, defeating hybrid on an auth-gated challenge page.
@@ -244,7 +260,46 @@ export async function createFirefoxPage(bidi, opts = {}) {
244
260
  async function setupSubscriptions() {
245
261
  await setupDialogs();
246
262
  await setupDownloads();
247
- await bidi.subscribe(['browsingContext.load']);
263
+ await bidi.subscribe(['browsingContext.load', 'network.responseCompleted', 'network.responseStarted']);
264
+ }
265
+
266
+ /**
267
+ * browsingContext.navigate(wait:'complete') plus the navigation's outcome:
268
+ * the LAST network.responseCompleted for its navigation id in the top
269
+ * context (one fires per redirect hop; the final hop is where we landed).
270
+ * A network failure is rewrapped as a typed NAV_FAILED error. NS_BINDING_ABORTED
271
+ * (a 204, or a download replacing nothing) is not a failure, as with CDP's
272
+ * ERR_ABORTED: no responseCompleted fires for it, so the status comes from
273
+ * the last network.responseStarted (a 204 reports 204, as on Chromium).
274
+ * @returns {Promise<import('./navigation.js').NavigationOutcome>}
275
+ */
276
+ async function navigateTo(url, timeout) {
277
+ const events = [];
278
+ const started = [];
279
+ const off = bidi.on('network.responseCompleted', (e) => {
280
+ if (e.navigation && e.context === topContext) events.push(e);
281
+ });
282
+ const offStarted = bidi.on('network.responseStarted', (e) => {
283
+ if (e.navigation && e.context === topContext) started.push(e);
284
+ });
285
+ try {
286
+ const res = await withTimeout(
287
+ bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
288
+ timeout, `goto(${url})`);
289
+ const last = events.filter((e) => e.navigation === res.navigation).pop();
290
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
291
+ } catch (err) {
292
+ const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
293
+ if (!m) throw err;
294
+ if (m[1] === 'NS_BINDING_ABORTED') {
295
+ const last = started.pop();
296
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
297
+ }
298
+ throw navFailed(url, m[1]);
299
+ } finally {
300
+ off();
301
+ offStarted();
302
+ }
248
303
  }
249
304
 
250
305
  /**
@@ -260,8 +315,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
260
315
  // the pre-dismiss tree avoids a second full AX reconstruction per goto.
261
316
  let root = null;
262
317
  try { root = await buildTree(); } catch { /* null → treated as challenge */ }
318
+ consentRecheck = null;
263
319
  if (consent && root) {
264
- try { await dismissConsentFirefox(root, (ref) => pointerClick(ref)); }
320
+ try { if (!(await dismissConsentFirefox(root, (ref) => pointerClick(ref)))) consentRecheck = await docKey(); }
265
321
  catch { /* consent dismissal is best-effort */ }
266
322
  }
267
323
  return isChallengePage(root, countNodes(root));
@@ -274,6 +330,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
274
330
  get context() { return topContext; },
275
331
  /** Whether the last goto() landed on a bot-challenge page (parity w/ CDP). */
276
332
  get botBlocked() { return botBlocked; },
333
+ /** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
334
+ get lastNavigation() { return lastNavigation; },
277
335
 
278
336
  /** @type {'firefox'} Engine driving this session (see also `capabilities`). */
279
337
  engine: 'firefox',
@@ -302,9 +360,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
302
360
  // view-source: and (optionally) private-network hosts before navigating.
303
361
  assertNavigable(url, urlGuard);
304
362
  refContexts = new Map(); // refs from the previous page are now stale
305
- await withTimeout(
306
- bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
307
- timeout, `goto(${url})`);
363
+ consentRecheck = null; // a throwing navigate() must not leave the old page's flag
364
+ lastNavigation = null;
365
+ lastNavigation = await navigateTo(url, timeout);
308
366
  // Settle + consent + challenge detection (mirrors the CDP path).
309
367
  botBlocked = await afterNavigate();
310
368
 
@@ -326,32 +384,43 @@ export async function createFirefoxPage(bidi, opts = {}) {
326
384
  if (lastInject) {
327
385
  try { await page.injectCookies(lastInject.url, lastInject.cookieOpts); } catch { /* best-effort */ }
328
386
  }
329
- await withTimeout(
330
- bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
331
- timeout, `goto(${url})`);
387
+ lastNavigation = await navigateTo(url, timeout);
332
388
  botBlocked = await afterNavigate();
333
389
  } catch {
334
390
  // Headed relaunch failed (no display?) — keep the headless result;
335
391
  // botBlocked stays true so the caller can see it didn't clear.
336
392
  }
337
393
  }
394
+ return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
338
395
  },
339
396
 
340
397
  /** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
341
398
  async snapshot(pruneOpts = {}) {
342
- const root = await buildTree();
399
+ let root = await buildTree();
400
+ if (root && consentRecheck) {
401
+ const key = consentRecheck;
402
+ consentRecheck = null; // once per navigation
403
+ try {
404
+ if (key === await docKey() && await dismissConsentFirefox(root, (ref) => pointerClick(ref))) root = await buildTree();
405
+ } catch { /* best-effort */ }
406
+ }
343
407
  if (!root) return '';
344
408
 
345
409
  const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
346
410
  const raw = formatTree(root);
347
- if (pruneOpts === false) return `url: ${pageUrl}\n` + raw;
411
+ const warn = httpWarning(lastNavigation?.status);
412
+ const links = !!((pruneOpts || undefined)?.links ?? opts.links);
413
+ const tree = links ? annotateLinks(root, pageUrl) : root;
414
+ if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
348
415
 
349
416
  const mode = pruneOpts?.mode || defaultPruneMode;
350
- const pruned = pruneTree(root, { mode });
351
- const out = pruned ? formatTree(pruned) : '';
352
- const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${out.length.toLocaleString()} chars`
353
- + ` (${raw.length ? Math.round((1 - out.length / raw.length) * 100) : 0}% pruned)`;
354
- return stats + '\n' + out;
417
+ let out;
418
+ if (mode === 'data') out = dataSnapshot(tree, { links });
419
+ else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
420
+ const sized = stripLegend(out); // the legend is not content
421
+ const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars`
422
+ + ` (${raw.length ? Math.round((1 - sized.length / raw.length) * 100) : 0}% pruned)`;
423
+ return stats + '\n' + warn + out;
355
424
  },
356
425
 
357
426
  async readable() {
@@ -492,6 +561,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
492
561
  bidi.send('browsingContext.reload', { context: topContext, wait: 'complete' }),
493
562
  30000, 'reload');
494
563
  refContexts = new Map();
564
+ consentRecheck = null;
565
+ lastNavigation = null; // the reload's status is not read; don't keep the old one
495
566
  await new Promise((r) => setTimeout(r, 300));
496
567
  },
497
568
 
@@ -528,6 +599,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
528
599
  if (index < 0 || index >= tree.length) throw new Error(`Tab index ${index} out of range (0-${tree.length - 1})`);
529
600
  topContext = tree[index].context;
530
601
  refContexts = new Map(); // refs from the previous tab are invalid
602
+ lastNavigation = null; // the outcome belonged to the tab we left
531
603
  await bidi.send('browsingContext.activate', { context: topContext }).catch(() => {});
532
604
  },
533
605
 
@@ -644,6 +716,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
644
716
  bidi.send('browsingContext.traverseHistory', { context: topContext, delta }),
645
717
  30000, 'history navigation');
646
718
  refContexts = new Map();
719
+ consentRecheck = null;
720
+ lastNavigation = null; // no outcome is known for the history entry we landed on
647
721
  await new Promise((r) => setTimeout(r, 500));
648
722
  }
649
723
 
package/src/index.js CHANGED
@@ -16,6 +16,8 @@ import { createFirefoxPage } from './firefox-page.js';
16
16
  import { formatTree } from './aria.js';
17
17
  import { authenticate } from './auth.js';
18
18
  import { prune as pruneTree } from './prune.js';
19
+ import { dataSnapshot, stripLegend } from './data.js';
20
+ import { annotateLinks } from './links.js';
19
21
  import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
20
22
  import { dismissConsent } from './consent.js';
21
23
  import { applyStealth } from './stealth.js';
@@ -27,6 +29,7 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
27
29
  import { isChallengePage } from './challenge.js';
28
30
  import { withTimeout } from './with-timeout.js';
29
31
  import { readable as extractReadable } from './readable.js';
32
+ import { navFailed, httpWarning } from './navigation.js';
30
33
  import { assertNavigable, assertUploadAllowed } from './url-guard.js';
31
34
  import { join as pathJoin } from 'node:path';
32
35
  import { chmodSync } from 'node:fs';
@@ -58,7 +61,8 @@ import { chmodSync } from 'node:fs';
58
61
  * @param {{width: number, height: number}} [opts.viewport] - Viewport dimensions.
59
62
  * @param {string} [opts.browser] - Source browser for cookie extraction.
60
63
  * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
61
- * @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Pruning mode.
64
+ * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
65
+ * @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
62
66
  * @returns {Promise<string>} ARIA snapshot text
63
67
  */
64
68
  export async function browse(url, opts = {}) {
@@ -105,7 +109,7 @@ export async function browse(url, opts = {}) {
105
109
  }
106
110
 
107
111
  // Step 4: Navigate and wait for load
108
- await navigate(page, url, timeout);
112
+ let outcome = await navigate(page, url, timeout);
109
113
 
110
114
  // Step 4.5: Auto-dismiss cookie consent dialogs
111
115
  if (opts.consent !== false) {
@@ -129,7 +133,7 @@ export async function browse(url, opts = {}) {
129
133
  if (!noAuth) {
130
134
  try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
131
135
  }
132
- await navigate(page, url, timeout);
136
+ outcome = await navigate(page, url, timeout);
133
137
  if (opts.consent !== false) await dismissConsent(page.session);
134
138
  ({ tree } = await ariaTree(page));
135
139
  } catch {
@@ -139,19 +143,27 @@ export async function browse(url, opts = {}) {
139
143
 
140
144
  // Step 6: Prune for agent consumption
141
145
  const raw = formatTree(tree);
146
+ const links = !!opts.links;
147
+ let linked = tree;
148
+ if (links) {
149
+ const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
150
+ linked = annotateLinks(tree, entries[currentIndex]?.url || url);
151
+ }
142
152
  let snapshot;
143
153
  if (opts.prune !== false) {
144
- const pruned = pruneTree(tree, { mode: opts.pruneMode || 'act' });
145
- snapshot = formatTree(pruned);
154
+ snapshot = opts.pruneMode === 'data'
155
+ ? dataSnapshot(linked, { links })
156
+ : formatTree(pruneTree(linked, { mode: opts.pruneMode || 'act', links }));
146
157
  } else {
147
- snapshot = raw;
158
+ snapshot = links ? formatTree(linked) : raw;
148
159
  }
149
- const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${snapshot.length.toLocaleString()} chars (${Math.round((1 - snapshot.length / raw.length) * 100)}% pruned)`;
160
+ const sized = stripLegend(snapshot); // the legend is not content
161
+ const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
150
162
  const actMode = !opts.pruneMode || opts.pruneMode === 'act';
151
163
  const hint = (actMode && raw.length > 5000 && snapshot.length < 500 && snapshot.length < raw.length * 0.05)
152
164
  ? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
153
165
  : '';
154
- snapshot = stats + '\n' + hint + snapshot;
166
+ snapshot = stats + '\n' + hint + httpWarning(outcome.status) + snapshot;
155
167
 
156
168
  // Step 7: Clean up
157
169
  await cdp.send('Target.closeTarget', { targetId: page.targetId });
@@ -201,10 +213,20 @@ export async function browse(url, opts = {}) {
201
213
  * @property {boolean} cookieInjection - Cookie extraction/injection is active (off in incognito)
202
214
  */
203
215
 
216
+ /**
217
+ * What `goto()` resolves to: the final URL (after redirects) and the main
218
+ * document's HTTP status (0 when no HTTP response was observed, e.g. an aborted navigation).
219
+ * Network failures reject with an Error whose `code` is 'NAV_FAILED' and
220
+ * `reason` the browser's raw error string.
221
+ * @typedef {import('./navigation.js').NavigationOutcome} NavigationOutcome
222
+ */
223
+
204
224
  /**
205
225
  * Options for `page.snapshot()`. Pass `false` instead of an object to skip
206
226
  * pruning and return the raw ARIA tree.
207
- * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read' }} SnapshotOptions
227
+ * `links: true` adds each link's absolute URL after its name (default off,
228
+ * or the `connect({ links })` default).
229
+ * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
208
230
  */
209
231
 
210
232
  /**
@@ -257,7 +279,8 @@ export async function browse(url, opts = {}) {
257
279
  * @param {boolean} [opts.incognito=false] - Clean, unauthenticated session:
258
280
  * skip storageState loading and make injectCookies() a no-op, so no auth
259
281
  * ever enters the session even if a caller injects unconditionally.
260
- * @param {'act'|'browse'|'navigate'|'full'|'read'} [opts.pruneMode='act'] - Pruning mode.
282
+ * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
283
+ * @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
261
284
  * @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
262
285
  * 'firefox' drives over WebDriver BiDi (a separate transport / page object);
263
286
  * 'chromium' (default) uses CDP.
@@ -364,6 +387,24 @@ export async function connect(opts = {}) {
364
387
  let page = await createPage(cdp, !currentlyHeaded, pageOpts);
365
388
  let refMap = new Map();
366
389
  let botBlocked = false;
390
+ // Outcome of the latest goto() ({url, status}); null before any goto.
391
+ /** @type {NavigationOutcome|null} */
392
+ let lastNavigation = null;
393
+ // Set when the consent pass after the latest goto() found nothing: the first
394
+ // snapshot() re-checks once, since some CMPs (Google Funding Choices) render
395
+ // their dialog 1-3s after load. Holds the document identity (tab + main-frame
396
+ // loaderId) the goto landed on, so a snapshot of any other document (click,
397
+ // redirect, switchTab) never re-checks. loaderId changes on every
398
+ // cross-document navigation but not on pushState/replaceState/hash change.
399
+ // Null when no re-check is pending.
400
+ // Also set in attach mode (goto() runs consent there too).
401
+ let consentRecheck = null;
402
+ async function navKey() {
403
+ try {
404
+ const { frameTree } = await page.session.send('Page.getFrameTree');
405
+ return `${page.targetId}:${frameTree.frame.loaderId}`;
406
+ } catch { return null; }
407
+ }
367
408
 
368
409
  // Suppress permission prompts. Skipped in attach mode — Browser.setPermission
369
410
  // is browser-wide (no origin scope here), so flipping permissions to denied
@@ -470,6 +511,8 @@ export async function connect(opts = {}) {
470
511
  // silently resolving to whatever backendNodeId happens to still be in
471
512
  // the map.
472
513
  refMap = new Map();
514
+ consentRecheck = null; // a throwing navigate() must not leave the old page's flag
515
+ lastNavigation = null;
473
516
  // Switch back to headless if we fell back to headed previously.
474
517
  // Not in attach mode — we never own the browser there, so there's
475
518
  // nothing to rewind.
@@ -486,9 +529,10 @@ export async function connect(opts = {}) {
486
529
  currentlyHeaded = false;
487
530
  }
488
531
 
489
- await navigate(page, url, timeout);
532
+ lastNavigation = await navigate(page, url, timeout);
490
533
  if (opts.consent !== false) {
491
- await dismissConsent(page.session);
534
+ const dismissed = await dismissConsent(page.session);
535
+ if (!dismissed) consentRecheck = await navKey();
492
536
  }
493
537
 
494
538
  // Check for bot challenge
@@ -509,8 +553,11 @@ export async function connect(opts = {}) {
509
553
  page = await createPage(cdp, false, pageOpts);
510
554
  setupDialogHandler(page.session);
511
555
  await suppressPermissions(cdp);
512
- await navigate(page, url, timeout);
513
- if (opts.consent !== false) await dismissConsent(page.session);
556
+ lastNavigation = await navigate(page, url, timeout);
557
+ consentRecheck = null;
558
+ if (opts.consent !== false && !(await dismissConsent(page.session))) {
559
+ consentRecheck = await navKey();
560
+ }
514
561
 
515
562
  // Re-check after headed fallback
516
563
  const after = await ariaTree(page);
@@ -520,6 +567,7 @@ export async function connect(opts = {}) {
520
567
  // Headed launch failed (no display?) — keep headless result, botBlocked stays true
521
568
  }
522
569
  }
570
+ return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
523
571
  },
524
572
 
525
573
  async goBack() {
@@ -529,6 +577,8 @@ export async function connect(opts = {}) {
529
577
  await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex - 1].id });
530
578
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
531
579
  refMap = new Map(); // refs from the previous page are now invalid
580
+ consentRecheck = null;
581
+ lastNavigation = null; // no outcome is known for the history entry we landed on
532
582
  },
533
583
 
534
584
  async goForward() {
@@ -538,6 +588,8 @@ export async function connect(opts = {}) {
538
588
  await page.session.send('Page.navigateToHistoryEntry', { entryId: entries[currentIndex + 1].id });
539
589
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
540
590
  refMap = new Map();
591
+ consentRecheck = null;
592
+ lastNavigation = null;
541
593
  },
542
594
 
543
595
  async reload(reloadOpts = {}) {
@@ -548,6 +600,8 @@ export async function connect(opts = {}) {
548
600
  });
549
601
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
550
602
  refMap = new Map(); // refs from the pre-reload page are invalid
603
+ consentRecheck = null;
604
+ lastNavigation = null; // the reload's status is not read; don't keep the old one
551
605
  },
552
606
 
553
607
  /** @param {string} url @param {CookieOptions} [cookieOpts] */
@@ -560,17 +614,28 @@ export async function connect(opts = {}) {
560
614
 
561
615
  /** @param {SnapshotOptions|false} [pruneOpts] */
562
616
  async snapshot(pruneOpts = {}) {
617
+ if (consentRecheck) {
618
+ const key = consentRecheck;
619
+ consentRecheck = null; // once per navigation
620
+ // Only if we are still on the document the goto landed on.
621
+ if (key === await navKey()) {
622
+ try { await dismissConsent(page.session); } catch { /* best-effort */ }
623
+ }
624
+ }
563
625
  const result = await ariaTree(page);
564
626
  refMap = result.refMap;
565
627
  const raw = formatTree(result.tree);
566
628
  const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
567
629
  const pageUrl = entries[currentIndex]?.url || '';
568
- const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '';
569
- if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + raw;
570
- const mode = pruneOpts?.mode || 'act';
571
- const pruned = pruneTree(result.tree, { mode });
572
- const out = formatTree(pruned);
573
- const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${out.length.toLocaleString()} chars (${Math.round((1 - out.length / raw.length) * 100)}% pruned)`;
630
+ const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
631
+ + httpWarning(lastNavigation?.status);
632
+ const links = !!((pruneOpts || undefined)?.links ?? opts.links);
633
+ const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
634
+ if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
635
+ const mode = pruneOpts?.mode || opts.pruneMode || 'act';
636
+ const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
637
+ const sized = stripLegend(out); // the legend is not content
638
+ const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
574
639
  const hint = (mode === 'act' && raw.length > 5000 && out.length < 500 && out.length < raw.length * 0.05)
575
640
  ? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
576
641
  : '';
@@ -668,6 +733,7 @@ export async function connect(opts = {}) {
668
733
  const oldSessionId = page.sessionId;
669
734
  page = await attachToExistingTarget(cdp, target.targetId, pageOpts);
670
735
  refMap = new Map(); // refs from the previous tab are no longer valid
736
+ lastNavigation = null; // the outcome belonged to the tab we left
671
737
  setupDialogHandler(page.session);
672
738
  try { await cdp.send('Target.detachFromTarget', { sessionId: oldSessionId }); } catch {}
673
739
  },
@@ -712,6 +778,8 @@ export async function connect(opts = {}) {
712
778
  },
713
779
 
714
780
  get botBlocked() { return botBlocked; },
781
+ /** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
782
+ get lastNavigation() { return lastNavigation; },
715
783
 
716
784
  /** @type {'chromium'} Engine driving this session (see also `capabilities`) */
717
785
  engine: 'chromium',
@@ -786,12 +854,13 @@ export async function connect(opts = {}) {
786
854
  return {
787
855
  async goto(url, timeout = 30000) {
788
856
  assertNavigable(url, urlGuard);
789
- await navigate(tab, url, timeout);
857
+ const outcome = await navigate(tab, url, timeout);
790
858
  if (opts.consent !== false) {
791
859
  await dismissConsent(tab.session);
792
860
  }
793
861
  const { tree, nodeCount } = await ariaTree(tab);
794
862
  tabBotBlocked = isChallengePage(tree, nodeCount);
863
+ return outcome;
795
864
  },
796
865
  get botBlocked() { return tabBotBlocked; },
797
866
  /** @param {string} url @param {CookieOptions} [cookieOpts] */
@@ -925,6 +994,7 @@ async function connectFirefox(opts) {
925
994
 
926
995
  const page = await createFirefoxPage(bidi, {
927
996
  pruneMode: opts.pruneMode,
997
+ links: opts.links,
928
998
  urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
929
999
  uploadDir: opts.uploadDir || null,
930
1000
  incognito: !!opts.incognito,
@@ -1092,14 +1162,45 @@ export function _resetBlocklistWarning() {
1092
1162
  }
1093
1163
 
1094
1164
  /**
1095
- * Navigate to a URL and wait for the page to load.
1165
+ * Navigate to a URL and wait for the page to load. Resolves to the outcome of
1166
+ * the main-document navigation (final URL after redirects, HTTP status);
1167
+ * rejects with a NAV_FAILED error when the browser could not reach a document.
1168
+ * @returns {Promise<NavigationOutcome>}
1096
1169
  */
1097
1170
  async function navigate(page, url, timeout = 30000) {
1098
- const loadPromise = page.session.once('Page.loadEventFired', timeout);
1099
- await page.session.send('Page.navigate', { url });
1100
- await loadPromise;
1101
- // Brief settle time for dynamic content
1102
- await new Promise((r) => setTimeout(r, 500));
1171
+ const { session } = page;
1172
+ // Main-document responses (every redirect hop fires one; the last wins).
1173
+ const docs = [];
1174
+ const offDoc = session.on('Network.responseReceived', (p) => { if (p.type === 'Document') docs.push(p); });
1175
+ // Load waiter, hand-rolled (not session.once) so it can be cancelled when
1176
+ // Page.navigate fails or aborts and no load event will ever come.
1177
+ let cancel = () => {};
1178
+ const loaded = /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
1179
+ const timer = setTimeout(() => { off(); reject(new Error('Timeout waiting for CDP event: Page.loadEventFired')); }, timeout);
1180
+ const off = session.on('Page.loadEventFired', () => { cancel(); resolve(undefined); });
1181
+ cancel = () => { clearTimeout(timer); off(); };
1182
+ }));
1183
+ loaded.catch(() => {}); // rejection is only read if we await it below
1184
+ try {
1185
+ const nav = await session.send('Page.navigate', { url });
1186
+ // ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
1187
+ // no document and fires no load event, so don't wait for one.
1188
+ const aborted = nav.errorText === 'net::ERR_ABORTED';
1189
+ // ERR_HTTP_RESPONSE_CODE_FAILURE is a 4xx/5xx with an empty body: the
1190
+ // document did load (Chromium's error page fires load, the Document
1191
+ // response carries the status), so report it like any other HTTP error.
1192
+ const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
1193
+ if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
1194
+ if (!aborted) await loaded;
1195
+ // Brief settle time for dynamic content
1196
+ await new Promise((r) => setTimeout(r, 500));
1197
+ // loaderId is shared by every redirect hop of one navigation.
1198
+ const last = docs.filter((d) => d.loaderId === nav.loaderId).pop();
1199
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
1200
+ } finally {
1201
+ cancel();
1202
+ offDoc();
1203
+ }
1103
1204
  }
1104
1205
 
1105
1206
  /**
package/src/links.js ADDED
@@ -0,0 +1,46 @@
1
+ /**
2
+ * links.js — opt-in link URLs for snapshots (`links: true`).
3
+ *
4
+ * `annotateLinks()` stamps `urls: [url]` on every link node that has a usable
5
+ * browser-serialized absolute URL. aria.js, prune.js and data.js print/carry
6
+ * `urls` — they never see the option, so with links off nothing changes.
7
+ */
8
+
9
+ /**
10
+ * Normalise a link's URL for a snapshot line. Returns '' when it should not be
11
+ * shown: no URL, `javascript:`, or a same-document link (the page URL, only
12
+ * the #fragment differs). A raw space, `<` or `>` (browsers encode them, but
13
+ * not always in the fragment) is percent-encoded so a line splits cleanly on
14
+ * ` <` and `> `.
15
+ *
16
+ * @param {string|undefined} url - Absolute URL as serialized by the browser
17
+ * @param {string} [pageUrl] - URL of the page the link is on
18
+ * @returns {string}
19
+ */
20
+ export function linkUrl(url, pageUrl = '') {
21
+ if (typeof url !== 'string') return '';
22
+ const u = url.trim();
23
+ if (!u || /^javascript:/i.test(u)) return '';
24
+ if (pageUrl && u.split('#')[0] === pageUrl.split('#')[0]) return '';
25
+ return u.replace(/ /g, '%20').replace(/</g, '%3C').replace(/>/g, '%3E');
26
+ }
27
+
28
+ /**
29
+ * Copy a tree with `urls` set on its link nodes (input is not mutated).
30
+ *
31
+ * @param {object|null} node - Root from buildTree()
32
+ * @param {string} pageUrl
33
+ * @returns {object|null}
34
+ */
35
+ export function annotateLinks(node, pageUrl) {
36
+ if (!node) return node;
37
+ const out = { ...node, children: node.children.map((c) => annotateLinks(c, pageUrl)) };
38
+ if (node.role === 'link') {
39
+ const u = linkUrl(node.properties?.url, pageUrl);
40
+ if (u) out.urls = [u];
41
+ }
42
+ return out;
43
+ }
44
+
45
+ /** ` <u1> <u2>` for a node's urls ('' when none). */
46
+ export const urlsText = (urls) => (urls || []).map((u) => ` <${u}>`).join('');
@@ -0,0 +1,42 @@
1
+ /**
2
+ * navigation.js — Shared pieces of the goto() outcome contract, so both
3
+ * engines (CDP in index.js, BiDi in firefox-page.js) report it identically.
4
+ */
5
+
6
+ /**
7
+ * What goto() resolves to: where the page ended up and how the server answered.
8
+ * `status` is 0 when no HTTP response was observed for the navigation (e.g. an
9
+ * aborted navigation that never got one).
10
+ * @typedef {{ url: string, status: number }} NavigationOutcome
11
+ */
12
+
13
+ /**
14
+ * Typed error for a navigation that never reached a document (DNS failure,
15
+ * connection refused, TLS error, ...). `code` is 'NAV_FAILED'; `reason` is the
16
+ * browser's raw error string (`net::ERR_NAME_NOT_RESOLVED`,
17
+ * `NS_ERROR_UNKNOWN_HOST`, ...).
18
+ * @param {string} url
19
+ * @param {string} reason
20
+ * @returns {Error & {code: 'NAV_FAILED', reason: string}}
21
+ */
22
+ export function navFailed(url, reason) {
23
+ return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
24
+ }
25
+
26
+ /**
27
+ * True for a final HTTP status that is not 2xx (0 = unknown is not an error).
28
+ * @param {number|undefined} status
29
+ */
30
+ export function isHttpError(status) {
31
+ return !!status && (status < 200 || status >= 300);
32
+ }
33
+
34
+ /**
35
+ * Warning line for a non-2xx final status ('' for 2xx and for unknown 0), in
36
+ * the style of the bot-challenge line, newline-terminated for prepending.
37
+ * @param {number|undefined} status
38
+ * @returns {string}
39
+ */
40
+ export function httpWarning(status) {
41
+ return isHttpError(status) ? `[HTTP ${status} — the page may be an error page]\n` : '';
42
+ }