barebrowse 0.25.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/data.js CHANGED
@@ -33,6 +33,8 @@
33
33
  * click/type resolve through the same refMap).
34
34
  */
35
35
 
36
+ import { urlsText } from './links.js';
37
+
36
38
  const SKIP = new Set(['InlineTextBox', 'LineBreak', 'ListMarker', 'image', 'img']);
37
39
  const CELLS = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);
38
40
  const TABLES = new Set(['table', 'grid', 'treegrid']);
@@ -47,13 +49,20 @@ const MIN_RUN = 3;
47
49
  /** First line of every data-mode snapshot: tells an agent with no docs how to read it. */
48
50
  export const LEGEND = '# data mode: one line per record. Tables: row "<label>": <column>: <value> · … Groups: <label>: a, b, c. A title line followed by detail lines is one block, up to the next title line. Main content only; use mode "act" for menus and page controls.';
49
51
 
52
+ /** Appended to the legend only when links are on. */
53
+ export const LINKS_NOTE = ' Links: "name" <url>.';
54
+
50
55
  // A name made only of private-use code points (icon-font glyphs, unreadable to an
51
56
  // agent) is no name. Glyphs mixed into real text are left as they are.
52
57
  const norm = (s) => (/^[\p{Co}\s]*$/u.test(s || '') ? '' : s.replace(/\s+/g, ' ').trim());
58
+ const allUrls = (n) => [...(n.urls || []), ...(n.children || []).flatMap(allUrls)];
53
59
  const hasInteractive = (n) => INTERACTIVE.has(n.role) || (n.children || []).some(hasInteractive);
54
60
  const isLeaf = (n) => n.text === undefined && !n.children.length && n.name;
55
61
  const showRef = (n) => !!n.ref && INTERACTIVE.has(n.role);
56
- const withRef = (n) => (showRef(n) ? `${n.name} [ref=${n.ref}]` : n.name);
62
+ // ` <url>` for a node's own links (already unique per node). A link line always
63
+ // carries its own URL: nothing is deduped across lines, items or cells.
64
+ const urlPart = (n) => urlsText(n.urls);
65
+ const withRef = (n) => (showRef(n) ? `${n.name}${urlPart(n)} [ref=${n.ref}]` : n.name);
57
66
  // Leaves carrying state (checked, level, value...) never merge into a run.
58
67
  const plain = (n) => !(n.props.checked !== undefined || n.props.expanded !== undefined || n.props.selected || n.props.level || n.props.required || (n.props.value !== undefined && n.props.value !== ''));
59
68
  // Run key: leaves with the same key are "the same kind of item". `g` = the
@@ -69,19 +78,20 @@ function inline(n, refs) {
69
78
  const sub = n.children.length ? n.children.map((c) => inline(c, refs)).filter(Boolean).join(' ') : n.name;
70
79
  const v = (n.name && !n.children.length) || INTERACTIVE.has(n.role) && n.name && refs ? n.name : sub;
71
80
  const out = n.children.length && n.name && n.role !== 'StaticText' && !sub ? n.name : v;
72
- return refs && n.ref && INTERACTIVE.has(n.role) ? `${out} [ref=${n.ref}]` : out;
81
+ return refs && n.ref && INTERACTIVE.has(n.role) ? `${out}${urlPart(n)} [ref=${n.ref}]` : out;
73
82
  }
74
83
 
75
84
  /** A data-mode result without its legend line, for size stats ('' stays ''). */
76
- export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(LEGEND.length + 1) : s);
85
+ export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(s.indexOf('\n') + 1) : s);
77
86
 
78
87
  /**
79
88
  * Render a raw ARIA tree in data mode.
80
89
  *
81
90
  * @param {object} root - Root node from buildTree() (CDP vocabulary)
91
+ * @param {{ links?: boolean }} [opts] - links: the tree's link nodes carry `urls` (annotateLinks); adds the legend note
82
92
  * @returns {string} LEGEND line, then one line per record, YAML-ish like formatTree(); '' when there are no records
83
93
  */
84
- export function dataSnapshot(root) {
94
+ export function dataSnapshot(root, opts = {}) {
85
95
  if (!root) return '';
86
96
  let gid = 0;
87
97
  const rawKids = (n) => n.children.flatMap((c) => (c.ignored ? rawKids(c) : [c]));
@@ -120,7 +130,7 @@ export function dataSnapshot(root) {
120
130
  if (!nm && kids.length === 0 && (n.role === 'link' || n.role === 'button')) return []; // N6
121
131
  if (!nm && kids.length === 0 && !INTERACTIVE.has(n.role) && !CELLS.has(n.role) && n.role !== 'row') return [];
122
132
  if (n.role === 'listitem' && !nm && kids.length === 1) return kids;
123
- return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig }];
133
+ return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig, urls: n.urls }];
124
134
  }
125
135
 
126
136
  // ---------- Transforms, bottom-up with the ancestor chain for labels ----------
@@ -134,7 +144,7 @@ export function dataSnapshot(root) {
134
144
  const out = [];
135
145
  for (const k of kids) {
136
146
  const p = out[out.length - 1];
137
- if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name) continue;
147
+ if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name && (p.urls || [])[0] === (k.urls || [])[0]) continue;
138
148
  out.push(k);
139
149
  }
140
150
  return out;
@@ -243,6 +253,7 @@ export function dataSnapshot(root) {
243
253
  const labelCells = cs.filter((c) => c.role === 'rowheader' || c.role === 'columnheader');
244
254
  const lcx = labelCells.map(xcell);
245
255
  const label = lcx.map((c) => inline(c, false)).filter(Boolean).join(' / ');
256
+ const labelUrls = urlsText([...new Set(lcx.flatMap(allUrls))]);
246
257
  const labelRef = lcx.map(firstRef).find(Boolean);
247
258
  const vals = [];
248
259
  cs.forEach((c, idx) => {
@@ -262,7 +273,7 @@ export function dataSnapshot(root) {
262
273
  }
263
274
  if (!vals.length && !label) continue; // header-only / empty rows
264
275
  const body = vals.map((x) => (x.h ? `${x.h}: ` : '') + x.v).join(' · ');
265
- out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
276
+ out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelUrls}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
266
277
  }
267
278
  return { ...t, children: out };
268
279
  }
@@ -278,10 +289,14 @@ export function dataSnapshot(root) {
278
289
  const lines = [];
279
290
  function emit(c, d) {
280
291
  const ind = ' '.repeat(d);
281
- if (c.text !== undefined) { lines.push(`${ind}- ${c.text}`); return; }
292
+ if (c.text !== undefined) {
293
+ lines.push(`${ind}- ${c.text}`);
294
+ return;
295
+ }
282
296
  if (isLeaf(c) && (c.role === 'StaticText' || c.role === 'generic')) { lines.push(`${ind}- ${c.name}`); return; }
283
297
  let l = `${ind}- ${c.role}`;
284
298
  if (c.name) l += ` "${c.name}"`;
299
+ l += urlPart(c);
285
300
  const p = c.props, pp = [];
286
301
  if (p.checked !== undefined) pp.push(`checked=${p.checked}`);
287
302
  if (p.disabled) pp.push('disabled');
@@ -296,5 +311,5 @@ export function dataSnapshot(root) {
296
311
  for (const k of c.children) emit(k, d + 1);
297
312
  }
298
313
  for (const c of res.children) emit(c, 0);
299
- return lines.length ? [LEGEND, ...lines].join('\n') : '';
314
+ return lines.length ? [LEGEND + (opts.links ? LINKS_NOTE : ''), ...lines].join('\n') : '';
300
315
  }
@@ -18,6 +18,7 @@
18
18
  import { formatTree } from './aria.js';
19
19
  import { prune as pruneTree } from './prune.js';
20
20
  import { dataSnapshot, stripLegend } from './data.js';
21
+ import { annotateLinks } from './links.js';
21
22
  import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
22
23
  import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
23
24
  import { scopedCookiesForUrl } from './auth.js';
@@ -27,6 +28,7 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
27
28
  import { decideDialog, dialogLogEntry } from './dialog.js';
28
29
  import { isChallengePage, countNodes } from './challenge.js';
29
30
  import { withTimeout } from './with-timeout.js';
31
+ import { navFailed, httpWarning } from './navigation.js';
30
32
 
31
33
  /** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
32
34
  const BIDI_KEYS = {
@@ -41,6 +43,7 @@ const BIDI_KEYS = {
41
43
  * @param {object} bidi - BiDi client from createBiDi()
42
44
  * @param {object} [opts]
43
45
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Default prune mode.
46
+ * @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
44
47
  * @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
45
48
  * @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
46
49
  * @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
@@ -71,6 +74,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
71
74
  const relaunchHeaded = opts.relaunchHeaded || null;
72
75
  let currentlyHeaded = !!opts.headed; // already headed → no fallback needed
73
76
  let botBlocked = false;
77
+ // Outcome of the latest goto() ({url, status}); null before any goto.
78
+ /** @type {import('./navigation.js').NavigationOutcome|null} */
79
+ let lastNavigation = null;
74
80
  // Last injectCookies(url, opts) args, so a hybrid relaunch (a brand-new
75
81
  // Firefox profile) can re-inject them — otherwise the headed retry loads
76
82
  // unauthenticated, defeating hybrid on an auth-gated challenge page.
@@ -254,7 +260,46 @@ export async function createFirefoxPage(bidi, opts = {}) {
254
260
  async function setupSubscriptions() {
255
261
  await setupDialogs();
256
262
  await setupDownloads();
257
- await bidi.subscribe(['browsingContext.load']);
263
+ await bidi.subscribe(['browsingContext.load', 'network.responseCompleted', 'network.responseStarted']);
264
+ }
265
+
266
+ /**
267
+ * browsingContext.navigate(wait:'complete') plus the navigation's outcome:
268
+ * the LAST network.responseCompleted for its navigation id in the top
269
+ * context (one fires per redirect hop; the final hop is where we landed).
270
+ * A network failure is rewrapped as a typed NAV_FAILED error. NS_BINDING_ABORTED
271
+ * (a 204, or a download replacing nothing) is not a failure, as with CDP's
272
+ * ERR_ABORTED: no responseCompleted fires for it, so the status comes from
273
+ * the last network.responseStarted (a 204 reports 204, as on Chromium).
274
+ * @returns {Promise<import('./navigation.js').NavigationOutcome>}
275
+ */
276
+ async function navigateTo(url, timeout) {
277
+ const events = [];
278
+ const started = [];
279
+ const off = bidi.on('network.responseCompleted', (e) => {
280
+ if (e.navigation && e.context === topContext) events.push(e);
281
+ });
282
+ const offStarted = bidi.on('network.responseStarted', (e) => {
283
+ if (e.navigation && e.context === topContext) started.push(e);
284
+ });
285
+ try {
286
+ const res = await withTimeout(
287
+ bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
288
+ timeout, `goto(${url})`);
289
+ const last = events.filter((e) => e.navigation === res.navigation).pop();
290
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
291
+ } catch (err) {
292
+ const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
293
+ if (!m) throw err;
294
+ if (m[1] === 'NS_BINDING_ABORTED') {
295
+ const last = started.pop();
296
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
297
+ }
298
+ throw navFailed(url, m[1]);
299
+ } finally {
300
+ off();
301
+ offStarted();
302
+ }
258
303
  }
259
304
 
260
305
  /**
@@ -285,6 +330,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
285
330
  get context() { return topContext; },
286
331
  /** Whether the last goto() landed on a bot-challenge page (parity w/ CDP). */
287
332
  get botBlocked() { return botBlocked; },
333
+ /** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
334
+ get lastNavigation() { return lastNavigation; },
288
335
 
289
336
  /** @type {'firefox'} Engine driving this session (see also `capabilities`). */
290
337
  engine: 'firefox',
@@ -314,9 +361,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
314
361
  assertNavigable(url, urlGuard);
315
362
  refContexts = new Map(); // refs from the previous page are now stale
316
363
  consentRecheck = null; // a throwing navigate() must not leave the old page's flag
317
- await withTimeout(
318
- bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
319
- timeout, `goto(${url})`);
364
+ lastNavigation = null;
365
+ lastNavigation = await navigateTo(url, timeout);
320
366
  // Settle + consent + challenge detection (mirrors the CDP path).
321
367
  botBlocked = await afterNavigate();
322
368
 
@@ -338,15 +384,14 @@ export async function createFirefoxPage(bidi, opts = {}) {
338
384
  if (lastInject) {
339
385
  try { await page.injectCookies(lastInject.url, lastInject.cookieOpts); } catch { /* best-effort */ }
340
386
  }
341
- await withTimeout(
342
- bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
343
- timeout, `goto(${url})`);
387
+ lastNavigation = await navigateTo(url, timeout);
344
388
  botBlocked = await afterNavigate();
345
389
  } catch {
346
390
  // Headed relaunch failed (no display?) — keep the headless result;
347
391
  // botBlocked stays true so the caller can see it didn't clear.
348
392
  }
349
393
  }
394
+ return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
350
395
  },
351
396
 
352
397
  /** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
@@ -363,16 +408,19 @@ export async function createFirefoxPage(bidi, opts = {}) {
363
408
 
364
409
  const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
365
410
  const raw = formatTree(root);
366
- if (pruneOpts === false) return `url: ${pageUrl}\n` + raw;
411
+ const warn = httpWarning(lastNavigation?.status);
412
+ const links = !!((pruneOpts || undefined)?.links ?? opts.links);
413
+ const tree = links ? annotateLinks(root, pageUrl) : root;
414
+ if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
367
415
 
368
416
  const mode = pruneOpts?.mode || defaultPruneMode;
369
417
  let out;
370
- if (mode === 'data') out = dataSnapshot(root);
371
- else { const pruned = pruneTree(root, { mode }); out = pruned ? formatTree(pruned) : ''; }
372
- const sized = mode === 'data' ? stripLegend(out) : out; // the legend is not content
418
+ if (mode === 'data') out = dataSnapshot(tree, { links });
419
+ else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
420
+ const sized = stripLegend(out); // the legend is not content
373
421
  const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars`
374
422
  + ` (${raw.length ? Math.round((1 - sized.length / raw.length) * 100) : 0}% pruned)`;
375
- return stats + '\n' + out;
423
+ return stats + '\n' + warn + out;
376
424
  },
377
425
 
378
426
  async readable() {
@@ -514,6 +562,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
514
562
  30000, 'reload');
515
563
  refContexts = new Map();
516
564
  consentRecheck = null;
565
+ lastNavigation = null; // the reload's status is not read; don't keep the old one
517
566
  await new Promise((r) => setTimeout(r, 300));
518
567
  },
519
568
 
@@ -550,6 +599,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
550
599
  if (index < 0 || index >= tree.length) throw new Error(`Tab index ${index} out of range (0-${tree.length - 1})`);
551
600
  topContext = tree[index].context;
552
601
  refContexts = new Map(); // refs from the previous tab are invalid
602
+ lastNavigation = null; // the outcome belonged to the tab we left
553
603
  await bidi.send('browsingContext.activate', { context: topContext }).catch(() => {});
554
604
  },
555
605
 
@@ -667,6 +717,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
667
717
  30000, 'history navigation');
668
718
  refContexts = new Map();
669
719
  consentRecheck = null;
720
+ lastNavigation = null; // no outcome is known for the history entry we landed on
670
721
  await new Promise((r) => setTimeout(r, 500));
671
722
  }
672
723
 
package/src/index.js CHANGED
@@ -17,6 +17,7 @@ import { formatTree } from './aria.js';
17
17
  import { authenticate } from './auth.js';
18
18
  import { prune as pruneTree } from './prune.js';
19
19
  import { dataSnapshot, stripLegend } from './data.js';
20
+ import { annotateLinks } from './links.js';
20
21
  import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
21
22
  import { dismissConsent } from './consent.js';
22
23
  import { applyStealth } from './stealth.js';
@@ -28,6 +29,7 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
28
29
  import { isChallengePage } from './challenge.js';
29
30
  import { withTimeout } from './with-timeout.js';
30
31
  import { readable as extractReadable } from './readable.js';
32
+ import { navFailed, httpWarning } from './navigation.js';
31
33
  import { assertNavigable, assertUploadAllowed } from './url-guard.js';
32
34
  import { join as pathJoin } from 'node:path';
33
35
  import { chmodSync } from 'node:fs';
@@ -60,6 +62,7 @@ import { chmodSync } from 'node:fs';
60
62
  * @param {string} [opts.browser] - Source browser for cookie extraction.
61
63
  * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
62
64
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
65
+ * @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
63
66
  * @returns {Promise<string>} ARIA snapshot text
64
67
  */
65
68
  export async function browse(url, opts = {}) {
@@ -106,7 +109,7 @@ export async function browse(url, opts = {}) {
106
109
  }
107
110
 
108
111
  // Step 4: Navigate and wait for load
109
- await navigate(page, url, timeout);
112
+ let outcome = await navigate(page, url, timeout);
110
113
 
111
114
  // Step 4.5: Auto-dismiss cookie consent dialogs
112
115
  if (opts.consent !== false) {
@@ -130,7 +133,7 @@ export async function browse(url, opts = {}) {
130
133
  if (!noAuth) {
131
134
  try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
132
135
  }
133
- await navigate(page, url, timeout);
136
+ outcome = await navigate(page, url, timeout);
134
137
  if (opts.consent !== false) await dismissConsent(page.session);
135
138
  ({ tree } = await ariaTree(page));
136
139
  } catch {
@@ -140,21 +143,27 @@ export async function browse(url, opts = {}) {
140
143
 
141
144
  // Step 6: Prune for agent consumption
142
145
  const raw = formatTree(tree);
146
+ const links = !!opts.links;
147
+ let linked = tree;
148
+ if (links) {
149
+ const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
150
+ linked = annotateLinks(tree, entries[currentIndex]?.url || url);
151
+ }
143
152
  let snapshot;
144
153
  if (opts.prune !== false) {
145
154
  snapshot = opts.pruneMode === 'data'
146
- ? dataSnapshot(tree)
147
- : formatTree(pruneTree(tree, { mode: opts.pruneMode || 'act' }));
155
+ ? dataSnapshot(linked, { links })
156
+ : formatTree(pruneTree(linked, { mode: opts.pruneMode || 'act', links }));
148
157
  } else {
149
- snapshot = raw;
158
+ snapshot = links ? formatTree(linked) : raw;
150
159
  }
151
- const sized = opts.pruneMode === 'data' ? stripLegend(snapshot) : snapshot; // the legend is not content
160
+ const sized = stripLegend(snapshot); // the legend is not content
152
161
  const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
153
162
  const actMode = !opts.pruneMode || opts.pruneMode === 'act';
154
163
  const hint = (actMode && raw.length > 5000 && snapshot.length < 500 && snapshot.length < raw.length * 0.05)
155
164
  ? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
156
165
  : '';
157
- snapshot = stats + '\n' + hint + snapshot;
166
+ snapshot = stats + '\n' + hint + httpWarning(outcome.status) + snapshot;
158
167
 
159
168
  // Step 7: Clean up
160
169
  await cdp.send('Target.closeTarget', { targetId: page.targetId });
@@ -204,10 +213,20 @@ export async function browse(url, opts = {}) {
204
213
  * @property {boolean} cookieInjection - Cookie extraction/injection is active (off in incognito)
205
214
  */
206
215
 
216
+ /**
217
+ * What `goto()` resolves to: the final URL (after redirects) and the main
218
+ * document's HTTP status (0 when no HTTP response was observed, e.g. an aborted navigation).
219
+ * Network failures reject with an Error whose `code` is 'NAV_FAILED' and
220
+ * `reason` the browser's raw error string.
221
+ * @typedef {import('./navigation.js').NavigationOutcome} NavigationOutcome
222
+ */
223
+
207
224
  /**
208
225
  * Options for `page.snapshot()`. Pass `false` instead of an object to skip
209
226
  * pruning and return the raw ARIA tree.
210
- * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data' }} SnapshotOptions
227
+ * `links: true` adds each link's absolute URL after its name (default off,
228
+ * or the `connect({ links })` default).
229
+ * @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
211
230
  */
212
231
 
213
232
  /**
@@ -261,6 +280,7 @@ export async function browse(url, opts = {}) {
261
280
  * skip storageState loading and make injectCookies() a no-op, so no auth
262
281
  * ever enters the session even if a caller injects unconditionally.
263
282
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
283
+ * @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
264
284
  * @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
265
285
  * 'firefox' drives over WebDriver BiDi (a separate transport / page object);
266
286
  * 'chromium' (default) uses CDP.
@@ -367,6 +387,9 @@ export async function connect(opts = {}) {
367
387
  let page = await createPage(cdp, !currentlyHeaded, pageOpts);
368
388
  let refMap = new Map();
369
389
  let botBlocked = false;
390
+ // Outcome of the latest goto() ({url, status}); null before any goto.
391
+ /** @type {NavigationOutcome|null} */
392
+ let lastNavigation = null;
370
393
  // Set when the consent pass after the latest goto() found nothing: the first
371
394
  // snapshot() re-checks once, since some CMPs (Google Funding Choices) render
372
395
  // their dialog 1-3s after load. Holds the document identity (tab + main-frame
@@ -489,6 +512,7 @@ export async function connect(opts = {}) {
489
512
  // the map.
490
513
  refMap = new Map();
491
514
  consentRecheck = null; // a throwing navigate() must not leave the old page's flag
515
+ lastNavigation = null;
492
516
  // Switch back to headless if we fell back to headed previously.
493
517
  // Not in attach mode — we never own the browser there, so there's
494
518
  // nothing to rewind.
@@ -505,7 +529,7 @@ export async function connect(opts = {}) {
505
529
  currentlyHeaded = false;
506
530
  }
507
531
 
508
- await navigate(page, url, timeout);
532
+ lastNavigation = await navigate(page, url, timeout);
509
533
  if (opts.consent !== false) {
510
534
  const dismissed = await dismissConsent(page.session);
511
535
  if (!dismissed) consentRecheck = await navKey();
@@ -529,7 +553,7 @@ export async function connect(opts = {}) {
529
553
  page = await createPage(cdp, false, pageOpts);
530
554
  setupDialogHandler(page.session);
531
555
  await suppressPermissions(cdp);
532
- await navigate(page, url, timeout);
556
+ lastNavigation = await navigate(page, url, timeout);
533
557
  consentRecheck = null;
534
558
  if (opts.consent !== false && !(await dismissConsent(page.session))) {
535
559
  consentRecheck = await navKey();
@@ -543,6 +567,7 @@ export async function connect(opts = {}) {
543
567
  // Headed launch failed (no display?) — keep headless result, botBlocked stays true
544
568
  }
545
569
  }
570
+ return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
546
571
  },
547
572
 
548
573
  async goBack() {
@@ -553,6 +578,7 @@ export async function connect(opts = {}) {
553
578
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
554
579
  refMap = new Map(); // refs from the previous page are now invalid
555
580
  consentRecheck = null;
581
+ lastNavigation = null; // no outcome is known for the history entry we landed on
556
582
  },
557
583
 
558
584
  async goForward() {
@@ -563,6 +589,7 @@ export async function connect(opts = {}) {
563
589
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
564
590
  refMap = new Map();
565
591
  consentRecheck = null;
592
+ lastNavigation = null;
566
593
  },
567
594
 
568
595
  async reload(reloadOpts = {}) {
@@ -574,6 +601,7 @@ export async function connect(opts = {}) {
574
601
  try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
575
602
  refMap = new Map(); // refs from the pre-reload page are invalid
576
603
  consentRecheck = null;
604
+ lastNavigation = null; // the reload's status is not read; don't keep the old one
577
605
  },
578
606
 
579
607
  /** @param {string} url @param {CookieOptions} [cookieOpts] */
@@ -599,11 +627,14 @@ export async function connect(opts = {}) {
599
627
  const raw = formatTree(result.tree);
600
628
  const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
601
629
  const pageUrl = entries[currentIndex]?.url || '';
602
- const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '';
603
- if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + raw;
630
+ const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
631
+ + httpWarning(lastNavigation?.status);
632
+ const links = !!((pruneOpts || undefined)?.links ?? opts.links);
633
+ const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
634
+ if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
604
635
  const mode = pruneOpts?.mode || opts.pruneMode || 'act';
605
- const out = mode === 'data' ? dataSnapshot(result.tree) : formatTree(pruneTree(result.tree, { mode }));
606
- const sized = mode === 'data' ? stripLegend(out) : out; // the legend is not content
636
+ const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
637
+ const sized = stripLegend(out); // the legend is not content
607
638
  const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
608
639
  const hint = (mode === 'act' && raw.length > 5000 && out.length < 500 && out.length < raw.length * 0.05)
609
640
  ? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
@@ -702,6 +733,7 @@ export async function connect(opts = {}) {
702
733
  const oldSessionId = page.sessionId;
703
734
  page = await attachToExistingTarget(cdp, target.targetId, pageOpts);
704
735
  refMap = new Map(); // refs from the previous tab are no longer valid
736
+ lastNavigation = null; // the outcome belonged to the tab we left
705
737
  setupDialogHandler(page.session);
706
738
  try { await cdp.send('Target.detachFromTarget', { sessionId: oldSessionId }); } catch {}
707
739
  },
@@ -746,6 +778,8 @@ export async function connect(opts = {}) {
746
778
  },
747
779
 
748
780
  get botBlocked() { return botBlocked; },
781
+ /** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
782
+ get lastNavigation() { return lastNavigation; },
749
783
 
750
784
  /** @type {'chromium'} Engine driving this session (see also `capabilities`) */
751
785
  engine: 'chromium',
@@ -820,12 +854,13 @@ export async function connect(opts = {}) {
820
854
  return {
821
855
  async goto(url, timeout = 30000) {
822
856
  assertNavigable(url, urlGuard);
823
- await navigate(tab, url, timeout);
857
+ const outcome = await navigate(tab, url, timeout);
824
858
  if (opts.consent !== false) {
825
859
  await dismissConsent(tab.session);
826
860
  }
827
861
  const { tree, nodeCount } = await ariaTree(tab);
828
862
  tabBotBlocked = isChallengePage(tree, nodeCount);
863
+ return outcome;
829
864
  },
830
865
  get botBlocked() { return tabBotBlocked; },
831
866
  /** @param {string} url @param {CookieOptions} [cookieOpts] */
@@ -959,6 +994,7 @@ async function connectFirefox(opts) {
959
994
 
960
995
  const page = await createFirefoxPage(bidi, {
961
996
  pruneMode: opts.pruneMode,
997
+ links: opts.links,
962
998
  urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
963
999
  uploadDir: opts.uploadDir || null,
964
1000
  incognito: !!opts.incognito,
@@ -1126,14 +1162,45 @@ export function _resetBlocklistWarning() {
1126
1162
  }
1127
1163
 
1128
1164
  /**
1129
- * Navigate to a URL and wait for the page to load.
1165
+ * Navigate to a URL and wait for the page to load. Resolves to the outcome of
1166
+ * the main-document navigation (final URL after redirects, HTTP status);
1167
+ * rejects with a NAV_FAILED error when the browser could not reach a document.
1168
+ * @returns {Promise<NavigationOutcome>}
1130
1169
  */
1131
1170
  async function navigate(page, url, timeout = 30000) {
1132
- const loadPromise = page.session.once('Page.loadEventFired', timeout);
1133
- await page.session.send('Page.navigate', { url });
1134
- await loadPromise;
1135
- // Brief settle time for dynamic content
1136
- await new Promise((r) => setTimeout(r, 500));
1171
+ const { session } = page;
1172
+ // Main-document responses (every redirect hop fires one; the last wins).
1173
+ const docs = [];
1174
+ const offDoc = session.on('Network.responseReceived', (p) => { if (p.type === 'Document') docs.push(p); });
1175
+ // Load waiter, hand-rolled (not session.once) so it can be cancelled when
1176
+ // Page.navigate fails or aborts and no load event will ever come.
1177
+ let cancel = () => {};
1178
+ const loaded = /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
1179
+ const timer = setTimeout(() => { off(); reject(new Error('Timeout waiting for CDP event: Page.loadEventFired')); }, timeout);
1180
+ const off = session.on('Page.loadEventFired', () => { cancel(); resolve(undefined); });
1181
+ cancel = () => { clearTimeout(timer); off(); };
1182
+ }));
1183
+ loaded.catch(() => {}); // rejection is only read if we await it below
1184
+ try {
1185
+ const nav = await session.send('Page.navigate', { url });
1186
+ // ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
1187
+ // no document and fires no load event, so don't wait for one.
1188
+ const aborted = nav.errorText === 'net::ERR_ABORTED';
1189
+ // ERR_HTTP_RESPONSE_CODE_FAILURE is a 4xx/5xx with an empty body: the
1190
+ // document did load (Chromium's error page fires load, the Document
1191
+ // response carries the status), so report it like any other HTTP error.
1192
+ const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
1193
+ if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
1194
+ if (!aborted) await loaded;
1195
+ // Brief settle time for dynamic content
1196
+ await new Promise((r) => setTimeout(r, 500));
1197
+ // loaderId is shared by every redirect hop of one navigation.
1198
+ const last = docs.filter((d) => d.loaderId === nav.loaderId).pop();
1199
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
1200
+ } finally {
1201
+ cancel();
1202
+ offDoc();
1203
+ }
1137
1204
  }
1138
1205
 
1139
1206
  /**
package/src/links.js ADDED
@@ -0,0 +1,46 @@
1
+ /**
2
+ * links.js — opt-in link URLs for snapshots (`links: true`).
3
+ *
4
+ * `annotateLinks()` stamps `urls: [url]` on every link node that has a usable
5
+ * browser-serialized absolute URL. aria.js, prune.js and data.js print/carry
6
+ * `urls` — they never see the option, so with links off nothing changes.
7
+ */
8
+
9
+ /**
10
+ * Normalise a link's URL for a snapshot line. Returns '' when it should not be
11
+ * shown: no URL, `javascript:`, or a same-document link (the page URL, only
12
+ * the #fragment differs). A raw space, `<` or `>` (browsers encode them, but
13
+ * not always in the fragment) is percent-encoded so a line splits cleanly on
14
+ * ` <` and `> `.
15
+ *
16
+ * @param {string|undefined} url - Absolute URL as serialized by the browser
17
+ * @param {string} [pageUrl] - URL of the page the link is on
18
+ * @returns {string}
19
+ */
20
+ export function linkUrl(url, pageUrl = '') {
21
+ if (typeof url !== 'string') return '';
22
+ const u = url.trim();
23
+ if (!u || /^javascript:/i.test(u)) return '';
24
+ if (pageUrl && u.split('#')[0] === pageUrl.split('#')[0]) return '';
25
+ return u.replace(/ /g, '%20').replace(/</g, '%3C').replace(/>/g, '%3E');
26
+ }
27
+
28
+ /**
29
+ * Copy a tree with `urls` set on its link nodes (input is not mutated).
30
+ *
31
+ * @param {object|null} node - Root from buildTree()
32
+ * @param {string} pageUrl
33
+ * @returns {object|null}
34
+ */
35
+ export function annotateLinks(node, pageUrl) {
36
+ if (!node) return node;
37
+ const out = { ...node, children: node.children.map((c) => annotateLinks(c, pageUrl)) };
38
+ if (node.role === 'link') {
39
+ const u = linkUrl(node.properties?.url, pageUrl);
40
+ if (u) out.urls = [u];
41
+ }
42
+ return out;
43
+ }
44
+
45
+ /** ` <u1> <u2>` for a node's urls ('' when none). */
46
+ export const urlsText = (urls) => (urls || []).map((u) => ` <${u}>`).join('');
@@ -0,0 +1,42 @@
1
+ /**
2
+ * navigation.js — Shared pieces of the goto() outcome contract, so both
3
+ * engines (CDP in index.js, BiDi in firefox-page.js) report it identically.
4
+ */
5
+
6
+ /**
7
+ * What goto() resolves to: where the page ended up and how the server answered.
8
+ * `status` is 0 when no HTTP response was observed for the navigation (e.g. an
9
+ * aborted navigation that never got one).
10
+ * @typedef {{ url: string, status: number }} NavigationOutcome
11
+ */
12
+
13
+ /**
14
+ * Typed error for a navigation that never reached a document (DNS failure,
15
+ * connection refused, TLS error, ...). `code` is 'NAV_FAILED'; `reason` is the
16
+ * browser's raw error string (`net::ERR_NAME_NOT_RESOLVED`,
17
+ * `NS_ERROR_UNKNOWN_HOST`, ...).
18
+ * @param {string} url
19
+ * @param {string} reason
20
+ * @returns {Error & {code: 'NAV_FAILED', reason: string}}
21
+ */
22
+ export function navFailed(url, reason) {
23
+ return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
24
+ }
25
+
26
+ /**
27
+ * True for a final HTTP status that is not 2xx (0 = unknown is not an error).
28
+ * @param {number|undefined} status
29
+ */
30
+ export function isHttpError(status) {
31
+ return !!status && (status < 200 || status >= 300);
32
+ }
33
+
34
+ /**
35
+ * Warning line for a non-2xx final status ('' for 2xx and for unknown 0), in
36
+ * the style of the bot-challenge line, newline-terminated for prepending.
37
+ * @param {number|undefined} status
38
+ * @returns {string}
39
+ */
40
+ export function httpWarning(status) {
41
+ return isHttpError(status) ? `[HTTP ${status} — the page may be an error page]\n` : '';
42
+ }