barebrowse 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +21 -0
- package/barebrowse.context.md +7 -3
- package/cli.js +6 -6
- package/commands/barebrowse/SKILL.md +141 -0
- package/commands/barebrowse.md +140 -0
- package/mcp-server.js +8 -5
- package/package.json +2 -1
- package/src/aria.js +5 -0
- package/src/ax-snapshot.js +1 -0
- package/src/bareagent.js +5 -3
- package/src/daemon.js +5 -4
- package/src/data.js +24 -9
- package/src/firefox-page.js +63 -12
- package/src/index.js +88 -21
- package/src/links.js +46 -0
- package/src/navigation.js +42 -0
- package/src/prune.js +11 -3
- package/types/aria.d.ts +0 -6
- package/types/data.d.ts +6 -1
- package/types/firefox-page.d.ts +5 -1
- package/types/index.d.ts +18 -3
- package/types/links.d.ts +28 -0
- package/types/navigation.d.ts +44 -0
- package/types/prune.d.ts +2 -0
package/src/data.js
CHANGED
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
* click/type resolve through the same refMap).
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
|
+
import { urlsText } from './links.js';
|
|
37
|
+
|
|
36
38
|
const SKIP = new Set(['InlineTextBox', 'LineBreak', 'ListMarker', 'image', 'img']);
|
|
37
39
|
const CELLS = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);
|
|
38
40
|
const TABLES = new Set(['table', 'grid', 'treegrid']);
|
|
@@ -47,13 +49,20 @@ const MIN_RUN = 3;
|
|
|
47
49
|
/** First line of every data-mode snapshot: tells an agent with no docs how to read it. */
|
|
48
50
|
export const LEGEND = '# data mode: one line per record. Tables: row "<label>": <column>: <value> · … Groups: <label>: a, b, c. A title line followed by detail lines is one block, up to the next title line. Main content only; use mode "act" for menus and page controls.';
|
|
49
51
|
|
|
52
|
+
/** Appended to the legend only when links are on. */
|
|
53
|
+
export const LINKS_NOTE = ' Links: "name" <url>.';
|
|
54
|
+
|
|
50
55
|
// A name made only of private-use code points (icon-font glyphs, unreadable to an
|
|
51
56
|
// agent) is no name. Glyphs mixed into real text are left as they are.
|
|
52
57
|
const norm = (s) => (/^[\p{Co}\s]*$/u.test(s || '') ? '' : s.replace(/\s+/g, ' ').trim());
|
|
58
|
+
const allUrls = (n) => [...(n.urls || []), ...(n.children || []).flatMap(allUrls)];
|
|
53
59
|
const hasInteractive = (n) => INTERACTIVE.has(n.role) || (n.children || []).some(hasInteractive);
|
|
54
60
|
const isLeaf = (n) => n.text === undefined && !n.children.length && n.name;
|
|
55
61
|
const showRef = (n) => !!n.ref && INTERACTIVE.has(n.role);
|
|
56
|
-
|
|
62
|
+
// ` <url>` for a node's own links (already unique per node). A link line always
|
|
63
|
+
// carries its own URL: nothing is deduped across lines, items or cells.
|
|
64
|
+
const urlPart = (n) => urlsText(n.urls);
|
|
65
|
+
const withRef = (n) => (showRef(n) ? `${n.name}${urlPart(n)} [ref=${n.ref}]` : n.name);
|
|
57
66
|
// Leaves carrying state (checked, level, value...) never merge into a run.
|
|
58
67
|
const plain = (n) => !(n.props.checked !== undefined || n.props.expanded !== undefined || n.props.selected || n.props.level || n.props.required || (n.props.value !== undefined && n.props.value !== ''));
|
|
59
68
|
// Run key: leaves with the same key are "the same kind of item". `g` = the
|
|
@@ -69,19 +78,20 @@ function inline(n, refs) {
|
|
|
69
78
|
const sub = n.children.length ? n.children.map((c) => inline(c, refs)).filter(Boolean).join(' ') : n.name;
|
|
70
79
|
const v = (n.name && !n.children.length) || INTERACTIVE.has(n.role) && n.name && refs ? n.name : sub;
|
|
71
80
|
const out = n.children.length && n.name && n.role !== 'StaticText' && !sub ? n.name : v;
|
|
72
|
-
return refs && n.ref && INTERACTIVE.has(n.role) ? `${out} [ref=${n.ref}]` : out;
|
|
81
|
+
return refs && n.ref && INTERACTIVE.has(n.role) ? `${out}${urlPart(n)} [ref=${n.ref}]` : out;
|
|
73
82
|
}
|
|
74
83
|
|
|
75
84
|
/** A data-mode result without its legend line, for size stats ('' stays ''). */
|
|
76
|
-
export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(
|
|
85
|
+
export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(s.indexOf('\n') + 1) : s);
|
|
77
86
|
|
|
78
87
|
/**
|
|
79
88
|
* Render a raw ARIA tree in data mode.
|
|
80
89
|
*
|
|
81
90
|
* @param {object} root - Root node from buildTree() (CDP vocabulary)
|
|
91
|
+
* @param {{ links?: boolean }} [opts] - links: the tree's link nodes carry `urls` (annotateLinks); adds the legend note
|
|
82
92
|
* @returns {string} LEGEND line, then one line per record, YAML-ish like formatTree(); '' when there are no records
|
|
83
93
|
*/
|
|
84
|
-
export function dataSnapshot(root) {
|
|
94
|
+
export function dataSnapshot(root, opts = {}) {
|
|
85
95
|
if (!root) return '';
|
|
86
96
|
let gid = 0;
|
|
87
97
|
const rawKids = (n) => n.children.flatMap((c) => (c.ignored ? rawKids(c) : [c]));
|
|
@@ -120,7 +130,7 @@ export function dataSnapshot(root) {
|
|
|
120
130
|
if (!nm && kids.length === 0 && (n.role === 'link' || n.role === 'button')) return []; // N6
|
|
121
131
|
if (!nm && kids.length === 0 && !INTERACTIVE.has(n.role) && !CELLS.has(n.role) && n.role !== 'row') return [];
|
|
122
132
|
if (n.role === 'listitem' && !nm && kids.length === 1) return kids;
|
|
123
|
-
return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig }];
|
|
133
|
+
return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig, urls: n.urls }];
|
|
124
134
|
}
|
|
125
135
|
|
|
126
136
|
// ---------- Transforms, bottom-up with the ancestor chain for labels ----------
|
|
@@ -134,7 +144,7 @@ export function dataSnapshot(root) {
|
|
|
134
144
|
const out = [];
|
|
135
145
|
for (const k of kids) {
|
|
136
146
|
const p = out[out.length - 1];
|
|
137
|
-
if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name) continue;
|
|
147
|
+
if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name && (p.urls || [])[0] === (k.urls || [])[0]) continue;
|
|
138
148
|
out.push(k);
|
|
139
149
|
}
|
|
140
150
|
return out;
|
|
@@ -243,6 +253,7 @@ export function dataSnapshot(root) {
|
|
|
243
253
|
const labelCells = cs.filter((c) => c.role === 'rowheader' || c.role === 'columnheader');
|
|
244
254
|
const lcx = labelCells.map(xcell);
|
|
245
255
|
const label = lcx.map((c) => inline(c, false)).filter(Boolean).join(' / ');
|
|
256
|
+
const labelUrls = urlsText([...new Set(lcx.flatMap(allUrls))]);
|
|
246
257
|
const labelRef = lcx.map(firstRef).find(Boolean);
|
|
247
258
|
const vals = [];
|
|
248
259
|
cs.forEach((c, idx) => {
|
|
@@ -262,7 +273,7 @@ export function dataSnapshot(root) {
|
|
|
262
273
|
}
|
|
263
274
|
if (!vals.length && !label) continue; // header-only / empty rows
|
|
264
275
|
const body = vals.map((x) => (x.h ? `${x.h}: ` : '') + x.v).join(' · ');
|
|
265
|
-
out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
|
|
276
|
+
out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelUrls}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
|
|
266
277
|
}
|
|
267
278
|
return { ...t, children: out };
|
|
268
279
|
}
|
|
@@ -278,10 +289,14 @@ export function dataSnapshot(root) {
|
|
|
278
289
|
const lines = [];
|
|
279
290
|
function emit(c, d) {
|
|
280
291
|
const ind = ' '.repeat(d);
|
|
281
|
-
if (c.text !== undefined) {
|
|
292
|
+
if (c.text !== undefined) {
|
|
293
|
+
lines.push(`${ind}- ${c.text}`);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
282
296
|
if (isLeaf(c) && (c.role === 'StaticText' || c.role === 'generic')) { lines.push(`${ind}- ${c.name}`); return; }
|
|
283
297
|
let l = `${ind}- ${c.role}`;
|
|
284
298
|
if (c.name) l += ` "${c.name}"`;
|
|
299
|
+
l += urlPart(c);
|
|
285
300
|
const p = c.props, pp = [];
|
|
286
301
|
if (p.checked !== undefined) pp.push(`checked=${p.checked}`);
|
|
287
302
|
if (p.disabled) pp.push('disabled');
|
|
@@ -296,5 +311,5 @@ export function dataSnapshot(root) {
|
|
|
296
311
|
for (const k of c.children) emit(k, d + 1);
|
|
297
312
|
}
|
|
298
313
|
for (const c of res.children) emit(c, 0);
|
|
299
|
-
return lines.length ? [LEGEND, ...lines].join('\n') : '';
|
|
314
|
+
return lines.length ? [LEGEND + (opts.links ? LINKS_NOTE : ''), ...lines].join('\n') : '';
|
|
300
315
|
}
|
package/src/firefox-page.js
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { formatTree } from './aria.js';
|
|
19
19
|
import { prune as pruneTree } from './prune.js';
|
|
20
20
|
import { dataSnapshot, stripLegend } from './data.js';
|
|
21
|
+
import { annotateLinks } from './links.js';
|
|
21
22
|
import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
|
|
22
23
|
import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
|
|
23
24
|
import { scopedCookiesForUrl } from './auth.js';
|
|
@@ -27,6 +28,7 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
|
|
|
27
28
|
import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
28
29
|
import { isChallengePage, countNodes } from './challenge.js';
|
|
29
30
|
import { withTimeout } from './with-timeout.js';
|
|
31
|
+
import { navFailed, httpWarning } from './navigation.js';
|
|
30
32
|
|
|
31
33
|
/** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
|
|
32
34
|
const BIDI_KEYS = {
|
|
@@ -41,6 +43,7 @@ const BIDI_KEYS = {
|
|
|
41
43
|
* @param {object} bidi - BiDi client from createBiDi()
|
|
42
44
|
* @param {object} [opts]
|
|
43
45
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Default prune mode.
|
|
46
|
+
* @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
|
|
44
47
|
* @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
|
|
45
48
|
* @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
|
|
46
49
|
* @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
|
|
@@ -71,6 +74,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
71
74
|
const relaunchHeaded = opts.relaunchHeaded || null;
|
|
72
75
|
let currentlyHeaded = !!opts.headed; // already headed → no fallback needed
|
|
73
76
|
let botBlocked = false;
|
|
77
|
+
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
78
|
+
/** @type {import('./navigation.js').NavigationOutcome|null} */
|
|
79
|
+
let lastNavigation = null;
|
|
74
80
|
// Last injectCookies(url, opts) args, so a hybrid relaunch (a brand-new
|
|
75
81
|
// Firefox profile) can re-inject them — otherwise the headed retry loads
|
|
76
82
|
// unauthenticated, defeating hybrid on an auth-gated challenge page.
|
|
@@ -254,7 +260,46 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
254
260
|
async function setupSubscriptions() {
|
|
255
261
|
await setupDialogs();
|
|
256
262
|
await setupDownloads();
|
|
257
|
-
await bidi.subscribe(['browsingContext.load']);
|
|
263
|
+
await bidi.subscribe(['browsingContext.load', 'network.responseCompleted', 'network.responseStarted']);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* browsingContext.navigate(wait:'complete') plus the navigation's outcome:
|
|
268
|
+
* the LAST network.responseCompleted for its navigation id in the top
|
|
269
|
+
* context (one fires per redirect hop; the final hop is where we landed).
|
|
270
|
+
* A network failure is rewrapped as a typed NAV_FAILED error. NS_BINDING_ABORTED
|
|
271
|
+
* (a 204, or a download replacing nothing) is not a failure, as with CDP's
|
|
272
|
+
* ERR_ABORTED: no responseCompleted fires for it, so the status comes from
|
|
273
|
+
* the last network.responseStarted (a 204 reports 204, as on Chromium).
|
|
274
|
+
* @returns {Promise<import('./navigation.js').NavigationOutcome>}
|
|
275
|
+
*/
|
|
276
|
+
async function navigateTo(url, timeout) {
|
|
277
|
+
const events = [];
|
|
278
|
+
const started = [];
|
|
279
|
+
const off = bidi.on('network.responseCompleted', (e) => {
|
|
280
|
+
if (e.navigation && e.context === topContext) events.push(e);
|
|
281
|
+
});
|
|
282
|
+
const offStarted = bidi.on('network.responseStarted', (e) => {
|
|
283
|
+
if (e.navigation && e.context === topContext) started.push(e);
|
|
284
|
+
});
|
|
285
|
+
try {
|
|
286
|
+
const res = await withTimeout(
|
|
287
|
+
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
288
|
+
timeout, `goto(${url})`);
|
|
289
|
+
const last = events.filter((e) => e.navigation === res.navigation).pop();
|
|
290
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
291
|
+
} catch (err) {
|
|
292
|
+
const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
|
|
293
|
+
if (!m) throw err;
|
|
294
|
+
if (m[1] === 'NS_BINDING_ABORTED') {
|
|
295
|
+
const last = started.pop();
|
|
296
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
297
|
+
}
|
|
298
|
+
throw navFailed(url, m[1]);
|
|
299
|
+
} finally {
|
|
300
|
+
off();
|
|
301
|
+
offStarted();
|
|
302
|
+
}
|
|
258
303
|
}
|
|
259
304
|
|
|
260
305
|
/**
|
|
@@ -285,6 +330,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
285
330
|
get context() { return topContext; },
|
|
286
331
|
/** Whether the last goto() landed on a bot-challenge page (parity w/ CDP). */
|
|
287
332
|
get botBlocked() { return botBlocked; },
|
|
333
|
+
/** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
|
|
334
|
+
get lastNavigation() { return lastNavigation; },
|
|
288
335
|
|
|
289
336
|
/** @type {'firefox'} Engine driving this session (see also `capabilities`). */
|
|
290
337
|
engine: 'firefox',
|
|
@@ -314,9 +361,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
314
361
|
assertNavigable(url, urlGuard);
|
|
315
362
|
refContexts = new Map(); // refs from the previous page are now stale
|
|
316
363
|
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
timeout, `goto(${url})`);
|
|
364
|
+
lastNavigation = null;
|
|
365
|
+
lastNavigation = await navigateTo(url, timeout);
|
|
320
366
|
// Settle + consent + challenge detection (mirrors the CDP path).
|
|
321
367
|
botBlocked = await afterNavigate();
|
|
322
368
|
|
|
@@ -338,15 +384,14 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
338
384
|
if (lastInject) {
|
|
339
385
|
try { await page.injectCookies(lastInject.url, lastInject.cookieOpts); } catch { /* best-effort */ }
|
|
340
386
|
}
|
|
341
|
-
await
|
|
342
|
-
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
343
|
-
timeout, `goto(${url})`);
|
|
387
|
+
lastNavigation = await navigateTo(url, timeout);
|
|
344
388
|
botBlocked = await afterNavigate();
|
|
345
389
|
} catch {
|
|
346
390
|
// Headed relaunch failed (no display?) — keep the headless result;
|
|
347
391
|
// botBlocked stays true so the caller can see it didn't clear.
|
|
348
392
|
}
|
|
349
393
|
}
|
|
394
|
+
return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
|
|
350
395
|
},
|
|
351
396
|
|
|
352
397
|
/** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
|
|
@@ -363,16 +408,19 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
363
408
|
|
|
364
409
|
const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
|
|
365
410
|
const raw = formatTree(root);
|
|
366
|
-
|
|
411
|
+
const warn = httpWarning(lastNavigation?.status);
|
|
412
|
+
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
413
|
+
const tree = links ? annotateLinks(root, pageUrl) : root;
|
|
414
|
+
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
367
415
|
|
|
368
416
|
const mode = pruneOpts?.mode || defaultPruneMode;
|
|
369
417
|
let out;
|
|
370
|
-
if (mode === 'data') out = dataSnapshot(
|
|
371
|
-
else { const pruned = pruneTree(
|
|
372
|
-
const sized =
|
|
418
|
+
if (mode === 'data') out = dataSnapshot(tree, { links });
|
|
419
|
+
else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
|
|
420
|
+
const sized = stripLegend(out); // the legend is not content
|
|
373
421
|
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars`
|
|
374
422
|
+ ` (${raw.length ? Math.round((1 - sized.length / raw.length) * 100) : 0}% pruned)`;
|
|
375
|
-
return stats + '\n' + out;
|
|
423
|
+
return stats + '\n' + warn + out;
|
|
376
424
|
},
|
|
377
425
|
|
|
378
426
|
async readable() {
|
|
@@ -514,6 +562,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
514
562
|
30000, 'reload');
|
|
515
563
|
refContexts = new Map();
|
|
516
564
|
consentRecheck = null;
|
|
565
|
+
lastNavigation = null; // the reload's status is not read; don't keep the old one
|
|
517
566
|
await new Promise((r) => setTimeout(r, 300));
|
|
518
567
|
},
|
|
519
568
|
|
|
@@ -550,6 +599,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
550
599
|
if (index < 0 || index >= tree.length) throw new Error(`Tab index ${index} out of range (0-${tree.length - 1})`);
|
|
551
600
|
topContext = tree[index].context;
|
|
552
601
|
refContexts = new Map(); // refs from the previous tab are invalid
|
|
602
|
+
lastNavigation = null; // the outcome belonged to the tab we left
|
|
553
603
|
await bidi.send('browsingContext.activate', { context: topContext }).catch(() => {});
|
|
554
604
|
},
|
|
555
605
|
|
|
@@ -667,6 +717,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
667
717
|
30000, 'history navigation');
|
|
668
718
|
refContexts = new Map();
|
|
669
719
|
consentRecheck = null;
|
|
720
|
+
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
670
721
|
await new Promise((r) => setTimeout(r, 500));
|
|
671
722
|
}
|
|
672
723
|
|
package/src/index.js
CHANGED
|
@@ -17,6 +17,7 @@ import { formatTree } from './aria.js';
|
|
|
17
17
|
import { authenticate } from './auth.js';
|
|
18
18
|
import { prune as pruneTree } from './prune.js';
|
|
19
19
|
import { dataSnapshot, stripLegend } from './data.js';
|
|
20
|
+
import { annotateLinks } from './links.js';
|
|
20
21
|
import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
|
|
21
22
|
import { dismissConsent } from './consent.js';
|
|
22
23
|
import { applyStealth } from './stealth.js';
|
|
@@ -28,6 +29,7 @@ import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
|
28
29
|
import { isChallengePage } from './challenge.js';
|
|
29
30
|
import { withTimeout } from './with-timeout.js';
|
|
30
31
|
import { readable as extractReadable } from './readable.js';
|
|
32
|
+
import { navFailed, httpWarning } from './navigation.js';
|
|
31
33
|
import { assertNavigable, assertUploadAllowed } from './url-guard.js';
|
|
32
34
|
import { join as pathJoin } from 'node:path';
|
|
33
35
|
import { chmodSync } from 'node:fs';
|
|
@@ -60,6 +62,7 @@ import { chmodSync } from 'node:fs';
|
|
|
60
62
|
* @param {string} [opts.browser] - Source browser for cookie extraction.
|
|
61
63
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs.
|
|
62
64
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
65
|
+
* @param {boolean} [opts.links=false] - Show link URLs: `link "name" <url> [ref=N]`.
|
|
63
66
|
* @returns {Promise<string>} ARIA snapshot text
|
|
64
67
|
*/
|
|
65
68
|
export async function browse(url, opts = {}) {
|
|
@@ -106,7 +109,7 @@ export async function browse(url, opts = {}) {
|
|
|
106
109
|
}
|
|
107
110
|
|
|
108
111
|
// Step 4: Navigate and wait for load
|
|
109
|
-
await navigate(page, url, timeout);
|
|
112
|
+
let outcome = await navigate(page, url, timeout);
|
|
110
113
|
|
|
111
114
|
// Step 4.5: Auto-dismiss cookie consent dialogs
|
|
112
115
|
if (opts.consent !== false) {
|
|
@@ -130,7 +133,7 @@ export async function browse(url, opts = {}) {
|
|
|
130
133
|
if (!noAuth) {
|
|
131
134
|
try { await authenticate(page.session, url, { browser: opts.browser }); } catch {}
|
|
132
135
|
}
|
|
133
|
-
await navigate(page, url, timeout);
|
|
136
|
+
outcome = await navigate(page, url, timeout);
|
|
134
137
|
if (opts.consent !== false) await dismissConsent(page.session);
|
|
135
138
|
({ tree } = await ariaTree(page));
|
|
136
139
|
} catch {
|
|
@@ -140,21 +143,27 @@ export async function browse(url, opts = {}) {
|
|
|
140
143
|
|
|
141
144
|
// Step 6: Prune for agent consumption
|
|
142
145
|
const raw = formatTree(tree);
|
|
146
|
+
const links = !!opts.links;
|
|
147
|
+
let linked = tree;
|
|
148
|
+
if (links) {
|
|
149
|
+
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
150
|
+
linked = annotateLinks(tree, entries[currentIndex]?.url || url);
|
|
151
|
+
}
|
|
143
152
|
let snapshot;
|
|
144
153
|
if (opts.prune !== false) {
|
|
145
154
|
snapshot = opts.pruneMode === 'data'
|
|
146
|
-
? dataSnapshot(
|
|
147
|
-
: formatTree(pruneTree(
|
|
155
|
+
? dataSnapshot(linked, { links })
|
|
156
|
+
: formatTree(pruneTree(linked, { mode: opts.pruneMode || 'act', links }));
|
|
148
157
|
} else {
|
|
149
|
-
snapshot = raw;
|
|
158
|
+
snapshot = links ? formatTree(linked) : raw;
|
|
150
159
|
}
|
|
151
|
-
const sized =
|
|
160
|
+
const sized = stripLegend(snapshot); // the legend is not content
|
|
152
161
|
const stats = `url: ${url}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
153
162
|
const actMode = !opts.pruneMode || opts.pruneMode === 'act';
|
|
154
163
|
const hint = (actMode && raw.length > 5000 && snapshot.length < 500 && snapshot.length < raw.length * 0.05)
|
|
155
164
|
? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
|
|
156
165
|
: '';
|
|
157
|
-
snapshot = stats + '\n' + hint + snapshot;
|
|
166
|
+
snapshot = stats + '\n' + hint + httpWarning(outcome.status) + snapshot;
|
|
158
167
|
|
|
159
168
|
// Step 7: Clean up
|
|
160
169
|
await cdp.send('Target.closeTarget', { targetId: page.targetId });
|
|
@@ -204,10 +213,20 @@ export async function browse(url, opts = {}) {
|
|
|
204
213
|
* @property {boolean} cookieInjection - Cookie extraction/injection is active (off in incognito)
|
|
205
214
|
*/
|
|
206
215
|
|
|
216
|
+
/**
|
|
217
|
+
* What `goto()` resolves to: the final URL (after redirects) and the main
|
|
218
|
+
* document's HTTP status (0 when no HTTP response was observed, e.g. an aborted navigation).
|
|
219
|
+
* Network failures reject with an Error whose `code` is 'NAV_FAILED' and
|
|
220
|
+
* `reason` the browser's raw error string.
|
|
221
|
+
* @typedef {import('./navigation.js').NavigationOutcome} NavigationOutcome
|
|
222
|
+
*/
|
|
223
|
+
|
|
207
224
|
/**
|
|
208
225
|
* Options for `page.snapshot()`. Pass `false` instead of an object to skip
|
|
209
226
|
* pruning and return the raw ARIA tree.
|
|
210
|
-
*
|
|
227
|
+
* `links: true` adds each link's absolute URL after its name (default off,
|
|
228
|
+
* or the `connect({ links })` default).
|
|
229
|
+
* @typedef {{ mode?: 'act'|'browse'|'navigate'|'full'|'read'|'data', links?: boolean }} SnapshotOptions
|
|
211
230
|
*/
|
|
212
231
|
|
|
213
232
|
/**
|
|
@@ -261,6 +280,7 @@ export async function browse(url, opts = {}) {
|
|
|
261
280
|
* skip storageState loading and make injectCookies() a no-op, so no auth
|
|
262
281
|
* ever enters the session even if a caller injects unconditionally.
|
|
263
282
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Pruning mode.
|
|
283
|
+
* @param {boolean} [opts.links=false] - Default for `snapshot({ links })`: show link URLs.
|
|
264
284
|
* @param {'chromium'|'firefox'} [opts.engine='chromium'] - Browser engine.
|
|
265
285
|
* 'firefox' drives over WebDriver BiDi (a separate transport / page object);
|
|
266
286
|
* 'chromium' (default) uses CDP.
|
|
@@ -367,6 +387,9 @@ export async function connect(opts = {}) {
|
|
|
367
387
|
let page = await createPage(cdp, !currentlyHeaded, pageOpts);
|
|
368
388
|
let refMap = new Map();
|
|
369
389
|
let botBlocked = false;
|
|
390
|
+
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
391
|
+
/** @type {NavigationOutcome|null} */
|
|
392
|
+
let lastNavigation = null;
|
|
370
393
|
// Set when the consent pass after the latest goto() found nothing: the first
|
|
371
394
|
// snapshot() re-checks once, since some CMPs (Google Funding Choices) render
|
|
372
395
|
// their dialog 1-3s after load. Holds the document identity (tab + main-frame
|
|
@@ -489,6 +512,7 @@ export async function connect(opts = {}) {
|
|
|
489
512
|
// the map.
|
|
490
513
|
refMap = new Map();
|
|
491
514
|
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
515
|
+
lastNavigation = null;
|
|
492
516
|
// Switch back to headless if we fell back to headed previously.
|
|
493
517
|
// Not in attach mode — we never own the browser there, so there's
|
|
494
518
|
// nothing to rewind.
|
|
@@ -505,7 +529,7 @@ export async function connect(opts = {}) {
|
|
|
505
529
|
currentlyHeaded = false;
|
|
506
530
|
}
|
|
507
531
|
|
|
508
|
-
await navigate(page, url, timeout);
|
|
532
|
+
lastNavigation = await navigate(page, url, timeout);
|
|
509
533
|
if (opts.consent !== false) {
|
|
510
534
|
const dismissed = await dismissConsent(page.session);
|
|
511
535
|
if (!dismissed) consentRecheck = await navKey();
|
|
@@ -529,7 +553,7 @@ export async function connect(opts = {}) {
|
|
|
529
553
|
page = await createPage(cdp, false, pageOpts);
|
|
530
554
|
setupDialogHandler(page.session);
|
|
531
555
|
await suppressPermissions(cdp);
|
|
532
|
-
await navigate(page, url, timeout);
|
|
556
|
+
lastNavigation = await navigate(page, url, timeout);
|
|
533
557
|
consentRecheck = null;
|
|
534
558
|
if (opts.consent !== false && !(await dismissConsent(page.session))) {
|
|
535
559
|
consentRecheck = await navKey();
|
|
@@ -543,6 +567,7 @@ export async function connect(opts = {}) {
|
|
|
543
567
|
// Headed launch failed (no display?) — keep headless result, botBlocked stays true
|
|
544
568
|
}
|
|
545
569
|
}
|
|
570
|
+
return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
|
|
546
571
|
},
|
|
547
572
|
|
|
548
573
|
async goBack() {
|
|
@@ -553,6 +578,7 @@ export async function connect(opts = {}) {
|
|
|
553
578
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
554
579
|
refMap = new Map(); // refs from the previous page are now invalid
|
|
555
580
|
consentRecheck = null;
|
|
581
|
+
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
556
582
|
},
|
|
557
583
|
|
|
558
584
|
async goForward() {
|
|
@@ -563,6 +589,7 @@ export async function connect(opts = {}) {
|
|
|
563
589
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
564
590
|
refMap = new Map();
|
|
565
591
|
consentRecheck = null;
|
|
592
|
+
lastNavigation = null;
|
|
566
593
|
},
|
|
567
594
|
|
|
568
595
|
async reload(reloadOpts = {}) {
|
|
@@ -574,6 +601,7 @@ export async function connect(opts = {}) {
|
|
|
574
601
|
try { await loadPromise; } catch { await new Promise((r) => setTimeout(r, 500)); }
|
|
575
602
|
refMap = new Map(); // refs from the pre-reload page are invalid
|
|
576
603
|
consentRecheck = null;
|
|
604
|
+
lastNavigation = null; // the reload's status is not read; don't keep the old one
|
|
577
605
|
},
|
|
578
606
|
|
|
579
607
|
/** @param {string} url @param {CookieOptions} [cookieOpts] */
|
|
@@ -599,11 +627,14 @@ export async function connect(opts = {}) {
|
|
|
599
627
|
const raw = formatTree(result.tree);
|
|
600
628
|
const { currentIndex, entries } = await page.session.send('Page.getNavigationHistory');
|
|
601
629
|
const pageUrl = entries[currentIndex]?.url || '';
|
|
602
|
-
const warn = botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : ''
|
|
603
|
-
|
|
630
|
+
const warn = (botBlocked ? '[BOT CHALLENGE DETECTED — page content may be incomplete or blocked]\n' : '')
|
|
631
|
+
+ httpWarning(lastNavigation?.status);
|
|
632
|
+
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
633
|
+
const tree = links ? annotateLinks(result.tree, pageUrl) : result.tree;
|
|
634
|
+
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
604
635
|
const mode = pruneOpts?.mode || opts.pruneMode || 'act';
|
|
605
|
-
const out = mode === 'data' ? dataSnapshot(
|
|
606
|
-
const sized =
|
|
636
|
+
const out = mode === 'data' ? dataSnapshot(tree, { links }) : formatTree(pruneTree(tree, { mode, links }));
|
|
637
|
+
const sized = stripLegend(out); // the legend is not content
|
|
607
638
|
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars (${Math.round((1 - sized.length / raw.length) * 100)}% pruned)`;
|
|
608
639
|
const hint = (mode === 'act' && raw.length > 5000 && out.length < 500 && out.length < raw.length * 0.05)
|
|
609
640
|
? `hint: act mode dropped most of the page — retry with pruneMode='read' for paragraphs and long text\n`
|
|
@@ -702,6 +733,7 @@ export async function connect(opts = {}) {
|
|
|
702
733
|
const oldSessionId = page.sessionId;
|
|
703
734
|
page = await attachToExistingTarget(cdp, target.targetId, pageOpts);
|
|
704
735
|
refMap = new Map(); // refs from the previous tab are no longer valid
|
|
736
|
+
lastNavigation = null; // the outcome belonged to the tab we left
|
|
705
737
|
setupDialogHandler(page.session);
|
|
706
738
|
try { await cdp.send('Target.detachFromTarget', { sessionId: oldSessionId }); } catch {}
|
|
707
739
|
},
|
|
@@ -746,6 +778,8 @@ export async function connect(opts = {}) {
|
|
|
746
778
|
},
|
|
747
779
|
|
|
748
780
|
get botBlocked() { return botBlocked; },
|
|
781
|
+
/** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
|
|
782
|
+
get lastNavigation() { return lastNavigation; },
|
|
749
783
|
|
|
750
784
|
/** @type {'chromium'} Engine driving this session (see also `capabilities`) */
|
|
751
785
|
engine: 'chromium',
|
|
@@ -820,12 +854,13 @@ export async function connect(opts = {}) {
|
|
|
820
854
|
return {
|
|
821
855
|
async goto(url, timeout = 30000) {
|
|
822
856
|
assertNavigable(url, urlGuard);
|
|
823
|
-
await navigate(tab, url, timeout);
|
|
857
|
+
const outcome = await navigate(tab, url, timeout);
|
|
824
858
|
if (opts.consent !== false) {
|
|
825
859
|
await dismissConsent(tab.session);
|
|
826
860
|
}
|
|
827
861
|
const { tree, nodeCount } = await ariaTree(tab);
|
|
828
862
|
tabBotBlocked = isChallengePage(tree, nodeCount);
|
|
863
|
+
return outcome;
|
|
829
864
|
},
|
|
830
865
|
get botBlocked() { return tabBotBlocked; },
|
|
831
866
|
/** @param {string} url @param {CookieOptions} [cookieOpts] */
|
|
@@ -959,6 +994,7 @@ async function connectFirefox(opts) {
|
|
|
959
994
|
|
|
960
995
|
const page = await createFirefoxPage(bidi, {
|
|
961
996
|
pruneMode: opts.pruneMode,
|
|
997
|
+
links: opts.links,
|
|
962
998
|
urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
|
|
963
999
|
uploadDir: opts.uploadDir || null,
|
|
964
1000
|
incognito: !!opts.incognito,
|
|
@@ -1126,14 +1162,45 @@ export function _resetBlocklistWarning() {
|
|
|
1126
1162
|
}
|
|
1127
1163
|
|
|
1128
1164
|
/**
|
|
1129
|
-
* Navigate to a URL and wait for the page to load.
|
|
1165
|
+
* Navigate to a URL and wait for the page to load. Resolves to the outcome of
|
|
1166
|
+
* the main-document navigation (final URL after redirects, HTTP status);
|
|
1167
|
+
* rejects with a NAV_FAILED error when the browser could not reach a document.
|
|
1168
|
+
* @returns {Promise<NavigationOutcome>}
|
|
1130
1169
|
*/
|
|
1131
1170
|
async function navigate(page, url, timeout = 30000) {
|
|
1132
|
-
const
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1171
|
+
const { session } = page;
|
|
1172
|
+
// Main-document responses (every redirect hop fires one; the last wins).
|
|
1173
|
+
const docs = [];
|
|
1174
|
+
const offDoc = session.on('Network.responseReceived', (p) => { if (p.type === 'Document') docs.push(p); });
|
|
1175
|
+
// Load waiter, hand-rolled (not session.once) so it can be cancelled when
|
|
1176
|
+
// Page.navigate fails or aborts and no load event will ever come.
|
|
1177
|
+
let cancel = () => {};
|
|
1178
|
+
const loaded = /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
|
|
1179
|
+
const timer = setTimeout(() => { off(); reject(new Error('Timeout waiting for CDP event: Page.loadEventFired')); }, timeout);
|
|
1180
|
+
const off = session.on('Page.loadEventFired', () => { cancel(); resolve(undefined); });
|
|
1181
|
+
cancel = () => { clearTimeout(timer); off(); };
|
|
1182
|
+
}));
|
|
1183
|
+
loaded.catch(() => {}); // rejection is only read if we await it below
|
|
1184
|
+
try {
|
|
1185
|
+
const nav = await session.send('Page.navigate', { url });
|
|
1186
|
+
// ERR_ABORTED is not a failure: a download (isDownload) or a 204 replaces
|
|
1187
|
+
// no document and fires no load event, so don't wait for one.
|
|
1188
|
+
const aborted = nav.errorText === 'net::ERR_ABORTED';
|
|
1189
|
+
// ERR_HTTP_RESPONSE_CODE_FAILURE is a 4xx/5xx with an empty body: the
|
|
1190
|
+
// document did load (Chromium's error page fires load, the Document
|
|
1191
|
+
// response carries the status), so report it like any other HTTP error.
|
|
1192
|
+
const httpError = nav.errorText === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE';
|
|
1193
|
+
if (nav.errorText && !aborted && !httpError) throw navFailed(url, nav.errorText);
|
|
1194
|
+
if (!aborted) await loaded;
|
|
1195
|
+
// Brief settle time for dynamic content
|
|
1196
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1197
|
+
// loaderId is shared by every redirect hop of one navigation.
|
|
1198
|
+
const last = docs.filter((d) => d.loaderId === nav.loaderId).pop();
|
|
1199
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
1200
|
+
} finally {
|
|
1201
|
+
cancel();
|
|
1202
|
+
offDoc();
|
|
1203
|
+
}
|
|
1137
1204
|
}
|
|
1138
1205
|
|
|
1139
1206
|
/**
|
package/src/links.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* links.js — opt-in link URLs for snapshots (`links: true`).
|
|
3
|
+
*
|
|
4
|
+
* `annotateLinks()` stamps `urls: [url]` on every link node that has a usable
|
|
5
|
+
* browser-serialized absolute URL. aria.js, prune.js and data.js print/carry
|
|
6
|
+
* `urls` — they never see the option, so with links off nothing changes.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Normalise a link's URL for a snapshot line. Returns '' when it should not be
|
|
11
|
+
* shown: no URL, `javascript:`, or a same-document link (the page URL, only
|
|
12
|
+
* the #fragment differs). A raw space, `<` or `>` (browsers encode them, but
|
|
13
|
+
* not always in the fragment) is percent-encoded so a line splits cleanly on
|
|
14
|
+
* ` <` and `> `.
|
|
15
|
+
*
|
|
16
|
+
* @param {string|undefined} url - Absolute URL as serialized by the browser
|
|
17
|
+
* @param {string} [pageUrl] - URL of the page the link is on
|
|
18
|
+
* @returns {string}
|
|
19
|
+
*/
|
|
20
|
+
export function linkUrl(url, pageUrl = '') {
|
|
21
|
+
if (typeof url !== 'string') return '';
|
|
22
|
+
const u = url.trim();
|
|
23
|
+
if (!u || /^javascript:/i.test(u)) return '';
|
|
24
|
+
if (pageUrl && u.split('#')[0] === pageUrl.split('#')[0]) return '';
|
|
25
|
+
return u.replace(/ /g, '%20').replace(/</g, '%3C').replace(/>/g, '%3E');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Copy a tree with `urls` set on its link nodes (input is not mutated).
|
|
30
|
+
*
|
|
31
|
+
* @param {object|null} node - Root from buildTree()
|
|
32
|
+
* @param {string} pageUrl
|
|
33
|
+
* @returns {object|null}
|
|
34
|
+
*/
|
|
35
|
+
export function annotateLinks(node, pageUrl) {
|
|
36
|
+
if (!node) return node;
|
|
37
|
+
const out = { ...node, children: node.children.map((c) => annotateLinks(c, pageUrl)) };
|
|
38
|
+
if (node.role === 'link') {
|
|
39
|
+
const u = linkUrl(node.properties?.url, pageUrl);
|
|
40
|
+
if (u) out.urls = [u];
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** ` <u1> <u2>` for a node's urls ('' when none). */
|
|
46
|
+
export const urlsText = (urls) => (urls || []).map((u) => ` <${u}>`).join('');
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* navigation.js — Shared pieces of the goto() outcome contract, so both
|
|
3
|
+
* engines (CDP in index.js, BiDi in firefox-page.js) report it identically.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What goto() resolves to: where the page ended up and how the server answered.
|
|
8
|
+
* `status` is 0 when no HTTP response was observed for the navigation (e.g. an
|
|
9
|
+
* aborted navigation that never got one).
|
|
10
|
+
* @typedef {{ url: string, status: number }} NavigationOutcome
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Typed error for a navigation that never reached a document (DNS failure,
|
|
15
|
+
* connection refused, TLS error, ...). `code` is 'NAV_FAILED'; `reason` is the
|
|
16
|
+
* browser's raw error string (`net::ERR_NAME_NOT_RESOLVED`,
|
|
17
|
+
* `NS_ERROR_UNKNOWN_HOST`, ...).
|
|
18
|
+
* @param {string} url
|
|
19
|
+
* @param {string} reason
|
|
20
|
+
* @returns {Error & {code: 'NAV_FAILED', reason: string}}
|
|
21
|
+
*/
|
|
22
|
+
export function navFailed(url, reason) {
|
|
23
|
+
return Object.assign(new Error(`Navigation to ${url} failed: ${reason}`), { code: /** @type {const} */ ('NAV_FAILED'), reason });
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* True for a final HTTP status that is not 2xx (0 = unknown is not an error).
|
|
28
|
+
* @param {number|undefined} status
|
|
29
|
+
*/
|
|
30
|
+
export function isHttpError(status) {
|
|
31
|
+
return !!status && (status < 200 || status >= 300);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Warning line for a non-2xx final status ('' for 2xx and for unknown 0), in
|
|
36
|
+
* the style of the bot-challenge line, newline-terminated for prepending.
|
|
37
|
+
* @param {number|undefined} status
|
|
38
|
+
* @returns {string}
|
|
39
|
+
*/
|
|
40
|
+
export function httpWarning(status) {
|
|
41
|
+
return isHttpError(status) ? `[HTTP ${status} — the page may be an error page]\n` : '';
|
|
42
|
+
}
|