barebrowse 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/README.md +32 -1
- package/barebrowse.context.md +10 -5
- package/cli.js +24 -6
- package/commands/barebrowse/SKILL.md +142 -0
- package/commands/barebrowse.md +141 -0
- package/mcp-server.js +78 -9
- package/package.json +2 -1
- package/src/aria.js +5 -0
- package/src/ax-snapshot.js +1 -0
- package/src/bareagent.js +5 -3
- package/src/blocklist-firefox.js +12 -4
- package/src/daemon.js +7 -4
- package/src/data.js +24 -9
- package/src/firefox-page.js +71 -12
- package/src/index.js +127 -24
- package/src/links.js +46 -0
- package/src/navigation.js +54 -0
- package/src/prune.js +11 -3
- package/src/site-lock.js +184 -0
- package/types/aria.d.ts +0 -6
- package/types/blocklist-firefox.d.ts +2 -1
- package/types/data.d.ts +6 -1
- package/types/firefox-page.d.ts +17 -1
- package/types/index.d.ts +22 -3
- package/types/links.d.ts +28 -0
- package/types/navigation.d.ts +56 -0
- package/types/prune.d.ts +2 -0
- package/types/site-lock.d.ts +70 -0
package/mcp-server.js
CHANGED
|
@@ -14,6 +14,8 @@ import { browse, connect } from './src/index.js';
|
|
|
14
14
|
import { formatReadable } from './src/readable.js';
|
|
15
15
|
import { cleanOld } from './src/output-clean.js';
|
|
16
16
|
import { scopedCookiesForUrl } from './src/auth.js';
|
|
17
|
+
import { isHttpError } from './src/navigation.js';
|
|
18
|
+
import { parseAllowHosts } from './src/site-lock.js';
|
|
17
19
|
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
|
|
18
20
|
import { join, dirname } from 'node:path';
|
|
19
21
|
import { pathToFileURL, fileURLToPath } from 'node:url';
|
|
@@ -216,6 +218,53 @@ function engineInfo() {
|
|
|
216
218
|
};
|
|
217
219
|
}
|
|
218
220
|
|
|
221
|
+
// --- Access tier + site lock (env, fixed at server start) ---
|
|
222
|
+
//
|
|
223
|
+
// BAREBROWSE_TIER = r | rw | rwx (default rwx = everything). r: read the web
|
|
224
|
+
// (navigate, observe); rw: + interact (click/type/...); rwx: + upload and eval.
|
|
225
|
+
// Every tool has an explicit tier; a tool missing from the map is rwx, so a
|
|
226
|
+
// new tool is hidden from restricted tiers until someone classifies it.
|
|
227
|
+
const TIER_RANK = { r: 0, rw: 1, rwx: 2 };
|
|
228
|
+
export const TOOL_TIER = {
|
|
229
|
+
browse: 'r', goto: 'r', back: 'r', forward: 'r', reload: 'r', scroll: 'r',
|
|
230
|
+
snapshot: 'r', readable: 'r', screenshot: 'r', pdf: 'r', wait_for: 'r', tabs: 'r',
|
|
231
|
+
capabilities: 'r', assess: 'r',
|
|
232
|
+
click: 'rw', type: 'rw', press: 'rw', select: 'rw', drag: 'rw', hover: 'rw', switch_engine: 'rw',
|
|
233
|
+
upload: 'rwx', eval: 'rwx',
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
/** @param {string} name */
|
|
237
|
+
export function toolTier(name) { return TOOL_TIER[name] ?? 'rwx'; }
|
|
238
|
+
|
|
239
|
+
/** Parse BAREBROWSE_TIER. Unset/empty = rwx; anything else must be exact, or it throws (never widen on a typo). */
|
|
240
|
+
export function parseTier(v) {
|
|
241
|
+
if (v === undefined || v === '') return 'rwx';
|
|
242
|
+
if (!Object.hasOwn(TIER_RANK, v)) throw new Error(`BAREBROWSE_TIER must be r, rw or rwx (got ${JSON.stringify(v)})`);
|
|
243
|
+
return v;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Parse BAREBROWSE_ALLOW (comma list of hosts). Unset/blank = no lock (null); a bad entry throws. */
|
|
247
|
+
export function parseAllowEnv(v) {
|
|
248
|
+
if (v === undefined || !v.trim()) return null;
|
|
249
|
+
try { return parseAllowHosts(v.split(',')); }
|
|
250
|
+
catch (e) { throw new Error(`BAREBROWSE_ALLOW: ${e.message}`); }
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Read + validate the tier/site-lock env. Throws on bad config. */
|
|
254
|
+
function readAccess() {
|
|
255
|
+
return { tier: parseTier(process.env.BAREBROWSE_TIER), allowHosts: parseAllowEnv(process.env.BAREBROWSE_ALLOW) };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** MCP tool hints derived from the tier: r = read-only; rw/rwx change state (hover and switch_engine non-destructively). */
|
|
259
|
+
function annotationsFor(name) {
|
|
260
|
+
if (toolTier(name) === 'r') {
|
|
261
|
+
const a = { readOnlyHint: true, openWorldHint: name !== 'capabilities' };
|
|
262
|
+
if (['snapshot', 'readable', 'screenshot', 'pdf', 'tabs', 'capabilities', 'wait_for'].includes(name)) a.idempotentHint = true;
|
|
263
|
+
return a;
|
|
264
|
+
}
|
|
265
|
+
return { readOnlyHint: false, destructiveHint: !['hover', 'switch_engine'].includes(name), openWorldHint: true };
|
|
266
|
+
}
|
|
267
|
+
|
|
219
268
|
async function getPage() {
|
|
220
269
|
if (_page) return _page;
|
|
221
270
|
if (_pageConnecting) return _pageConnecting;
|
|
@@ -229,9 +278,10 @@ async function getPage() {
|
|
|
229
278
|
// across tool calls, so it's an env var (BAREBROWSE_INCOGNITO=1), not a
|
|
230
279
|
// per-request flag. Skips all auth injection for a clean, logged-out session.
|
|
231
280
|
const incognito = process.env.BAREBROWSE_INCOGNITO === '1';
|
|
281
|
+
const { allowHosts } = readAccess();
|
|
232
282
|
const connectOpts = _engine === 'firefox'
|
|
233
|
-
? { engine: 'firefox', mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito }
|
|
234
|
-
: { mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito };
|
|
283
|
+
? { engine: 'firefox', mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito, allowHosts }
|
|
284
|
+
: { mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito, allowHosts };
|
|
235
285
|
_pageConnecting = connect(connectOpts);
|
|
236
286
|
try {
|
|
237
287
|
_page = await _pageConnecting;
|
|
@@ -268,6 +318,7 @@ export const TOOLS = [
|
|
|
268
318
|
mode: { type: 'string', enum: ['headless', 'headed', 'hybrid'], description: 'Browser mode (default: headless)' },
|
|
269
319
|
pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: 'Pruning mode. "act" (default) keeps interactive elements and short labels — best for clicking/filling. "read" keeps paragraphs, headings, and long text — best for articles, docs, and content extraction. If the page is content-heavy and act-mode returns mostly empty, retry with "read". "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
|
|
270
320
|
incognito: { type: 'boolean', description: 'Clean, unauthenticated session: skip cookie injection so the page loads logged-out (default: false).' },
|
|
321
|
+
links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
|
|
271
322
|
maxChars: { type: 'number', description: 'Max chars to return inline. Larger snapshots are saved to .barebrowse/ and a file path is returned instead. Default: 30000.' },
|
|
272
323
|
},
|
|
273
324
|
required: ['url'],
|
|
@@ -275,7 +326,7 @@ export const TOOLS = [
|
|
|
275
326
|
},
|
|
276
327
|
{
|
|
277
328
|
name: 'goto',
|
|
278
|
-
description: 'Open URL in a persistent interactive browser session (pair with snapshot/click/type/press for multi-step flows). Use when the task needs clicking, typing, or form submission. Injects auth cookies. Returns ok — call snapshot to observe.',
|
|
329
|
+
description: 'Open URL in a persistent interactive browser session (pair with snapshot/click/type/press for multi-step flows). Use when the task needs clicking, typing, or form submission. Injects auth cookies. Returns ok — call snapshot to observe. A non-2xx page answers `ok (HTTP 404)` and the snapshot starts with an [HTTP …] warning line; an unreachable host (DNS, refused, TLS) is an error.',
|
|
279
330
|
inputSchema: {
|
|
280
331
|
type: 'object',
|
|
281
332
|
properties: {
|
|
@@ -291,6 +342,7 @@ export const TOOLS = [
|
|
|
291
342
|
type: 'object',
|
|
292
343
|
properties: {
|
|
293
344
|
pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: 'Pruning mode. "act" (default) keeps interactive elements and short labels — best for clicking/filling. "read" keeps paragraphs, headings, and long text — best for articles, docs, and content extraction. If a previous snapshot looked empty on a content-heavy page, retry with "read". "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
|
|
345
|
+
links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
|
|
294
346
|
maxChars: { type: 'number', description: 'Max chars to return inline. Larger snapshots are saved to .barebrowse/ and a file path is returned instead. Default: 30000.' },
|
|
295
347
|
},
|
|
296
348
|
},
|
|
@@ -514,12 +566,17 @@ if (assessFn) {
|
|
|
514
566
|
});
|
|
515
567
|
}
|
|
516
568
|
|
|
569
|
+
for (const t of TOOLS) t.annotations = annotationsFor(t.name);
|
|
570
|
+
|
|
517
571
|
async function handleToolCall(name, args) {
|
|
518
572
|
switch (name) {
|
|
519
573
|
case 'capabilities': {
|
|
520
574
|
// Introspection only — must NOT launch a browser. Report the configured
|
|
521
575
|
// engine from env; if a page is already connected, attach its live flags.
|
|
522
576
|
const info = engineInfo();
|
|
577
|
+
const access = readAccess();
|
|
578
|
+
info.tier = access.tier;
|
|
579
|
+
info.allowHosts = access.allowHosts;
|
|
523
580
|
if (_page) {
|
|
524
581
|
info.connected = true;
|
|
525
582
|
info.capabilities = _page.capabilities;
|
|
@@ -544,7 +601,7 @@ async function handleToolCall(name, args) {
|
|
|
544
601
|
case 'browse': {
|
|
545
602
|
let timer;
|
|
546
603
|
const text = await Promise.race([
|
|
547
|
-
browse(args.url, { mode: args.mode, pruneMode: args.pruneMode, incognito: args.incognito }),
|
|
604
|
+
browse(args.url, { mode: args.mode, pruneMode: args.pruneMode, incognito: args.incognito, links: args.links, allowHosts: readAccess().allowHosts ?? undefined }),
|
|
548
605
|
new Promise((_, rej) => { timer = setTimeout(() => rej(new Error('browse timed out after 60s')), 60000); }),
|
|
549
606
|
]);
|
|
550
607
|
clearTimeout(timer);
|
|
@@ -566,12 +623,12 @@ async function handleToolCall(name, args) {
|
|
|
566
623
|
try { await page.injectCookies(args.url); } catch (e) { error ??= e.message; }
|
|
567
624
|
_lastCookies = cookieReport(args.url, { cookies, error });
|
|
568
625
|
}
|
|
569
|
-
await page.goto(args.url);
|
|
570
|
-
return 'ok';
|
|
626
|
+
const { status } = await page.goto(args.url);
|
|
627
|
+
return isHttpError(status) ? `ok (HTTP ${status})` : 'ok';
|
|
571
628
|
}, TIMEOUTS.goto);
|
|
572
629
|
case 'snapshot': return withRetry(async () => {
|
|
573
630
|
const page = await getPage();
|
|
574
|
-
const text = await page.snapshot(args.pruneMode ? { mode: args.pruneMode } : undefined);
|
|
631
|
+
const text = await page.snapshot(args.pruneMode || args.links ? { mode: args.pruneMode, links: args.links } : undefined);
|
|
575
632
|
const limit = args.maxChars ?? MAX_CHARS_DEFAULT;
|
|
576
633
|
if (text.length > limit) {
|
|
577
634
|
const file = saveSnapshot(text);
|
|
@@ -768,6 +825,10 @@ async function handleMessage(msg) {
|
|
|
768
825
|
|
|
769
826
|
if (method === 'initialize') {
|
|
770
827
|
const e = engineInfo();
|
|
828
|
+
const { tier, allowHosts } = readAccess();
|
|
829
|
+
const access = (tier === 'r' ? ' Access tier: read-only (no clicking, typing or uploading).'
|
|
830
|
+
: tier === 'rw' ? ' Access tier: read/write (no upload or eval).' : '')
|
|
831
|
+
+ (allowHosts ? ` Navigation is restricted to: ${allowHosts.join(', ')}.` : '');
|
|
771
832
|
return jsonrpcResponse(id, {
|
|
772
833
|
protocolVersion: '2024-11-05',
|
|
773
834
|
capabilities: { tools: {} },
|
|
@@ -777,7 +838,7 @@ async function handleMessage(msg) {
|
|
|
777
838
|
instructions:
|
|
778
839
|
`barebrowse drives a real browser. Active engine: ${e.activeEngine} (mode: ${e.mode}). `
|
|
779
840
|
+ `Both chromium and firefox are supported; ${e.switchEngine} `
|
|
780
|
-
+ `Call the "capabilities" tool for the live engine + feature set
|
|
841
|
+
+ `Call the "capabilities" tool for the live engine + feature set.${access}`,
|
|
781
842
|
});
|
|
782
843
|
}
|
|
783
844
|
|
|
@@ -790,7 +851,8 @@ async function handleMessage(msg) {
|
|
|
790
851
|
// active engine (and that both are supported) is visible without a call.
|
|
791
852
|
const e = engineInfo();
|
|
792
853
|
const note = ` [Active engine: ${e.activeEngine}; chromium + firefox both supported — ${e.switchEngine}]`;
|
|
793
|
-
const
|
|
854
|
+
const { tier } = readAccess();
|
|
855
|
+
const tools = TOOLS.filter((t) => TIER_RANK[toolTier(t.name)] <= TIER_RANK[tier]).map((t) =>
|
|
794
856
|
t.name === 'browse' ? { ...t, description: t.description + note } : t);
|
|
795
857
|
return jsonrpcResponse(id, { tools });
|
|
796
858
|
}
|
|
@@ -800,6 +862,8 @@ async function handleMessage(msg) {
|
|
|
800
862
|
_inflight++;
|
|
801
863
|
clearTimeout(_idleTimer);
|
|
802
864
|
try {
|
|
865
|
+
// A tool above the tier answers exactly like one that doesn't exist.
|
|
866
|
+
if (TIER_RANK[toolTier(name)] > TIER_RANK[readAccess().tier]) throw new Error(`Unknown tool: ${name}`);
|
|
803
867
|
const result = await handleToolCall(name, args || {});
|
|
804
868
|
return jsonrpcResponse(id, {
|
|
805
869
|
content: [{ type: 'text', text: typeof result === 'string' ? result : JSON.stringify(result) }],
|
|
@@ -830,6 +894,11 @@ async function handleMessage(msg) {
|
|
|
830
894
|
// runStdio() explicitly. Tests import TIMEOUTS/TOOLS without calling it.
|
|
831
895
|
|
|
832
896
|
export function runStdio() {
|
|
897
|
+
// Fail closed: a bad tier or allow list must stop the server, never fall back to rwx / no lock.
|
|
898
|
+
try { readAccess(); } catch (e) {
|
|
899
|
+
process.stderr.write(`barebrowse mcp: ${e.message}\n`);
|
|
900
|
+
process.exit(1);
|
|
901
|
+
}
|
|
833
902
|
cleanOld(OUTPUT_DIR);
|
|
834
903
|
// One-line startup banner to stderr (stderr because stdout is the JSON-RPC
|
|
835
904
|
// channel — must not contain non-JSON-RPC bytes). Captured by Claude Code's
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "barebrowse",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.27.0",
|
|
4
4
|
"description": "Authenticated web browsing for autonomous agents via CDP. URL in, pruned ARIA snapshot out.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"src/",
|
|
29
29
|
"types/",
|
|
30
30
|
"cli.js",
|
|
31
|
+
"commands/",
|
|
31
32
|
"mcp-server.js",
|
|
32
33
|
"barebrowse.context.md",
|
|
33
34
|
"README.md",
|
package/src/aria.js
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* and formats it as readable YAML-like text, similar to Playwright's ariaSnapshot.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import { urlsText } from './links.js';
|
|
9
|
+
|
|
8
10
|
/**
|
|
9
11
|
* Format a nested ARIA tree as readable text output.
|
|
10
12
|
*
|
|
@@ -37,6 +39,9 @@ export function formatTree(node, depth = 0) {
|
|
|
37
39
|
line += ` "${node.name}"`;
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
// Link URLs (opt-in links: true): right after the name, before props and ref
|
|
43
|
+
line += urlsText(node.urls);
|
|
44
|
+
|
|
40
45
|
// Notable properties that agents care about
|
|
41
46
|
const props = node.properties || {};
|
|
42
47
|
const propParts = [];
|
package/src/ax-snapshot.js
CHANGED
|
@@ -166,6 +166,7 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
|
|
|
166
166
|
const exp = el.getAttribute('aria-expanded');
|
|
167
167
|
if (exp !== null) p.expanded = exp === 'true';
|
|
168
168
|
if (role === 'heading') p.level = Number(el.getAttribute('aria-level')) || Number(el.tagName[1]) || 2;
|
|
169
|
+
if (role === 'link' && el.href) p.url = el.href; // browser-serialized absolute URL (CDP's `url` property)
|
|
169
170
|
if (el.getAttribute('aria-selected') === 'true') p.selected = true;
|
|
170
171
|
if (el.required || el.getAttribute('aria-required') === 'true') p.required = true;
|
|
171
172
|
if ((role === 'textbox' || role === 'searchbox' || role === 'combobox' || role === 'spinbutton') && el.value) {
|
package/src/bareagent.js
CHANGED
|
@@ -63,10 +63,11 @@ export function createBrowseTools(opts = {}) {
|
|
|
63
63
|
properties: {
|
|
64
64
|
url: { type: 'string', description: 'URL to browse' },
|
|
65
65
|
pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: '"act" (default) for interactive elements only; "read" for paragraphs and long text (articles/docs); "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
|
|
66
|
+
links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
|
|
66
67
|
},
|
|
67
68
|
required: ['url'],
|
|
68
69
|
},
|
|
69
|
-
execute: async ({ url, pruneMode }) => await browse(url, pruneMode ? { ...opts, pruneMode } : opts),
|
|
70
|
+
execute: async ({ url, pruneMode, links }) => await browse(url, pruneMode || links ? { ...opts, pruneMode, links } : opts),
|
|
70
71
|
},
|
|
71
72
|
{
|
|
72
73
|
name: 'goto',
|
|
@@ -87,11 +88,12 @@ export function createBrowseTools(opts = {}) {
|
|
|
87
88
|
type: 'object',
|
|
88
89
|
properties: {
|
|
89
90
|
pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: '"act" (default) for interactive elements only; "read" for paragraphs and long text (articles/docs); "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
|
|
91
|
+
links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
|
|
90
92
|
},
|
|
91
93
|
},
|
|
92
|
-
execute: async (/** @type {{ pruneMode?: string }} */ { pruneMode } = {}) => {
|
|
94
|
+
execute: async (/** @type {{ pruneMode?: string, links?: boolean }} */ { pruneMode, links } = {}) => {
|
|
93
95
|
const page = await getPage();
|
|
94
|
-
return await page.snapshot(pruneMode ? { mode: pruneMode } : undefined);
|
|
96
|
+
return await page.snapshot(pruneMode || links ? { mode: pruneMode, links } : undefined);
|
|
95
97
|
},
|
|
96
98
|
},
|
|
97
99
|
{
|
package/src/blocklist-firefox.js
CHANGED
|
@@ -13,9 +13,15 @@
|
|
|
13
13
|
* request pauses for one continue/fail round-trip to Node. On a typical page
|
|
14
14
|
* (tens–low-hundreds of requests over a local socket) that's negligible, and
|
|
15
15
|
* it's the only route that preserves glob parity without a second list.
|
|
16
|
+
*
|
|
17
|
+
* The site lock (`allowHosts`, site-lock.js) plugs into this same handler: a
|
|
18
|
+
* second catch-all handler would race this one on the same paused request, so
|
|
19
|
+
* there is exactly one continue/fail decision per request, and the intercept
|
|
20
|
+
* exists whenever either the blocklist or the lock is active.
|
|
16
21
|
*/
|
|
17
22
|
|
|
18
23
|
import { resolveBlocklistPatterns, makeBlockMatcher } from './blocklist.js';
|
|
24
|
+
import { createFirefoxLockCheck } from './site-lock.js';
|
|
19
25
|
|
|
20
26
|
/**
|
|
21
27
|
* Install ad/tracker blocking on a BiDi session. Idempotent per session is the
|
|
@@ -26,13 +32,15 @@ import { resolveBlocklistPatterns, makeBlockMatcher } from './blocklist.js';
|
|
|
26
32
|
* @param {boolean} [pageOpts.blockAds] - false disables the default list.
|
|
27
33
|
* @param {string[]} [pageOpts.blockUrls] - extra CDP-format globs; extend the
|
|
28
34
|
* default unless blockAds is false, in which case they're the whole list.
|
|
35
|
+
* @param {import('./site-lock.js').SiteLock|null} [lock] - Site lock to enforce in the same handler.
|
|
29
36
|
* @returns {Promise<void>}
|
|
30
37
|
*/
|
|
31
|
-
export async function applyFirefoxBlocklist(bidi, pageOpts = {}) {
|
|
38
|
+
export async function applyFirefoxBlocklist(bidi, pageOpts = {}, lock = null) {
|
|
32
39
|
const patterns = resolveBlocklistPatterns(pageOpts);
|
|
33
|
-
if (!patterns.length) return;
|
|
40
|
+
if (!patterns.length && !lock) return;
|
|
34
41
|
|
|
35
|
-
const isBlocked = makeBlockMatcher(patterns);
|
|
42
|
+
const isBlocked = patterns.length ? makeBlockMatcher(patterns) : () => false;
|
|
43
|
+
const lockBlocks = lock ? await createFirefoxLockCheck(bidi, lock) : null;
|
|
36
44
|
|
|
37
45
|
await bidi.send('network.addIntercept', {
|
|
38
46
|
phases: ['beforeRequestSent'],
|
|
@@ -47,7 +55,7 @@ export async function applyFirefoxBlocklist(bidi, pageOpts = {}) {
|
|
|
47
55
|
const id = e.request?.request;
|
|
48
56
|
if (!id) return;
|
|
49
57
|
try {
|
|
50
|
-
if (isBlocked(e.request.url)) {
|
|
58
|
+
if (isBlocked(e.request.url) || (lockBlocks && await lockBlocks(e))) {
|
|
51
59
|
await bidi.send('network.failRequest', { request: id });
|
|
52
60
|
} else {
|
|
53
61
|
await bidi.send('network.continueRequest', { request: id });
|
package/src/daemon.js
CHANGED
|
@@ -13,6 +13,7 @@ import { join, resolve } from 'node:path';
|
|
|
13
13
|
import { connect } from './index.js';
|
|
14
14
|
import { formatReadable } from './readable.js';
|
|
15
15
|
import { cleanOld } from './output-clean.js';
|
|
16
|
+
import { isHttpError } from './navigation.js';
|
|
16
17
|
|
|
17
18
|
/** Owner-only file write helper — daemon artifacts can hold authenticated content. */
|
|
18
19
|
function writeFilePrivate(path, data) {
|
|
@@ -65,6 +66,7 @@ export function buildDaemonArgs(opts, absDir, initialUrl, cliPath) {
|
|
|
65
66
|
}
|
|
66
67
|
if (opts.blockPrivateNetwork) args.push('--block-private-network');
|
|
67
68
|
if (opts.uploadDir) args.push('--upload-dir', opts.uploadDir);
|
|
69
|
+
if (opts.allowHosts) args.push('--allow-hosts', opts.allowHosts.join(','));
|
|
68
70
|
return args;
|
|
69
71
|
}
|
|
70
72
|
|
|
@@ -234,6 +236,7 @@ export async function runDaemon(opts, outputDir, initialUrl) {
|
|
|
234
236
|
blockUrls: opts.blockUrls,
|
|
235
237
|
blockPrivateNetwork: opts.blockPrivateNetwork,
|
|
236
238
|
uploadDir: opts.uploadDir,
|
|
239
|
+
allowHosts: opts.allowHosts,
|
|
237
240
|
// incognito neuters injectCookies() session-wide; cookies:false (below)
|
|
238
241
|
// is the legacy per-open equivalent. Either yields a logged-out session.
|
|
239
242
|
incognito: opts.incognito || opts.cookies === false,
|
|
@@ -313,13 +316,13 @@ export async function runDaemon(opts, outputDir, initialUrl) {
|
|
|
313
316
|
// Command handlers
|
|
314
317
|
const handlers = {
|
|
315
318
|
async goto({ url, timeout }) {
|
|
316
|
-
await page.goto(url, timeout || 30000);
|
|
317
|
-
return { ok: true };
|
|
319
|
+
const { status } = await page.goto(url, timeout || 30000);
|
|
320
|
+
return isHttpError(status) ? { ok: true, status } : { ok: true };
|
|
318
321
|
},
|
|
319
322
|
|
|
320
|
-
async snapshot({ mode }) {
|
|
323
|
+
async snapshot({ mode, links }) {
|
|
321
324
|
const pruneMode = mode || defaultPruneMode;
|
|
322
|
-
const text = await page.snapshot({ mode: pruneMode });
|
|
325
|
+
const text = await page.snapshot({ mode: pruneMode, links });
|
|
323
326
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
324
327
|
const file = join(absDir, `page-${ts}.yml`);
|
|
325
328
|
writeFilePrivate(file, text);
|
package/src/data.js
CHANGED
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
* click/type resolve through the same refMap).
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
|
+
import { urlsText } from './links.js';
|
|
37
|
+
|
|
36
38
|
const SKIP = new Set(['InlineTextBox', 'LineBreak', 'ListMarker', 'image', 'img']);
|
|
37
39
|
const CELLS = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);
|
|
38
40
|
const TABLES = new Set(['table', 'grid', 'treegrid']);
|
|
@@ -47,13 +49,20 @@ const MIN_RUN = 3;
|
|
|
47
49
|
/** First line of every data-mode snapshot: tells an agent with no docs how to read it. */
|
|
48
50
|
export const LEGEND = '# data mode: one line per record. Tables: row "<label>": <column>: <value> · … Groups: <label>: a, b, c. A title line followed by detail lines is one block, up to the next title line. Main content only; use mode "act" for menus and page controls.';
|
|
49
51
|
|
|
52
|
+
/** Appended to the legend only when links are on. */
|
|
53
|
+
export const LINKS_NOTE = ' Links: "name" <url>.';
|
|
54
|
+
|
|
50
55
|
// A name made only of private-use code points (icon-font glyphs, unreadable to an
|
|
51
56
|
// agent) is no name. Glyphs mixed into real text are left as they are.
|
|
52
57
|
const norm = (s) => (/^[\p{Co}\s]*$/u.test(s || '') ? '' : s.replace(/\s+/g, ' ').trim());
|
|
58
|
+
const allUrls = (n) => [...(n.urls || []), ...(n.children || []).flatMap(allUrls)];
|
|
53
59
|
const hasInteractive = (n) => INTERACTIVE.has(n.role) || (n.children || []).some(hasInteractive);
|
|
54
60
|
const isLeaf = (n) => n.text === undefined && !n.children.length && n.name;
|
|
55
61
|
const showRef = (n) => !!n.ref && INTERACTIVE.has(n.role);
|
|
56
|
-
|
|
62
|
+
// ` <url>` for a node's own links (already unique per node). A link line always
|
|
63
|
+
// carries its own URL: nothing is deduped across lines, items or cells.
|
|
64
|
+
const urlPart = (n) => urlsText(n.urls);
|
|
65
|
+
const withRef = (n) => (showRef(n) ? `${n.name}${urlPart(n)} [ref=${n.ref}]` : n.name);
|
|
57
66
|
// Leaves carrying state (checked, level, value...) never merge into a run.
|
|
58
67
|
const plain = (n) => !(n.props.checked !== undefined || n.props.expanded !== undefined || n.props.selected || n.props.level || n.props.required || (n.props.value !== undefined && n.props.value !== ''));
|
|
59
68
|
// Run key: leaves with the same key are "the same kind of item". `g` = the
|
|
@@ -69,19 +78,20 @@ function inline(n, refs) {
|
|
|
69
78
|
const sub = n.children.length ? n.children.map((c) => inline(c, refs)).filter(Boolean).join(' ') : n.name;
|
|
70
79
|
const v = (n.name && !n.children.length) || INTERACTIVE.has(n.role) && n.name && refs ? n.name : sub;
|
|
71
80
|
const out = n.children.length && n.name && n.role !== 'StaticText' && !sub ? n.name : v;
|
|
72
|
-
return refs && n.ref && INTERACTIVE.has(n.role) ? `${out} [ref=${n.ref}]` : out;
|
|
81
|
+
return refs && n.ref && INTERACTIVE.has(n.role) ? `${out}${urlPart(n)} [ref=${n.ref}]` : out;
|
|
73
82
|
}
|
|
74
83
|
|
|
75
84
|
/** A data-mode result without its legend line, for size stats ('' stays ''). */
|
|
76
|
-
export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(
|
|
85
|
+
export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(s.indexOf('\n') + 1) : s);
|
|
77
86
|
|
|
78
87
|
/**
|
|
79
88
|
* Render a raw ARIA tree in data mode.
|
|
80
89
|
*
|
|
81
90
|
* @param {object} root - Root node from buildTree() (CDP vocabulary)
|
|
91
|
+
* @param {{ links?: boolean }} [opts] - links: the tree's link nodes carry `urls` (annotateLinks); adds the legend note
|
|
82
92
|
* @returns {string} LEGEND line, then one line per record, YAML-ish like formatTree(); '' when there are no records
|
|
83
93
|
*/
|
|
84
|
-
export function dataSnapshot(root) {
|
|
94
|
+
export function dataSnapshot(root, opts = {}) {
|
|
85
95
|
if (!root) return '';
|
|
86
96
|
let gid = 0;
|
|
87
97
|
const rawKids = (n) => n.children.flatMap((c) => (c.ignored ? rawKids(c) : [c]));
|
|
@@ -120,7 +130,7 @@ export function dataSnapshot(root) {
|
|
|
120
130
|
if (!nm && kids.length === 0 && (n.role === 'link' || n.role === 'button')) return []; // N6
|
|
121
131
|
if (!nm && kids.length === 0 && !INTERACTIVE.has(n.role) && !CELLS.has(n.role) && n.role !== 'row') return [];
|
|
122
132
|
if (n.role === 'listitem' && !nm && kids.length === 1) return kids;
|
|
123
|
-
return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig }];
|
|
133
|
+
return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig, urls: n.urls }];
|
|
124
134
|
}
|
|
125
135
|
|
|
126
136
|
// ---------- Transforms, bottom-up with the ancestor chain for labels ----------
|
|
@@ -134,7 +144,7 @@ export function dataSnapshot(root) {
|
|
|
134
144
|
const out = [];
|
|
135
145
|
for (const k of kids) {
|
|
136
146
|
const p = out[out.length - 1];
|
|
137
|
-
if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name) continue;
|
|
147
|
+
if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name && (p.urls || [])[0] === (k.urls || [])[0]) continue;
|
|
138
148
|
out.push(k);
|
|
139
149
|
}
|
|
140
150
|
return out;
|
|
@@ -243,6 +253,7 @@ export function dataSnapshot(root) {
|
|
|
243
253
|
const labelCells = cs.filter((c) => c.role === 'rowheader' || c.role === 'columnheader');
|
|
244
254
|
const lcx = labelCells.map(xcell);
|
|
245
255
|
const label = lcx.map((c) => inline(c, false)).filter(Boolean).join(' / ');
|
|
256
|
+
const labelUrls = urlsText([...new Set(lcx.flatMap(allUrls))]);
|
|
246
257
|
const labelRef = lcx.map(firstRef).find(Boolean);
|
|
247
258
|
const vals = [];
|
|
248
259
|
cs.forEach((c, idx) => {
|
|
@@ -262,7 +273,7 @@ export function dataSnapshot(root) {
|
|
|
262
273
|
}
|
|
263
274
|
if (!vals.length && !label) continue; // header-only / empty rows
|
|
264
275
|
const body = vals.map((x) => (x.h ? `${x.h}: ` : '') + x.v).join(' · ');
|
|
265
|
-
out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
|
|
276
|
+
out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelUrls}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
|
|
266
277
|
}
|
|
267
278
|
return { ...t, children: out };
|
|
268
279
|
}
|
|
@@ -278,10 +289,14 @@ export function dataSnapshot(root) {
|
|
|
278
289
|
const lines = [];
|
|
279
290
|
function emit(c, d) {
|
|
280
291
|
const ind = ' '.repeat(d);
|
|
281
|
-
if (c.text !== undefined) {
|
|
292
|
+
if (c.text !== undefined) {
|
|
293
|
+
lines.push(`${ind}- ${c.text}`);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
282
296
|
if (isLeaf(c) && (c.role === 'StaticText' || c.role === 'generic')) { lines.push(`${ind}- ${c.name}`); return; }
|
|
283
297
|
let l = `${ind}- ${c.role}`;
|
|
284
298
|
if (c.name) l += ` "${c.name}"`;
|
|
299
|
+
l += urlPart(c);
|
|
285
300
|
const p = c.props, pp = [];
|
|
286
301
|
if (p.checked !== undefined) pp.push(`checked=${p.checked}`);
|
|
287
302
|
if (p.disabled) pp.push('disabled');
|
|
@@ -296,5 +311,5 @@ export function dataSnapshot(root) {
|
|
|
296
311
|
for (const k of c.children) emit(k, d + 1);
|
|
297
312
|
}
|
|
298
313
|
for (const c of res.children) emit(c, 0);
|
|
299
|
-
return lines.length ? [LEGEND, ...lines].join('\n') : '';
|
|
314
|
+
return lines.length ? [LEGEND + (opts.links ? LINKS_NOTE : ''), ...lines].join('\n') : '';
|
|
300
315
|
}
|
package/src/firefox-page.js
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { formatTree } from './aria.js';
|
|
19
19
|
import { prune as pruneTree } from './prune.js';
|
|
20
20
|
import { dataSnapshot, stripLegend } from './data.js';
|
|
21
|
+
import { annotateLinks } from './links.js';
|
|
21
22
|
import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
|
|
22
23
|
import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
|
|
23
24
|
import { scopedCookiesForUrl } from './auth.js';
|
|
@@ -27,6 +28,7 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
|
|
|
27
28
|
import { decideDialog, dialogLogEntry } from './dialog.js';
|
|
28
29
|
import { isChallengePage, countNodes } from './challenge.js';
|
|
29
30
|
import { withTimeout } from './with-timeout.js';
|
|
31
|
+
import { navFailed, navBlocked, httpWarning } from './navigation.js';
|
|
30
32
|
|
|
31
33
|
/** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
|
|
32
34
|
const BIDI_KEYS = {
|
|
@@ -41,8 +43,10 @@ const BIDI_KEYS = {
|
|
|
41
43
|
* @param {object} bidi - BiDi client from createBiDi()
|
|
42
44
|
* @param {object} [opts]
|
|
43
45
|
* @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Default prune mode.
|
|
46
|
+
* @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
|
|
44
47
|
* @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
|
|
45
48
|
* @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
|
|
49
|
+
* @param {?import('./site-lock.js').SiteLock} [opts.siteLock] - Site lock (allowHosts) state; the request-level enforcement is installed by applyFirefoxBlocklist, this side maps it to NAV_BLOCKED and snapshot notes.
|
|
46
50
|
* @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
|
|
47
51
|
* @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs after goto().
|
|
48
52
|
* @param {boolean} [opts.hybrid=false] - Relaunch headed on a bot-challenge page and retry.
|
|
@@ -53,6 +57,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
53
57
|
const defaultPruneMode = opts.pruneMode || 'act';
|
|
54
58
|
const urlGuard = opts.urlGuard || {};
|
|
55
59
|
const uploadDir = opts.uploadDir || null;
|
|
60
|
+
const lock = opts.siteLock || null;
|
|
56
61
|
const incognito = !!opts.incognito;
|
|
57
62
|
const consent = opts.consent !== false;
|
|
58
63
|
// True when the consent pass after the latest goto() found nothing: the first
|
|
@@ -71,6 +76,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
71
76
|
const relaunchHeaded = opts.relaunchHeaded || null;
|
|
72
77
|
let currentlyHeaded = !!opts.headed; // already headed → no fallback needed
|
|
73
78
|
let botBlocked = false;
|
|
79
|
+
// Outcome of the latest goto() ({url, status}); null before any goto.
|
|
80
|
+
/** @type {import('./navigation.js').NavigationOutcome|null} */
|
|
81
|
+
let lastNavigation = null;
|
|
74
82
|
// Last injectCookies(url, opts) args, so a hybrid relaunch (a brand-new
|
|
75
83
|
// Firefox profile) can re-inject them — otherwise the headed retry loads
|
|
76
84
|
// unauthenticated, defeating hybrid on an auth-gated challenge page.
|
|
@@ -254,7 +262,51 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
254
262
|
async function setupSubscriptions() {
|
|
255
263
|
await setupDialogs();
|
|
256
264
|
await setupDownloads();
|
|
257
|
-
await bidi.subscribe(['browsingContext.load']);
|
|
265
|
+
await bidi.subscribe(['browsingContext.load', 'network.responseCompleted', 'network.responseStarted']);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* browsingContext.navigate(wait:'complete') plus the navigation's outcome:
|
|
270
|
+
* the LAST network.responseCompleted for its navigation id in the top
|
|
271
|
+
* context (one fires per redirect hop; the final hop is where we landed).
|
|
272
|
+
* A network failure is rewrapped as a typed NAV_FAILED error. NS_BINDING_ABORTED
|
|
273
|
+
* (a 204, or a download replacing nothing) is not a failure, as with CDP's
|
|
274
|
+
* ERR_ABORTED: no responseCompleted fires for it, so the status comes from
|
|
275
|
+
* the last network.responseStarted (a 204 reports 204, as on Chromium).
|
|
276
|
+
* @returns {Promise<import('./navigation.js').NavigationOutcome>}
|
|
277
|
+
*/
|
|
278
|
+
async function navigateTo(url, timeout) {
|
|
279
|
+
const events = [];
|
|
280
|
+
const started = [];
|
|
281
|
+
const off = bidi.on('network.responseCompleted', (e) => {
|
|
282
|
+
if (e.navigation && e.context === topContext) events.push(e);
|
|
283
|
+
});
|
|
284
|
+
const offStarted = bidi.on('network.responseStarted', (e) => {
|
|
285
|
+
if (e.navigation && e.context === topContext) started.push(e);
|
|
286
|
+
});
|
|
287
|
+
lock?.startGoto(topContext);
|
|
288
|
+
try {
|
|
289
|
+
const res = await withTimeout(
|
|
290
|
+
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
291
|
+
timeout, `goto(${url})`);
|
|
292
|
+
const last = events.filter((e) => e.navigation === res.navigation).pop();
|
|
293
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
294
|
+
} catch (err) {
|
|
295
|
+
// A lock-refused document (the URL or a redirect hop) surfaces as
|
|
296
|
+
// NS_ERROR_ABORT; report it as NAV_BLOCKED, not NAV_FAILED.
|
|
297
|
+
const blocked = lock?.takeGotoBlock(topContext);
|
|
298
|
+
if (blocked) throw navBlocked(blocked.url, blocked.host);
|
|
299
|
+
const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
|
|
300
|
+
if (!m) throw err;
|
|
301
|
+
if (m[1] === 'NS_BINDING_ABORTED') {
|
|
302
|
+
const last = started.pop();
|
|
303
|
+
return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
|
|
304
|
+
}
|
|
305
|
+
throw navFailed(url, m[1]);
|
|
306
|
+
} finally {
|
|
307
|
+
off();
|
|
308
|
+
offStarted();
|
|
309
|
+
}
|
|
258
310
|
}
|
|
259
311
|
|
|
260
312
|
/**
|
|
@@ -285,6 +337,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
285
337
|
get context() { return topContext; },
|
|
286
338
|
/** Whether the last goto() landed on a bot-challenge page (parity w/ CDP). */
|
|
287
339
|
get botBlocked() { return botBlocked; },
|
|
340
|
+
/** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
|
|
341
|
+
get lastNavigation() { return lastNavigation; },
|
|
288
342
|
|
|
289
343
|
/** @type {'firefox'} Engine driving this session (see also `capabilities`). */
|
|
290
344
|
engine: 'firefox',
|
|
@@ -312,11 +366,11 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
312
366
|
// Same navigation guard the CDP path enforces — block file:/chrome:/
|
|
313
367
|
// view-source: and (optionally) private-network hosts before navigating.
|
|
314
368
|
assertNavigable(url, urlGuard);
|
|
369
|
+
lock?.assertAllowed(url);
|
|
315
370
|
refContexts = new Map(); // refs from the previous page are now stale
|
|
316
371
|
consentRecheck = null; // a throwing navigate() must not leave the old page's flag
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
timeout, `goto(${url})`);
|
|
372
|
+
lastNavigation = null;
|
|
373
|
+
lastNavigation = await navigateTo(url, timeout);
|
|
320
374
|
// Settle + consent + challenge detection (mirrors the CDP path).
|
|
321
375
|
botBlocked = await afterNavigate();
|
|
322
376
|
|
|
@@ -338,15 +392,14 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
338
392
|
if (lastInject) {
|
|
339
393
|
try { await page.injectCookies(lastInject.url, lastInject.cookieOpts); } catch { /* best-effort */ }
|
|
340
394
|
}
|
|
341
|
-
await
|
|
342
|
-
bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
|
|
343
|
-
timeout, `goto(${url})`);
|
|
395
|
+
lastNavigation = await navigateTo(url, timeout);
|
|
344
396
|
botBlocked = await afterNavigate();
|
|
345
397
|
} catch {
|
|
346
398
|
// Headed relaunch failed (no display?) — keep the headless result;
|
|
347
399
|
// botBlocked stays true so the caller can see it didn't clear.
|
|
348
400
|
}
|
|
349
401
|
}
|
|
402
|
+
return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
|
|
350
403
|
},
|
|
351
404
|
|
|
352
405
|
/** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
|
|
@@ -363,16 +416,19 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
363
416
|
|
|
364
417
|
const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
|
|
365
418
|
const raw = formatTree(root);
|
|
366
|
-
|
|
419
|
+
const warn = httpWarning(lastNavigation?.status) + (lock?.takeNotes(topContext) ?? '');
|
|
420
|
+
const links = !!((pruneOpts || undefined)?.links ?? opts.links);
|
|
421
|
+
const tree = links ? annotateLinks(root, pageUrl) : root;
|
|
422
|
+
if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
|
|
367
423
|
|
|
368
424
|
const mode = pruneOpts?.mode || defaultPruneMode;
|
|
369
425
|
let out;
|
|
370
|
-
if (mode === 'data') out = dataSnapshot(
|
|
371
|
-
else { const pruned = pruneTree(
|
|
372
|
-
const sized =
|
|
426
|
+
if (mode === 'data') out = dataSnapshot(tree, { links });
|
|
427
|
+
else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
|
|
428
|
+
const sized = stripLegend(out); // the legend is not content
|
|
373
429
|
const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars`
|
|
374
430
|
+ ` (${raw.length ? Math.round((1 - sized.length / raw.length) * 100) : 0}% pruned)`;
|
|
375
|
-
return stats + '\n' + out;
|
|
431
|
+
return stats + '\n' + warn + out;
|
|
376
432
|
},
|
|
377
433
|
|
|
378
434
|
async readable() {
|
|
@@ -514,6 +570,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
514
570
|
30000, 'reload');
|
|
515
571
|
refContexts = new Map();
|
|
516
572
|
consentRecheck = null;
|
|
573
|
+
lastNavigation = null; // the reload's status is not read; don't keep the old one
|
|
517
574
|
await new Promise((r) => setTimeout(r, 300));
|
|
518
575
|
},
|
|
519
576
|
|
|
@@ -550,6 +607,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
550
607
|
if (index < 0 || index >= tree.length) throw new Error(`Tab index ${index} out of range (0-${tree.length - 1})`);
|
|
551
608
|
topContext = tree[index].context;
|
|
552
609
|
refContexts = new Map(); // refs from the previous tab are invalid
|
|
610
|
+
lastNavigation = null; // the outcome belonged to the tab we left
|
|
553
611
|
await bidi.send('browsingContext.activate', { context: topContext }).catch(() => {});
|
|
554
612
|
},
|
|
555
613
|
|
|
@@ -667,6 +725,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
|
|
|
667
725
|
30000, 'history navigation');
|
|
668
726
|
refContexts = new Map();
|
|
669
727
|
consentRecheck = null;
|
|
728
|
+
lastNavigation = null; // no outcome is known for the history entry we landed on
|
|
670
729
|
await new Promise((r) => setTimeout(r, 500));
|
|
671
730
|
}
|
|
672
731
|
|