barebrowse 0.25.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/mcp-server.js CHANGED
@@ -14,6 +14,8 @@ import { browse, connect } from './src/index.js';
14
14
  import { formatReadable } from './src/readable.js';
15
15
  import { cleanOld } from './src/output-clean.js';
16
16
  import { scopedCookiesForUrl } from './src/auth.js';
17
+ import { isHttpError } from './src/navigation.js';
18
+ import { parseAllowHosts } from './src/site-lock.js';
17
19
  import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
18
20
  import { join, dirname } from 'node:path';
19
21
  import { pathToFileURL, fileURLToPath } from 'node:url';
@@ -216,6 +218,53 @@ function engineInfo() {
216
218
  };
217
219
  }
218
220
 
221
+ // --- Access tier + site lock (env, fixed at server start) ---
222
+ //
223
+ // BAREBROWSE_TIER = r | rw | rwx (default rwx = everything). r: read the web
224
+ // (navigate, observe); rw: + interact (click/type/...); rwx: + upload and eval.
225
+ // Every tool has an explicit tier; a tool missing from the map is rwx, so a
226
+ // new tool is hidden from restricted tiers until someone classifies it.
227
+ const TIER_RANK = { r: 0, rw: 1, rwx: 2 };
228
+ export const TOOL_TIER = {
229
+ browse: 'r', goto: 'r', back: 'r', forward: 'r', reload: 'r', scroll: 'r',
230
+ snapshot: 'r', readable: 'r', screenshot: 'r', pdf: 'r', wait_for: 'r', tabs: 'r',
231
+ capabilities: 'r', assess: 'r',
232
+ click: 'rw', type: 'rw', press: 'rw', select: 'rw', drag: 'rw', hover: 'rw', switch_engine: 'rw',
233
+ upload: 'rwx', eval: 'rwx',
234
+ };
235
+
236
+ /** @param {string} name */
237
+ export function toolTier(name) { return TOOL_TIER[name] ?? 'rwx'; }
238
+
239
+ /** Parse BAREBROWSE_TIER. Unset/empty = rwx; anything else must be exact, or it throws (never widen on a typo). */
240
+ export function parseTier(v) {
241
+ if (v === undefined || v === '') return 'rwx';
242
+ if (!Object.hasOwn(TIER_RANK, v)) throw new Error(`BAREBROWSE_TIER must be r, rw or rwx (got ${JSON.stringify(v)})`);
243
+ return v;
244
+ }
245
+
246
+ /** Parse BAREBROWSE_ALLOW (comma list of hosts). Unset/blank = no lock (null); a bad entry throws. */
247
+ export function parseAllowEnv(v) {
248
+ if (v === undefined || !v.trim()) return null;
249
+ try { return parseAllowHosts(v.split(',')); }
250
+ catch (e) { throw new Error(`BAREBROWSE_ALLOW: ${e.message}`); }
251
+ }
252
+
253
+ /** Read + validate the tier/site-lock env. Throws on bad config. */
254
+ function readAccess() {
255
+ return { tier: parseTier(process.env.BAREBROWSE_TIER), allowHosts: parseAllowEnv(process.env.BAREBROWSE_ALLOW) };
256
+ }
257
+
258
+ /** MCP tool hints derived from the tier: r = read-only; rw/rwx change state (hover and switch_engine non-destructively). */
259
+ function annotationsFor(name) {
260
+ if (toolTier(name) === 'r') {
261
+ const a = { readOnlyHint: true, openWorldHint: name !== 'capabilities' };
262
+ if (['snapshot', 'readable', 'screenshot', 'pdf', 'tabs', 'capabilities', 'wait_for'].includes(name)) a.idempotentHint = true;
263
+ return a;
264
+ }
265
+ return { readOnlyHint: false, destructiveHint: !['hover', 'switch_engine'].includes(name), openWorldHint: true };
266
+ }
267
+
219
268
  async function getPage() {
220
269
  if (_page) return _page;
221
270
  if (_pageConnecting) return _pageConnecting;
@@ -229,9 +278,10 @@ async function getPage() {
229
278
  // across tool calls, so it's an env var (BAREBROWSE_INCOGNITO=1), not a
230
279
  // per-request flag. Skips all auth injection for a clean, logged-out session.
231
280
  const incognito = process.env.BAREBROWSE_INCOGNITO === '1';
281
+ const { allowHosts } = readAccess();
232
282
  const connectOpts = _engine === 'firefox'
233
- ? { engine: 'firefox', mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito }
234
- : { mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito };
283
+ ? { engine: 'firefox', mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito, allowHosts }
284
+ : { mode: process.env.BAREBROWSE_MODE || 'hybrid', incognito, allowHosts };
235
285
  _pageConnecting = connect(connectOpts);
236
286
  try {
237
287
  _page = await _pageConnecting;
@@ -268,6 +318,7 @@ export const TOOLS = [
268
318
  mode: { type: 'string', enum: ['headless', 'headed', 'hybrid'], description: 'Browser mode (default: headless)' },
269
319
  pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: 'Pruning mode. "act" (default) keeps interactive elements and short labels — best for clicking/filling. "read" keeps paragraphs, headings, and long text — best for articles, docs, and content extraction. If the page is content-heavy and act-mode returns mostly empty, retry with "read". "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
270
320
  incognito: { type: 'boolean', description: 'Clean, unauthenticated session: skip cookie injection so the page loads logged-out (default: false).' },
321
+ links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
271
322
  maxChars: { type: 'number', description: 'Max chars to return inline. Larger snapshots are saved to .barebrowse/ and a file path is returned instead. Default: 30000.' },
272
323
  },
273
324
  required: ['url'],
@@ -275,7 +326,7 @@ export const TOOLS = [
275
326
  },
276
327
  {
277
328
  name: 'goto',
278
- description: 'Open URL in a persistent interactive browser session (pair with snapshot/click/type/press for multi-step flows). Use when the task needs clicking, typing, or form submission. Injects auth cookies. Returns ok — call snapshot to observe.',
329
+ description: 'Open URL in a persistent interactive browser session (pair with snapshot/click/type/press for multi-step flows). Use when the task needs clicking, typing, or form submission. Injects auth cookies. Returns ok — call snapshot to observe. A non-2xx page answers `ok (HTTP 404)` and the snapshot starts with an [HTTP …] warning line; an unreachable host (DNS, refused, TLS) is an error.',
279
330
  inputSchema: {
280
331
  type: 'object',
281
332
  properties: {
@@ -291,6 +342,7 @@ export const TOOLS = [
291
342
  type: 'object',
292
343
  properties: {
293
344
  pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: 'Pruning mode. "act" (default) keeps interactive elements and short labels — best for clicking/filling. "read" keeps paragraphs, headings, and long text — best for articles, docs, and content extraction. If a previous snapshot looked empty on a content-heavy page, retry with "read". "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
345
+ links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
294
346
  maxChars: { type: 'number', description: 'Max chars to return inline. Larger snapshots are saved to .barebrowse/ and a file path is returned instead. Default: 30000.' },
295
347
  },
296
348
  },
@@ -514,12 +566,17 @@ if (assessFn) {
514
566
  });
515
567
  }
516
568
 
569
+ for (const t of TOOLS) t.annotations = annotationsFor(t.name);
570
+
517
571
  async function handleToolCall(name, args) {
518
572
  switch (name) {
519
573
  case 'capabilities': {
520
574
  // Introspection only — must NOT launch a browser. Report the configured
521
575
  // engine from env; if a page is already connected, attach its live flags.
522
576
  const info = engineInfo();
577
+ const access = readAccess();
578
+ info.tier = access.tier;
579
+ info.allowHosts = access.allowHosts;
523
580
  if (_page) {
524
581
  info.connected = true;
525
582
  info.capabilities = _page.capabilities;
@@ -544,7 +601,7 @@ async function handleToolCall(name, args) {
544
601
  case 'browse': {
545
602
  let timer;
546
603
  const text = await Promise.race([
547
- browse(args.url, { mode: args.mode, pruneMode: args.pruneMode, incognito: args.incognito }),
604
+ browse(args.url, { mode: args.mode, pruneMode: args.pruneMode, incognito: args.incognito, links: args.links, allowHosts: readAccess().allowHosts ?? undefined }),
548
605
  new Promise((_, rej) => { timer = setTimeout(() => rej(new Error('browse timed out after 60s')), 60000); }),
549
606
  ]);
550
607
  clearTimeout(timer);
@@ -566,12 +623,12 @@ async function handleToolCall(name, args) {
566
623
  try { await page.injectCookies(args.url); } catch (e) { error ??= e.message; }
567
624
  _lastCookies = cookieReport(args.url, { cookies, error });
568
625
  }
569
- await page.goto(args.url);
570
- return 'ok';
626
+ const { status } = await page.goto(args.url);
627
+ return isHttpError(status) ? `ok (HTTP ${status})` : 'ok';
571
628
  }, TIMEOUTS.goto);
572
629
  case 'snapshot': return withRetry(async () => {
573
630
  const page = await getPage();
574
- const text = await page.snapshot(args.pruneMode ? { mode: args.pruneMode } : undefined);
631
+ const text = await page.snapshot(args.pruneMode || args.links ? { mode: args.pruneMode, links: args.links } : undefined);
575
632
  const limit = args.maxChars ?? MAX_CHARS_DEFAULT;
576
633
  if (text.length > limit) {
577
634
  const file = saveSnapshot(text);
@@ -768,6 +825,10 @@ async function handleMessage(msg) {
768
825
 
769
826
  if (method === 'initialize') {
770
827
  const e = engineInfo();
828
+ const { tier, allowHosts } = readAccess();
829
+ const access = (tier === 'r' ? ' Access tier: read-only (no clicking, typing or uploading).'
830
+ : tier === 'rw' ? ' Access tier: read/write (no upload or eval).' : '')
831
+ + (allowHosts ? ` Navigation is restricted to: ${allowHosts.join(', ')}.` : '');
771
832
  return jsonrpcResponse(id, {
772
833
  protocolVersion: '2024-11-05',
773
834
  capabilities: { tools: {} },
@@ -777,7 +838,7 @@ async function handleMessage(msg) {
777
838
  instructions:
778
839
  `barebrowse drives a real browser. Active engine: ${e.activeEngine} (mode: ${e.mode}). `
779
840
  + `Both chromium and firefox are supported; ${e.switchEngine} `
780
- + `Call the "capabilities" tool for the live engine + feature set.`,
841
+ + `Call the "capabilities" tool for the live engine + feature set.${access}`,
781
842
  });
782
843
  }
783
844
 
@@ -790,7 +851,8 @@ async function handleMessage(msg) {
790
851
  // active engine (and that both are supported) is visible without a call.
791
852
  const e = engineInfo();
792
853
  const note = ` [Active engine: ${e.activeEngine}; chromium + firefox both supported — ${e.switchEngine}]`;
793
- const tools = TOOLS.map((t) =>
854
+ const { tier } = readAccess();
855
+ const tools = TOOLS.filter((t) => TIER_RANK[toolTier(t.name)] <= TIER_RANK[tier]).map((t) =>
794
856
  t.name === 'browse' ? { ...t, description: t.description + note } : t);
795
857
  return jsonrpcResponse(id, { tools });
796
858
  }
@@ -800,6 +862,8 @@ async function handleMessage(msg) {
800
862
  _inflight++;
801
863
  clearTimeout(_idleTimer);
802
864
  try {
865
+ // A tool above the tier answers exactly like one that doesn't exist.
866
+ if (TIER_RANK[toolTier(name)] > TIER_RANK[readAccess().tier]) throw new Error(`Unknown tool: ${name}`);
803
867
  const result = await handleToolCall(name, args || {});
804
868
  return jsonrpcResponse(id, {
805
869
  content: [{ type: 'text', text: typeof result === 'string' ? result : JSON.stringify(result) }],
@@ -830,6 +894,11 @@ async function handleMessage(msg) {
830
894
  // runStdio() explicitly. Tests import TIMEOUTS/TOOLS without calling it.
831
895
 
832
896
  export function runStdio() {
897
+ // Fail closed: a bad tier or allow list must stop the server, never fall back to rwx / no lock.
898
+ try { readAccess(); } catch (e) {
899
+ process.stderr.write(`barebrowse mcp: ${e.message}\n`);
900
+ process.exit(1);
901
+ }
833
902
  cleanOld(OUTPUT_DIR);
834
903
  // One-line startup banner to stderr (stderr because stdout is the JSON-RPC
835
904
  // channel — must not contain non-JSON-RPC bytes). Captured by Claude Code's
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "barebrowse",
3
- "version": "0.25.0",
3
+ "version": "0.27.0",
4
4
  "description": "Authenticated web browsing for autonomous agents via CDP. URL in, pruned ARIA snapshot out.",
5
5
  "repository": {
6
6
  "type": "git",
@@ -28,6 +28,7 @@
28
28
  "src/",
29
29
  "types/",
30
30
  "cli.js",
31
+ "commands/",
31
32
  "mcp-server.js",
32
33
  "barebrowse.context.md",
33
34
  "README.md",
package/src/aria.js CHANGED
@@ -5,6 +5,8 @@
5
5
  * and formats it as readable YAML-like text, similar to Playwright's ariaSnapshot.
6
6
  */
7
7
 
8
+ import { urlsText } from './links.js';
9
+
8
10
  /**
9
11
  * Format a nested ARIA tree as readable text output.
10
12
  *
@@ -37,6 +39,9 @@ export function formatTree(node, depth = 0) {
37
39
  line += ` "${node.name}"`;
38
40
  }
39
41
 
42
+ // Link URLs (opt-in links: true): right after the name, before props and ref
43
+ line += urlsText(node.urls);
44
+
40
45
  // Notable properties that agents care about
41
46
  const props = node.properties || {};
42
47
  const propParts = [];
@@ -166,6 +166,7 @@ const AX_FN = function reconstructAX(base, REF_ATTR) {
166
166
  const exp = el.getAttribute('aria-expanded');
167
167
  if (exp !== null) p.expanded = exp === 'true';
168
168
  if (role === 'heading') p.level = Number(el.getAttribute('aria-level')) || Number(el.tagName[1]) || 2;
169
+ if (role === 'link' && el.href) p.url = el.href; // browser-serialized absolute URL (CDP's `url` property)
169
170
  if (el.getAttribute('aria-selected') === 'true') p.selected = true;
170
171
  if (el.required || el.getAttribute('aria-required') === 'true') p.required = true;
171
172
  if ((role === 'textbox' || role === 'searchbox' || role === 'combobox' || role === 'spinbutton') && el.value) {
package/src/bareagent.js CHANGED
@@ -63,10 +63,11 @@ export function createBrowseTools(opts = {}) {
63
63
  properties: {
64
64
  url: { type: 'string', description: 'URL to browse' },
65
65
  pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: '"act" (default) for interactive elements only; "read" for paragraphs and long text (articles/docs); "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
66
+ links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
66
67
  },
67
68
  required: ['url'],
68
69
  },
69
- execute: async ({ url, pruneMode }) => await browse(url, pruneMode ? { ...opts, pruneMode } : opts),
70
+ execute: async ({ url, pruneMode, links }) => await browse(url, pruneMode || links ? { ...opts, pruneMode, links } : opts),
70
71
  },
71
72
  {
72
73
  name: 'goto',
@@ -87,11 +88,12 @@ export function createBrowseTools(opts = {}) {
87
88
  type: 'object',
88
89
  properties: {
89
90
  pruneMode: { type: 'string', enum: ['act', 'read', 'data'], description: '"act" (default) for interactive elements only; "read" for paragraphs and long text (articles/docs); "data": one line per record — tables with headers, repeated items grouped; for data pages, not articles. Output starts with a legend line; read it as: table rows `row "<label>": <column>: <value> · …`, groups `<label>: a, b, c`, and a title line followed by detail lines is one block up to the next title line. Main content only: use "act" for menus and page controls.' },
91
+ links: { type: 'boolean', description: 'Show each link\'s absolute URL after its name: link "name" <url> [ref=N]. Default off.' },
90
92
  },
91
93
  },
92
- execute: async (/** @type {{ pruneMode?: string }} */ { pruneMode } = {}) => {
94
+ execute: async (/** @type {{ pruneMode?: string, links?: boolean }} */ { pruneMode, links } = {}) => {
93
95
  const page = await getPage();
94
- return await page.snapshot(pruneMode ? { mode: pruneMode } : undefined);
96
+ return await page.snapshot(pruneMode || links ? { mode: pruneMode, links } : undefined);
95
97
  },
96
98
  },
97
99
  {
@@ -13,9 +13,15 @@
13
13
  * request pauses for one continue/fail round-trip to Node. On a typical page
14
14
  * (tens–low-hundreds of requests over a local socket) that's negligible, and
15
15
  * it's the only route that preserves glob parity without a second list.
16
+ *
17
+ * The site lock (`allowHosts`, site-lock.js) plugs into this same handler: a
18
+ * second catch-all handler would race this one on the same paused request, so
19
+ * there is exactly one continue/fail decision per request, and the intercept
20
+ * exists whenever either the blocklist or the lock is active.
16
21
  */
17
22
 
18
23
  import { resolveBlocklistPatterns, makeBlockMatcher } from './blocklist.js';
24
+ import { createFirefoxLockCheck } from './site-lock.js';
19
25
 
20
26
  /**
21
27
  * Install ad/tracker blocking on a BiDi session. Idempotent per session is the
@@ -26,13 +32,15 @@ import { resolveBlocklistPatterns, makeBlockMatcher } from './blocklist.js';
26
32
  * @param {boolean} [pageOpts.blockAds] - false disables the default list.
27
33
  * @param {string[]} [pageOpts.blockUrls] - extra CDP-format globs; extend the
28
34
  * default unless blockAds is false, in which case they're the whole list.
35
+ * @param {import('./site-lock.js').SiteLock|null} [lock] - Site lock to enforce in the same handler.
29
36
  * @returns {Promise<void>}
30
37
  */
31
- export async function applyFirefoxBlocklist(bidi, pageOpts = {}) {
38
+ export async function applyFirefoxBlocklist(bidi, pageOpts = {}, lock = null) {
32
39
  const patterns = resolveBlocklistPatterns(pageOpts);
33
- if (!patterns.length) return;
40
+ if (!patterns.length && !lock) return;
34
41
 
35
- const isBlocked = makeBlockMatcher(patterns);
42
+ const isBlocked = patterns.length ? makeBlockMatcher(patterns) : () => false;
43
+ const lockBlocks = lock ? await createFirefoxLockCheck(bidi, lock) : null;
36
44
 
37
45
  await bidi.send('network.addIntercept', {
38
46
  phases: ['beforeRequestSent'],
@@ -47,7 +55,7 @@ export async function applyFirefoxBlocklist(bidi, pageOpts = {}) {
47
55
  const id = e.request?.request;
48
56
  if (!id) return;
49
57
  try {
50
- if (isBlocked(e.request.url)) {
58
+ if (isBlocked(e.request.url) || (lockBlocks && await lockBlocks(e))) {
51
59
  await bidi.send('network.failRequest', { request: id });
52
60
  } else {
53
61
  await bidi.send('network.continueRequest', { request: id });
package/src/daemon.js CHANGED
@@ -13,6 +13,7 @@ import { join, resolve } from 'node:path';
13
13
  import { connect } from './index.js';
14
14
  import { formatReadable } from './readable.js';
15
15
  import { cleanOld } from './output-clean.js';
16
+ import { isHttpError } from './navigation.js';
16
17
 
17
18
  /** Owner-only file write helper — daemon artifacts can hold authenticated content. */
18
19
  function writeFilePrivate(path, data) {
@@ -65,6 +66,7 @@ export function buildDaemonArgs(opts, absDir, initialUrl, cliPath) {
65
66
  }
66
67
  if (opts.blockPrivateNetwork) args.push('--block-private-network');
67
68
  if (opts.uploadDir) args.push('--upload-dir', opts.uploadDir);
69
+ if (opts.allowHosts) args.push('--allow-hosts', opts.allowHosts.join(','));
68
70
  return args;
69
71
  }
70
72
 
@@ -234,6 +236,7 @@ export async function runDaemon(opts, outputDir, initialUrl) {
234
236
  blockUrls: opts.blockUrls,
235
237
  blockPrivateNetwork: opts.blockPrivateNetwork,
236
238
  uploadDir: opts.uploadDir,
239
+ allowHosts: opts.allowHosts,
237
240
  // incognito neuters injectCookies() session-wide; cookies:false (below)
238
241
  // is the legacy per-open equivalent. Either yields a logged-out session.
239
242
  incognito: opts.incognito || opts.cookies === false,
@@ -313,13 +316,13 @@ export async function runDaemon(opts, outputDir, initialUrl) {
313
316
  // Command handlers
314
317
  const handlers = {
315
318
  async goto({ url, timeout }) {
316
- await page.goto(url, timeout || 30000);
317
- return { ok: true };
319
+ const { status } = await page.goto(url, timeout || 30000);
320
+ return isHttpError(status) ? { ok: true, status } : { ok: true };
318
321
  },
319
322
 
320
- async snapshot({ mode }) {
323
+ async snapshot({ mode, links }) {
321
324
  const pruneMode = mode || defaultPruneMode;
322
- const text = await page.snapshot({ mode: pruneMode });
325
+ const text = await page.snapshot({ mode: pruneMode, links });
323
326
  const ts = new Date().toISOString().replace(/[:.]/g, '-');
324
327
  const file = join(absDir, `page-${ts}.yml`);
325
328
  writeFilePrivate(file, text);
package/src/data.js CHANGED
@@ -33,6 +33,8 @@
33
33
  * click/type resolve through the same refMap).
34
34
  */
35
35
 
36
+ import { urlsText } from './links.js';
37
+
36
38
  const SKIP = new Set(['InlineTextBox', 'LineBreak', 'ListMarker', 'image', 'img']);
37
39
  const CELLS = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);
38
40
  const TABLES = new Set(['table', 'grid', 'treegrid']);
@@ -47,13 +49,20 @@ const MIN_RUN = 3;
47
49
  /** First line of every data-mode snapshot: tells an agent with no docs how to read it. */
48
50
  export const LEGEND = '# data mode: one line per record. Tables: row "<label>": <column>: <value> · … Groups: <label>: a, b, c. A title line followed by detail lines is one block, up to the next title line. Main content only; use mode "act" for menus and page controls.';
49
51
 
52
+ /** Appended to the legend only when links are on. */
53
+ export const LINKS_NOTE = ' Links: "name" <url>.';
54
+
50
55
  // A name made only of private-use code points (icon-font glyphs, unreadable to an
51
56
  // agent) is no name. Glyphs mixed into real text are left as they are.
52
57
  const norm = (s) => (/^[\p{Co}\s]*$/u.test(s || '') ? '' : s.replace(/\s+/g, ' ').trim());
58
+ const allUrls = (n) => [...(n.urls || []), ...(n.children || []).flatMap(allUrls)];
53
59
  const hasInteractive = (n) => INTERACTIVE.has(n.role) || (n.children || []).some(hasInteractive);
54
60
  const isLeaf = (n) => n.text === undefined && !n.children.length && n.name;
55
61
  const showRef = (n) => !!n.ref && INTERACTIVE.has(n.role);
56
- const withRef = (n) => (showRef(n) ? `${n.name} [ref=${n.ref}]` : n.name);
62
+ // ` <url>` for a node's own links (already unique per node). A link line always
63
+ // carries its own URL: nothing is deduped across lines, items or cells.
64
+ const urlPart = (n) => urlsText(n.urls);
65
+ const withRef = (n) => (showRef(n) ? `${n.name}${urlPart(n)} [ref=${n.ref}]` : n.name);
57
66
  // Leaves carrying state (checked, level, value...) never merge into a run.
58
67
  const plain = (n) => !(n.props.checked !== undefined || n.props.expanded !== undefined || n.props.selected || n.props.level || n.props.required || (n.props.value !== undefined && n.props.value !== ''));
59
68
  // Run key: leaves with the same key are "the same kind of item". `g` = the
@@ -69,19 +78,20 @@ function inline(n, refs) {
69
78
  const sub = n.children.length ? n.children.map((c) => inline(c, refs)).filter(Boolean).join(' ') : n.name;
70
79
  const v = (n.name && !n.children.length) || INTERACTIVE.has(n.role) && n.name && refs ? n.name : sub;
71
80
  const out = n.children.length && n.name && n.role !== 'StaticText' && !sub ? n.name : v;
72
- return refs && n.ref && INTERACTIVE.has(n.role) ? `${out} [ref=${n.ref}]` : out;
81
+ return refs && n.ref && INTERACTIVE.has(n.role) ? `${out}${urlPart(n)} [ref=${n.ref}]` : out;
73
82
  }
74
83
 
75
84
  /** A data-mode result without its legend line, for size stats ('' stays ''). */
76
- export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(LEGEND.length + 1) : s);
85
+ export const stripLegend = (s) => (s.startsWith(LEGEND) ? s.slice(s.indexOf('\n') + 1) : s);
77
86
 
78
87
  /**
79
88
  * Render a raw ARIA tree in data mode.
80
89
  *
81
90
  * @param {object} root - Root node from buildTree() (CDP vocabulary)
91
+ * @param {{ links?: boolean }} [opts] - links: the tree's link nodes carry `urls` (annotateLinks); adds the legend note
82
92
  * @returns {string} LEGEND line, then one line per record, YAML-ish like formatTree(); '' when there are no records
83
93
  */
84
- export function dataSnapshot(root) {
94
+ export function dataSnapshot(root, opts = {}) {
85
95
  if (!root) return '';
86
96
  let gid = 0;
87
97
  const rawKids = (n) => n.children.flatMap((c) => (c.ignored ? rawKids(c) : [c]));
@@ -120,7 +130,7 @@ export function dataSnapshot(root) {
120
130
  if (!nm && kids.length === 0 && (n.role === 'link' || n.role === 'button')) return []; // N6
121
131
  if (!nm && kids.length === 0 && !INTERACTIVE.has(n.role) && !CELLS.has(n.role) && n.role !== 'row') return [];
122
132
  if (n.role === 'listitem' && !nm && kids.length === 1) return kids;
123
- return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig }];
133
+ return [{ role: n.role, name: nm, children: kids, props, ref: n.nodeId, sig, urls: n.urls }];
124
134
  }
125
135
 
126
136
  // ---------- Transforms, bottom-up with the ancestor chain for labels ----------
@@ -134,7 +144,7 @@ export function dataSnapshot(root) {
134
144
  const out = [];
135
145
  for (const k of kids) {
136
146
  const p = out[out.length - 1];
137
- if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name) continue;
147
+ if (p && isLeaf(p) && isLeaf(k) && p.role === k.role && p.name === k.name && (p.urls || [])[0] === (k.urls || [])[0]) continue;
138
148
  out.push(k);
139
149
  }
140
150
  return out;
@@ -243,6 +253,7 @@ export function dataSnapshot(root) {
243
253
  const labelCells = cs.filter((c) => c.role === 'rowheader' || c.role === 'columnheader');
244
254
  const lcx = labelCells.map(xcell);
245
255
  const label = lcx.map((c) => inline(c, false)).filter(Boolean).join(' / ');
256
+ const labelUrls = urlsText([...new Set(lcx.flatMap(allUrls))]);
246
257
  const labelRef = lcx.map(firstRef).find(Boolean);
247
258
  const vals = [];
248
259
  cs.forEach((c, idx) => {
@@ -262,7 +273,7 @@ export function dataSnapshot(root) {
262
273
  }
263
274
  if (!vals.length && !label) continue; // header-only / empty rows
264
275
  const body = vals.map((x) => (x.h ? `${x.h}: ` : '') + x.v).join(' · ');
265
- out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
276
+ out.push({ role: '_line', children: [], props: {}, text: `row${label ? ` "${label}"` : ''}${labelUrls}${labelRef ? ` [ref=${labelRef}]` : ''}${tail}${body ? ': ' + body : ''}` });
266
277
  }
267
278
  return { ...t, children: out };
268
279
  }
@@ -278,10 +289,14 @@ export function dataSnapshot(root) {
278
289
  const lines = [];
279
290
  function emit(c, d) {
280
291
  const ind = ' '.repeat(d);
281
- if (c.text !== undefined) { lines.push(`${ind}- ${c.text}`); return; }
292
+ if (c.text !== undefined) {
293
+ lines.push(`${ind}- ${c.text}`);
294
+ return;
295
+ }
282
296
  if (isLeaf(c) && (c.role === 'StaticText' || c.role === 'generic')) { lines.push(`${ind}- ${c.name}`); return; }
283
297
  let l = `${ind}- ${c.role}`;
284
298
  if (c.name) l += ` "${c.name}"`;
299
+ l += urlPart(c);
285
300
  const p = c.props, pp = [];
286
301
  if (p.checked !== undefined) pp.push(`checked=${p.checked}`);
287
302
  if (p.disabled) pp.push('disabled');
@@ -296,5 +311,5 @@ export function dataSnapshot(root) {
296
311
  for (const k of c.children) emit(k, d + 1);
297
312
  }
298
313
  for (const c of res.children) emit(c, 0);
299
- return lines.length ? [LEGEND, ...lines].join('\n') : '';
314
+ return lines.length ? [LEGEND + (opts.links ? LINKS_NOTE : ''), ...lines].join('\n') : '';
300
315
  }
@@ -18,6 +18,7 @@
18
18
  import { formatTree } from './aria.js';
19
19
  import { prune as pruneTree } from './prune.js';
20
20
  import { dataSnapshot, stripLegend } from './data.js';
21
+ import { annotateLinks } from './links.js';
21
22
  import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
22
23
  import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
23
24
  import { scopedCookiesForUrl } from './auth.js';
@@ -27,6 +28,7 @@ import { waitForNetworkIdleBiDi } from './network-idle.js';
27
28
  import { decideDialog, dialogLogEntry } from './dialog.js';
28
29
  import { isChallengePage, countNodes } from './challenge.js';
29
30
  import { withTimeout } from './with-timeout.js';
31
+ import { navFailed, navBlocked, httpWarning } from './navigation.js';
30
32
 
31
33
  /** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
32
34
  const BIDI_KEYS = {
@@ -41,8 +43,10 @@ const BIDI_KEYS = {
41
43
  * @param {object} bidi - BiDi client from createBiDi()
42
44
  * @param {object} [opts]
43
45
  * @param {'act'|'browse'|'navigate'|'full'|'read'|'data'} [opts.pruneMode='act'] - Default prune mode.
46
+ * @param {boolean} [opts.links=false] - Default for snapshot({ links }): show link URLs.
44
47
  * @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
45
48
  * @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
49
+ * @param {?import('./site-lock.js').SiteLock} [opts.siteLock] - Site lock (allowHosts) state; the request-level enforcement is installed by applyFirefoxBlocklist, this side maps it to NAV_BLOCKED and snapshot notes.
46
50
  * @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
47
51
  * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs after goto().
48
52
  * @param {boolean} [opts.hybrid=false] - Relaunch headed on a bot-challenge page and retry.
@@ -53,6 +57,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
53
57
  const defaultPruneMode = opts.pruneMode || 'act';
54
58
  const urlGuard = opts.urlGuard || {};
55
59
  const uploadDir = opts.uploadDir || null;
60
+ const lock = opts.siteLock || null;
56
61
  const incognito = !!opts.incognito;
57
62
  const consent = opts.consent !== false;
58
63
  // True when the consent pass after the latest goto() found nothing: the first
@@ -71,6 +76,9 @@ export async function createFirefoxPage(bidi, opts = {}) {
71
76
  const relaunchHeaded = opts.relaunchHeaded || null;
72
77
  let currentlyHeaded = !!opts.headed; // already headed → no fallback needed
73
78
  let botBlocked = false;
79
+ // Outcome of the latest goto() ({url, status}); null before any goto.
80
+ /** @type {import('./navigation.js').NavigationOutcome|null} */
81
+ let lastNavigation = null;
74
82
  // Last injectCookies(url, opts) args, so a hybrid relaunch (a brand-new
75
83
  // Firefox profile) can re-inject them — otherwise the headed retry loads
76
84
  // unauthenticated, defeating hybrid on an auth-gated challenge page.
@@ -254,7 +262,51 @@ export async function createFirefoxPage(bidi, opts = {}) {
254
262
  async function setupSubscriptions() {
255
263
  await setupDialogs();
256
264
  await setupDownloads();
257
- await bidi.subscribe(['browsingContext.load']);
265
+ await bidi.subscribe(['browsingContext.load', 'network.responseCompleted', 'network.responseStarted']);
266
+ }
267
+
268
+ /**
269
+ * browsingContext.navigate(wait:'complete') plus the navigation's outcome:
270
+ * the LAST network.responseCompleted for its navigation id in the top
271
+ * context (one fires per redirect hop; the final hop is where we landed).
272
+ * A network failure is rewrapped as a typed NAV_FAILED error. NS_BINDING_ABORTED
273
+ * (a 204, or a download replacing nothing) is not a failure, as with CDP's
274
+ * ERR_ABORTED: no responseCompleted fires for it, so the status comes from
275
+ * the last network.responseStarted (a 204 reports 204, as on Chromium).
276
+ * @returns {Promise<import('./navigation.js').NavigationOutcome>}
277
+ */
278
+ async function navigateTo(url, timeout) {
279
+ const events = [];
280
+ const started = [];
281
+ const off = bidi.on('network.responseCompleted', (e) => {
282
+ if (e.navigation && e.context === topContext) events.push(e);
283
+ });
284
+ const offStarted = bidi.on('network.responseStarted', (e) => {
285
+ if (e.navigation && e.context === topContext) started.push(e);
286
+ });
287
+ lock?.startGoto(topContext);
288
+ try {
289
+ const res = await withTimeout(
290
+ bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
291
+ timeout, `goto(${url})`);
292
+ const last = events.filter((e) => e.navigation === res.navigation).pop();
293
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
294
+ } catch (err) {
295
+ // A lock-refused document (the URL or a redirect hop) surfaces as
296
+ // NS_ERROR_ABORT; report it as NAV_BLOCKED, not NAV_FAILED.
297
+ const blocked = lock?.takeGotoBlock(topContext);
298
+ if (blocked) throw navBlocked(blocked.url, blocked.host);
299
+ const m = /^BiDi error: unknown error — (?:Error: )?(.+)$/.exec(err.message);
300
+ if (!m) throw err;
301
+ if (m[1] === 'NS_BINDING_ABORTED') {
302
+ const last = started.pop();
303
+ return { url: last ? last.response.url : url, status: last ? last.response.status : 0 };
304
+ }
305
+ throw navFailed(url, m[1]);
306
+ } finally {
307
+ off();
308
+ offStarted();
309
+ }
258
310
  }
259
311
 
260
312
  /**
@@ -285,6 +337,8 @@ export async function createFirefoxPage(bidi, opts = {}) {
285
337
  get context() { return topContext; },
286
338
  /** Whether the last goto() landed on a bot-challenge page (parity w/ CDP). */
287
339
  get botBlocked() { return botBlocked; },
340
+ /** Outcome of the latest goto(): `{url, status}`, or null before any goto. */
341
+ get lastNavigation() { return lastNavigation; },
288
342
 
289
343
  /** @type {'firefox'} Engine driving this session (see also `capabilities`). */
290
344
  engine: 'firefox',
@@ -312,11 +366,11 @@ export async function createFirefoxPage(bidi, opts = {}) {
312
366
  // Same navigation guard the CDP path enforces — block file:/chrome:/
313
367
  // view-source: and (optionally) private-network hosts before navigating.
314
368
  assertNavigable(url, urlGuard);
369
+ lock?.assertAllowed(url);
315
370
  refContexts = new Map(); // refs from the previous page are now stale
316
371
  consentRecheck = null; // a throwing navigate() must not leave the old page's flag
317
- await withTimeout(
318
- bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
319
- timeout, `goto(${url})`);
372
+ lastNavigation = null;
373
+ lastNavigation = await navigateTo(url, timeout);
320
374
  // Settle + consent + challenge detection (mirrors the CDP path).
321
375
  botBlocked = await afterNavigate();
322
376
 
@@ -338,15 +392,14 @@ export async function createFirefoxPage(bidi, opts = {}) {
338
392
  if (lastInject) {
339
393
  try { await page.injectCookies(lastInject.url, lastInject.cookieOpts); } catch { /* best-effort */ }
340
394
  }
341
- await withTimeout(
342
- bidi.send('browsingContext.navigate', { context: topContext, url, wait: 'complete' }),
343
- timeout, `goto(${url})`);
395
+ lastNavigation = await navigateTo(url, timeout);
344
396
  botBlocked = await afterNavigate();
345
397
  } catch {
346
398
  // Headed relaunch failed (no display?) — keep the headless result;
347
399
  // botBlocked stays true so the caller can see it didn't clear.
348
400
  }
349
401
  }
402
+ return /** @type {import('./navigation.js').NavigationOutcome} */ (lastNavigation); // set by the navigate above
350
403
  },
351
404
 
352
405
  /** @param {import('./index.js').SnapshotOptions|false} [pruneOpts] */
@@ -363,16 +416,19 @@ export async function createFirefoxPage(bidi, opts = {}) {
363
416
 
364
417
  const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
365
418
  const raw = formatTree(root);
366
- if (pruneOpts === false) return `url: ${pageUrl}\n` + raw;
419
+ const warn = httpWarning(lastNavigation?.status) + (lock?.takeNotes(topContext) ?? '');
420
+ const links = !!((pruneOpts || undefined)?.links ?? opts.links);
421
+ const tree = links ? annotateLinks(root, pageUrl) : root;
422
+ if (pruneOpts === false) return `url: ${pageUrl}\n` + warn + (links ? formatTree(tree) : raw);
367
423
 
368
424
  const mode = pruneOpts?.mode || defaultPruneMode;
369
425
  let out;
370
- if (mode === 'data') out = dataSnapshot(root);
371
- else { const pruned = pruneTree(root, { mode }); out = pruned ? formatTree(pruned) : ''; }
372
- const sized = mode === 'data' ? stripLegend(out) : out; // the legend is not content
426
+ if (mode === 'data') out = dataSnapshot(tree, { links });
427
+ else { const pruned = pruneTree(tree, { mode, links }); out = pruned ? formatTree(pruned) : ''; }
428
+ const sized = stripLegend(out); // the legend is not content
373
429
  const stats = `url: ${pageUrl}\n${raw.length.toLocaleString()} chars → ${sized.length.toLocaleString()} chars`
374
430
  + ` (${raw.length ? Math.round((1 - sized.length / raw.length) * 100) : 0}% pruned)`;
375
- return stats + '\n' + out;
431
+ return stats + '\n' + warn + out;
376
432
  },
377
433
 
378
434
  async readable() {
@@ -514,6 +570,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
514
570
  30000, 'reload');
515
571
  refContexts = new Map();
516
572
  consentRecheck = null;
573
+ lastNavigation = null; // the reload's status is not read; don't keep the old one
517
574
  await new Promise((r) => setTimeout(r, 300));
518
575
  },
519
576
 
@@ -550,6 +607,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
550
607
  if (index < 0 || index >= tree.length) throw new Error(`Tab index ${index} out of range (0-${tree.length - 1})`);
551
608
  topContext = tree[index].context;
552
609
  refContexts = new Map(); // refs from the previous tab are invalid
610
+ lastNavigation = null; // the outcome belonged to the tab we left
553
611
  await bidi.send('browsingContext.activate', { context: topContext }).catch(() => {});
554
612
  },
555
613
 
@@ -667,6 +725,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
667
725
  30000, 'history navigation');
668
726
  refContexts = new Map();
669
727
  consentRecheck = null;
728
+ lastNavigation = null; // no outcome is known for the history entry we landed on
670
729
  await new Promise((r) => setTimeout(r, 500));
671
730
  }
672
731