barebrowse 0.14.0 → 0.17.0

package/src/consent-patterns.js

@@ -0,0 +1,154 @@
+/**
+ * consent-patterns.js — Shared cookie-consent detection data.
+ *
+ * The engine-agnostic half of consent handling: the multilingual regex sets
+ * and dialog roles used to recognise a consent dialog and its "accept" button.
+ * Both the CDP walker (consent.js) and the BiDi walker (consent-firefox.js)
+ * import these so the language coverage is single-sourced — a new language or
+ * pattern is added once and both engines get it.
+ */
+
+// Button text patterns that mean "accept all" / "I agree" across common languages.
+// Order matters: more specific patterns first to avoid false positives.
+export const ACCEPT_PATTERNS = [
+  // English
+  /\baccept\s*all\b/i,
+  /\ballow\s*all\b/i,
+  /\bagree\s*to\s*all\b/i,
+  /\byes,?\s*i\s*agree\b/i,
+  /\bi\s*agree\b/i,
+  /\baccept\s*cookies?\b/i,
+  /\ballow\s*cookies?\b/i,
+  /\bgot\s*it\b/i,
+  // Dutch
+  /\balles\s*accepteren\b/i,
+  /\balles\s*toestaan\b/i,
+  /\baccepteren\b/i,
+  /\bakkoord\b/i,
+  // German
+  /\balle\s*akzeptieren\b/i,
+  /\ballem\s*zustimmen\b/i,
+  /\balle\s*cookies?\s*akzeptieren\b/i,
+  // French
+  /\btout\s*accepter\b/i,
+  /\baccepter\s*tout\b/i,
+  /\bj['']accepte\b/i,
+  // Spanish
+  /\baceptar\s*todo\b/i,
+  /\baceptar\s*todas?\b/i,
+  // Italian
+  /\baccetta\s*tutto\b/i,
+  /\baccetto\b/i,
+  // Portuguese
+  /\baceitar\s*tudo\b/i,
+  // Russian
+  /принять\s*все/i,
+  /принять/i,
+  /согласен/i,
+  // Ukrainian
+  /прийняти\s*все/i,
+  /прийняти/i,
+  // Polish
+  /zaakceptuj\s*wszystk/i,
+  /akceptuj\s*wszystk/i,
+  /zgadzam\s*się/i,
+  // Czech
+  /přijmout\s*vše/i,
+  /souhlasím/i,
+  // Turkish
+  /tümünü\s*kabul\s*et/i,
+  /kabul\s*et/i,
+  /kabul\s*ediyorum/i,
+  // Romanian
+  /acceptă\s*tot/i,
+  /accept\s*toate/i,
+  // Hungarian
+  /összes\s*elfogadás/i,
+  /elfogad/i,
+  // Greek
+  /αποδοχή\s*όλων/i,
+  /αποδέχομαι/i,
+  // Swedish
+  /acceptera\s*alla/i,
+  /godkänn\s*alla/i,
+  // Danish
+  /accepter\s*alle/i,
+  /acceptér\s*alle/i,
+  // Norwegian
+  /godta\s*alle/i,
+  /aksepter\s*alle/i,
+  // Finnish
+  /hyväksy\s*kaikki/i,
+  /hyväksyn/i,
+  // Arabic
+  /قبول\s*الكل/,
+  /قبول\s*الجميع/,
+  /موافق/,
+  /قبول/,
+  // Persian
+  /پذیرش\s*همه/,
+  /موافقم/,
+  /پذیرش/,
+  // Chinese (Simplified + Traditional)
+  /全部接受/,
+  /接受所有/,
+  /接受全部/,
+  /同意并继续/,
+  /全部接受/,
+  /接受/,
+  /同意/,
+  // Japanese
+  /すべて受け入れ/,
+  /すべて許可/,
+  /同意する/,
+  /同意します/,
+  // Korean
+  /모두\s*수락/,
+  /모두\s*동의/,
+  /동의합니다/,
+  /수락/,
+  // Vietnamese
+  /chấp\s*nhận\s*tất\s*cả/i,
+  /đồng\s*ý\s*tất\s*cả/i,
+  /đồng\s*ý/i,
+  // Thai
+  /ยอมรับทั้งหมด/,
+  /ยอมรับ/,
+  // Hindi
+  /सभी\s*स्वीकार/,
+  /स्वीकार\s*करें/,
+  /सहमत/,
+  // Indonesian / Malay
+  /terima\s*semua/i,
+  /setuju/i,
+  // Generic single-word fallbacks (only matched inside dialogs)
+  /^accept$/i,
+  /^agree$/i,
+  /^ok$/i,
+];
+
+// Roles that indicate a consent dialog container.
+export const DIALOG_ROLES = new Set(['dialog', 'alertdialog']);
+
+// Text patterns in dialog names/headings that confirm it's about consent.
+export const CONSENT_DIALOG_HINTS = [
+  /cookie/i,
+  /consent/i,
+  /privacy/i,
+  /before\s*you\s*continue/i,
+  /voordat\s*je\s*verdergaat/i,  // Dutch
+  /bevor\s*du\s*fortf/i,         // German
+  /avant\s*de\s*continuer/i,     // French
+  /antes\s*de\s*continuar/i,     // Spanish / Portuguese
+  /prima\s*di\s*continuare/i,    // Italian
+  /zanim\s*przejdziesz/i,        // Polish
+  /прежде\s*чем\s*продолжить/i,  // Russian
+  /devam\s*etmeden\s*önce/i,     // Turkish
+  /続行する前に/,                  // Japanese
+  /继续前/,                        // Chinese Simplified
+  /繼續前/,                        // Chinese Traditional
+  /계속하기\s*전에/,                // Korean
+  /trước\s*khi\s*tiếp\s*tục/i,   // Vietnamese
+  /ملفات\s*تعريف\s*الارتباط/,    // Arabic: cookies
+  /คุกกี้/,                        // Thai: cookies
+];

package/src/consent.js

@@ -4,152 +4,13 @@
  * Scans the page ARIA tree for consent dialogs and clicks the "accept" button.
  * Works across languages by matching common accept/agree patterns.
  * Runs once after page load — no polling, no mutation observers.
+ *
+ * The multilingual pattern sets live in consent-patterns.js (shared with the
+ * Firefox/BiDi walker in consent-firefox.js); this file owns the CDP-specific
+ * tree walk + click.
  */
 
-// Button text patterns that mean "accept all" / "I agree" across common languages.
-// Order matters: more specific patterns first to avoid false positives.
-const ACCEPT_PATTERNS = [
-  // English
-  /\baccept\s*all\b/i,
-  /\ballow\s*all\b/i,
-  /\bagree\s*to\s*all\b/i,
-  /\byes,?\s*i\s*agree\b/i,
-  /\bi\s*agree\b/i,
-  /\baccept\s*cookies?\b/i,
-  /\ballow\s*cookies?\b/i,
-  /\bgot\s*it\b/i,
-  // Dutch
-  /\balles\s*accepteren\b/i,
-  /\balles\s*toestaan\b/i,
-  /\baccepteren\b/i,
-  /\bakkoord\b/i,
-  // German
-  /\balle\s*akzeptieren\b/i,
-  /\ballem\s*zustimmen\b/i,
-  /\balle\s*cookies?\s*akzeptieren\b/i,
-  // French
-  /\btout\s*accepter\b/i,
-  /\baccepter\s*tout\b/i,
-  /\bj['']accepte\b/i,
-  // Spanish
-  /\baceptar\s*todo\b/i,
-  /\baceptar\s*todas?\b/i,
-  // Italian
-  /\baccetta\s*tutto\b/i,
-  /\baccetto\b/i,
-  // Portuguese
-  /\baceitar\s*tudo\b/i,
-  // Russian
-  /принять\s*все/i,
-  /принять/i,
-  /согласен/i,
-  // Ukrainian
-  /прийняти\s*все/i,
-  /прийняти/i,
-  // Polish
-  /zaakceptuj\s*wszystk/i,
-  /akceptuj\s*wszystk/i,
-  /zgadzam\s*się/i,
-  // Czech
-  /přijmout\s*vše/i,
-  /souhlasím/i,
-  // Turkish
-  /tümünü\s*kabul\s*et/i,
-  /kabul\s*et/i,
-  /kabul\s*ediyorum/i,
-  // Romanian
-  /acceptă\s*tot/i,
-  /accept\s*toate/i,
-  // Hungarian
-  /összes\s*elfogadás/i,
-  /elfogad/i,
-  // Greek
-  /αποδοχή\s*όλων/i,
-  /αποδέχομαι/i,
-  // Swedish
-  /acceptera\s*alla/i,
-  /godkänn\s*alla/i,
-  // Danish
-  /accepter\s*alle/i,
-  /acceptér\s*alle/i,
-  // Norwegian
-  /godta\s*alle/i,
-  /aksepter\s*alle/i,
-  // Finnish
-  /hyväksy\s*kaikki/i,
-  /hyväksyn/i,
-  // Arabic
-  /قبول\s*الكل/,
-  /قبول\s*الجميع/,
-  /موافق/,
-  /قبول/,
-  // Persian
-  /پذیرش\s*همه/,
-  /موافقم/,
-  /پذیرش/,
-  // Chinese (Simplified + Traditional)
-  /全部接受/,
-  /接受所有/,
-  /接受全部/,
-  /同意并继续/,
-  /全部接受/,
-  /接受/,
-  /同意/,
-  // Japanese
-  /すべて受け入れ/,
-  /すべて許可/,
-  /同意する/,
-  /同意します/,
-  // Korean
-  /모두\s*수락/,
-  /모두\s*동의/,
-  /동의합니다/,
-  /수락/,
-  // Vietnamese
-  /chấp\s*nhận\s*tất\s*cả/i,
-  /đồng\s*ý\s*tất\s*cả/i,
-  /đồng\s*ý/i,
-  // Thai
-  /ยอมรับทั้งหมด/,
-  /ยอมรับ/,
-  // Hindi
-  /सभी\s*स्वीकार/,
-  /स्वीकार\s*करें/,
-  /सहमत/,
-  // Indonesian / Malay
-  /terima\s*semua/i,
-  /setuju/i,
-  // Generic single-word fallbacks (only matched inside dialogs)
-  /^accept$/i,
-  /^agree$/i,
-  /^ok$/i,
-];
-
-// Roles that indicate a consent dialog container.
-const DIALOG_ROLES = new Set(['dialog', 'alertdialog']);
-
-// Text patterns in dialog names/headings that confirm it's about consent.
-const CONSENT_DIALOG_HINTS = [
-  /cookie/i,
-  /consent/i,
-  /privacy/i,
-  /before\s*you\s*continue/i,
-  /voordat\s*je\s*verdergaat/i,  // Dutch
-  /bevor\s*du\s*fortf/i,         // German
-  /avant\s*de\s*continuer/i,     // French
-  /antes\s*de\s*continuar/i,     // Spanish / Portuguese
-  /prima\s*di\s*continuare/i,    // Italian
-  /zanim\s*przejdziesz/i,        // Polish
-  /прежде\s*чем\s*продолжить/i,  // Russian
-  /devam\s*etmeden\s*önce/i,     // Turkish
-  /続行する前に/,                  // Japanese
-  /继续前/,                        // Chinese Simplified
-  /繼續前/,                        // Chinese Traditional
-  /계속하기\s*전에/,                // Korean
-  /trước\s*khi\s*tiếp\s*tục/i,   // Vietnamese
-  /ملفات\s*تعريف\s*الارتباط/,    // Arabic: cookies
-  /คุกกี้/,                        // Thai: cookies
-];
+import { ACCEPT_PATTERNS, DIALOG_ROLES, CONSENT_DIALOG_HINTS } from './consent-patterns.js';
 
 /**
  * Click a node via JavaScript .click() instead of mouse events.

package/src/daemon.js

@@ -31,24 +31,25 @@ function tokenMatches(expected, got) {
 const SESSION_FILE = 'session.json';
 
 /**
- * Spawn a detached child process that runs the daemon.
- * Parent polls for session.json, then exits.
+ * Build the argv for the detached daemon child. Pure + exported so the
+ * flag-forwarding contract is unit-testable: a forward that's silently
+ * dropped here (as `--incognito` was in v0.15.0, turning `open --incognito`
+ * into a fully-authenticated session) becomes a failing test, not a leak.
+ * @param {object} opts - the session options parsed in cli.js cmdOpen
+ * @param {string} absDir - resolved output directory
+ * @param {string|undefined} initialUrl - URL to navigate on start (may be undefined)
+ * @param {string} cliPath - path to cli.js for the child to run
+ * @returns {string[]}
  */
-export async function startDaemon(opts, outputDir, initialUrl) {
-  const absDir = resolve(outputDir);
-  mkdirSync(absDir, { recursive: true, mode: 0o700 });
-
-  // Clean stale session
-  const sessionPath = join(absDir, SESSION_FILE);
-  if (existsSync(sessionPath)) unlinkSync(sessionPath);
-
-  // Build child args
-  const args = [join(import.meta.dirname, '..', 'cli.js'), '--daemon-internal'];
+export function buildDaemonArgs(opts, absDir, initialUrl, cliPath) {
+  const args = [cliPath, '--daemon-internal'];
   args.push('--output-dir', absDir);
   if (initialUrl) args.push('--url', initialUrl);
+  if (opts.engine) args.push('--engine', opts.engine);
   if (opts.mode) args.push('--mode', opts.mode);
   if (opts.port) args.push('--port', String(opts.port));
   if (opts.cookies === false) args.push('--no-cookies');
+  if (opts.incognito) args.push('--incognito');
   if (opts.browser) args.push('--browser', opts.browser);
   if (opts.timeout) args.push('--timeout', String(opts.timeout));
   if (opts.pruneMode) args.push('--prune-mode', opts.pruneMode);
@@ -63,6 +64,91 @@ export async function startDaemon(opts, outputDir, initialUrl) {
   }
   if (opts.blockPrivateNetwork) args.push('--block-private-network');
   if (opts.uploadDir) args.push('--upload-dir', opts.uploadDir);
+  return args;
+}
+
+/**
+ * Wire Firefox/BiDi console + network capture onto the daemon's log arrays.
+ * The BiDi analogue of the CDP `Runtime.consoleAPICalled` / `Network.*`
+ * capture — measured event shapes: `log.entryAdded` carries {method, level,
+ * args:[{type,value}], text}; the network events key in-flight requests by
+ * `request.request` and expose `response.{status,statusText,mimeType}` /
+ * top-level `errorText`. Pure + exported so the shape mapping (warn→warning
+ * normalization, arg extraction, orphan-safe response/error pairing) is
+ * unit-testable against a fake bidi without launching Firefox.
+ *
+ * @param {object} bidi - BiDi client (async .subscribe(), .on(method, cb))
+ * @param {object} sinks
+ * @param {Array} sinks.consoleLogs - push-target for console entries
+ * @param {Array} sinks.networkLogs - push-target for completed/failed requests
+ * @param {Map} sinks.pendingRequests - in-flight request bookkeeping
+ * @returns {Promise<void>}
+ */
+export async function attachBiDiCapture(bidi, { consoleLogs, networkLogs, pendingRequests }) {
+  await bidi.subscribe([
+    'log.entryAdded',
+    'network.beforeRequestSent',
+    'network.responseCompleted',
+    'network.fetchError',
+  ]);
+
+  bidi.on('log.entryAdded', (e) => {
+    consoleLogs.push({
+      // Map BiDi's console method (or level for uncaught JS errors) to CDP's
+      // `type` vocabulary so `console-logs --level warning` matches on both
+      // engines — BiDi says 'warn', CDP says 'warning'.
+      type: e.method === 'warn' ? 'warning' : (e.method || e.level),
+      timestamp: new Date().toISOString(),
+      args: Array.isArray(e.args) && e.args.length
+        ? e.args.map((a) => a.value ?? a.type)
+        : [e.text],
+    });
+  });
+
+  bidi.on('network.beforeRequestSent', (e) => {
+    pendingRequests.set(e.request.request, {
+      url: e.request.url,
+      method: e.request.method,
+      timestamp: new Date().toISOString(),
+    });
+  });
+
+  bidi.on('network.responseCompleted', (e) => {
+    const req = pendingRequests.get(e.request.request);
+    if (req) {
+      networkLogs.push({
+        ...req,
+        status: e.response.status,
+        statusText: e.response.statusText,
+        mimeType: e.response.mimeType,
+      });
+      pendingRequests.delete(e.request.request);
+    }
+  });
+
+  bidi.on('network.fetchError', (e) => {
+    const req = pendingRequests.get(e.request.request);
+    if (req) {
+      networkLogs.push({ ...req, status: 0, error: e.errorText });
+      pendingRequests.delete(e.request.request);
+    }
+  });
+}
+
+/**
+ * Spawn a detached child process that runs the daemon.
+ * Parent polls for session.json, then exits.
+ */
+export async function startDaemon(opts, outputDir, initialUrl) {
+  const absDir = resolve(outputDir);
+  mkdirSync(absDir, { recursive: true, mode: 0o700 });
+
+  // Clean stale session
+  const sessionPath = join(absDir, SESSION_FILE);
+  if (existsSync(sessionPath)) unlinkSync(sessionPath);
+
+  // Build child args (pure + testable — see buildDaemonArgs).
+  const args = buildDaemonArgs(opts, absDir, initialUrl, join(import.meta.dirname, '..', 'cli.js'));
 
   const child = spawn(process.execPath, args, {
     detached: true,
@@ -104,6 +190,7 @@ export async function runDaemon(opts, outputDir, initialUrl) {
 
   // Connect to browser
   const page = await connect({
+    engine: opts.engine,
     mode: opts.mode || 'headless',
     port: opts.port ? Number(opts.port) : undefined,
     consent: opts.consent,
@@ -115,55 +202,65 @@ export async function runDaemon(opts, outputDir, initialUrl) {
     blockUrls: opts.blockUrls,
     blockPrivateNetwork: opts.blockPrivateNetwork,
     uploadDir: opts.uploadDir,
+    // incognito neuters injectCookies() session-wide; cookies:false (below)
+    // is the legacy per-open equivalent. Either yields a logged-out session.
+    incognito: opts.incognito || opts.cookies === false,
   });
 
-  // Console log capture
+  // Console + network log capture. CDP uses Runtime/Network events off
+  // page.cdp; the Firefox/BiDi engine (page.bidi) captures the same shape over
+  // log.entryAdded / network.* events (Phase 2 — attachBiDiCapture). Either
+  // way console-logs / network-log / wait-idle behave the same for both.
   const consoleLogs = [];
-  await page.cdp.send('Runtime.enable');
-  page.cdp.on('Runtime.consoleAPICalled', (params) => {
-    consoleLogs.push({
-      type: params.type,
-      timestamp: new Date().toISOString(),
-      args: params.args.map((a) => a.value ?? a.description ?? a.type),
-    });
-  });
-
-  // Network log capture (Network.enable already called by connect)
   const networkLogs = [];
   const pendingRequests = new Map();
 
-  page.cdp.on('Network.requestWillBeSent', (params) => {
-    pendingRequests.set(params.requestId, {
-      url: params.request.url,
-      method: params.request.method,
-      timestamp: new Date().toISOString(),
+  if (page.cdp) {
+    await page.cdp.send('Runtime.enable');
+    page.cdp.on('Runtime.consoleAPICalled', (params) => {
+      consoleLogs.push({
+        type: params.type,
+        timestamp: new Date().toISOString(),
+        args: params.args.map((a) => a.value ?? a.description ?? a.type),
+      });
     });
-  });
 
-  page.cdp.on('Network.responseReceived', (params) => {
-    const req = pendingRequests.get(params.requestId);
-    if (req) {
-      networkLogs.push({
-        ...req,
-        status: params.response.status,
-        statusText: params.response.statusText,
-        mimeType: params.response.mimeType,
+    // Network log capture (Network.enable already called by connect)
+    page.cdp.on('Network.requestWillBeSent', (params) => {
+      pendingRequests.set(params.requestId, {
+        url: params.request.url,
+        method: params.request.method,
+        timestamp: new Date().toISOString(),
       });
-      pendingRequests.delete(params.requestId);
-    }
-  });
+    });
 
-  page.cdp.on('Network.loadingFailed', (params) => {
-    const req = pendingRequests.get(params.requestId);
-    if (req) {
-      networkLogs.push({
-        ...req,
-        status: 0,
-        error: params.errorText,
-      });
-      pendingRequests.delete(params.requestId);
-    }
-  });
+    page.cdp.on('Network.responseReceived', (params) => {
+      const req = pendingRequests.get(params.requestId);
+      if (req) {
+        networkLogs.push({
+          ...req,
+          status: params.response.status,
+          statusText: params.response.statusText,
+          mimeType: params.response.mimeType,
+        });
+        pendingRequests.delete(params.requestId);
+      }
+    });
+
+    page.cdp.on('Network.loadingFailed', (params) => {
+      const req = pendingRequests.get(params.requestId);
+      if (req) {
+        networkLogs.push({
+          ...req,
+          status: 0,
+          error: params.errorText,
+        });
+        pendingRequests.delete(params.requestId);
+      }
+    });
+  } else if (page.bidi) {
+    await attachBiDiCapture(page.bidi, { consoleLogs, networkLogs, pendingRequests });
+  }
 
   // Navigate to initial URL if provided
   if (initialUrl) {
@@ -316,6 +413,20 @@ export async function runDaemon(opts, outputDir, initialUrl) {
     },
 
     async eval({ expression }) {
+      // Firefox/BiDi has no page.cdp — evaluate over BiDi in the active context.
+      if (!page.cdp) {
+        try {
+          // BiDi's raw result.value serializes objects into a {type,value}
+          // entries structure, not a plain object (CDP's returnByValue does).
+          // Stringify + parse in-page so an object/array eval matches the CDP
+          // shape (same trick readable() uses). `undefined` → JSON `null`.
+          const wrapped = `(async () => { const v = await (${expression}); return JSON.stringify(v === undefined ? null : v); })()`;
+          const raw = await page.bidi.evaluate(page.context, wrapped, true);
+          return { ok: true, value: raw == null ? null : JSON.parse(raw) };
+        } catch (err) {
+          return { ok: false, error: err.message || 'eval error' };
+        }
+      }
       const result = await page.cdp.send('Runtime.evaluate', {
         expression,
         returnByValue: true,