barebrowse 0.15.0 → 0.17.0

package/src/daemon.js

@@ -67,6 +67,74 @@ export function buildDaemonArgs(opts, absDir, initialUrl, cliPath) {
   return args;
 }
 
+/**
+ * Wire Firefox/BiDi console + network capture onto the daemon's log arrays.
+ * The BiDi analogue of the CDP `Runtime.consoleAPICalled` / `Network.*`
+ * capture — measured event shapes: `log.entryAdded` carries {method, level,
+ * args:[{type,value}], text}; the network events key in-flight requests by
+ * `request.request` and expose `response.{status,statusText,mimeType}` /
+ * top-level `errorText`. Pure + exported so the shape mapping (warn→warning
+ * normalization, arg extraction, orphan-safe response/error pairing) is
+ * unit-testable against a fake bidi without launching Firefox.
+ *
+ * @param {object} bidi - BiDi client (async .subscribe(), .on(method, cb))
+ * @param {object} sinks
+ * @param {Array} sinks.consoleLogs - push-target for console entries
+ * @param {Array} sinks.networkLogs - push-target for completed/failed requests
+ * @param {Map} sinks.pendingRequests - in-flight request bookkeeping
+ * @returns {Promise<void>}
+ */
+export async function attachBiDiCapture(bidi, { consoleLogs, networkLogs, pendingRequests }) {
+  await bidi.subscribe([
+    'log.entryAdded',
+    'network.beforeRequestSent',
+    'network.responseCompleted',
+    'network.fetchError',
+  ]);
+
+  bidi.on('log.entryAdded', (e) => {
+    consoleLogs.push({
+      // Map BiDi's console method (or level for uncaught JS errors) to CDP's
+      // `type` vocabulary so `console-logs --level warning` matches on both
+      // engines — BiDi says 'warn', CDP says 'warning'.
+      type: e.method === 'warn' ? 'warning' : (e.method || e.level),
+      timestamp: new Date().toISOString(),
+      args: Array.isArray(e.args) && e.args.length
+        ? e.args.map((a) => a.value ?? a.type)
+        : [e.text],
+    });
+  });
+
+  bidi.on('network.beforeRequestSent', (e) => {
+    pendingRequests.set(e.request.request, {
+      url: e.request.url,
+      method: e.request.method,
+      timestamp: new Date().toISOString(),
+    });
+  });
+
+  bidi.on('network.responseCompleted', (e) => {
+    const req = pendingRequests.get(e.request.request);
+    if (req) {
+      networkLogs.push({
+        ...req,
+        status: e.response.status,
+        statusText: e.response.statusText,
+        mimeType: e.response.mimeType,
+      });
+      pendingRequests.delete(e.request.request);
+    }
+  });
+
+  bidi.on('network.fetchError', (e) => {
+    const req = pendingRequests.get(e.request.request);
+    if (req) {
+      networkLogs.push({ ...req, status: 0, error: e.errorText });
+      pendingRequests.delete(e.request.request);
+    }
+  });
+}
+
 /**
  * Spawn a detached child process that runs the daemon.
  * Parent polls for session.json, then exits.
@@ -139,10 +207,10 @@ export async function runDaemon(opts, outputDir, initialUrl) {
     incognito: opts.incognito || opts.cookies === false,
   });
 
-  // Console + network log capture is CDP-specific (page.cdp). The Firefox/BiDi
-  // engine exposes page.bidi instead, so these captures are skipped there —
-  // console-logs / network-log commands return empty for a Firefox session.
-  // (BiDi log.entryAdded / network.* events could back these later.)
+  // Console + network log capture. CDP uses Runtime/Network events off
+  // page.cdp; the Firefox/BiDi engine (page.bidi) captures the same shape over
+  // log.entryAdded / network.* events (Phase 2 — attachBiDiCapture). Either
+  // way console-logs / network-log / wait-idle behave the same for both.
   const consoleLogs = [];
   const networkLogs = [];
   const pendingRequests = new Map();
@@ -190,6 +258,8 @@ export async function runDaemon(opts, outputDir, initialUrl) {
         pendingRequests.delete(params.requestId);
       }
     });
+  } else if (page.bidi) {
+    await attachBiDiCapture(page.bidi, { consoleLogs, networkLogs, pendingRequests });
   }
 
   // Navigate to initial URL if provided

package/src/firefox-page.js

@@ -21,6 +21,8 @@ import { axSnapshotExpression, REF_ATTR } from './ax-snapshot.js';
 import { EXTRACT_EXPRESSION, finalizeReadable } from './readable.js';
 import { scopedCookiesForUrl } from './auth.js';
 import { assertNavigable, assertUploadAllowed } from './url-guard.js';
+import { dismissConsentFirefox } from './consent-firefox.js';
+import { waitForNetworkIdleBiDi } from './network-idle.js';
 
 /** BiDi/WebDriver normalized key values for named keys (U+E000 block). */
 const BIDI_KEYS = {
@@ -38,6 +40,7 @@ const BIDI_KEYS = {
  * @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
  * @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
  * @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
+ * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs after goto().
  * @returns {Promise<object>} page object
  */
 export async function createFirefoxPage(bidi, opts = {}) {
@@ -45,6 +48,7 @@ export async function createFirefoxPage(bidi, opts = {}) {
   const urlGuard = opts.urlGuard || {};
   const uploadDir = opts.uploadDir || null;
   const incognito = !!opts.incognito;
+  const consent = opts.consent !== false;
   // The active browsing context. Starts at the initial tab; switchTab() points
   // it at another top-level context, so it's mutable and read via a getter.
   const { contexts } = await bidi.send('browsingContext.getTree', {});
@@ -71,6 +75,53 @@ export async function createFirefoxPage(bidi, opts = {}) {
     return flat;
   }
 
+  /**
+   * Build the spliced AX tree for the active tab (main frame + child frames)
+   * and (re)populate refContexts so a subsequent click routes to the right
+   * frame. Returns the top context's root node, or null if it couldn't be
+   * snapshotted. Shared by snapshot() and the consent auto-dismiss so the
+   * iframe-splice logic lives in one place.
+   */
+  async function buildTree() {
+    const contextIds = await allContexts();
+    refContexts = new Map();
+
+    // Snapshot each context, assigning refs from a shared running counter so
+    // they're globally unique (matching CDP's flat integer refs).
+    let base = 0;
+    const treesByContext = new Map();
+    for (const ctx of contextIds) {
+      let raw;
+      try {
+        raw = await bidi.evaluate(ctx, axSnapshotExpression(base), false);
+      } catch { continue; } // frame navigated mid-snapshot — skip it
+      const { tree, count } = JSON.parse(raw);
+      for (let r = base + 1; r <= base + count; r++) refContexts.set(String(r), ctx);
+      base += count;
+      treesByContext.set(ctx, tree);
+    }
+
+    // Splice each child frame's tree under an <iframe> (role 'Iframe')
+    // placeholder in its parent, matched by document order — BiDi getTree
+    // lists children in frame order, and the AX walk emits Iframe nodes in
+    // the same order.
+    const iframeNodes = (tree) => {
+      const out = [];
+      (function walk(n) { if (n.role === 'Iframe') out.push(n); for (const c of n.children) walk(c); })(tree);
+      return out;
+    };
+    for (let i = 1; i < contextIds.length; i++) {
+      const childTree = treesByContext.get(contextIds[i]);
+      if (!childTree) continue;
+      // Attach to the next unfilled Iframe placeholder in the top tree.
+      const holders = iframeNodes(treesByContext.get(topContext) || { role: '', children: [] });
+      const holder = holders[i - 1];
+      if (holder) holder.children = [childTree];
+    }
+
+    return treesByContext.get(topContext) || null;
+  }
+
   /** Resolve a ref to a BiDi element sharedId in its owning context. */
   async function resolveRef(ref) {
     const context = refContexts.get(String(ref));
@@ -134,46 +185,19 @@ export async function createFirefoxPage(bidi, opts = {}) {
         timeout, `goto(${url})`);
       // Brief settle for dynamic/SPA content, matching the CDP path.
       await new Promise((r) => setTimeout(r, 500));
-    },
-
-    async snapshot(pruneOpts) {
-      const contextIds = await allContexts();
-      refContexts = new Map();
-
-      // Snapshot each context, assigning refs from a shared running counter so
-      // they're globally unique (matching CDP's flat integer refs).
-      let base = 0;
-      const treesByContext = new Map();
-      for (const ctx of contextIds) {
-        let raw;
+      // Auto-dismiss cookie consent, mirroring the CDP path (index.js runs
+      // dismissConsent right after navigate). Best-effort: a failure here must
+      // never fail the navigation.
+      if (consent) {
         try {
-          raw = await bidi.evaluate(ctx, axSnapshotExpression(base), false);
-        } catch { continue; } // frame navigated mid-snapshot — skip it
-        const { tree, count } = JSON.parse(raw);
-        for (let r = base + 1; r <= base + count; r++) refContexts.set(String(r), ctx);
-        base += count;
-        treesByContext.set(ctx, tree);
-      }
-
-      // Splice each child frame's tree under an <iframe> (role 'Iframe')
-      // placeholder in its parent, matched by document order — BiDi getTree
-      // lists children in frame order, and the AX walk emits Iframe nodes in
-      // the same order.
-      const iframeNodes = (tree) => {
-        const out = [];
-        (function walk(n) { if (n.role === 'Iframe') out.push(n); for (const c of n.children) walk(c); })(tree);
-        return out;
-      };
-      for (let i = 1; i < contextIds.length; i++) {
-        const childTree = treesByContext.get(contextIds[i]);
-        if (!childTree) continue;
-        // Attach to the next unfilled Iframe placeholder in the top tree.
-        const holders = iframeNodes(treesByContext.get(topContext) || { role: '', children: [] });
-        const holder = holders[i - 1];
-        if (holder) holder.children = [childTree];
+          const root = await buildTree();
+          if (root) await dismissConsentFirefox(root, (ref) => pointerClick(ref));
+        } catch { /* consent dismissal is best-effort */ }
       }
+    },
 
-      const root = treesByContext.get(topContext);
+    async snapshot(pruneOpts) {
+      const root = await buildTree();
       if (!root) return '';
 
       const pageUrl = await bidi.evaluate(topContext, 'location.href', false).catch(() => '');
@@ -379,12 +403,20 @@ export async function createFirefoxPage(bidi, opts = {}) {
       throw new Error(`waitFor timed out after ${timeout}ms`);
     },
 
+    /**
+     * Wait until the network has been idle for `idle` ms, over BiDi
+     * network.* events. Parity with the CDP page.waitForNetworkIdle (Phase 2).
+     */
+    async waitForNetworkIdle(idleOpts = {}) {
+      return waitForNetworkIdleBiDi(bidi, idleOpts);
+    },
+
     // --- CDP-only surfaces, stubbed for daemon parity ---------------------
     // The daemon (src/daemon.js) dispatches these unconditionally. On the
-    // Firefox/BiDi engine download tracking and dialog capture aren't wired,
-    // so the two logs are genuinely empty; the three actions are CDP-only and
-    // fail with a clear, intentional message instead of an incidental
-    // TypeError. Documented as a known gap in CHANGELOG (Firefox engine).
+    // Firefox/BiDi engine download tracking and dialog capture aren't wired
+    // (Phase 3/4), so the two logs are genuinely empty; the two actions are
+    // CDP-only and fail with a clear, intentional message instead of an
+    // incidental TypeError. Documented as a known gap in CHANGELOG.
     get downloads() { return []; },
     get dialogLog() { return []; },
     async saveState() {
@@ -393,9 +425,6 @@ export async function createFirefoxPage(bidi, opts = {}) {
     async waitForNavigation() {
       throw new Error('waitForNavigation() is not supported on the Firefox/BiDi engine (CDP-only)');
     },
-    async waitForNetworkIdle() {
-      throw new Error('waitForNetworkIdle() is not supported on the Firefox/BiDi engine (CDP-only)');
-    },
 
     async close() {
       try { await bidi.send('browsingContext.close', { context: topContext }); } catch {}

package/src/index.js

@@ -19,6 +19,7 @@ import { prune as pruneTree } from './prune.js';
 import { click as cdpClick, type as cdpType, scroll as cdpScroll, press as cdpPress, hover as cdpHover, select as cdpSelect, drag as cdpDrag, upload as cdpUpload } from './interact.js';
 import { dismissConsent } from './consent.js';
 import { applyStealth } from './stealth.js';
+import { applyFirefoxStealth } from './stealth-firefox.js';
 import { DEFAULT_BLOCKLIST } from './blocklist.js';
 import { waitForNetworkIdle } from './network-idle.js';
 import { readable as extractReadable } from './readable.js';
@@ -746,12 +747,17 @@ async function suppressPermissions(cdp) {
  * ref model, and AX-tree source all differ (see firefox-page.js). close() is
  * wrapped to also reap the Firefox process + temp profile.
  *
- * Chromium-only options NOT applied on this path: `consent` (auto-dismiss),
- * `blockAds`/`blockUrls`, stealth patches, and `hybrid` mode. `saveState`,
- * `waitForNavigation`, `waitForNetworkIdle`, and download/dialog capture are
- * CDP-only too (the page object stubs them — see firefox-page.js). The
- * navigation guard (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir`
- * sandbox, `incognito`, `proxy`, `viewport`, and `pruneMode` DO apply.
+ * Chromium-only options NOT applied on this path: `blockAds`/`blockUrls` and
+ * `hybrid` mode. `saveState`, `waitForNavigation`, and download/dialog capture
+ * are CDP-only too (the page object stubs them — see firefox-page.js).
+ * `waitForNetworkIdle` and daemon console/network capture DO apply as of Phase
+ * 2 (over BiDi `network.*` and `log.entryAdded` events). `consent`
+ * (auto-dismiss) and
+ * stealth patches DO apply here as of v0.16.0 (stealth headless-only, via BiDi
+ * preload script — see stealth-firefox.js / consent-firefox.js), alongside the
+ * navigation guard
+ * (`allowLocalUrls`/`blockPrivateNetwork`), `uploadDir` sandbox, `incognito`,
+ * `proxy`, `viewport`, and `pruneMode`.
  * @param {object} opts - connect() options ({ mode, proxy, binary, viewport, pruneMode, urlGuard, uploadDir, incognito })
  * @returns {Promise<object>} Firefox page object
  */
@@ -763,11 +769,18 @@ async function connectFirefox(opts) {
     viewport: opts.viewport,
   });
   const bidi = await createBiDi(browser.wsUrl);
+  // Stealth patches (headless only, mirroring the CDP path's `mode !== 'headed'`
+  // gate). Registered before the first navigation via script.addPreloadScript.
+  // Firefox needs a much smaller script than Chromium — see stealth-firefox.js.
+  if (opts.mode !== 'headed') {
+    await applyFirefoxStealth(bidi);
+  }
   const page = await createFirefoxPage(bidi, {
     pruneMode: opts.pruneMode,
     urlGuard: { allowLocalUrls: opts.allowLocalUrls, blockPrivateNetwork: opts.blockPrivateNetwork },
     uploadDir: opts.uploadDir || null,
     incognito: !!opts.incognito,
+    consent: opts.consent,
   });
   const closePage = page.close.bind(page);
   page.close = async () => {

package/src/network-idle.js

@@ -2,24 +2,27 @@
  * network-idle.js — wait until the page's network has been idle for N ms.
  *
  * Tracks in-flight requests by requestId in a Set, so an orphan
- * loadingFinished/Failed (event for a request whose requestWillBeSent
- * arrived before our listener attached) is a harmless no-op instead of
- * driving a counter negative and resolving prematurely.
+ * finish/fail event (for a request whose start event arrived before our
+ * listener attached) is a harmless no-op instead of driving a counter
+ * negative and resolving prematurely.
+ *
+ * Two engines, one core: `waitForNetworkIdle` wires the CDP `Network.*`
+ * events, `waitForNetworkIdleBiDi` wires the WebDriver BiDi `network.*`
+ * events. Both feed the same orphan-resilient idle waiter (`idleWaiter`).
  */
 
 /**
- * @param {object} session - CDP session-scoped handle with .on() returning unsub
- * @param {object} [opts]
- * @param {number} [opts.timeout=30000] - Max wait time before reject
- * @param {number} [opts.idle=500] - Required idle duration before resolve
+ * Shared idle-detection loop. `attach` registers the engine-specific event
+ * listeners, calling onStart(id) when a request begins and onEnd(id) when it
+ * finishes/fails, and pushing each listener's unsub fn into `unsubs`.
+ * @param {object} cfg
+ * @param {number} cfg.timeout - Max wait before reject
+ * @param {number} cfg.idle - Required quiet duration before resolve
+ * @param {(hooks: {onStart: (id:any)=>void, onEnd: (id:any)=>void, unsubs: Array<()=>void>}) => void} cfg.attach
  * @returns {Promise<void>}
  */
-export function waitForNetworkIdle(session, opts = {}) {
-  const timeout = opts.timeout || 30000;
-  const idle = opts.idle || 500;
-
-  /** @type {Promise<void>} */
-  const settled = new Promise((resolve, reject) => {
+function idleWaiter({ timeout, idle, attach }) {
+  return new Promise((resolve, reject) => {
     const pending = new Set();
     let timer = null;
     const unsubs = [];
@@ -38,20 +41,12 @@ export function waitForNetworkIdle(session, opts = {}) {
       }
     };
 
-    unsubs.push(session.on('Network.requestWillBeSent', (p) => {
-      pending.add(p.requestId);
-      clearTimeout(timer);
-    }));
-    unsubs.push(session.on('Network.loadingFinished', (p) => {
-      // delete() on a Set is a no-op for unknown keys — orphan events from
-      // requests started before we attached the listener can't push us negative.
-      pending.delete(p.requestId);
-      check();
-    }));
-    unsubs.push(session.on('Network.loadingFailed', (p) => {
-      pending.delete(p.requestId);
-      check();
-    }));
+    const onStart = (id) => { pending.add(id); clearTimeout(timer); };
+    // delete() on a Set is a no-op for unknown keys — orphan finish/fail
+    // events from requests started before we attached can't push us negative.
+    const onEnd = (id) => { pending.delete(id); check(); };
+
+    attach({ onStart, onEnd, unsubs });
 
     const deadlineTimer = setTimeout(() => {
       for (const unsub of unsubs) unsub();
@@ -61,5 +56,48 @@ export function waitForNetworkIdle(session, opts = {}) {
     // Start check immediately (might already be idle)
     check();
   });
-  return settled;
+}
+
+/**
+ * CDP: wait for network idle over `Network.*` events.
+ * @param {object} session - CDP session-scoped handle with .on() returning unsub
+ * @param {object} [opts]
+ * @param {number} [opts.timeout=30000] - Max wait time before reject
+ * @param {number} [opts.idle=500] - Required idle duration before resolve
+ * @returns {Promise<void>}
+ */
+export function waitForNetworkIdle(session, opts = {}) {
+  return idleWaiter({
+    timeout: opts.timeout || 30000,
+    idle: opts.idle || 500,
+    attach: ({ onStart, onEnd, unsubs }) => {
+      unsubs.push(session.on('Network.requestWillBeSent', (p) => onStart(p.requestId)));
+      unsubs.push(session.on('Network.loadingFinished', (p) => onEnd(p.requestId)));
+      unsubs.push(session.on('Network.loadingFailed', (p) => onEnd(p.requestId)));
+    },
+  });
+}
+
+/**
+ * Firefox/BiDi: wait for network idle over `network.*` events. Subscribes the
+ * events first (idempotent — safe even if the daemon already subscribed for
+ * log capture), then runs the same orphan-resilient idle loop. The in-flight
+ * key is `params.request.request` (the BiDi request id).
+ * @param {object} bidi - BiDi client with async .subscribe() and .on() returning unsub
+ * @param {object} [opts]
+ * @param {number} [opts.timeout=30000] - Max wait time before reject
+ * @param {number} [opts.idle=500] - Required idle duration before resolve
+ * @returns {Promise<void>}
+ */
+export async function waitForNetworkIdleBiDi(bidi, opts = {}) {
+  await bidi.subscribe(['network.beforeRequestSent', 'network.responseCompleted', 'network.fetchError']);
+  return idleWaiter({
+    timeout: opts.timeout || 30000,
+    idle: opts.idle || 500,
+    attach: ({ onStart, onEnd, unsubs }) => {
+      unsubs.push(bidi.on('network.beforeRequestSent', (p) => onStart(p.request.request)));
+      unsubs.push(bidi.on('network.responseCompleted', (p) => onEnd(p.request.request)));
+      unsubs.push(bidi.on('network.fetchError', (p) => onEnd(p.request.request)));
+    },
+  });
 }

package/src/stealth-firefox.js

@@ -0,0 +1,42 @@
+/**
+ * stealth-firefox.js — Anti-detection patches for headless Firefox (BiDi).
+ *
+ * Deliberately MUCH smaller than the Chromium stealth.js. That isn't laziness —
+ * it's what a POC measured stock Firefox-under-BiDi to actually expose:
+ *
+ *   navigator.webdriver  → true   (the tell — we hide it)
+ *   window.chrome        → absent (correct; ADDING it would be a spoof tell)
+ *   navigator.plugins    → 5      (realistic PDF set; already normal)
+ *   languages / hardwareConcurrency → normal (en-US,en / 8)
+ *   User-Agent           → real Firefox, NO "Headless" marker (no rewrite needed)
+ *   WebGL vendor/renderer → the REAL GPU, not SwiftShader (spoofing would
+ *                           replace a real value with a fake one)
+ *
+ * So porting Chromium's STEALTH_SCRIPT verbatim would have made Firefox look
+ * like a spoofed browser (window.chrome + Chrome plugins on Firefox = obvious
+ * tell) — a worse fingerprint than the one removed. The only genuinely
+ * engine-agnostic pieces are webdriver-hiding and canvas noise, both reused
+ * verbatim from stealth.js so their fixes stay single-sourced.
+ *
+ * BiDi's script.addPreloadScript is the equivalent of CDP's
+ * Page.addScriptToEvaluateOnNewDocument — the POC confirmed it runs BEFORE any
+ * page script (navigator.webdriver read at parse time already saw `undefined`).
+ */
+
+import { WEBDRIVER_PATCH, CANVAS_NOISE_PATCH } from './stealth.js';
+
+const FIREFOX_STEALTH_SCRIPT = `${WEBDRIVER_PATCH}\n${CANVAS_NOISE_PATCH}`;
+
+/**
+ * Register the Firefox stealth patches so they run before any page script.
+ * Must be called before the first navigation. addPreloadScript is global
+ * (applies to every browsing context / navigation), so a single registration
+ * covers the whole session.
+ *
+ * @param {object} bidi - BiDi client from createBiDi()
+ */
+export async function applyFirefoxStealth(bidi) {
+  await bidi.send('script.addPreloadScript', {
+    functionDeclaration: `() => { ${FIREFOX_STEALTH_SCRIPT} }`,
+  });
+}