barebrowse 0.15.0 → 0.17.0

package/src/stealth.js

@@ -5,9 +5,114 @@
  * any page scripts (unlike Runtime.evaluate which runs after).
  */
 
+/**
+ * navigator.webdriver hiding — the one automation tell shared by every engine
+ * (Chromium headless AND Firefox-under-BiDi both report `true`). Split out so
+ * the Firefox stealth path (stealth-firefox.js) reuses the exact same patch
+ * instead of duplicating it. Measured baseline on both engines: `true`.
+ *
+ * A naive `Object.defineProperty(navigator, 'webdriver', …)` hides the *value*
+ * but leaves an advanced tell: it creates an OWN property on the instance, so
+ * `navigator.hasOwnProperty('webdriver')` becomes `true` where a real browser
+ * (property lives on Navigator.prototype) reports `false` — sophisticated
+ * anti-bot checks (e.g. sannysoft's "WebDriver New") catch that inconsistency.
+ * So we DELETE the getter off the prototype instead: then `navigator.webdriver`
+ * is `undefined` AND `'webdriver' in navigator` AND `hasOwnProperty` are both
+ * `false`, matching a stock browser. POC-verified on Chromium and Firefox
+ * (own/in both false, prototype descriptor gone). The defineProperty fallback
+ * only fires if the descriptor is somehow non-configurable (delete a no-op).
+ */
+export const WEBDRIVER_PATCH = `
+  try {
+    const proto = Object.getPrototypeOf(navigator);
+    const desc = proto && Object.getOwnPropertyDescriptor(proto, 'webdriver');
+    if (desc && desc.configurable) delete proto.webdriver;
+    if (navigator.webdriver) {
+      Object.defineProperty(navigator, 'webdriver', { get: () => undefined, configurable: true });
+    }
+  } catch {}
+`;
+
+/**
+ * Canvas-fingerprint noise — browser-agnostic (standard Canvas2D API), so both
+ * the Chromium and Firefox stealth scripts compose it. This block carries the
+ * subtle double-XOR-cancellation fixes (see git history / project memory); keep
+ * it SINGLE-SOURCED here so a fix never has to be made in two places.
+ */
+export const CANVAS_NOISE_PATCH = `
+  // Canvas fingerprinting — sites render standard text/shapes, then read
+  // pixels via toDataURL or getImageData. The output is stable per machine
+  // (GPU, font rasterizer, anti-aliasing) but unique across machines, which
+  // makes it the second-most-common fingerprint after WebGL. Defense: nudge
+  // a few RGB channels by ±1 per session so the hash changes each visit
+  // while the canvas still looks identical to the human eye. The per-tab
+  // seed keeps reads stable within a session so legitimate canvas use
+  // (image processing, games) doesn't flicker.
+  // crypto.getRandomValues is guaranteed unique per browsing context; using
+  // Math.random alone can collide when two fresh V8 contexts start within
+  // microseconds of each other (real-world: parallel tests, real-world hit:
+  // we observed it). performance.now adds a wall-clock anchor as a belt-and-
+  // braces guard against contexts where crypto is somehow stubbed.
+  const _seedBuf = new Uint32Array(1);
+  crypto.getRandomValues(_seedBuf);
+  const CANVAS_SEED = (_seedBuf[0] ^ ((performance.now() * 1e6) | 0)) >>> 0;
+  function shiftPixels(data) {
+    // Touch ~1 byte per 64-byte stride. The bit we XOR with is taken from a
+    // position-dependent SLICE of a seed-mixed hash, not its low bit — a
+    // naive 'mix & 1' collapses to only two possible outputs per seed
+    // parity because every stride index is even (the position multiplier
+    // is odd, so the low bit only depends on seed parity). Indexing the
+    // hash by (i/64) mod 32 makes every stride position sample a different
+    // bit, so two distinct seeds produce different mask patterns.
+    for (let i = 0; i < data.length; i += 64) {
+      const h = ((CANVAS_SEED * 2654435761) ^ (i * 1597334677)) >>> 0;
+      const bit = (h >>> ((i >>> 6) & 31)) & 1;
+      data[i] = (data[i] ^ bit) & 0xff;
+    }
+    return data;
+  }
+  // Capture originals BEFORE replacing — toDataURL must read pixels via the
+  // native getImageData (not the patched one), otherwise the patch double-
+  // applies and the second XOR cancels the first, leaving output unchanged.
+  const origGetImageData = CanvasRenderingContext2D.prototype.getImageData;
+  const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
+  HTMLCanvasElement.prototype.toDataURL = function() {
+    const ctx = this.getContext('2d');
+    if (ctx && this.width > 0 && this.height > 0) {
+      try {
+        const img = origGetImageData.call(ctx, 0, 0, this.width, this.height);
+        // Snapshot the original bytes so we can restore them after encoding.
+        // Without this, repeated toDataURL() alternates noisy/clean: call 1
+        // XORs the canvas in place, call 2 reads the noisy canvas and XORs
+        // again (self-inverse), call 3 again, etc. Same XOR-cancellation
+        // class as the earlier double-apply bug, just through canvas state
+        // rather than method composition. The restore also keeps the
+        // bitmap idempotent for any downstream legitimate canvas reads.
+        const original = new Uint8ClampedArray(img.data);
+        shiftPixels(img.data);
+        ctx.putImageData(img, 0, 0);
+        const result = origToDataURL.apply(this, arguments);
+        img.data.set(original);
+        ctx.putImageData(img, 0, 0);
+        return result;
+      } catch {
+        // Tainted canvas (cross-origin image) — can't read; skip the nudge
+        // and fall through to the original call so the page sees the
+        // expected SecurityError instead of silent corruption.
+      }
+    }
+    return origToDataURL.apply(this, arguments);
+  };
+  CanvasRenderingContext2D.prototype.getImageData = function() {
+    const img = origGetImageData.apply(this, arguments);
+    shiftPixels(img.data);
+    return img;
+  };
+`;
+
 const STEALTH_SCRIPT = `
   // Hide webdriver flag
-  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+  ${WEBDRIVER_PATCH}
 
   // Fake plugins (headless has 0)
   Object.defineProperty(navigator, 'plugins', {
@@ -93,74 +198,7 @@ const STEALTH_SCRIPT = `
     };
   }
 
-  // Canvas fingerprinting — sites render standard text/shapes, then read
-  // pixels via toDataURL or getImageData. The output is stable per machine
-  // (GPU, font rasterizer, anti-aliasing) but unique across machines, which
-  // makes it the second-most-common fingerprint after WebGL. Defense: nudge
-  // a few RGB channels by ±1 per session so the hash changes each visit
-  // while the canvas still looks identical to the human eye. The per-tab
-  // seed keeps reads stable within a session so legitimate canvas use
-  // (image processing, games) doesn't flicker.
-  // crypto.getRandomValues is guaranteed unique per browsing context; using
-  // Math.random alone can collide when two fresh V8 contexts start within
-  // microseconds of each other (real-world: parallel tests, real-world hit:
-  // we observed it). performance.now adds a wall-clock anchor as a belt-and-
-  // braces guard against contexts where crypto is somehow stubbed.
-  const _seedBuf = new Uint32Array(1);
-  crypto.getRandomValues(_seedBuf);
-  const CANVAS_SEED = (_seedBuf[0] ^ ((performance.now() * 1e6) | 0)) >>> 0;
-  function shiftPixels(data) {
-    // Touch ~1 byte per 64-byte stride. The bit we XOR with is taken from a
-    // position-dependent SLICE of a seed-mixed hash, not its low bit — a
-    // naive 'mix & 1' collapses to only two possible outputs per seed
-    // parity because every stride index is even (the position multiplier
-    // is odd, so the low bit only depends on seed parity). Indexing the
-    // hash by (i/64) mod 32 makes every stride position sample a different
-    // bit, so two distinct seeds produce different mask patterns.
-    for (let i = 0; i < data.length; i += 64) {
-      const h = ((CANVAS_SEED * 2654435761) ^ (i * 1597334677)) >>> 0;
-      const bit = (h >>> ((i >>> 6) & 31)) & 1;
-      data[i] = (data[i] ^ bit) & 0xff;
-    }
-    return data;
-  }
-  // Capture originals BEFORE replacing — toDataURL must read pixels via the
-  // native getImageData (not the patched one), otherwise the patch double-
-  // applies and the second XOR cancels the first, leaving output unchanged.
-  const origGetImageData = CanvasRenderingContext2D.prototype.getImageData;
-  const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
-  HTMLCanvasElement.prototype.toDataURL = function() {
-    const ctx = this.getContext('2d');
-    if (ctx && this.width > 0 && this.height > 0) {
-      try {
-        const img = origGetImageData.call(ctx, 0, 0, this.width, this.height);
-        // Snapshot the original bytes so we can restore them after encoding.
-        // Without this, repeated toDataURL() alternates noisy/clean: call 1
-        // XORs the canvas in place, call 2 reads the noisy canvas and XORs
-        // again (self-inverse), call 3 again, etc. Same XOR-cancellation
-        // class as the earlier double-apply bug, just through canvas state
-        // rather than method composition. The restore also keeps the
-        // bitmap idempotent for any downstream legitimate canvas reads.
-        const original = new Uint8ClampedArray(img.data);
-        shiftPixels(img.data);
-        ctx.putImageData(img, 0, 0);
-        const result = origToDataURL.apply(this, arguments);
-        img.data.set(original);
-        ctx.putImageData(img, 0, 0);
-        return result;
-      } catch {
-        // Tainted canvas (cross-origin image) — can't read; skip the nudge
-        // and fall through to the original call so the page sees the
-        // expected SecurityError instead of silent corruption.
-      }
-    }
-    return origToDataURL.apply(this, arguments);
-  };
-  CanvasRenderingContext2D.prototype.getImageData = function() {
-    const img = origGetImageData.apply(this, arguments);
-    shiftPixels(img.data);
-    return img;
-  };
+  ${CANVAS_NOISE_PATCH}
 `;
 
 /**

package/types/consent-firefox.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Try to dismiss a cookie consent dialog in a reconstructed Firefox AX tree.
+ *
+ * @param {object} root - Spliced AX tree root (from firefox-page's buildTree).
+ * @param {(ref: string) => Promise<void>} click - Clicks the element for a ref
+ *   (firefox-page injects one that routes to pointerClick / performActions).
+ * @returns {Promise<boolean>} true if an accept button was found and clicked.
+ */
+export function dismissConsentFirefox(root: object, click: (ref: string) => Promise<void>): Promise<boolean>;

package/types/consent-patterns.d.ts

@@ -0,0 +1,12 @@
+/**
+ * consent-patterns.js — Shared cookie-consent detection data.
+ *
+ * The engine-agnostic half of consent handling: the multilingual regex sets
+ * and dialog roles used to recognise a consent dialog and its "accept" button.
+ * Both the CDP walker (consent.js) and the BiDi walker (consent-firefox.js)
+ * import these so the language coverage is single-sourced — a new language or
+ * pattern is added once and both engines get it.
+ */
+export const ACCEPT_PATTERNS: RegExp[];
+export const DIALOG_ROLES: Set<string>;
+export const CONSENT_DIALOG_HINTS: RegExp[];

package/types/daemon.d.ts

@@ -10,6 +10,28 @@
  * @returns {string[]}
  */
 export function buildDaemonArgs(opts: object, absDir: string, initialUrl: string | undefined, cliPath: string): string[];
+/**
+ * Wire Firefox/BiDi console + network capture onto the daemon's log arrays.
+ * The BiDi analogue of the CDP `Runtime.consoleAPICalled` / `Network.*`
+ * capture — measured event shapes: `log.entryAdded` carries {method, level,
+ * args:[{type,value}], text}; the network events key in-flight requests by
+ * `request.request` and expose `response.{status,statusText,mimeType}` /
+ * top-level `errorText`. Pure + exported so the shape mapping (warn→warning
+ * normalization, arg extraction, orphan-safe response/error pairing) is
+ * unit-testable against a fake bidi without launching Firefox.
+ *
+ * @param {object} bidi - BiDi client (async .subscribe(), .on(method, cb))
+ * @param {object} sinks
+ * @param {Array} sinks.consoleLogs - push-target for console entries
+ * @param {Array} sinks.networkLogs - push-target for completed/failed requests
+ * @param {Map} sinks.pendingRequests - in-flight request bookkeeping
+ * @returns {Promise<void>}
+ */
+export function attachBiDiCapture(bidi: object, { consoleLogs, networkLogs, pendingRequests }: {
+    consoleLogs: any[];
+    networkLogs: any[];
+    pendingRequests: Map<any, any>;
+}): Promise<void>;
 /**
  * Spawn a detached child process that runs the daemon.
  * Parent polls for session.json, then exits.

package/types/firefox-page.d.ts

@@ -6,6 +6,7 @@
  * @param {{allowLocalUrls?: boolean, blockPrivateNetwork?: boolean}} [opts.urlGuard] - Navigation safety policy, applied on every goto().
  * @param {string} [opts.uploadDir] - When set, upload() rejects files outside this dir.
  * @param {boolean} [opts.incognito=false] - Clean session: injectCookies() is a no-op.
+ * @param {boolean} [opts.consent=true] - Auto-dismiss cookie consent dialogs after goto().
  * @returns {Promise<object>} page object
  */
 export function createFirefoxPage(bidi: object, opts?: {
@@ -16,4 +17,5 @@ export function createFirefoxPage(bidi: object, opts?: {
     } | undefined;
     uploadDir?: string | undefined;
     incognito?: boolean | undefined;
+    consent?: boolean | undefined;
 }): Promise<object>;

package/types/network-idle.d.ts

@@ -1,12 +1,5 @@
 /**
- * network-idle.js — wait until the page's network has been idle for N ms.
- *
- * Tracks in-flight requests by requestId in a Set, so an orphan
- * loadingFinished/Failed (event for a request whose requestWillBeSent
- * arrived before our listener attached) is a harmless no-op instead of
- * driving a counter negative and resolving prematurely.
- */
-/**
+ * CDP: wait for network idle over `Network.*` events.
  * @param {object} session - CDP session-scoped handle with .on() returning unsub
  * @param {object} [opts]
  * @param {number} [opts.timeout=30000] - Max wait time before reject
@@ -17,3 +10,18 @@ export function waitForNetworkIdle(session: object, opts?: {
     timeout?: number | undefined;
     idle?: number | undefined;
 }): Promise<void>;
+/**
+ * Firefox/BiDi: wait for network idle over `network.*` events. Subscribes the
+ * events first (idempotent — safe even if the daemon already subscribed for
+ * log capture), then runs the same orphan-resilient idle loop. The in-flight
+ * key is `params.request.request` (the BiDi request id).
+ * @param {object} bidi - BiDi client with async .subscribe() and .on() returning unsub
+ * @param {object} [opts]
+ * @param {number} [opts.timeout=30000] - Max wait time before reject
+ * @param {number} [opts.idle=500] - Required idle duration before resolve
+ * @returns {Promise<void>}
+ */
+export function waitForNetworkIdleBiDi(bidi: object, opts?: {
+    timeout?: number | undefined;
+    idle?: number | undefined;
+}): Promise<void>;

package/types/stealth-firefox.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Register the Firefox stealth patches so they run before any page script.
+ * Must be called before the first navigation. addPreloadScript is global
+ * (applies to every browsing context / navigation), so a single registration
+ * covers the whole session.
+ *
+ * @param {object} bidi - BiDi client from createBiDi()
+ */
+export function applyFirefoxStealth(bidi: object): Promise<void>;

package/types/stealth.d.ts

@@ -12,3 +12,34 @@
  * @param {object} session - Session-scoped CDP handle
  */
 export function applyStealth(session: object): Promise<void>;
+/**
+ * stealth.js — Anti-detection patches for headless Chromium.
+ *
+ * Uses Page.addScriptToEvaluateOnNewDocument so patches run before
+ * any page scripts (unlike Runtime.evaluate which runs after).
+ */
+/**
+ * navigator.webdriver hiding — the one automation tell shared by every engine
+ * (Chromium headless AND Firefox-under-BiDi both report `true`). Split out so
+ * the Firefox stealth path (stealth-firefox.js) reuses the exact same patch
+ * instead of duplicating it. Measured baseline on both engines: `true`.
+ *
+ * A naive `Object.defineProperty(navigator, 'webdriver', …)` hides the *value*
+ * but leaves an advanced tell: it creates an OWN property on the instance, so
+ * `navigator.hasOwnProperty('webdriver')` becomes `true` where a real browser
+ * (property lives on Navigator.prototype) reports `false` — sophisticated
+ * anti-bot checks (e.g. sannysoft's "WebDriver New") catch that inconsistency.
+ * So we DELETE the getter off the prototype instead: then `navigator.webdriver`
+ * is `undefined` AND `'webdriver' in navigator` AND `hasOwnProperty` are both
+ * `false`, matching a stock browser. POC-verified on Chromium and Firefox
+ * (own/in both false, prototype descriptor gone). The defineProperty fallback
+ * only fires if the descriptor is somehow non-configurable (delete a no-op).
+ */
+export const WEBDRIVER_PATCH: "\n  try {\n    const proto = Object.getPrototypeOf(navigator);\n    const desc = proto && Object.getOwnPropertyDescriptor(proto, 'webdriver');\n    if (desc && desc.configurable) delete proto.webdriver;\n    if (navigator.webdriver) {\n      Object.defineProperty(navigator, 'webdriver', { get: () => undefined, configurable: true });\n    }\n  } catch {}\n";
+/**
+ * Canvas-fingerprint noise — browser-agnostic (standard Canvas2D API), so both
+ * the Chromium and Firefox stealth scripts compose it. This block carries the
+ * subtle double-XOR-cancellation fixes (see git history / project memory); keep
+ * it SINGLE-SOURCED here so a fix never has to be made in two places.
+ */
+export const CANVAS_NOISE_PATCH: "\n  // Canvas fingerprinting \u2014 sites render standard text/shapes, then read\n  // pixels via toDataURL or getImageData. The output is stable per machine\n  // (GPU, font rasterizer, anti-aliasing) but unique across machines, which\n  // makes it the second-most-common fingerprint after WebGL. Defense: nudge\n  // a few RGB channels by \u00B11 per session so the hash changes each visit\n  // while the canvas still looks identical to the human eye. The per-tab\n  // seed keeps reads stable within a session so legitimate canvas use\n  // (image processing, games) doesn't flicker.\n  // crypto.getRandomValues is guaranteed unique per browsing context; using\n  // Math.random alone can collide when two fresh V8 contexts start within\n  // microseconds of each other (real-world: parallel tests, real-world hit:\n  // we observed it). performance.now adds a wall-clock anchor as a belt-and-\n  // braces guard against contexts where crypto is somehow stubbed.\n  const _seedBuf = new Uint32Array(1);\n  crypto.getRandomValues(_seedBuf);\n  const CANVAS_SEED = (_seedBuf[0] ^ ((performance.now() * 1e6) | 0)) >>> 0;\n  function shiftPixels(data) {\n    // Touch ~1 byte per 64-byte stride. The bit we XOR with is taken from a\n    // position-dependent SLICE of a seed-mixed hash, not its low bit \u2014 a\n    // naive 'mix & 1' collapses to only two possible outputs per seed\n    // parity because every stride index is even (the position multiplier\n    // is odd, so the low bit only depends on seed parity). Indexing the\n    // hash by (i/64) mod 32 makes every stride position sample a different\n    // bit, so two distinct seeds produce different mask patterns.\n    for (let i = 0; i < data.length; i += 64) {\n      const h = ((CANVAS_SEED * 2654435761) ^ (i * 1597334677)) >>> 0;\n      const bit = (h >>> ((i >>> 6) & 31)) & 1;\n      data[i] = (data[i] ^ bit) & 0xff;\n    }\n    return data;\n  }\n  // Capture originals BEFORE replacing \u2014 toDataURL must read pixels via the\n  // native getImageData (not the patched one), otherwise the patch double-\n  // applies and the second XOR cancels the first, leaving output unchanged.\n  const origGetImageData = CanvasRenderingContext2D.prototype.getImageData;\n  const origToDataURL = HTMLCanvasElement.prototype.toDataURL;\n  HTMLCanvasElement.prototype.toDataURL = function() {\n    const ctx = this.getContext('2d');\n    if (ctx && this.width > 0 && this.height > 0) {\n      try {\n        const img = origGetImageData.call(ctx, 0, 0, this.width, this.height);\n        // Snapshot the original bytes so we can restore them after encoding.\n        // Without this, repeated toDataURL() alternates noisy/clean: call 1\n        // XORs the canvas in place, call 2 reads the noisy canvas and XORs\n        // again (self-inverse), call 3 again, etc. Same XOR-cancellation\n        // class as the earlier double-apply bug, just through canvas state\n        // rather than method composition. The restore also keeps the\n        // bitmap idempotent for any downstream legitimate canvas reads.\n        const original = new Uint8ClampedArray(img.data);\n        shiftPixels(img.data);\n        ctx.putImageData(img, 0, 0);\n        const result = origToDataURL.apply(this, arguments);\n        img.data.set(original);\n        ctx.putImageData(img, 0, 0);\n        return result;\n      } catch {\n        // Tainted canvas (cross-origin image) \u2014 can't read; skip the nudge\n        // and fall through to the original call so the page sees the\n        // expected SecurityError instead of silent corruption.\n      }\n    }\n    return origToDataURL.apply(this, arguments);\n  };\n  CanvasRenderingContext2D.prototype.getImageData = function() {\n    const img = origGetImageData.apply(this, arguments);\n    shiftPixels(img.data);\n    return img;\n  };\n";