@browserless/goto 10.11.4 → 10.12.7

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Navigate to web pages with built-in ad blocking, device emulation, and optimized loading for faster automation.",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "10.11.4",
+  "version": "10.12.7",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -31,18 +31,18 @@
   ],
   "dependencies": {
     "@browserless/devices": "^10.11.3",
-    "@duckduckgo/autoconsent": "~14.56.0",
+    "@duckduckgo/autoconsent": "~14.67.0",
     "@ghostery/adblocker-puppeteer": "~2.14.1",
-    "debug-logfmt": "~1.4.8",
+    "debug-logfmt": "~1.4.10",
     "got": "~11.8.6",
     "is-url-http": "~2.3.13",
     "p-reflect": "~2.1.0",
     "p-timeout": "~4.1.0",
     "shallow-equal": "~3.1.0",
-    "tough-cookie": "~6.0.0"
+    "tough-cookie": "~6.0.1"
   },
   "devDependencies": {
-    "@browserless/test": "^10.11.4",
+    "@browserless/test": "^10.12.6",
     "ava": "5",
     "p-wait-for": "3"
   },
@@ -66,5 +66,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "9ff2a0f0dad20d64a5c47a04a989ee77d1ee5ae8"
+  "gitHead": "398978a3237780fa44ee70e9ffeb30759cab50a9"
 }

package/src/adblock.js

@@ -1,60 +1,67 @@
 'use strict'
 
 const { PuppeteerBlocker } = require('@ghostery/adblocker-puppeteer')
+const { randomUUID } = require('crypto')
+const pTimeout = require('p-timeout')
 const fs = require('fs/promises')
 const path = require('path')
 
 const debug = require('debug-logfmt')('browserless:goto:adblock')
 
-let enginePromise
+const lazy = fn => {
+  let p
+  return () => (p ??= fn())
+}
 
-const getEngine = () => {
-  if (enginePromise) return enginePromise
+const autoconsentDir = path.dirname(require.resolve('@duckduckgo/autoconsent'))
 
-  enginePromise = fs.readFile(path.resolve(__dirname, './engine.bin')).then(buffer => {
+const getEngine = lazy(() =>
+  fs.readFile(path.resolve(__dirname, './engine.bin')).then(buffer => {
     const engine = PuppeteerBlocker.deserialize(new Uint8Array(buffer))
     engine.on('request-blocked', ({ url }) => debug('block', url))
     engine.on('request-redirected', ({ url }) => debug('redirect', url))
     return engine
   })
-
-  return enginePromise
-}
+)
 
 /**
  * autoconsent.playwright.js is the only browser-injectable IIFE bundle in the package.
  * It is not in the package's "exports" map, so pin @duckduckgo/autoconsent with ~ to
  * avoid breakage from internal restructuring on minor/patch bumps.
  */
-let autoconsentPlaywrightScriptPromise
-
-const getAutoconsentPlaywrightScript = () => {
-  if (autoconsentPlaywrightScriptPromise) return autoconsentPlaywrightScriptPromise
+const getAutoconsentPlaywrightScript = lazy(() =>
+  fs.readFile(path.resolve(autoconsentDir, 'autoconsent.playwright.js'), 'utf8')
+)
 
-  autoconsentPlaywrightScriptPromise = fs.readFile(
-    path.resolve(
-      path.dirname(require.resolve('@duckduckgo/autoconsent')),
-      'autoconsent.playwright.js'
-    ),
-    'utf8'
-  )
-
-  return autoconsentPlaywrightScriptPromise
-}
+const getAutoconsentRules = lazy(() =>
+  fs.readFile(path.resolve(autoconsentDir, '../rules/compact-rules.json'), 'utf8').then(JSON.parse)
+)
 
 /* Configuration passed to autoconsent's `initResp` message.
-   See https://github.com/duckduckgo/autoconsent/blob/main/api.md */
+   See https://github.com/duckduckgo/autoconsent/blob/main/docs/api.md */
 const autoconsentConfig = Object.freeze({
   /* activate consent rule matching */
   enabled: true,
   /* automatically reject (opt-out) all cookies */
   autoAction: 'optOut',
+  /* skip these CMPs even if detected */
+  disabledCmps: [],
   /* hide banners early via CSS before detection finishes */
   enablePrehide: true,
   /* apply CSS-only rules that hide popups lacking a reject button */
   enableCosmeticRules: true,
+  /* enable rules auto-generated from common CMP patterns */
+  enableGeneratedRules: true,
   /* skip bundled ABP/uBO cosmetic filter list (saves bundle size) */
   enableFilterList: false,
+  /* detect CMPs using heuristics when no specific rule matches */
+  enableHeuristicDetection: true,
+  /* fall back to heuristic click when no specific rule matches */
+  enableHeuristicAction: true,
+  /* run in the page's main world (false = isolated world) */
+  isMainWorld: false,
+  /* max ms to keep prehide CSS applied before removing it */
+  prehideTimeout: 2000,
   /* how many times to retry CMP detection (~50 ms apart) */
   detectRetries: 20,
   logs: {
@@ -62,12 +69,16 @@ const autoconsentConfig = Object.freeze({
     lifecycle: false,
     /* individual rule step execution */
     rulesteps: false,
+    /* CMP detection step details */
+    detectionsteps: false,
     /* eval snippet calls */
     evals: false,
     /* rule errors */
     errors: false,
     /* background ↔ content-script messages */
-    messages: false
+    messages: false,
+    /* wait/delay step timing */
+    waits: false
   }
 })
 
@@ -80,29 +91,63 @@ const sendMessage = (page, message) =>
     }, message)
     .catch(() => {})
 
-const setupAutoConsent = async page => {
+const setupAutoConsent = async (page, timeout) => {
   if (page._autoconsentSetup) return
   const autoconsentPlaywrightScript = await getAutoconsentPlaywrightScript()
+  const nonce = randomUUID()
 
   await page.exposeFunction('autoconsentSendMessage', async message => {
     if (!message || typeof message !== 'object') return
+    if (message.__nonce !== nonce) return
 
-    if (message.type === 'init') {
-      return sendMessage(page, { type: 'initResp', config: autoconsentConfig })
-    }
+    switch (message.type) {
+      case 'init': {
+        page._autoconsentInitDone = true
+        const rules = await getAutoconsentRules()
+        return sendMessage(page, { type: 'initResp', config: autoconsentConfig, rules })
+      }
+
+      case 'eval': {
+        let result = false
+        try {
+          result = await pTimeout(page.evaluate(message.code), timeout)
+        } catch {}
+        return sendMessage(page, { type: 'evalResp', id: message.id, result })
+      }
+
+      case 'cmpDetected':
+      case 'popupFound':
+      case 'autoconsentDone':
+        debug(message.type, { cmp: message.cmp })
+        break
 
-    if (message.type === 'eval') {
-      return sendMessage(page, { type: 'evalResp', id: message.id, result: false })
+      case 'optOutResult':
+        debug(message.type, { result: message.result })
+        break
+
+      case 'autoconsentError':
+        debug(message.type, { details: message.details })
+        break
     }
   })
 
-  await page.evaluateOnNewDocument(autoconsentPlaywrightScript)
+  /* Single injection: wrap the binding in the top frame so every outgoing
+     message carries the nonce, then run the autoconsent script. Child frames
+     keep the raw CDP binding which lacks the nonce, so their messages are
+     silently rejected. */
+  const nonceGuard = `(function(n){if(window.self!==window.top)return;var raw=window.autoconsentSendMessage;if(raw)window.autoconsentSendMessage=function(msg){return raw(Object.assign({},msg,{__nonce:n}))}})(${JSON.stringify(nonce)});`
+  await page.evaluateOnNewDocument(nonceGuard + autoconsentPlaywrightScript)
   page._autoconsentSetup = true
 }
 
-const runAutoConsent = async page => page.evaluate(await getAutoconsentPlaywrightScript())
+const runAutoConsent = async page => {
+  if (page._autoconsentInitDone) return
+  return page.evaluate(await getAutoconsentPlaywrightScript())
+}
+
+const enableBlockingInPage = (page, run, timeout) => {
+  getAutoconsentRules().catch(() => {})
 
-const enableBlockingInPage = (page, run, actionTimeout) => {
   page.disableAdblock = () =>
     getEngine()
       .then(engine => engine.disableBlockingInPage(page, { keepRequestInterception: true }))
@@ -111,13 +156,13 @@ const enableBlockingInPage = (page, run, actionTimeout) => {
 
   return [
     run({
-      fn: setupAutoConsent(page),
-      timeout: actionTimeout,
+      fn: setupAutoConsent(page, timeout),
+      timeout,
       debug: 'autoconsent:setup'
     }),
     run({
       fn: getEngine().then(engine => engine.enableBlockingInPage(page)),
-      timeout: actionTimeout,
+      timeout,
       debug: 'adblock'
     })
   ]