@browserless/goto 10.9.18 → 10.10.1

package/README.md

@@ -113,6 +113,8 @@ await goto(page, { url: 'https://example.com', adblock: false })
 page.disableAdblock()
 ```
 
+Cookie consent handling (opt-out) is included as part of `adblock: true` and powered by [duckduckgo/autoconsent](https://github.com/duckduckgo/autoconsent).
+
 ### Script and style injection
 
 Inject external resources or inline code:

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Navigate to web pages with built-in ad blocking, device emulation, and optimized loading for faster automation.",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "10.9.18",
+  "version": "10.10.1",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -31,6 +31,7 @@
   ],
   "dependencies": {
     "@browserless/devices": "^10.9.18",
+    "@duckduckgo/autoconsent": "~14.53.0",
     "@ghostery/adblocker-puppeteer": "~2.13.4",
     "debug-logfmt": "~1.4.7",
     "got": "~11.8.6",
@@ -65,5 +66,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "f5e8cd0788e4bad3b3ad9b007943754f96653817"
+  "gitHead": "32f6e72bcb489a83ac9659520a3961aeb97c47b7"
 }

package/src/adblock.js

@@ -0,0 +1,126 @@
+'use strict'
+
+const { PuppeteerBlocker } = require('@ghostery/adblocker-puppeteer')
+const path = require('path')
+const fs = require('fs')
+
+const debug = require('debug-logfmt')('browserless:goto:adblock')
+
+let enginePromise
+
+const getEngine = () => {
+  if (enginePromise) return enginePromise
+
+  enginePromise = fs.promises.readFile(path.resolve(__dirname, './engine.bin')).then(buffer => {
+    const engine = PuppeteerBlocker.deserialize(new Uint8Array(buffer))
+    engine.on('request-blocked', ({ url }) => debug('block', url))
+    engine.on('request-redirected', ({ url }) => debug('redirect', url))
+    return engine
+  })
+
+  return enginePromise
+}
+
+/**
+ * autoconsent.playwright.js is the only browser-injectable IIFE bundle in the package.
+ * It is not in the package's "exports" map, so pin @duckduckgo/autoconsent with ~ to
+ * avoid breakage from internal restructuring on minor/patch bumps.
+ */
+let autoconsentPlaywrightScriptPromise
+
+const getAutoconsentPlaywrightScript = () => {
+  if (autoconsentPlaywrightScriptPromise) return autoconsentPlaywrightScriptPromise
+
+  autoconsentPlaywrightScriptPromise = fs.promises.readFile(
+    path.resolve(
+      path.dirname(require.resolve('@duckduckgo/autoconsent')),
+      'autoconsent.playwright.js'
+    ),
+    'utf8'
+  )
+
+  return autoconsentPlaywrightScriptPromise
+}
+
+/* Configuration passed to autoconsent's `initResp` message.
+   See https://github.com/duckduckgo/autoconsent/blob/main/api.md */
+const autoconsentConfig = Object.freeze({
+  /* activate consent rule matching */
+  enabled: true,
+  /* automatically reject (opt-out) all cookies */
+  autoAction: 'optOut',
+  /* hide banners early via CSS before detection finishes */
+  enablePrehide: true,
+  /* apply CSS-only rules that hide popups lacking a reject button */
+  enableCosmeticRules: true,
+  /* skip bundled ABP/uBO cosmetic filter list (saves bundle size) */
+  enableFilterList: false,
+  /* how many times to retry CMP detection (~50 ms apart) */
+  detectRetries: 20,
+  logs: {
+    /* CMP detection / opt-out lifecycle events */
+    lifecycle: false,
+    /* individual rule step execution */
+    rulesteps: false,
+    /* eval snippet calls */
+    evals: false,
+    /* rule errors */
+    errors: false,
+    /* background ↔ content-script messages */
+    messages: false
+  }
+})
+
+const sendMessage = (page, message) =>
+  page
+    .evaluate(msg => {
+      if (window.autoconsentReceiveMessage) {
+        return window.autoconsentReceiveMessage(msg)
+      }
+    }, message)
+    .catch(() => {})
+
+const setupAutoConsent = async page => {
+  if (page._autoconsentSetup) return
+  const autoconsentPlaywrightScript = await getAutoconsentPlaywrightScript()
+
+  await page.exposeFunction('autoconsentSendMessage', async message => {
+    if (!message || typeof message !== 'object') return
+
+    if (message.type === 'init') {
+      return sendMessage(page, { type: 'initResp', config: autoconsentConfig })
+    }
+
+    if (message.type === 'eval') {
+      return sendMessage(page, { type: 'evalResp', id: message.id, result: false })
+    }
+  })
+
+  await page.evaluateOnNewDocument(autoconsentPlaywrightScript)
+  page._autoconsentSetup = true
+}
+
+const runAutoConsent = async page => page.evaluate(await getAutoconsentPlaywrightScript())
+
+const enableBlockingInPage = (page, run, actionTimeout) => {
+  page.disableAdblock = () =>
+    getEngine()
+      .then(engine => engine.disableBlockingInPage(page, { keepRequestInterception: true }))
+      .then(() => debug('disabled'))
+      .catch(() => {})
+
+  return [
+    run({
+      fn: setupAutoConsent(page),
+      timeout: actionTimeout,
+      debug: 'autoconsent:setup'
+    }),
+    run({
+      fn: getEngine().then(engine => engine.enableBlockingInPage(page)),
+      timeout: actionTimeout,
+      debug: 'adblock'
+    })
+  ]
+}
+
+module.exports = { enableBlockingInPage, runAutoConsent }

package/src/index.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const { PuppeteerBlocker } = require('@ghostery/adblocker-puppeteer')
 const { shallowEqualObjects } = require('shallow-equal')
 const { setTimeout } = require('node:timers/promises')
 const createDevices = require('@browserless/devices')
@@ -8,25 +7,17 @@ const toughCookie = require('tough-cookie')
 const pReflect = require('p-reflect')
 const pTimeout = require('p-timeout')
 const isUrl = require('is-url-http')
-const path = require('path')
-const fs = require('fs')
 
 const { DEFAULT_INTERCEPT_RESOLUTION_PRIORITY } = require('puppeteer')
 
+const adblock = require('./adblock')
+
 const debug = require('debug-logfmt')('browserless:goto')
 debug.continue = require('debug-logfmt')('browserless:goto:continue')
 debug.abort = require('debug-logfmt')('browserless:goto:abort')
-debug.adblock = require('debug-logfmt')('browserless:goto:adblock')
 
 const truncate = (str, n = 80) => (str.length > n ? str.substr(0, n - 1) + '…' : str)
 
-const engine = PuppeteerBlocker.deserialize(
-  new Uint8Array(fs.readFileSync(path.resolve(__dirname, './engine.bin')))
-)
-
-engine.on('request-blocked', ({ url }) => debug.adblock('block', url))
-engine.on('request-redirected', ({ url }) => debug.adblock('redirect', url))
-
 const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
 const castArray = value => [].concat(value).filter(Boolean)
@@ -39,15 +30,13 @@ const run = async ({ fn, timeout, debug: props }) => {
   return result
 }
 
-const parseCookies = (url, str) =>
-  str.split(';').reduce((acc, cookieStr) => {
-    const jar = new toughCookie.CookieJar(undefined, { rejectPublicSuffixes: false })
-    jar.setCookieSync(cookieStr.trim(), url)
-    const parsedCookie = jar.serializeSync().cookies[0]
+const parseCookies = (url, str) => {
+  const jar = new toughCookie.CookieJar(undefined, { rejectPublicSuffixes: false })
 
-    // Use this instead of the above when the following issue is fixed:
-    // https://github.com/salesforce/tough-cookie/issues/149
-    // const ret = toughCookie.parse(cookie).serializeSync();
+  return str.split(';').reduce((acc, cookieStr) => {
+    const cookie = jar.setCookieSync(cookieStr.trim(), url)
+    if (!cookie) return acc
+    const parsedCookie = cookie.toJSON()
 
     parsedCookie.name = parsedCookie.key
     delete parsedCookie.key
@@ -59,6 +48,7 @@ const parseCookies = (url, str) =>
     acc.push(parsedCookie)
     return acc
   }, [])
+}
 
 const getMediaFeatures = ({ animations, colorScheme }) => {
   const prefers = []
@@ -181,7 +171,7 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
     page,
     {
       abortTypes = [],
-      adblock = true,
+      adblock: withAdblock = true,
       animations = false,
       authenticate,
       click,
@@ -268,33 +258,8 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       })
     }
 
-    if (adblock) {
-      let adblockContext
-
-      page.disableAdblock = () => {
-        // TODO: drop this when https://github.com/ghostery/adblocker/pull/5161 is merged
-
-        engine.contexts.delete(page)
-
-        if (adblockContext.blocker.config.loadNetworkFilters) {
-          adblockContext.page.off('request', adblockContext.onRequest)
-        }
-
-        if (adblockContext.blocker.config.loadCosmeticFilters) {
-          adblockContext.page.off('frameattached', adblockContext.onFrameNavigated)
-          adblockContext.page.off('domcontentloaded', adblockContext.onDomContentLoaded)
-        }
-
-        debug.adblock('disabled')
-      }
-
-      prePromises.push(
-        run({
-          fn: engine.enableBlockingInPage(page).then(context => (adblockContext = context)),
-          timeout: actionTimeout,
-          debug: 'adblock'
-        })
-      )
+    if (withAdblock) {
+      prePromises.push(...adblock.enableBlockingInPage(page, run, actionTimeout))
     }
 
     if (javascript === false) {
@@ -398,6 +363,14 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       debug: { fn: html ? 'html' : 'url', waitUntil }
     })
 
+    if (withAdblock) {
+      await run({
+        fn: adblock.runAutoConsent(page),
+        timeout: actionTimeout,
+        debug: 'autoconsent:run'
+      })
+    }
+
     for (const [key, value] of Object.entries({
       waitForSelector,
       waitForFunction