@browserless/goto 10.9.16 → 10.10.0

package/README.md

@@ -1,14 +1,18 @@
 <div align="center">
-  <br>
   <img style="width: 500px; margin:3rem 0 1.5rem;" src="https://github.com/microlinkhq/browserless/raw/master/static/logo-banner.png#gh-light-mode-only" alt="browserless">
   <img style="width: 500px; margin:3rem 0 1.5rem;" src="https://github.com/microlinkhq/browserless/raw/master/static/logo-banner-light.png#gh-dark-mode-only" alt="browserless">
-  <br>
-  <br>
-  <p align="center"><strong>@browserless/goto</strong>: Go to a page aborting unnecessary requests.</p>
-  <p align="center">See <a href="https://browserless.js.org/#%2F%3Fid=gotopage-options" target='_blank' rel='noopener noreferrer'>goto</a> section our <a href="https://browserless.js.org" target='_blank' rel='noopener noreferrer'>website</a> for more information.</p>
-  <br>
+  <br><br>
+  <a href="https://microlink.io"><img src="https://img.shields.io/badge/powered_by-microlink.io-blue?style=flat-square&color=%23EA407B" alt="Powered by microlink.io"></a>
+  <img src="https://img.shields.io/github/tag/microlinkhq/browserless.svg?style=flat-square" alt="Last version">
+  <a href="https://coveralls.io/github/microlinkhq/browserless"><img src="https://img.shields.io/coveralls/microlinkhq/browserless.svg?style=flat-square" alt="Coverage Status"></a>
+  <a href="https://www.npmjs.org/package/browserless"><img src="https://img.shields.io/npm/dm/browserless.svg?style=flat-square" alt="NPM Status"></a>
+  <br><br>
 </div>
 
+> @browserless/goto: Go to a page aborting unnecessary requests.
+
+See the [Go To section](https://browserless.js.org/#/?id=gotopage-options) on our website for more details.
+
 ## Install
 
 Using npm:
@@ -17,6 +21,192 @@ Using npm:
 npm install @browserless/goto --save
 ```
 
+## About
+
+This package provides **advanced page navigation** with built-in ad blocking, smart waiting strategies, and extensive customization options. It's the core navigation engine that powers all browserless page loading operations.
+
+### What this package does
+
+The `@browserless/goto` package allows you to:
+
+- **Navigate to pages** with optimized loading and smart waiting strategies
+- **Block ads and trackers** using a precompiled Ghostery ad-blocker engine
+- **Inject scripts, modules, and styles** into pages
+- **Emulate devices** with viewport, user agent, and media features
+- **Intercept and abort requests** by resource type
+- **Handle cookies and authentication** seamlessly
+
+### Usage
+
+```js
+const createGoto = require('@browserless/goto')
+const puppeteer = require('puppeteer')
+
+const goto = createGoto({ 
+  timeout: 30000,
+  defaultDevice: 'Macbook Pro 13'
+})
+
+const browser = await puppeteer.launch()
+const page = await browser.newPage()
+
+const { response, device, error } = await goto(page, {
+  url: 'https://example.com',
+  adblock: true,
+  waitUntil: 'auto'
+})
+```
+
+### Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `url` | `string` | — | Target URL to navigate to |
+| `html` | `string` | — | HTML content to render (instead of URL) |
+| `adblock` | `boolean` | `true` | Enable built-in ad blocker |
+| `waitUntil` | `string\|string[]` | `'auto'` | Navigation wait condition |
+| `timeout` | `number` | `30000` | Navigation timeout in ms |
+| `device` | `string` | `'Macbook Pro 13'` | Device to emulate |
+| `headers` | `object` | `{}` | Extra HTTP headers |
+| `javascript` | `boolean` | `true` | Enable/disable JavaScript |
+| `animations` | `boolean` | `false` | Enable CSS animations |
+| `colorScheme` | `string` | — | `'light'` or `'dark'` preference |
+| `mediaType` | `string` | — | CSS media type (`'screen'`, `'print'`) |
+| `timezone` | `string` | — | Timezone to emulate |
+| `authenticate` | `object` | — | HTTP authentication credentials |
+| `scripts` | `string\|string[]` | — | Scripts to inject |
+| `modules` | `string\|string[]` | — | ES modules to inject |
+| `styles` | `string\|string[]` | — | Stylesheets to inject |
+| `click` | `string\|string[]` | — | CSS selectors to click |
+| `scroll` | `string` | — | CSS selector to scroll into view |
+| `abortTypes` | `string[]` | `[]` | Resource types to abort |
+| `waitForSelector` | `string` | — | Wait for selector to appear |
+| `waitForFunction` | `string` | — | Wait for function to return truthy |
+| `waitForTimeout` | `number` | — | Wait for specified milliseconds |
+| `onPageRequest` | `function` | — | Request interception handler |
+
+### Smart waiting with `waitUntil: 'auto'`
+
+The default `'auto'` mode intelligently waits for page readiness:
+
+```js
+// Auto mode combines 'load' with 'networkidle2' smartly
+await goto(page, { url: 'https://example.com', waitUntil: 'auto' })
+
+// Standard Puppeteer wait conditions also supported
+await goto(page, { url: 'https://example.com', waitUntil: 'networkidle0' })
+await goto(page, { url: 'https://example.com', waitUntil: ['load', 'domcontentloaded'] })
+```
+
+### Built-in Ad Blocker
+
+The package includes a precompiled [Ghostery ad-blocker](https://github.com/ghostery/adblocker) engine that blocks ads and trackers automatically:
+
+```js
+// Enabled by default
+await goto(page, { url: 'https://example.com', adblock: true })
+
+// Disable for specific pages
+await goto(page, { url: 'https://example.com', adblock: false })
+
+// The adblocker can be disabled mid-session
+page.disableAdblock()
+```
+
+Cookie consent handling (opt-out) is included as part of `adblock: true` and powered by [duckduckgo/autoconsent](https://github.com/duckduckgo/autoconsent).
+
+### Script and style injection
+
+Inject external resources or inline code:
+
+```js
+await goto(page, {
+  url: 'https://example.com',
+  // External URLs
+  scripts: ['https://cdn.example.com/library.js'],
+  // ES modules
+  modules: ['https://cdn.example.com/module.mjs'],
+  // CSS (URLs, paths, or inline)
+  styles: [
+    'https://cdn.example.com/styles.css',
+    'body { background: red; }'
+  ]
+})
+```
+
+### Request interception
+
+Abort specific resource types to speed up navigation:
+
+```js
+await goto(page, {
+  url: 'https://example.com',
+  abortTypes: ['image', 'stylesheet', 'font', 'media'],
+  onPageRequest: (request, page) => {
+    console.log('Request:', request.url())
+  }
+})
+```
+
+### Device emulation
+
+```js
+// Use preset device
+await goto(page, { url: 'https://example.com', device: 'iPhone 13' })
+
+// Custom viewport
+await goto(page, {
+  url: 'https://example.com',
+  viewport: { width: 1920, height: 1080 }
+})
+
+// Custom headers
+await goto(page, {
+  url: 'https://example.com',
+  headers: {
+    'user-agent': 'custom-agent',
+    'cookie': 'session=abc123'
+  }
+})
+```
+
+### Return value
+
+The `goto` function returns:
+
+```js
+const { response, device, error } = await goto(page, { url })
+
+// response: Puppeteer Response object (or undefined if navigation failed)
+// device: { userAgent, viewport } used for the request
+// error: Error object if navigation failed
+```
+
+### How it fits in the monorepo
+
+This is the **core navigation engine** used by the entire browserless ecosystem:
+
+| Consumer | Purpose |
+|----------|---------|
+| `browserless` (core) | Powers `.goto()`, `.html()`, `.text()`, `.pdf()`, `.screenshot()` |
+| `@browserless/screenshot` | Navigation before capturing screenshots |
+| `@browserless/pdf` | Navigation before generating PDFs |
+| `@browserless/function` | Navigation for sandboxed function execution |
+| `@browserless/lighthouse` | Navigation for Lighthouse audits |
+
+### Dependencies
+
+| Package | Purpose |
+|---------|---------|
+| `@browserless/devices` | Device descriptor lookups and emulation |
+| `@ghostery/adblocker-puppeteer` | Ad and tracker blocking |
+| `debug-logfmt` | Structured debug logging |
+| `got` | HTTP client for postinstall script |
+| `is-url-http` | Detect if value is URL for injection |
+| `p-reflect` / `p-timeout` | Promise utilities for timeouts |
+| `shallow-equal` | Viewport comparison optimization |
+| `tough-cookie` | Cookie string parsing |
+
 ## License
 
 **@browserless/goto** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/browserless/blob/master/LICENSE.md) License.<br>

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@browserless/goto",
-  "description": "Go to a page aborting unnecessary requests",
+  "description": "Navigate to web pages with built-in ad blocking, device emulation, and optimized loading for faster automation.",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "10.9.16",
+  "version": "10.10.0",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -18,18 +18,20 @@
     "url": "https://github.com/microlinkhq/browserless/issues"
   },
   "keywords": [
-    "browser",
     "browserless",
-    "chrome",
-    "chromeless",
-    "core",
     "goto",
+    "navigation",
     "headless",
-    "page",
-    "puppeteer"
+    "chrome",
+    "puppeteer",
+    "adblock",
+    "automation",
+    "device-emulation",
+    "performance"
   ],
   "dependencies": {
-    "@browserless/devices": "^10.7.13",
+    "@browserless/devices": "^10.9.18",
+    "@duckduckgo/autoconsent": "~14.53.0",
     "@ghostery/adblocker-puppeteer": "~2.13.4",
     "debug-logfmt": "~1.4.7",
     "got": "~11.8.6",
@@ -40,7 +42,7 @@
     "tough-cookie": "~6.0.0"
   },
   "devDependencies": {
-    "@browserless/test": "^10.9.16",
+    "@browserless/test": "^10.9.18",
     "ava": "5",
     "p-wait-for": "3"
   },
@@ -64,5 +66,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "b40c9ffbb5d346ff1f09857ca12fa1c876b92849"
+  "gitHead": "9b80e677418f2defb804d39043680fe65e3e277b"
 }

package/src/adblock.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const { PuppeteerBlocker } = require('@ghostery/adblocker-puppeteer')
+const path = require('path')
+const fs = require('fs')
+
+const debug = require('debug-logfmt')('browserless:goto:adblock')
+
+const engine = PuppeteerBlocker.deserialize(
+  new Uint8Array(fs.readFileSync(path.resolve(__dirname, './engine.bin')))
+)
+
+engine.on('request-blocked', ({ url }) => debug('block', url))
+engine.on('request-redirected', ({ url }) => debug('redirect', url))
+
+/**
+ * autoconsent.playwright.js is the only browser-injectable IIFE bundle in the package.
+ * It is not in the package's "exports" map, so pin @duckduckgo/autoconsent with ~ to
+ * avoid breakage from internal restructuring on minor/patch bumps.
+ */
+const autoconsentPlaywrightScript = fs.readFileSync(
+  path.resolve(
+    path.dirname(require.resolve('@duckduckgo/autoconsent')),
+    'autoconsent.playwright.js'
+  ),
+  'utf8'
+)
+
+/* Configuration passed to autoconsent's `initResp` message.
+   See https://github.com/duckduckgo/autoconsent/blob/main/api.md */
+const autoconsentConfig = Object.freeze({
+  /* activate consent rule matching */
+  enabled: true,
+  /* automatically reject (opt-out) all cookies */
+  autoAction: 'optOut',
+  /* hide banners early via CSS before detection finishes */
+  enablePrehide: true,
+  /* apply CSS-only rules that hide popups lacking a reject button */
+  enableCosmeticRules: true,
+  /* skip bundled ABP/uBO cosmetic filter list (saves bundle size) */
+  enableFilterList: false,
+  /* how many times to retry CMP detection (~50 ms apart) */
+  detectRetries: 20,
+  logs: {
+    /* CMP detection / opt-out lifecycle events */
+    lifecycle: false,
+    /* individual rule step execution */
+    rulesteps: false,
+    /* eval snippet calls */
+    evals: false,
+    /* rule errors */
+    errors: false,
+    /* background ↔ content-script messages */
+    messages: false
+  }
+})
+
+const sendMessage = (page, message) =>
+  page
+    .evaluate(msg => {
+      if (window.autoconsentReceiveMessage) {
+        return window.autoconsentReceiveMessage(msg)
+      }
+    }, message)
+    .catch(() => {})
+
+const setupAutoConsent = async page => {
+  if (page._autoconsentSetup) return
+
+  await page.exposeFunction('autoconsentSendMessage', async message => {
+    if (!message || typeof message !== 'object') return
+
+    if (message.type === 'init') {
+      return sendMessage(page, { type: 'initResp', config: autoconsentConfig })
+    }
+
+    if (message.type === 'eval') {
+      return sendMessage(page, { type: 'evalResp', id: message.id, result: false })
+    }
+  })
+
+  await page.evaluateOnNewDocument(autoconsentPlaywrightScript)
+  page._autoconsentSetup = true
+}
+
+const runAutoConsent = page => page.evaluate(autoconsentPlaywrightScript)
+
+const enableBlockingInPage = (page, run, actionTimeout) => {
+  page.disableAdblock = () =>
+    engine
+      .disableBlockingInPage(page, { keepRequestInterception: true })
+      .then(() => debug('disabled'))
+      .catch(() => {})
+
+  return [
+    run({
+      fn: setupAutoConsent(page),
+      timeout: actionTimeout,
+      debug: 'autoconsent:setup'
+    }),
+    run({
+      fn: engine.enableBlockingInPage(page),
+      timeout: actionTimeout,
+      debug: 'adblock'
+    })
+  ]
+}
+
+module.exports = { enableBlockingInPage, runAutoConsent }

package/src/index.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const { PuppeteerBlocker } = require('@ghostery/adblocker-puppeteer')
 const { shallowEqualObjects } = require('shallow-equal')
 const { setTimeout } = require('node:timers/promises')
 const createDevices = require('@browserless/devices')
@@ -8,25 +7,17 @@ const toughCookie = require('tough-cookie')
 const pReflect = require('p-reflect')
 const pTimeout = require('p-timeout')
 const isUrl = require('is-url-http')
-const path = require('path')
-const fs = require('fs')
 
 const { DEFAULT_INTERCEPT_RESOLUTION_PRIORITY } = require('puppeteer')
 
+const adblock = require('./adblock')
+
 const debug = require('debug-logfmt')('browserless:goto')
 debug.continue = require('debug-logfmt')('browserless:goto:continue')
 debug.abort = require('debug-logfmt')('browserless:goto:abort')
-debug.adblock = require('debug-logfmt')('browserless:goto:adblock')
 
 const truncate = (str, n = 80) => (str.length > n ? str.substr(0, n - 1) + '…' : str)
 
-const engine = PuppeteerBlocker.deserialize(
-  new Uint8Array(fs.readFileSync(path.resolve(__dirname, './engine.bin')))
-)
-
-engine.on('request-blocked', ({ url }) => debug.adblock('block', url))
-engine.on('request-redirected', ({ url }) => debug.adblock('redirect', url))
-
 const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
 const castArray = value => [].concat(value).filter(Boolean)
@@ -181,7 +172,7 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
     page,
     {
       abortTypes = [],
-      adblock = true,
+      adblock: withAdblock = true,
       animations = false,
       authenticate,
       click,
@@ -268,33 +259,8 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       })
     }
 
-    if (adblock) {
-      let adblockContext
-
-      page.disableAdblock = () => {
-        // TODO: drop this when https://github.com/ghostery/adblocker/pull/5161 is merged
-
-        engine.contexts.delete(page)
-
-        if (adblockContext.blocker.config.loadNetworkFilters) {
-          adblockContext.page.off('request', adblockContext.onRequest)
-        }
-
-        if (adblockContext.blocker.config.loadCosmeticFilters) {
-          adblockContext.page.off('frameattached', adblockContext.onFrameNavigated)
-          adblockContext.page.off('domcontentloaded', adblockContext.onDomContentLoaded)
-        }
-
-        debug.adblock('disabled')
-      }
-
-      prePromises.push(
-        run({
-          fn: engine.enableBlockingInPage(page).then(context => (adblockContext = context)),
-          timeout: actionTimeout,
-          debug: 'adblock'
-        })
-      )
+    if (withAdblock) {
+      prePromises.push(...adblock.enableBlockingInPage(page, run, actionTimeout))
     }
 
     if (javascript === false) {
@@ -398,6 +364,14 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       debug: { fn: html ? 'html' : 'url', waitUntil }
     })
 
+    if (withAdblock) {
+      await run({
+        fn: adblock.runAutoConsent(page),
+        timeout: actionTimeout,
+        debug: 'autoconsent:run'
+      })
+    }
+
     for (const [key, value] of Object.entries({
       waitForSelector,
       waitForFunction