@browserless/goto 10.10.1 → 10.11.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Navigate to web pages with built-in ad blocking, device emulation, and optimized loading for faster automation.",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "10.10.1",
+  "version": "10.11.0",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -66,5 +66,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "32f6e72bcb489a83ac9659520a3961aeb97c47b7"
+  "gitHead": "e6bd03bae91b08baaa6f1c0c0c9743f6c7de3d29"
 }

package/src/adblock.js

@@ -1,8 +1,8 @@
 'use strict'
 
 const { PuppeteerBlocker } = require('@ghostery/adblocker-puppeteer')
+const fs = require('fs/promises')
 const path = require('path')
-const fs = require('fs')
 
 const debug = require('debug-logfmt')('browserless:goto:adblock')
 
@@ -11,7 +11,7 @@ let enginePromise
 const getEngine = () => {
   if (enginePromise) return enginePromise
 
-  enginePromise = fs.promises.readFile(path.resolve(__dirname, './engine.bin')).then(buffer => {
+  enginePromise = fs.readFile(path.resolve(__dirname, './engine.bin')).then(buffer => {
     const engine = PuppeteerBlocker.deserialize(new Uint8Array(buffer))
     engine.on('request-blocked', ({ url }) => debug('block', url))
     engine.on('request-redirected', ({ url }) => debug('redirect', url))
@@ -31,7 +31,7 @@ let autoconsentPlaywrightScriptPromise
 const getAutoconsentPlaywrightScript = () => {
   if (autoconsentPlaywrightScriptPromise) return autoconsentPlaywrightScriptPromise
 
-  autoconsentPlaywrightScriptPromise = fs.promises.readFile(
+  autoconsentPlaywrightScriptPromise = fs.readFile(
     path.resolve(
       path.dirname(require.resolve('@duckduckgo/autoconsent')),
       'autoconsent.playwright.js'

package/src/index.js

@@ -22,15 +22,15 @@ const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
 const castArray = value => [].concat(value).filter(Boolean)
 
-const run = async ({ fn, timeout, debug: props }) => {
-  const duration = debug.duration()
-  const result = await pReflect(timeout ? pTimeout(fn, timeout) : fn)
-  const errorProps = result.isRejected ? { error: result.reason.message || result.reason } : {}
-  duration(props, errorProps)
-  return result
+const getDefaultPath = pathname => {
+  if (!pathname || pathname[0] !== '/') return '/'
+  if (pathname === '/') return '/'
+
+  const rightSlash = pathname.lastIndexOf('/')
+  return rightSlash === 0 ? '/' : pathname.slice(0, rightSlash)
 }
 
-const parseCookies = (url, str) => {
+const parseCookiesWithJar = (url, str) => {
   const jar = new toughCookie.CookieJar(undefined, { rejectPublicSuffixes: false })
 
   return str.split(';').reduce((acc, cookieStr) => {
@@ -50,6 +50,81 @@ const parseCookies = (url, str) => {
   }, [])
 }
 
+const run = async ({ fn, timeout, debug: props }) => {
+  const duration = debug.duration()
+  const result = await pReflect(timeout ? pTimeout(fn, timeout) : fn)
+  const errorProps = result.isRejected ? { error: result.reason.message || result.reason } : {}
+  duration(props, errorProps)
+  return result
+}
+
+const stopLoadingOnTimeout = (page, timeout) => {
+  let timeoutId
+
+  return {
+    promise: new Promise(resolve => {
+      timeoutId = globalThis.setTimeout(() => {
+        pReflect(page._client().send('Page.stopLoading')).then(resolve)
+      }, timeout)
+
+      if (typeof timeoutId.unref === 'function') timeoutId.unref()
+    }),
+    clear: () => {
+      if (timeoutId) clearTimeout(timeoutId)
+    }
+  }
+}
+
+const parseCookies = (url, str) => {
+  let parsedURL
+
+  try {
+    parsedURL = new URL(url)
+  } catch {
+    return parseCookiesWithJar(url, str)
+  }
+
+  const domain = parsedURL.hostname
+
+  if (!domain) {
+    return parseCookiesWithJar(url, str)
+  }
+
+  const path = getDefaultPath(parsedURL.pathname)
+  const chunks = str.split(';')
+  const cookies = new Array(chunks.length)
+  let index = 0
+
+  for (const chunk of chunks) {
+    const cookieStr = chunk.trim()
+
+    if (cookieStr.length === 0) {
+      return parseCookiesWithJar(url, str)
+    }
+
+    const separatorIndex = cookieStr.indexOf('=')
+
+    if (separatorIndex === -1) {
+      return parseCookiesWithJar(url, str)
+    }
+
+    const name = cookieStr.slice(0, separatorIndex).trim()
+
+    if (name.length === 0) {
+      return parseCookiesWithJar(url, str)
+    }
+
+    cookies[index++] = {
+      name,
+      value: cookieStr.slice(separatorIndex + 1).trim(),
+      domain,
+      path
+    }
+  }
+
+  return cookies
+}
+
 const getMediaFeatures = ({ animations, colorScheme }) => {
   const prefers = []
   if (animations === false) prefers.push({ name: 'prefers-reduced-motion', value: 'reduce' })
@@ -225,37 +300,51 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       )
     }
 
-    const enableInterception =
-      (onPageRequest || abortTypes.length > 0) &&
-      run({
-        fn: page.setRequestInterception(true),
-        debug: 'enableInterception'
-      })
+    const abortTypesSet = abortTypes.length > 0 ? new Set(abortTypes) : null
+
+    const requestHandlers = []
+    let abortTypesHandler
+    let disableInterceptionForAbortTypes = false
 
     if (onPageRequest) {
-      Promise.resolve(enableInterception).then(() =>
-        page.on('request', req => onPageRequest(req, page))
-      )
+      const onPageRequestHandler = req => onPageRequest(req, page)
+      page.on('request', onPageRequestHandler)
+      requestHandlers.push(onPageRequestHandler)
     }
 
     if (abortTypes.length > 0) {
-      Promise.resolve(enableInterception).then(() => {
-        page.on('request', req => {
-          if (req.isInterceptResolutionHandled()) return
-          const resourceType = req.resourceType()
-          const url = truncate(req.url())
-
-          if (!abortTypes.includes(resourceType)) {
-            debug.continue({ url, resourceType })
-            return req.continue(
-              req.continueRequestOverrides(),
-              DEFAULT_INTERCEPT_RESOLUTION_PRIORITY
-            )
+      abortTypesHandler = req => {
+        if (req.isInterceptResolutionHandled()) return
+        const resourceType = req.resourceType()
+        const url = truncate(req.url())
+
+        if (!abortTypesSet.has(resourceType)) {
+          debug.continue({ url, resourceType })
+          return req.continue(req.continueRequestOverrides(), DEFAULT_INTERCEPT_RESOLUTION_PRIORITY)
+        }
+        debug.abort({ url, resourceType })
+        return req.abort('blockedbyclient', DEFAULT_INTERCEPT_RESOLUTION_PRIORITY)
+      }
+
+      page.on('request', abortTypesHandler)
+      requestHandlers.push(abortTypesHandler)
+    }
+
+    if (requestHandlers.length > 0) {
+      prePromises.push(
+        run({
+          fn: page.setRequestInterception(true),
+          debug: 'enableInterception'
+        }).then(result => {
+          // If interception setup fails, remove handlers to avoid keeping dead listeners.
+          if (result.isRejected) {
+            requestHandlers.forEach(handler => page.off('request', handler))
+          } else if (abortTypesHandler && !withAdblock && !onPageRequest) {
+            disableInterceptionForAbortTypes = true
           }
-          debug.abort({ url, resourceType })
-          return req.abort('blockedbyclient', DEFAULT_INTERCEPT_RESOLUTION_PRIORITY)
+          return result
         })
-      })
+      )
     }
 
     if (withAdblock) {
@@ -272,7 +361,11 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       )
     }
 
-    const device = getDevice({ headers, ...args, device: args.device ?? defaultDevice })
+    const device = getDevice({
+      headers,
+      device: args.device ?? defaultDevice,
+      viewport: args.viewport
+    })
 
     if (device.userAgent && !headers['user-agent']) {
       headers['user-agent'] = device.userAgent
@@ -291,10 +384,11 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
     const headersKeys = Object.keys(headers)
 
     if (headersKeys.length > 0) {
-      const { cookie, ...headersWithoutCookie } = headers
+      const cookie = headers.cookie
+      const userAgent = headers['user-agent']
 
-      if (headers.cookie) {
-        const cookies = parseCookies(url, headers.cookie)
+      if (cookie) {
+        const cookies = parseCookies(url, cookie)
         prePromises.push(
           run({
             fn: page.setCookie(...cookies),
@@ -304,25 +398,41 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
         )
       }
 
-      const extraHTTPHeaders = headers.cookie ? headersWithoutCookie : headers
-      const extraHTTPHeadersKeys = Object.keys(extraHTTPHeaders)
-
-      if (headers['user-agent']) {
+      if (userAgent) {
         prePromises.push(
           run({
-            fn: page.setUserAgent(headers['user-agent']),
+            fn: page.setUserAgent(userAgent),
             timeout: actionTimeout,
-            debug: { 'user-agent': headers['user-agent'] }
+            debug: { 'user-agent': userAgent }
           })
         )
       }
 
-      if (extraHTTPHeadersKeys.length > 0) {
+      if (cookie) {
+        const extraHTTPHeaders = {}
+        const extraHTTPHeadersKeys = []
+
+        for (const key of headersKeys) {
+          if (key === 'cookie') continue
+          extraHTTPHeaders[key] = headers[key]
+          extraHTTPHeadersKeys.push(key)
+        }
+
+        if (extraHTTPHeadersKeys.length > 0) {
+          prePromises.push(
+            run({
+              fn: page.setExtraHTTPHeaders(extraHTTPHeaders),
+              timeout: actionTimeout,
+              debug: { headers: extraHTTPHeadersKeys }
+            })
+          )
+        }
+      } else if (!(userAgent && headersKeys.length === 1)) {
         prePromises.push(
           run({
-            fn: page.setExtraHTTPHeaders(extraHTTPHeaders),
+            fn: page.setExtraHTTPHeaders(headers),
             timeout: actionTimeout,
-            debug: { headers: extraHTTPHeadersKeys }
+            debug: { headers: headersKeys }
           })
         )
       }
@@ -350,72 +460,91 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       )
     }
 
-    await Promise.all(prePromises)
+    try {
+      await Promise.all(prePromises)
 
-    const { value: response, reason: error } = await run({
-      fn: html
+      let clearStopLoadingTimer = () => {}
+      const navigationPromise = html
         ? page.setContent(html, { waitUntil, ...args })
-        : Promise.race([
-          page.goto(url, { waitUntil, ...args }),
-          setTimeout(gotoTimeout).then(() => page._client().send('Page.stopLoading'))
-        ]),
-      timeout: gotoTimeout,
-      debug: { fn: html ? 'html' : 'url', waitUntil }
-    })
-
-    if (withAdblock) {
-      await run({
-        fn: adblock.runAutoConsent(page),
-        timeout: actionTimeout,
-        debug: 'autoconsent:run'
+        : (() => {
+            const { promise, clear } = stopLoadingOnTimeout(page, gotoTimeout)
+            clearStopLoadingTimer = clear
+            return Promise.race([page.goto(url, { waitUntil, ...args }), promise])
+          })()
+
+      const { value: response, reason: error } = await run({
+        fn: navigationPromise,
+        timeout: gotoTimeout,
+        debug: { fn: html ? 'html' : 'url', waitUntil }
       })
-    }
+      clearStopLoadingTimer()
 
-    for (const [key, value] of Object.entries({
-      waitForSelector,
-      waitForFunction
-    })) {
-      if (value) {
-        await run({ fn: page[key](value), timeout: gotoTimeout, debug: { [key]: value } })
+      if (withAdblock) {
+        await run({
+          fn: adblock.runAutoConsent(page),
+          timeout: actionTimeout,
+          debug: 'autoconsent:run'
+        })
       }
-    }
 
-    if (waitForTimeout) {
-      await setTimeout(waitForTimeout)
-    }
-
-    await inject(page, {
-      timeout: actionTimeout,
-      mediaType,
-      animations,
-      modules,
-      scripts,
-      styles
-    })
+      if (waitForSelector) {
+        await run({
+          fn: page.waitForSelector(waitForSelector),
+          timeout: gotoTimeout,
+          debug: { waitForSelector }
+        })
+      }
 
-    if (click) {
-      for (const selector of castArray(click)) {
+      if (waitForFunction) {
         await run({
-          fn: page.click(selector),
-          timeout: actionTimeout,
-          debug: { click: selector }
+          fn: page.waitForFunction(waitForFunction),
+          timeout: gotoTimeout,
+          debug: { waitForFunction }
         })
       }
-    }
 
-    if (scroll) {
-      await run({
-        fn: page.$eval(scroll, el => el.scrollIntoView()),
+      if (waitForTimeout) {
+        await setTimeout(waitForTimeout)
+      }
+
+      await inject(page, {
         timeout: actionTimeout,
-        debug: { scroll }
+        mediaType,
+        animations,
+        modules,
+        scripts,
+        styles
       })
-    }
 
-    if (isWaitUntilAuto) {
-      await waitUntilAuto(page, { response, timeout: actionTimeout * 2 })
-    }
+      if (click) {
+        for (const selector of castArray(click)) {
+          await run({
+            fn: page.click(selector),
+            timeout: actionTimeout,
+            debug: { click: selector }
+          })
+        }
+      }
+
+      if (scroll) {
+        await run({
+          fn: page.$eval(scroll, el => el.scrollIntoView()),
+          timeout: actionTimeout,
+          debug: { scroll }
+        })
+      }
+
+      if (isWaitUntilAuto) {
+        await waitUntilAuto(page, { response, timeout: actionTimeout * 2 })
+      }
 
-    return { response, device, error }
+      return { response, device, error }
+    } finally {
+      if (abortTypesHandler) page.off('request', abortTypesHandler)
+      if (disableInterceptionForAbortTypes) {
+        await pReflect(page.setRequestInterception(false))
+      }
+    }
   }
 
   goto.getDevice = getDevice