@browserless/goto 10.10.0 → 10.10.2

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Navigate to web pages with built-in ad blocking, device emulation, and optimized loading for faster automation.",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "10.10.0",
+  "version": "10.10.2",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -66,5 +66,5 @@
     "timeout": "2m",
     "workerThreads": false
   },
-  "gitHead": "9b80e677418f2defb804d39043680fe65e3e277b"
+  "gitHead": "ec6a614923a1a692bd717ecc8e6f1b09417801d9"
 }

package/src/adblock.js

@@ -6,25 +6,41 @@ const fs = require('fs')
 
 const debug = require('debug-logfmt')('browserless:goto:adblock')
 
-const engine = PuppeteerBlocker.deserialize(
-  new Uint8Array(fs.readFileSync(path.resolve(__dirname, './engine.bin')))
-)
+let enginePromise
 
-engine.on('request-blocked', ({ url }) => debug('block', url))
-engine.on('request-redirected', ({ url }) => debug('redirect', url))
+const getEngine = () => {
+  if (enginePromise) return enginePromise
+
+  enginePromise = fs.promises.readFile(path.resolve(__dirname, './engine.bin')).then(buffer => {
+    const engine = PuppeteerBlocker.deserialize(new Uint8Array(buffer))
+    engine.on('request-blocked', ({ url }) => debug('block', url))
+    engine.on('request-redirected', ({ url }) => debug('redirect', url))
+    return engine
+  })
+
+  return enginePromise
+}
 
 /**
  * autoconsent.playwright.js is the only browser-injectable IIFE bundle in the package.
  * It is not in the package's "exports" map, so pin @duckduckgo/autoconsent with ~ to
  * avoid breakage from internal restructuring on minor/patch bumps.
  */
-const autoconsentPlaywrightScript = fs.readFileSync(
-  path.resolve(
-    path.dirname(require.resolve('@duckduckgo/autoconsent')),
-    'autoconsent.playwright.js'
-  ),
-  'utf8'
-)
+let autoconsentPlaywrightScriptPromise
+
+const getAutoconsentPlaywrightScript = () => {
+  if (autoconsentPlaywrightScriptPromise) return autoconsentPlaywrightScriptPromise
+
+  autoconsentPlaywrightScriptPromise = fs.promises.readFile(
+    path.resolve(
+      path.dirname(require.resolve('@duckduckgo/autoconsent')),
+      'autoconsent.playwright.js'
+    ),
+    'utf8'
+  )
+
+  return autoconsentPlaywrightScriptPromise
+}
 
 /* Configuration passed to autoconsent's `initResp` message.
    See https://github.com/duckduckgo/autoconsent/blob/main/api.md */
@@ -66,6 +82,7 @@ const sendMessage = (page, message) =>
 
 const setupAutoConsent = async page => {
   if (page._autoconsentSetup) return
+  const autoconsentPlaywrightScript = await getAutoconsentPlaywrightScript()
 
   await page.exposeFunction('autoconsentSendMessage', async message => {
     if (!message || typeof message !== 'object') return
@@ -83,12 +100,12 @@ const setupAutoConsent = async page => {
   page._autoconsentSetup = true
 }
 
-const runAutoConsent = page => page.evaluate(autoconsentPlaywrightScript)
+const runAutoConsent = async page => page.evaluate(await getAutoconsentPlaywrightScript())
 
 const enableBlockingInPage = (page, run, actionTimeout) => {
   page.disableAdblock = () =>
-    engine
-      .disableBlockingInPage(page, { keepRequestInterception: true })
+    getEngine()
+      .then(engine => engine.disableBlockingInPage(page, { keepRequestInterception: true }))
       .then(() => debug('disabled'))
       .catch(() => {})
 
@@ -99,7 +116,7 @@ const enableBlockingInPage = (page, run, actionTimeout) => {
       debug: 'autoconsent:setup'
     }),
     run({
-      fn: engine.enableBlockingInPage(page),
+      fn: getEngine().then(engine => engine.enableBlockingInPage(page)),
       timeout: actionTimeout,
       debug: 'adblock'
     })

package/src/index.js

@@ -22,23 +22,21 @@ const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
 const castArray = value => [].concat(value).filter(Boolean)
 
-const run = async ({ fn, timeout, debug: props }) => {
-  const duration = debug.duration()
-  const result = await pReflect(timeout ? pTimeout(fn, timeout) : fn)
-  const errorProps = result.isRejected ? { error: result.reason.message || result.reason } : {}
-  duration(props, errorProps)
-  return result
+const getDefaultPath = pathname => {
+  if (!pathname || pathname[0] !== '/') return '/'
+  if (pathname === '/') return '/'
+
+  const rightSlash = pathname.lastIndexOf('/')
+  return rightSlash === 0 ? '/' : pathname.slice(0, rightSlash)
 }
 
-const parseCookies = (url, str) =>
-  str.split(';').reduce((acc, cookieStr) => {
-    const jar = new toughCookie.CookieJar(undefined, { rejectPublicSuffixes: false })
-    jar.setCookieSync(cookieStr.trim(), url)
-    const parsedCookie = jar.serializeSync().cookies[0]
+const parseCookiesWithJar = (url, str) => {
+  const jar = new toughCookie.CookieJar(undefined, { rejectPublicSuffixes: false })
 
-    // Use this instead of the above when the following issue is fixed:
-    // https://github.com/salesforce/tough-cookie/issues/149
-    // const ret = toughCookie.parse(cookie).serializeSync();
+  return str.split(';').reduce((acc, cookieStr) => {
+    const cookie = jar.setCookieSync(cookieStr.trim(), url)
+    if (!cookie) return acc
+    const parsedCookie = cookie.toJSON()
 
     parsedCookie.name = parsedCookie.key
     delete parsedCookie.key
@@ -50,6 +48,82 @@ const parseCookies = (url, str) =>
     acc.push(parsedCookie)
     return acc
   }, [])
+}
+
+const run = async ({ fn, timeout, debug: props }) => {
+  const duration = debug.duration()
+  const result = await pReflect(timeout ? pTimeout(fn, timeout) : fn)
+  const errorProps = result.isRejected ? { error: result.reason.message || result.reason } : {}
+  duration(props, errorProps)
+  return result
+}
+
+const stopLoadingOnTimeout = (page, timeout) => {
+  let timeoutId
+
+  return {
+    promise: new Promise(resolve => {
+      timeoutId = globalThis.setTimeout(() => {
+        pReflect(page._client().send('Page.stopLoading')).then(resolve)
+      }, timeout)
+
+      if (typeof timeoutId.unref === 'function') timeoutId.unref()
+    }),
+    clear: () => {
+      if (timeoutId) clearTimeout(timeoutId)
+    }
+  }
+}
+
+const parseCookies = (url, str) => {
+  let parsedURL
+
+  try {
+    parsedURL = new URL(url)
+  } catch {
+    return parseCookiesWithJar(url, str)
+  }
+
+  const domain = parsedURL.hostname
+
+  if (!domain) {
+    return parseCookiesWithJar(url, str)
+  }
+
+  const path = getDefaultPath(parsedURL.pathname)
+  const chunks = str.split(';')
+  const cookies = new Array(chunks.length)
+  let index = 0
+
+  for (const chunk of chunks) {
+    const cookieStr = chunk.trim()
+
+    if (cookieStr.length === 0) {
+      return parseCookiesWithJar(url, str)
+    }
+
+    const separatorIndex = cookieStr.indexOf('=')
+
+    if (separatorIndex === -1) {
+      return parseCookiesWithJar(url, str)
+    }
+
+    const name = cookieStr.slice(0, separatorIndex).trim()
+
+    if (name.length === 0) {
+      return parseCookiesWithJar(url, str)
+    }
+
+    cookies[index++] = {
+      name,
+      value: cookieStr.slice(separatorIndex + 1).trim(),
+      domain,
+      path
+    }
+  }
+
+  return cookies
+}
 
 const getMediaFeatures = ({ animations, colorScheme }) => {
   const prefers = []
@@ -226,37 +300,51 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       )
     }
 
-    const enableInterception =
-      (onPageRequest || abortTypes.length > 0) &&
-      run({
-        fn: page.setRequestInterception(true),
-        debug: 'enableInterception'
-      })
+    const abortTypesSet = abortTypes.length > 0 ? new Set(abortTypes) : null
+
+    const requestHandlers = []
+    let abortTypesHandler
+    let disableInterceptionForAbortTypes = false
 
     if (onPageRequest) {
-      Promise.resolve(enableInterception).then(() =>
-        page.on('request', req => onPageRequest(req, page))
-      )
+      const onPageRequestHandler = req => onPageRequest(req, page)
+      page.on('request', onPageRequestHandler)
+      requestHandlers.push(onPageRequestHandler)
     }
 
     if (abortTypes.length > 0) {
-      Promise.resolve(enableInterception).then(() => {
-        page.on('request', req => {
-          if (req.isInterceptResolutionHandled()) return
-          const resourceType = req.resourceType()
-          const url = truncate(req.url())
-
-          if (!abortTypes.includes(resourceType)) {
-            debug.continue({ url, resourceType })
-            return req.continue(
-              req.continueRequestOverrides(),
-              DEFAULT_INTERCEPT_RESOLUTION_PRIORITY
-            )
+      abortTypesHandler = req => {
+        if (req.isInterceptResolutionHandled()) return
+        const resourceType = req.resourceType()
+        const url = truncate(req.url())
+
+        if (!abortTypesSet.has(resourceType)) {
+          debug.continue({ url, resourceType })
+          return req.continue(req.continueRequestOverrides(), DEFAULT_INTERCEPT_RESOLUTION_PRIORITY)
+        }
+        debug.abort({ url, resourceType })
+        return req.abort('blockedbyclient', DEFAULT_INTERCEPT_RESOLUTION_PRIORITY)
+      }
+
+      page.on('request', abortTypesHandler)
+      requestHandlers.push(abortTypesHandler)
+    }
+
+    if (requestHandlers.length > 0) {
+      prePromises.push(
+        run({
+          fn: page.setRequestInterception(true),
+          debug: 'enableInterception'
+        }).then(result => {
+          // If interception setup fails, remove handlers to avoid keeping dead listeners.
+          if (result.isRejected) {
+            requestHandlers.forEach(handler => page.off('request', handler))
+          } else if (abortTypesHandler && !withAdblock && !onPageRequest) {
+            disableInterceptionForAbortTypes = true
           }
-          debug.abort({ url, resourceType })
-          return req.abort('blockedbyclient', DEFAULT_INTERCEPT_RESOLUTION_PRIORITY)
+          return result
         })
-      })
+      )
     }
 
     if (withAdblock) {
@@ -273,7 +361,11 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       )
     }
 
-    const device = getDevice({ headers, ...args, device: args.device ?? defaultDevice })
+    const device = getDevice({
+      headers,
+      device: args.device ?? defaultDevice,
+      viewport: args.viewport
+    })
 
     if (device.userAgent && !headers['user-agent']) {
       headers['user-agent'] = device.userAgent
@@ -292,10 +384,11 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
     const headersKeys = Object.keys(headers)
 
     if (headersKeys.length > 0) {
-      const { cookie, ...headersWithoutCookie } = headers
+      const cookie = headers.cookie
+      const userAgent = headers['user-agent']
 
-      if (headers.cookie) {
-        const cookies = parseCookies(url, headers.cookie)
+      if (cookie) {
+        const cookies = parseCookies(url, cookie)
         prePromises.push(
           run({
             fn: page.setCookie(...cookies),
@@ -305,25 +398,41 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
         )
       }
 
-      const extraHTTPHeaders = headers.cookie ? headersWithoutCookie : headers
-      const extraHTTPHeadersKeys = Object.keys(extraHTTPHeaders)
-
-      if (headers['user-agent']) {
+      if (userAgent) {
         prePromises.push(
           run({
-            fn: page.setUserAgent(headers['user-agent']),
+            fn: page.setUserAgent(userAgent),
             timeout: actionTimeout,
-            debug: { 'user-agent': headers['user-agent'] }
+            debug: { 'user-agent': userAgent }
           })
         )
       }
 
-      if (extraHTTPHeadersKeys.length > 0) {
+      if (cookie) {
+        const extraHTTPHeaders = {}
+        const extraHTTPHeadersKeys = []
+
+        for (const key of headersKeys) {
+          if (key === 'cookie') continue
+          extraHTTPHeaders[key] = headers[key]
+          extraHTTPHeadersKeys.push(key)
+        }
+
+        if (extraHTTPHeadersKeys.length > 0) {
+          prePromises.push(
+            run({
+              fn: page.setExtraHTTPHeaders(extraHTTPHeaders),
+              timeout: actionTimeout,
+              debug: { headers: extraHTTPHeadersKeys }
+            })
+          )
+        }
+      } else if (!(userAgent && headersKeys.length === 1)) {
         prePromises.push(
           run({
-            fn: page.setExtraHTTPHeaders(extraHTTPHeaders),
+            fn: page.setExtraHTTPHeaders(headers),
             timeout: actionTimeout,
-            debug: { headers: extraHTTPHeadersKeys }
+            debug: { headers: headersKeys }
           })
         )
       }
@@ -351,72 +460,91 @@ module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ..
       )
     }
 
-    await Promise.all(prePromises)
+    try {
+      await Promise.all(prePromises)
 
-    const { value: response, reason: error } = await run({
-      fn: html
+      let clearStopLoadingTimer = () => {}
+      const navigationPromise = html
         ? page.setContent(html, { waitUntil, ...args })
-        : Promise.race([
-          page.goto(url, { waitUntil, ...args }),
-          setTimeout(gotoTimeout).then(() => page._client().send('Page.stopLoading'))
-        ]),
-      timeout: gotoTimeout,
-      debug: { fn: html ? 'html' : 'url', waitUntil }
-    })
-
-    if (withAdblock) {
-      await run({
-        fn: adblock.runAutoConsent(page),
-        timeout: actionTimeout,
-        debug: 'autoconsent:run'
+        : (() => {
+            const { promise, clear } = stopLoadingOnTimeout(page, gotoTimeout)
+            clearStopLoadingTimer = clear
+            return Promise.race([page.goto(url, { waitUntil, ...args }), promise])
+          })()
+
+      const { value: response, reason: error } = await run({
+        fn: navigationPromise,
+        timeout: gotoTimeout,
+        debug: { fn: html ? 'html' : 'url', waitUntil }
       })
-    }
+      clearStopLoadingTimer()
 
-    for (const [key, value] of Object.entries({
-      waitForSelector,
-      waitForFunction
-    })) {
-      if (value) {
-        await run({ fn: page[key](value), timeout: gotoTimeout, debug: { [key]: value } })
+      if (withAdblock) {
+        await run({
+          fn: adblock.runAutoConsent(page),
+          timeout: actionTimeout,
+          debug: 'autoconsent:run'
+        })
       }
-    }
 
-    if (waitForTimeout) {
-      await setTimeout(waitForTimeout)
-    }
-
-    await inject(page, {
-      timeout: actionTimeout,
-      mediaType,
-      animations,
-      modules,
-      scripts,
-      styles
-    })
+      if (waitForSelector) {
+        await run({
+          fn: page.waitForSelector(waitForSelector),
+          timeout: gotoTimeout,
+          debug: { waitForSelector }
+        })
+      }
 
-    if (click) {
-      for (const selector of castArray(click)) {
+      if (waitForFunction) {
         await run({
-          fn: page.click(selector),
-          timeout: actionTimeout,
-          debug: { click: selector }
+          fn: page.waitForFunction(waitForFunction),
+          timeout: gotoTimeout,
+          debug: { waitForFunction }
         })
       }
-    }
 
-    if (scroll) {
-      await run({
-        fn: page.$eval(scroll, el => el.scrollIntoView()),
+      if (waitForTimeout) {
+        await setTimeout(waitForTimeout)
+      }
+
+      await inject(page, {
         timeout: actionTimeout,
-        debug: { scroll }
+        mediaType,
+        animations,
+        modules,
+        scripts,
+        styles
       })
-    }
 
-    if (isWaitUntilAuto) {
-      await waitUntilAuto(page, { response, timeout: actionTimeout * 2 })
-    }
+      if (click) {
+        for (const selector of castArray(click)) {
+          await run({
+            fn: page.click(selector),
+            timeout: actionTimeout,
+            debug: { click: selector }
+          })
+        }
+      }
+
+      if (scroll) {
+        await run({
+          fn: page.$eval(scroll, el => el.scrollIntoView()),
+          timeout: actionTimeout,
+          debug: { scroll }
+        })
+      }
+
+      if (isWaitUntilAuto) {
+        await waitUntilAuto(page, { response, timeout: actionTimeout * 2 })
+      }
 
-    return { response, device, error }
+      return { response, device, error }
+    } finally {
+      if (abortTypesHandler) page.off('request', abortTypesHandler)
+      if (disableInterceptionForAbortTypes) {
+        await pReflect(page.setRequestInterception(false))
+      }
+    }
   }
 
   goto.getDevice = getDevice