@browserless/goto 9.3.0-beta.9 → 9.3.3

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Go to a page aborting unnecessary requests",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "9.3.0-beta.9",
+  "version": "9.3.3",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -29,22 +29,22 @@
     "puppeteer"
   ],
   "dependencies": {
-    "@browserless/devices": "^9.2.22",
-    "@cliqz/adblocker-puppeteer": "~1.23.0",
+    "@browserless/devices": "^9.3.3",
+    "@cliqz/adblocker-puppeteer": "~1.23.3",
     "debug-logfmt": "~1.0.4",
     "got": "~11.8.3",
-    "is-url-http": "~2.2.5",
+    "is-url-http": "~2.2.7",
     "p-reflect": "~2.1.0",
     "p-timeout": "~4.1.0",
     "pretty-ms": "~7.0.1",
     "shallow-equal": "~1.2.1",
     "time-span": "~4.0.0",
-    "top-user-agents": "~1.0.39",
+    "top-user-agents": "~1.0.40",
     "tough-cookie": "~4.0.0",
     "unique-random-array": "~2.0.0"
   },
   "devDependencies": {
-    "ava": "latest",
+    "ava": "3",
     "browserless": "latest",
     "puppeteer": "latest",
     "signal-exit": "latest"
@@ -68,5 +68,5 @@
     "timeout": "2m",
     "verbose": true
   },
-  "gitHead": "5917cbb975fba39b208fe7820d16a5655fa75420"
+  "gitHead": "e15c8a9fea38ad134170d52c9b8058010a3c7cd1"
 }

package/src/index.js

@@ -24,7 +24,6 @@ const engine = PuppeteerBlocker.deserialize(
 
 engine.on('request-blocked', ({ url }) => debugAdblock('block', url))
 engine.on('request-redirected', ({ url }) => debugAdblock('redirect', url))
-engine.setRequestInterceptionPriority(1)
 
 const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
@@ -32,7 +31,7 @@ const castArray = value => [].concat(value).filter(Boolean)
 
 const run = async ({ fn, timeout, debug: props }) => {
   const debugProps = { duration: timeSpan() }
-  const result = await pReflect(pTimeout(fn, timeout))
+  const result = await pReflect(timeout ? pTimeout(fn, timeout) : fn())
   debugProps.duration = prettyMs(debugProps.duration())
   if (result.isRejected) debugProps.error = result.reason.message || result.reason
   debug(props, debugProps)
@@ -92,22 +91,85 @@ const disableAnimations = `
   }
 `.trim()
 
-// related https://github.com/puppeteer/puppeteer/issues/1353
-const createWaitUntilAuto = defaultOpts => (page, opts) => {
-  const { timeout } = { ...defaultOpts, ...opts }
-
-  return Promise.all(
-    [
-      {
-        fn: () => page.waitForNavigation({ waitUntil: 'networkidle2' }),
-        debug: 'waitUntilAuto:waitForNavigation'
-      },
-      {
-        fn: () => page.evaluate(() => window.history.pushState(null, null, null)),
-        debug: 'waitUntilAuto:pushState'
-      }
-    ].map(({ fn, ...opts }) => run({ fn: fn(), timeout, ...opts }))
-  )
+const inject = async (
+  page,
+  { timeout, mediaType, animations, hide, remove, modules, scripts, styles }
+) => {
+  const postPromises = []
+
+  if (mediaType) {
+    postPromises.push(
+      run({
+        fn: page.emulateMediaType(mediaType),
+        timeout,
+        debug: { mediaType }
+      })
+    )
+  }
+
+  if (animations === false) {
+    postPromises.push(
+      run({
+        fn: injectStyle(page, disableAnimations),
+        timeout,
+        debug: 'disableAnimations'
+      })
+    )
+  }
+
+  if (hide) {
+    postPromises.push(
+      run({
+        fn: injectStyle(page, `${castArray(hide).join(', ')} { visibility: hidden !important; }`),
+        timeout,
+        debug: 'hide'
+      })
+    )
+  }
+
+  if (remove) {
+    postPromises.push(
+      run({
+        fn: injectStyle(page, `${castArray(remove).join(', ')} { display: none !important; }`),
+        timeout,
+        debug: 'remove'
+      })
+    )
+  }
+
+  if (modules) {
+    postPromises.push(
+      run({
+        fn: Promise.all(
+          castArray(modules).map(value => injectScript(page, value, { type: 'modules' }))
+        ),
+        timeout,
+        debug: 'modules'
+      })
+    )
+  }
+
+  if (scripts) {
+    postPromises.push(
+      run({
+        fn: Promise.all(castArray(scripts).map(value => injectScript(page, value))),
+        timeout,
+        debug: 'scripts'
+      })
+    )
+  }
+
+  if (styles) {
+    postPromises.push(
+      run({
+        fn: Promise.all(castArray(styles).map(style => injectStyle(page, style))),
+        timeout,
+        debug: 'styles'
+      })
+    )
+  }
+
+  return Promise.all(postPromises)
 }
 
 module.exports = ({
@@ -116,14 +178,32 @@ module.exports = ({
   timeout: globalTimeout,
   ...deviceOpts
 }) => {
-  const baseTimeout = globalTimeout * (1 / 2)
-  const gotoTimeout = timeout * 0.8
-  const actionTimeout = baseTimeout * (1 / 8)
-
   const getDevice = createDevices(deviceOpts)
   const { viewport: defaultViewport } = getDevice.findDevice(defaultDevice)
 
-  const _waitUntilAuto = createWaitUntilAuto({ timeout: actionTimeout })
+  const timeouts = {
+    base: (milliseconds = globalTimeout) => milliseconds * (2 / 3),
+    action: (milliseconds = globalTimeout) => milliseconds * (1 / 11),
+    goto: (milliseconds = globalTimeout) => milliseconds * (7 / 8)
+  }
+
+  // related https://github.com/puppeteer/puppeteer/issues/1353
+  const _waitUntilAuto = (page, opts = {}) => {
+    const timeout = timeouts.action(opts.timeout)
+
+    return Promise.all(
+      [
+        {
+          fn: () => page.waitForNavigation({ waitUntil: 'networkidle2' }),
+          debug: 'waitUntilAuto:waitForNavigation'
+        },
+        {
+          fn: () => page.evaluate(() => window.history.pushState(null, null, null)),
+          debug: 'waitUntilAuto:pushState'
+        }
+      ].map(({ fn, ...opts }) => run({ fn: fn(), timeout, ...opts }))
+    )
+  }
 
   const goto = async (
     page,
@@ -143,7 +223,7 @@ module.exports = ({
       scripts,
       scroll,
       styles,
-      timeout = baseTimeout,
+      timeout = globalTimeout,
       timezone,
       url,
       waitForFunction,
@@ -155,6 +235,10 @@ module.exports = ({
       ...args
     }
   ) => {
+    const baseTimeout = timeouts.base(globalTimeout)
+    const actionTimeout = timeouts.action(baseTimeout)
+    const gotoTimeout = timeouts.goto(baseTimeout)
+
     const isWaitUntilAuto = waitUntil === 'auto'
     if (isWaitUntilAuto) waitUntil = 'load'
 
@@ -163,13 +247,15 @@ module.exports = ({
     if (abortTypes.length > 0) {
       await page.setRequestInterception(true)
       page.on('request', req => {
+        if (req.isInterceptResolutionHandled()) return
+
         const resourceType = req.resourceType()
         if (!abortTypes.includes(resourceType)) {
           debug('continue', { url: req.url(), resourceType })
           return req.continue(req.continueRequestOverrides(), 0)
         }
         debug('abort', { url: req.url(), resourceType })
-        return req.abort('blockedbyclient', 2)
+        return req.abort('blockedbyclient', 0)
       })
     }
 
@@ -276,7 +362,7 @@ module.exports = ({
 
     await Promise.all(prePromises.concat(applyEvasions))
 
-    const { value } = await run({
+    const { value: response } = await run({
       fn: html ? page.setContent(html, args) : page.goto(url, args),
       timeout: gotoTimeout,
       debug: html ? 'html' : 'url'
@@ -293,85 +379,24 @@ module.exports = ({
       }
     }
 
-    const postPromises = []
-
-    if (mediaType) {
-      postPromises.push(
-        run({
-          fn: page.emulateMediaType(mediaType),
-          timeout: actionTimeout,
-          debug: { mediaType }
-        })
-      )
-    }
-
-    if (animations === false) {
-      postPromises.push(
-        run({
-          fn: injectStyle(page, disableAnimations),
-          timeout: actionTimeout,
-          debug: 'disableAnimations'
-        })
-      )
-    }
-
-    if (hide) {
-      postPromises.push(
-        run({
-          fn: injectStyle(page, `${castArray(hide).join(', ')} { visibility: hidden !important; }`),
-          timeout: actionTimeout,
-          debug: 'hide'
-        })
-      )
-    }
-
-    if (remove) {
-      postPromises.push(
-        run({
-          fn: injectStyle(page, `${castArray(remove).join(', ')} { display: none !important; }`),
-          timeout: actionTimeout,
-          debug: 'remove'
-        })
-      )
-    }
-
-    if (modules) {
-      postPromises.push(
-        run({
-          fn: Promise.all(
-            castArray(modules).map(value => injectScript(page, value, { type: 'modules' }))
-          ),
-          timeout: actionTimeout,
-          debug: 'modules'
-        })
-      )
-    }
-
-    if (scripts) {
-      postPromises.push(
-        run({
-          fn: Promise.all(castArray(scripts).map(value => injectScript(page, value))),
-          timeout: actionTimeout,
-          debug: 'scripts'
-        })
-      )
-    }
-
-    if (styles) {
-      postPromises.push(
-        run({
-          fn: Promise.all(castArray(styles).map(style => injectStyle(page, style))),
-          timeout: actionTimeout,
-          debug: 'styles'
-        })
-      )
-    }
-
-    await Promise.all(postPromises)
+    await inject(page, {
+      timeout: actionTimeout,
+      mediaType,
+      animations,
+      hide,
+      remove,
+      modules,
+      scripts,
+      styles
+    })
 
     if (click) {
       for (const selector of castArray(click)) {
-        await run({ fn: page.click(selector), timeout: actionTimeout, debug: { click: selector } })
+        await run({
+          fn: page.click(selector),
+          timeout: actionTimeout,
+          debug: { click: selector }
+        })
       }
     }
 
@@ -384,10 +409,10 @@ module.exports = ({
     }
 
     if (isWaitUntilAuto) {
-      await waitUntilAuto(page, { response: value, timeout: actionTimeout * 2 })
+      await waitUntilAuto(page, { response, timeout: actionTimeout * 2 })
     }
 
-    return { response: value, device }
+    return { response, device }
   }
 
   goto.getDevice = getDevice
@@ -396,14 +421,12 @@ module.exports = ({
   goto.deviceDescriptors = getDevice.deviceDescriptors
   goto.defaultViewport = defaultViewport
   goto.waitUntilAuto = _waitUntilAuto
-  goto.timeout = baseTimeout
-  goto.actionTimeout = actionTimeout
+  goto.timeouts = timeouts
   goto.run = run
 
   return goto
 }
 
 module.exports.parseCookies = parseCookies
-module.exports.injectScript = injectScript
-module.exports.injectStyle = injectStyle
+module.exports.inject = inject
 module.exports.evasions = ALL_EVASIONS_KEYS