@browserless/goto 9.2.20 → 9.3.0-beta.11

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Go to a page aborting unnecessary requests",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "9.2.20",
+  "version": "9.3.0-beta.11",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -29,17 +29,17 @@
     "puppeteer"
   ],
   "dependencies": {
-    "@browserless/devices": "^9.1.6",
+    "@browserless/devices": "^9.2.22",
     "@cliqz/adblocker-puppeteer": "~1.23.0",
     "debug-logfmt": "~1.0.4",
-    "got": "~11.8.2",
-    "is-url-http": "~2.2.4",
+    "got": "~11.8.3",
+    "is-url-http": "~2.2.5",
     "p-reflect": "~2.1.0",
     "p-timeout": "~4.1.0",
     "pretty-ms": "~7.0.1",
     "shallow-equal": "~1.2.1",
     "time-span": "~4.0.0",
-    "top-user-agents": "~1.0.38",
+    "top-user-agents": "~1.0.39",
     "tough-cookie": "~4.0.0",
     "unique-random-array": "~2.0.0"
   },
@@ -68,5 +68,5 @@
     "timeout": "2m",
     "verbose": true
   },
-  "gitHead": "58b992afb012bb270f3dfe27c2d57a1f91f8051f"
+  "gitHead": "5a93704c7fdb292fdf3a867299b3bfa4b095e701"
 }

package/src/index.js

@@ -24,6 +24,7 @@ const engine = PuppeteerBlocker.deserialize(
 
 engine.on('request-blocked', ({ url }) => debugAdblock('block', url))
 engine.on('request-redirected', ({ url }) => debugAdblock('redirect', url))
+engine.setRequestInterceptionPriority(1)
 
 const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
@@ -91,37 +92,38 @@ const disableAnimations = `
   }
 `.trim()
 
-// related https://github.com/puppeteer/puppeteer/issues/1353
-const createWaitUntilAuto = defaultOpts => (page, opts) => {
-  const { timeout } = { ...defaultOpts, ...opts }
-
-  return Promise.all(
-    [
-      {
-        fn: () => page.waitForNavigation({ waitUntil: 'networkidle2' }),
-        debug: 'waitUntilAuto:waitForNavigation'
-      },
-      {
-        fn: () => page.evaluate(() => window.history.pushState(null, null, null)),
-        debug: 'waitUntilAuto:pushState'
-      }
-    ].map(({ fn, ...opts }) => run({ fn: fn(), timeout, ...opts }))
-  )
-}
-
 module.exports = ({
   evasions = ALL_EVASIONS_KEYS,
   defaultDevice = 'Macbook Pro 13',
   timeout: globalTimeout,
   ...deviceOpts
 }) => {
-  const baseTimeout = globalTimeout * (1 / 2)
-  const actionTimeout = baseTimeout * (1 / 8)
-
   const getDevice = createDevices(deviceOpts)
   const { viewport: defaultViewport } = getDevice.findDevice(defaultDevice)
 
-  const _waitUntilAuto = createWaitUntilAuto({ timeout: actionTimeout })
+  const timeouts = {
+    base: (milliseconds = globalTimeout) => milliseconds * (2 / 3),
+    action: (milliseconds = globalTimeout) => milliseconds * (1 / 11),
+    goto: (milliseconds = globalTimeout) => milliseconds * (7 / 8)
+  }
+
+  // related https://github.com/puppeteer/puppeteer/issues/1353
+  const _waitUntilAuto = (page, opts = {}) => {
+    const timeout = timeouts.action(opts.timeout)
+
+    return Promise.all(
+      [
+        {
+          fn: () => page.waitForNavigation({ waitUntil: 'networkidle2' }),
+          debug: 'waitUntilAuto:waitForNavigation'
+        },
+        {
+          fn: () => page.evaluate(() => window.history.pushState(null, null, null)),
+          debug: 'waitUntilAuto:pushState'
+        }
+      ].map(({ fn, ...opts }) => run({ fn: fn(), timeout, ...opts }))
+    )
+  }
 
   const goto = async (
     page,
@@ -141,7 +143,7 @@ module.exports = ({
       scripts,
       scroll,
       styles,
-      timeout = baseTimeout,
+      timeout = globalTimeout,
       timezone,
       url,
       waitForFunction,
@@ -153,11 +155,28 @@ module.exports = ({
       ...args
     }
   ) => {
+    const baseTimeout = timeouts.base(globalTimeout)
+    const actionTimeout = timeouts.action(baseTimeout)
+    const gotoTimeout = timeouts.goto(baseTimeout)
+
     const isWaitUntilAuto = waitUntil === 'auto'
     if (isWaitUntilAuto) waitUntil = 'load'
 
     const prePromises = []
 
+    if (abortTypes.length > 0) {
+      await page.setRequestInterception(true)
+      page.on('request', req => {
+        const resourceType = req.resourceType()
+        if (!abortTypes.includes(resourceType)) {
+          debug('continue', { url: req.url(), resourceType })
+          return req.continue(req.continueRequestOverrides(), 0)
+        }
+        debug('abort', { url: req.url(), resourceType })
+        return req.abort('blockedbyclient', 2)
+      })
+    }
+
     if (adblock) {
       prePromises.push(
         run({
@@ -227,16 +246,6 @@ module.exports = ({
       )
     }
 
-    if (mediaType) {
-      prePromises.push(
-        run({
-          fn: page.emulateMediaType(mediaType),
-          timeout: actionTimeout,
-          debug: { mediaType }
-        })
-      )
-    }
-
     if (timezone) {
       prePromises.push(
         run({
@@ -271,19 +280,9 @@ module.exports = ({
 
     await Promise.all(prePromises.concat(applyEvasions))
 
-    if (abortTypes.length > 0) {
-      await page.setRequestInterception(true)
-      page.on('request', req => {
-        const resourceType = req.resourceType()
-        if (!abortTypes.includes(resourceType)) return req.continue()
-        debug('abort', { url: req.url(), resourceType })
-        return req.abort('blockedbyclient')
-      })
-    }
-
     const { value } = await run({
       fn: html ? page.setContent(html, args) : page.goto(url, args),
-      timeout,
+      timeout: gotoTimeout,
       debug: html ? 'html' : 'url'
     })
 
@@ -294,12 +293,22 @@ module.exports = ({
       waitForTimeout
     })) {
       if (value) {
-        await run({ fn: page[key](value), timeout: actionTimeout, debug: { [key]: value } })
+        await run({ fn: page[key](value), timeout: gotoTimeout, debug: { [key]: value } })
       }
     }
 
     const postPromises = []
 
+    if (mediaType) {
+      postPromises.push(
+        run({
+          fn: page.emulateMediaType(mediaType),
+          timeout: actionTimeout,
+          debug: { mediaType }
+        })
+      )
+    }
+
     if (animations === false) {
       postPromises.push(
         run({
@@ -366,7 +375,11 @@ module.exports = ({
 
     if (click) {
       for (const selector of castArray(click)) {
-        await run({ fn: page.click(selector), timeout: actionTimeout, debug: { click: selector } })
+        await run({
+          fn: page.click(selector),
+          timeout: actionTimeout,
+          debug: { click: selector }
+        })
       }
     }
 
@@ -391,8 +404,7 @@ module.exports = ({
   goto.deviceDescriptors = getDevice.deviceDescriptors
   goto.defaultViewport = defaultViewport
   goto.waitUntilAuto = _waitUntilAuto
-  goto.timeout = baseTimeout
-  goto.actionTimeout = actionTimeout
+  goto.timeouts = timeouts
   goto.run = run
 
   return goto