@browserless/goto 9.12.4 → 10.0.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "@browserless/goto",
   "description": "Go to a page aborting unnecessary requests",
   "homepage": "https://browserless.js.org/#/?id=gotopage-options",
-  "version": "9.12.4",
+  "version": "10.0.0",
   "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
@@ -29,7 +29,7 @@
     "puppeteer"
   ],
   "dependencies": {
-    "@browserless/devices": "^9.10.2",
+    "@browserless/devices": "^10.0.0",
     "@cliqz/adblocker-puppeteer": "~1.26.6",
     "debug-logfmt": "~1.0.4",
     "got": "~11.8.6",
@@ -39,15 +39,12 @@
     "pretty-ms": "~7.0.1",
     "shallow-equal": "~3.1.0",
     "time-span": "~4.0.0",
-    "top-user-agents": "~1.0.57",
-    "tough-cookie": "~4.1.3",
-    "unique-random-array": "~2.0.0"
+    "tough-cookie": "~4.1.3"
   },
   "devDependencies": {
-    "@browserless/test": "^9.11.0",
+    "@browserless/test": "^10.0.0",
     "ava": "latest",
-    "browserless": "latest",
-    "fpscanner": "latest",
+    "is-ci": "latest",
     "p-wait-for": "3"
   },
   "engines": {
@@ -57,10 +54,6 @@
     "scripts",
     "src"
   ],
-  "scripts": {
-    "postinstall": "node scripts/postinstall",
-    "test": "ava"
-  },
   "license": "MIT",
   "ava": {
     "files": [
@@ -70,5 +63,9 @@
     "timeout": "30s",
     "workerThreads": false
   },
-  "gitHead": "5b81d2e12abab0726176894365f4fb4b4efd6580"
-}
+  "gitHead": "9085ee165094687aaf15c2d9a429d1c6acb9fe83",
+  "scripts": {
+    "postinstall": "node scripts/postinstall",
+    "test": "ava"
+  }
+}

package/src/index.js

@@ -19,10 +19,6 @@ debug.adblock = require('debug-logfmt')('browserless:goto:adblock')
 
 const truncate = (str, n = 80) => (str.length > n ? str.substr(0, n - 1) + '…' : str)
 
-const EVASIONS = require('./evasions')
-
-const ALL_EVASIONS_KEYS = Object.keys(EVASIONS)
-
 const engine = PuppeteerBlocker.deserialize(
   new Uint8Array(fs.readFileSync(path.resolve(__dirname, './engine.bin')))
 )
@@ -155,12 +151,7 @@ const inject = async (page, { timeout, mediaType, animations, modules, scripts,
   return Promise.all(postPromises)
 }
 
-module.exports = ({
-  evasions = ALL_EVASIONS_KEYS,
-  defaultDevice = 'Macbook Pro 13',
-  timeout: globalTimeout,
-  ...deviceOpts
-}) => {
+module.exports = ({ defaultDevice = 'Macbook Pro 13', timeout: globalTimeout, ...deviceOpts }) => {
   const getDevice = createDevices(deviceOpts)
   const { viewport: defaultViewport } = getDevice.findDevice(defaultDevice)
 
@@ -249,7 +240,8 @@ module.exports = ({
     }
 
     const enableInterception =
-      (onPageRequest || abortTypes.length > 0) && run({ fn: page.setRequestInterception(true), debug: 'enableInterception' })
+      (onPageRequest || abortTypes.length > 0) &&
+      run({ fn: page.setRequestInterception(true), debug: 'enableInterception' })
 
     if (onPageRequest) {
       Promise.resolve(enableInterception).then(() => page.on('request', onPageRequest))
@@ -363,18 +355,6 @@ module.exports = ({
       )
     }
 
-    const applyEvasions = castArray(evasions)
-      .filter(Boolean)
-      .map(key =>
-        run({
-          fn: EVASIONS[key](page),
-          timeout: actionTimeout,
-          debug: key
-        })
-      )
-
-    await Promise.all(prePromises.concat(applyEvasions))
-
     const { value: response, reason: error } = await run({
       fn: html
         ? page.setContent(html, { waitUntil, ...args })
@@ -442,4 +422,3 @@ module.exports = ({
 
 module.exports.parseCookies = parseCookies
 module.exports.inject = inject
-module.exports.evasions = ALL_EVASIONS_KEYS

package/src/evasions/chrome-runtime.js

@@ -1,69 +0,0 @@
-'use strict'
-
-/**
- * Pass the Chrome Test.
- *
- * This will work for iframes as well, except for `srcdoc` iframes:
- * https://github.com/puppeteer/puppeteer/issues/1106
- *
- * Could be mocked further.
- */
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    window.chrome = {
-      app: {
-        isInstalled: false,
-        InstallState: {
-          DISABLED: 'disabled',
-          INSTALLED: 'installed',
-          NOT_INSTALLED: 'not_installed'
-        },
-        RunningState: {
-          CANNOT_RUN: 'cannot_run',
-          READY_TO_RUN: 'ready_to_run',
-          RUNNING: 'running'
-        }
-      },
-      runtime: {
-        OnInstalledReason: {
-          CHROME_UPDATE: 'chrome_update',
-          INSTALL: 'install',
-          SHARED_MODULE_UPDATE: 'shared_module_update',
-          UPDATE: 'update'
-        },
-        OnRestartRequiredReason: {
-          APP_UPDATE: 'app_update',
-          OS_UPDATE: 'os_update',
-          PERIODIC: 'periodic'
-        },
-        PlatformArch: {
-          ARM: 'arm',
-          ARM64: 'arm64',
-          MIPS: 'mips',
-          MIPS64: 'mips64',
-          X86_32: 'x86-32',
-          X86_64: 'x86-64'
-        },
-        PlatformNaclArch: {
-          ARM: 'arm',
-          MIPS: 'mips',
-          MIPS64: 'mips64',
-          X86_32: 'x86-32',
-          X86_64: 'x86-64'
-        },
-        PlatformOs: {
-          ANDROID: 'android',
-          CROS: 'cros',
-          LINUX: 'linux',
-          MAC: 'mac',
-          OPENBSD: 'openbsd',
-          WIN: 'win'
-        },
-        RequestUpdateCheckStatus: {
-          NO_UPDATE: 'no_update',
-          THROTTLED: 'throttled',
-          UPDATE_AVAILABLE: 'update_available'
-        }
-      }
-    }
-  })

package/src/evasions/index.js

@@ -1,13 +0,0 @@
-'use strict'
-
-module.exports = {
-  chromeRuntime: require('./chrome-runtime'),
-  mediaCodecs: require('./media-codecs'),
-  navigatorPermissions: require('./navigator-permissions'),
-  navigatorPlugins: require('./navigator-plugins'),
-  navigatorVendor: require('./navigator-vendor'),
-  randomizeUserAgent: require('./randomize-user-agent'),
-  stackTraces: require('./stack-traces'),
-  webglVendor: require('./webgl-vendor'),
-  windowFrame: require('./window-frame')
-}

package/src/evasions/media-codecs.js

@@ -1,76 +0,0 @@
-'use strict'
-
-/**
- * Fix Chromium not reporting "probably" to codecs like `videoEl.canPlayType('video/mp4; codecs="avc1.42E01E"')`.
- * (Chromium doesn't support proprietary codecs, only Chrome does)
- */
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    try {
-      /**
-       * Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing.
-       *
-       * @example
-       * video/webm; codecs="vp8, vorbis"
-       * video/mp4; codecs="avc1.42E01E"
-       * audio/x-m4a;
-       * audio/ogg; codecs="vorbis"
-       * @param {String} arg
-       */
-      const parseInput = arg => {
-        const [mime, codecStr] = arg.trim().split(';')
-        let codecs = []
-        if (codecStr && codecStr.includes('codecs="')) {
-          codecs = codecStr
-            .trim()
-            .replace('codecs="', '')
-            .replace('"', '')
-            .trim()
-            .split(',')
-            .filter(x => !!x)
-            .map(x => x.trim())
-        }
-        return { mime, codecStr, codecs }
-      }
-
-      /* global HTMLMediaElement */
-      const canPlayType = {
-        // Make toString() native
-        get (target, key) {
-          // Mitigate Chromium bug (#130)
-          if (typeof target[key] === 'function') {
-            return target[key].bind(target)
-          }
-          return Reflect.get(target, key)
-        },
-        // Intercept certain requests
-        apply: function (target, ctx, args) {
-          if (!args || !args.length) {
-            return target.apply(ctx, args)
-          }
-          const { mime, codecs } = parseInput(args[0])
-          // This specific mp4 codec is missing in Chromium
-          if (mime === 'video/mp4') {
-            if (codecs.includes('avc1.42E01E')) {
-              return 'probably'
-            }
-          }
-          // This mimetype is only supported if no codecs are specified
-          if (mime === 'audio/x-m4a' && !codecs.length) {
-            return 'maybe'
-          }
-
-          // This mimetype is only supported if no codecs are specified
-          if (mime === 'audio/aac' && !codecs.length) {
-            return 'probably'
-          }
-          // Everything else as usual
-          return target.apply(ctx, args)
-        }
-      }
-      HTMLMediaElement.prototype.canPlayType = new Proxy(
-        HTMLMediaElement.prototype.canPlayType,
-        canPlayType
-      )
-    } catch (err) {}
-  })

package/src/evasions/navigator-permissions.js

@@ -1,43 +0,0 @@
-'use strict'
-
-/**
- * Pass the Permissions Test.
- */
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    if (!window.Notification) {
-      window.Notification = {
-        permission: 'denied'
-      }
-    }
-
-    const originalQuery = window.navigator.permissions.query
-    // eslint-disable-next-line
-    window.navigator.permissions.__proto__.query = parameters =>
-      parameters.name === 'notifications'
-        ? Promise.resolve({ state: window.Notification.permission })
-        : originalQuery(parameters)
-
-    // Inspired by: https://github.com/ikarienator/phantomjs_hide_and_seek/blob/master/5.spoofFunctionBind.js
-    const oldCall = Function.prototype.call
-    function call () {
-      return oldCall.apply(this, arguments)
-    }
-    // eslint-disable-next-line
-    Function.prototype.call = call
-
-    const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
-    const oldToString = Function.prototype.toString
-
-    function functionToString () {
-      if (this === window.navigator.permissions.query) {
-        return 'function query() { [native code] }'
-      }
-      if (this === functionToString) {
-        return nativeToStringFunctionString
-      }
-      return oldCall.call(oldToString, this)
-    }
-    // eslint-disable-next-line
-    Function.prototype.toString = functionToString
-  })

package/src/evasions/navigator-plugins.js

@@ -1,199 +0,0 @@
-'use strict'
-
-/**
- * In headless mode `navigator.mimeTypes` and `navigator.plugins` are empty.
- * This plugin quite emulates both of these to match regular headful Chrome.
- * We even go so far as to mock functional methods, instance types and `.toString` properties. :D
- */
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    function mockPluginsAndMimeTypes () {
-      /* global MimeType MimeTypeArray PluginArray */
-
-      // Disguise custom functions as being native
-      const makeFnsNative = (fns = []) => {
-        const oldCall = Function.prototype.call
-        function call () {
-          return oldCall.apply(this, arguments)
-        }
-        // eslint-disable-next-line
-        Function.prototype.call = call
-
-        const nativeToStringFunctionString = Error.toString().replace(/Error/g, 'toString')
-        const oldToString = Function.prototype.toString
-
-        function functionToString () {
-          for (const fn of fns) {
-            if (this === fn.ref) {
-              return `function ${fn.name}() { [native code] }`
-            }
-          }
-
-          if (this === functionToString) {
-            return nativeToStringFunctionString
-          }
-          return oldCall.call(oldToString, this)
-        }
-        // eslint-disable-next-line
-        Function.prototype.toString = functionToString
-      }
-
-      const mockedFns = []
-
-      const fakeData = {
-        mimeTypes: [
-          {
-            type: 'application/pdf',
-            suffixes: 'pdf',
-            description: '',
-            __pluginName: 'Chrome PDF Viewer'
-          },
-          {
-            type: 'application/x-google-chrome-pdf',
-            suffixes: 'pdf',
-            description: 'Portable Document Format',
-            __pluginName: 'Chrome PDF Plugin'
-          },
-          {
-            type: 'application/x-nacl',
-            suffixes: '',
-            description: 'Native Client Executable',
-            // eslint-disable-next-line
-            enabledPlugin: Plugin,
-            __pluginName: 'Native Client'
-          },
-          {
-            type: 'application/x-pnacl',
-            suffixes: '',
-            description: 'Portable Native Client Executable',
-            __pluginName: 'Native Client'
-          }
-        ],
-        plugins: [
-          {
-            name: 'Chrome PDF Plugin',
-            filename: 'internal-pdf-viewer',
-            description: 'Portable Document Format'
-          },
-          {
-            name: 'Chrome PDF Viewer',
-            filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai',
-            description: ''
-          },
-          {
-            name: 'Native Client',
-            filename: 'internal-nacl-plugin',
-            description: ''
-          }
-        ],
-        fns: {
-          namedItem: instanceName => {
-            // Returns the Plugin/MimeType with the specified name.
-            const fn = function (name) {
-              if (!arguments.length) {
-                throw new TypeError(
-                  `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.`
-                )
-              }
-              return this[name] || null
-            }
-            mockedFns.push({ ref: fn, name: 'namedItem' })
-            return fn
-          },
-          item: instanceName => {
-            // Returns the Plugin/MimeType at the specified index into the array.
-            const fn = function (index) {
-              if (!arguments.length) {
-                throw new TypeError(
-                  `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.`
-                )
-              }
-              return this[index] || null
-            }
-            mockedFns.push({ ref: fn, name: 'item' })
-            return fn
-          },
-          refresh: instanceName => {
-            // Refreshes all plugins on the current page, optionally reloading documents.
-            const fn = function () {
-              return undefined
-            }
-            mockedFns.push({ ref: fn, name: 'refresh' })
-            return fn
-          }
-        }
-      }
-      // Poor mans _.pluck
-      const getSubset = (keys, obj) => keys.reduce((a, c) => ({ ...a, [c]: obj[c] }), {})
-
-      function generateMimeTypeArray () {
-        const arr = fakeData.mimeTypes
-          .map(obj => getSubset(['type', 'suffixes', 'description'], obj))
-          .map(obj => Object.setPrototypeOf(obj, MimeType.prototype))
-        arr.forEach(obj => {
-          arr[obj.type] = obj
-        })
-
-        // Mock functions
-        arr.namedItem = fakeData.fns.namedItem('MimeTypeArray')
-        arr.item = fakeData.fns.item('MimeTypeArray')
-
-        return Object.setPrototypeOf(arr, MimeTypeArray.prototype)
-      }
-
-      const mimeTypeArray = generateMimeTypeArray()
-      Object.defineProperty(navigator, 'mimeTypes', {
-        get: () => mimeTypeArray
-      })
-
-      function generatePluginArray () {
-        const arr = fakeData.plugins
-          .map(obj => getSubset(['name', 'filename', 'description'], obj))
-          .map(obj => {
-            const mimes = fakeData.mimeTypes.filter(m => m.__pluginName === obj.name)
-            // Add mimetypes
-            mimes.forEach((mime, index) => {
-              navigator.mimeTypes[mime.type].enabledPlugin = obj
-              obj[mime.type] = navigator.mimeTypes[mime.type]
-              obj[index] = navigator.mimeTypes[mime.type]
-            })
-            obj.length = mimes.length
-            return obj
-          })
-          .map(obj => {
-            // Mock functions
-            obj.namedItem = fakeData.fns.namedItem('Plugin')
-            obj.item = fakeData.fns.item('Plugin')
-            return obj
-          })
-          // eslint-disable-next-line
-          .map(obj => Object.setPrototypeOf(obj, Plugin.prototype))
-        arr.forEach(obj => {
-          arr[obj.name] = obj
-        })
-
-        // Mock functions
-        arr.namedItem = fakeData.fns.namedItem('PluginArray')
-        arr.item = fakeData.fns.item('PluginArray')
-        arr.refresh = fakeData.fns.refresh('PluginArray')
-
-        return Object.setPrototypeOf(arr, PluginArray.prototype)
-      }
-
-      const pluginArray = generatePluginArray()
-      Object.defineProperty(navigator, 'plugins', {
-        get: () => pluginArray
-      })
-
-      // Make mockedFns toString() representation resemble a native function
-      makeFnsNative(mockedFns)
-    }
-    try {
-      const isPluginArray = navigator.plugins instanceof PluginArray
-      const hasPlugins = isPluginArray && navigator.plugins.length > 0
-      if (isPluginArray && hasPlugins) {
-        return // nothing to do here
-      }
-      mockPluginsAndMimeTypes()
-    } catch (err) {}
-  })

package/src/evasions/navigator-vendor.js

@@ -1,25 +0,0 @@
-module.exports = page =>
-  page.evaluateOnNewDocument(async () => {
-    const getUserAgentVendor = userAgent => {
-      let vendor
-
-      if (userAgent.includes('Firefox')) {
-        vendor = 'firefox'
-      } else if (userAgent.includes('Chrome')) {
-        vendor = 'chrome'
-      } else {
-        vendor = 'safari'
-      }
-
-      return vendor
-    }
-
-    const userAgent = navigator.userAgent
-    const userAgentVendor = getUserAgentVendor(userAgent)
-
-    if (userAgentVendor === 'chrome') return
-
-    Object.defineProperty(navigator, 'vendor', {
-      value: userAgentVendor === 'safari' ? 'Apple Computer, Inc.' : ''
-    })
-  })

package/src/evasions/randomize-user-agent.js

@@ -1,8 +0,0 @@
-'use strict'
-
-const uniqueRandomArray = require('unique-random-array')
-const userAgents = require('top-user-agents')
-
-const randomUserAgent = uniqueRandomArray(userAgents)
-
-module.exports = page => page.setUserAgent(randomUserAgent())

package/src/evasions/stack-traces.js

@@ -1,51 +0,0 @@
-'use strict'
-
-/**
- *
- * Prevent detect Puppeteer via variable name
- *
- * References:
- * - https://github.com/digitalhurricane-io/puppeteer-detection-100-percent
- * - https://github.com/berstend/puppeteer-extra/issues/209
- */
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    const errors = {
-      Error,
-      EvalError,
-      RangeError,
-      ReferenceError,
-      SyntaxError,
-      TypeError,
-      URIError
-    }
-    for (const name in errors) {
-      // eslint-disable-next-line
-      globalThis[name] = (function (NativeError) {
-        return function (message) {
-          const err = new NativeError(message)
-          const stub = {
-            message: err.message,
-            name: err.name,
-            toString: () => err.toString(),
-            get stack () {
-              const lines = err.stack.split('\n')
-              lines.splice(1, 1) // remove anonymous function above
-              lines.pop() // remove puppeteer line
-              return lines.join('\n')
-            }
-          }
-          // eslint-disable-next-line
-          if (this === globalThis) {
-            // called as function, not constructor
-            // eslint-disable-next-line
-            stub.__proto__ = NativeError
-            return stub
-          }
-          Object.assign(this, stub)
-          // eslint-disable-next-line
-          this.__proto__ = NativeError
-        }
-      })(errors[name])
-    }
-  })

package/src/evasions/webgl-vendor.js

@@ -1,50 +0,0 @@
-/**
- * Fix WebGL Vendor/Renderer being set to Google in headless mode
- */
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    // Remove traces of our Proxy ;-)
-    const stripErrorStack = stack =>
-      stack
-        .split('\n')
-        .filter(line => !line.includes('at Object.apply'))
-        .filter(line => !line.includes('at Object.get'))
-        .join('\n')
-
-    const getParameterProxyHandler = {
-      get (target, key) {
-        try {
-          // Mitigate Chromium bug (#130)
-          if (typeof target[key] === 'function') {
-            return target[key].bind(target)
-          }
-          return Reflect.get(target, key)
-        } catch (err) {
-          err.stack = stripErrorStack(err.stack)
-          throw err
-        }
-      },
-      apply: function (target, thisArg, args) {
-        const param = (args || [])[0]
-        // UNMASKED_VENDOR_WEBGL
-        if (param === 37445) return 'Intel Inc.'
-        // UNMASKED_RENDERER_WEBGL
-        if (param === 37446) return 'Intel(R) Iris(TM) Plus Graphics 640'
-        try {
-          return Reflect.apply(target, thisArg, args)
-        } catch (err) {
-          err.stack = stripErrorStack(err.stack)
-          throw err
-        }
-      }
-    }
-
-    ;['WebGLRenderingContext', 'WebGL2RenderingContext'].forEach(function (ctx) {
-      Object.defineProperty(window[ctx].prototype, 'getParameter', {
-        configurable: true,
-        enumerable: false,
-        writable: false,
-        value: new Proxy(window[ctx].prototype.getParameter, getParameterProxyHandler)
-      })
-    })
-  })

package/src/evasions/window-frame.js

@@ -1,9 +0,0 @@
-'use strict'
-
-module.exports = page =>
-  page.evaluateOnNewDocument(() => {
-    if (window.outerWidth && window.outerHeight) return
-    const windowFrame = 85
-    window.outerWidth = window.innerWidth
-    window.outerHeight = window.innerHeight + windowFrame
-  })