@zorilla/puppeteer-extra-plugin-stealth 1.0.0

package/src/evasions/navigator.vendor/index.js

@@ -0,0 +1,64 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+import withUtils from '../_utils/withUtils.js';
+
+/**
+ * By default puppeteer will have a fixed `navigator.vendor` property.
+ *
+ * This plugin makes it possible to change this property.
+ *
+ * @example
+ * const puppeteer = require("puppeteer-extra")
+ *
+ * const StealthPlugin = require("puppeteer-extra-plugin-stealth")
+ * const stealth = StealthPlugin()
+ * // Remove this specific stealth plugin from the default set
+ * stealth.enabledEvasions.delete("navigator.vendor")
+ * puppeteer.use(stealth)
+ *
+ * // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
+ * const NavigatorVendorPlugin = require("puppeteer-extra-plugin-stealth/evasions/navigator.vendor")
+ * const nvp = NavigatorVendorPlugin({ vendor: 'Apple Computer, Inc.' }) // Custom vendor
+ * puppeteer.use(nvp)
+ *
+ * @param {Object} [opts] - Options
+ * @param {string} [opts.vendor] - The vendor to use in `navigator.vendor` (default: `Google Inc.`)
+ *
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/navigator.vendor';
+  }
+
+  get defaults() {
+    return {
+      vendor: 'Google Inc.',
+    };
+  }
+
+  async onPageCreated(page) {
+    this.debug('onPageCreated', {
+      opts: this.opts,
+    });
+
+    await withUtils(page).evaluateOnNewDocument(
+      (utils, { opts }) => {
+        utils.replaceGetterWithProxy(
+          Object.getPrototypeOf(navigator),
+          'vendor',
+          utils.makeHandler().getterValue(opts.vendor)
+        );
+      },
+      {
+        opts: this.opts,
+      }
+    );
+  } // onPageCreated
+}
+
+const defaultExport = opts => new Plugin(opts);
+module.exports = defaultExport;

package/src/evasions/navigator.vendor/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/navigator.webdriver/README.md

@@ -0,0 +1,17 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.js#L9-L23)
+
+- `opts` (optional, default `{}`)
+
+**Extends: PuppeteerExtraPlugin**
+
+Pass the Webdriver Test.
+Will delete `navigator.webdriver` property.
+
+---

package/src/evasions/navigator.webdriver/index.js

@@ -0,0 +1,48 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+/**
+ * Pass the Webdriver Test.
+ * Will delete `navigator.webdriver` property.
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/navigator.webdriver';
+  }
+
+  async onPageCreated(page) {
+    await page.evaluateOnNewDocument(() => {
+      if (navigator.webdriver === false) {
+        // Post Chrome 89.0.4339.0 and already good
+      } else if (navigator.webdriver === undefined) {
+        // Pre Chrome 89.0.4339.0 and already good
+      } else {
+        // Pre Chrome 88.0.4291.0 and needs patching
+        delete Object.getPrototypeOf(navigator).webdriver;
+      }
+    });
+  }
+
+  // Post Chrome 88.0.4291.0
+  // Note: this will add an infobar to Chrome with a warning that an unsupported flag is set
+  // To remove this bar on Linux, run: mkdir -p /etc/opt/chrome/policies/managed && echo '{ "CommandLineFlagSecurityWarningsEnabled": false }' > /etc/opt/chrome/policies/managed/managed_policies.json
+  async beforeLaunch(options) {
+    // If disable-blink-features is already passed, append the AutomationControlled switch
+    const idx = options.args.findIndex(arg =>
+      arg.startsWith('--disable-blink-features=')
+    );
+    if (idx !== -1) {
+      const arg = options.args[idx];
+      options.args[idx] = `${arg},AutomationControlled`;
+    } else {
+      options.args.push('--disable-blink-features=AutomationControlled');
+    }
+  }
+}
+
+export default function (pluginConfig) {
+  return new Plugin(pluginConfig);
+}

package/src/evasions/navigator.webdriver/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/sourceurl/README.md

@@ -0,0 +1,17 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.js#L9-L58)
+
+- `opts` (optional, default `{}`)
+
+**Extends: PuppeteerExtraPlugin**
+
+Strip sourceURL from scripts injected by puppeteer.
+It can be used to identify the presence of pptr via stacktraces.
+
+---

package/src/evasions/sourceurl/_fixtures/test.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Page Title</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+  </head>
+  <body>
+    <h1 id="result">Please use `document.querySelector`..</h1>
+
+    <script>
+      function test() {
+        const err = new Error('Test Error')
+        const isPptr = err.stack
+          .toString()
+          .includes('__puppeteer_evaluation_script')
+
+        document.getElementById('result').innerHTML = isPptr ? 'FAIL' : 'PASS'
+        console.log({ err, isPptr })
+      }
+
+      function overrideFunction(item) {
+        item.obj[item.propName] = ((orig) => function() {
+            test()
+            return orig.apply(this, arguments)
+          })(item.obj[item.propName])
+      }
+
+      overrideFunction({
+        propName: 'querySelector',
+        obj: document
+      })
+    </script>
+  </body>
+</html>

package/src/evasions/sourceurl/index.js

@@ -0,0 +1,82 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+/**
+ * Strip sourceURL from scripts injected by puppeteer.
+ * It can be used to identify the presence of pptr via stacktraces.
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/sourceurl';
+  }
+
+  async onPageCreated(page) {
+    const client =
+      page && typeof page._client === 'function'
+        ? page._client()
+        : page._client;
+    if (!client) {
+      this.debug('Warning, missing properties to intercept CDP.', { page });
+      return;
+    }
+
+    // Intercept CDP commands and strip identifying and unnecessary sourceURL
+    // https://github.com/puppeteer/puppeteer/blob/9b3005c105995cd267fdc7fb95b78aceab82cf0e/new-docs/puppeteer.cdpsession.md
+    const debug = this.debug;
+    client.send = ((originalMethod, context) =>
+      async function () {
+        const [method, paramArgs] = arguments || [];
+        const next = async () => {
+          try {
+            return await originalMethod.apply(context, [method, paramArgs]);
+          } catch (error) {
+            // This seems to happen sometimes when redirects cause other outstanding requests to be cut short
+            if (
+              error instanceof Error &&
+              error.message.includes(
+                `Protocol error (Network.getResponseBody): No resource with given identifier found`
+              )
+            ) {
+              debug(
+                `Caught and ignored an error about a missing network resource.`,
+                { error }
+              );
+            } else {
+              throw error;
+            }
+          }
+        };
+
+        if (!method || !paramArgs) {
+          return next();
+        }
+
+        // To find the methods/props in question check `_evaluateInternal` at:
+        // https://github.com/puppeteer/puppeteer/blob/main/src/common/ExecutionContext.ts#L186
+        const methodsToPatch = {
+          'Runtime.evaluate': 'expression',
+          'Runtime.callFunctionOn': 'functionDeclaration',
+        };
+        const SOURCE_URL_SUFFIX =
+          '//# sourceURL=__puppeteer_evaluation_script__';
+
+        if (!methodsToPatch[method] || !paramArgs[methodsToPatch[method]]) {
+          return next();
+        }
+
+        debug('Stripping sourceURL', { method });
+        paramArgs[methodsToPatch[method]] = paramArgs[
+          methodsToPatch[method]
+        ].replace(SOURCE_URL_SUFFIX, '');
+
+        return next();
+      })(client.send, client);
+  }
+}
+
+export default function (pluginConfig) {
+  return new Plugin(pluginConfig);
+}

package/src/evasions/sourceurl/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/user-agent-override/README.md

@@ -0,0 +1,53 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/ab0047d1af7dc38412744abdb61bcfc35c42dc34/packages/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js#L42-L203)
+
+- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
+  - `opts.userAgent` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The user agent to use (default: browser.userAgent())
+  - `opts.locale` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The locale to use in `Accept-Language` header and in `navigator.languages` (default: `en-US,en`)
+  - `opts.maskLinux` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)?** Wether to hide Linux as platform in the user agent or not - true by default
+
+**Extends: PuppeteerExtraPlugin**
+
+Fixes the UserAgent info (composed of UA string, Accept-Language, Platform, and UA hints).
+
+If you don't provide any values this plugin will default to using the regular UserAgent string (while stripping the headless part).
+Default language is set to "en-US,en", the other settings match the UserAgent string.
+If you are running on Linux, it will mask the settins to look like Windows. This behavior can be disabled with the `maskLinux` option.
+
+By default puppeteer will not set a `Accept-Language` header in headless:
+It's (theoretically) possible to fix that using either `page.setExtraHTTPHeaders` or a `--lang` launch arg.
+Unfortunately `page.setExtraHTTPHeaders` will lowercase everything and launch args are not always available. :)
+
+In addition, the `navigator.platform` property is always set to the host value, e.g. `Linux` which makes detection very easy.
+
+Note: You cannot use the regular `page.setUserAgent()` puppeteer call in your code,
+as it will reset the language and platform values you set with this plugin.
+
+Example:
+
+```javascript
+const puppeteer = require('@zorilla/puppeteer-extra')
+
+const StealthPlugin = require('@zorilla/puppeteer-extra-plugin-stealth')
+const stealth = StealthPlugin()
+// Remove this specific stealth plugin from the default set
+stealth.enabledEvasions.delete('user-agent-override')
+puppeteer.use(stealth)
+
+// Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
+const UserAgentOverride = require('@zorilla/puppeteer-extra-plugin-stealth/evasions/user-agent-override')
+// Define custom UA and locale
+const ua = UserAgentOverride({
+  userAgent: 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',
+  locale: 'de-DE,de'
+})
+puppeteer.use(ua)
+```
+
+---

package/src/evasions/user-agent-override/index.js

@@ -0,0 +1,206 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+/**
+ * Fixes the UserAgent info (composed of UA string, Accept-Language, Platform, and UA hints).
+ *
+ * If you don't provide any values this plugin will default to using the regular UserAgent string (while stripping the headless part).
+ * Default language is set to "en-US,en", the other settings match the UserAgent string.
+ * If you are running on Linux, it will mask the settins to look like Windows. This behavior can be disabled with the `maskLinux` option.
+ *
+ * By default puppeteer will not set a `Accept-Language` header in headless:
+ * It's (theoretically) possible to fix that using either `page.setExtraHTTPHeaders` or a `--lang` launch arg.
+ * Unfortunately `page.setExtraHTTPHeaders` will lowercase everything and launch args are not always available. :)
+ *
+ * In addition, the `navigator.platform` property is always set to the host value, e.g. `Linux` which makes detection very easy.
+ *
+ * Note: You cannot use the regular `page.setUserAgent()` puppeteer call in your code,
+ * as it will reset the language and platform values you set with this plugin.
+ *
+ * @example
+ * const puppeteer = require("puppeteer-extra")
+ *
+ * const StealthPlugin = require("puppeteer-extra-plugin-stealth")
+ * const stealth = StealthPlugin()
+ * // Remove this specific stealth plugin from the default set
+ * stealth.enabledEvasions.delete("user-agent-override")
+ * puppeteer.use(stealth)
+ *
+ * // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
+ * const UserAgentOverride = require("puppeteer-extra-plugin-stealth/evasions/user-agent-override")
+ * // Define custom UA and locale
+ * const ua = UserAgentOverride({ userAgent: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)", locale: "de-DE,de" })
+ * puppeteer.use(ua)
+ *
+ * @param {Object} [opts] - Options
+ * @param {string} [opts.userAgent] - The user agent to use (default: browser.userAgent())
+ * @param {string} [opts.locale] - The locale to use in `Accept-Language` header and in `navigator.languages` (default: `en-US,en`)
+ * @param {boolean} [opts.maskLinux] - Wether to hide Linux as platform in the user agent or not - true by default
+ *
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+
+    this._headless = false;
+  }
+
+  get name() {
+    return 'stealth/evasions/user-agent-override';
+  }
+
+  get dependencies() {
+    return new Set(['user-preferences']);
+  }
+
+  get defaults() {
+    return {
+      userAgent: null,
+      locale: 'en-US,en',
+      maskLinux: true,
+    };
+  }
+
+  async onPageCreated(page) {
+    // Determine the full user agent string, strip the "Headless" part
+    let ua =
+      this.opts.userAgent ||
+      (await page.browser().userAgent()).replace('HeadlessChrome/', 'Chrome/');
+
+    if (
+      this.opts.maskLinux &&
+      ua.includes('Linux') &&
+      !ua.includes('Android') // Skip Android user agents since they also contain Linux
+    ) {
+      ua = ua.replace(/\(([^)]+)\)/, '(Windows NT 10.0; Win64; x64)'); // Replace the first part in parentheses with Windows data
+    }
+
+    // Full version number from Chrome
+    const uaVersion = ua.includes('Chrome/')
+      ? ua.match(/Chrome\/([\d|.]+)/)[1]
+      : (await page.browser().version()).match(/\/([\d|.]+)/)[1];
+
+    // Get platform identifier (short or long version)
+    const _getPlatform = (extended = false) => {
+      if (ua.includes('Mac OS X')) {
+        return extended ? 'Mac OS X' : 'MacIntel';
+      } else if (ua.includes('Android')) {
+        return 'Android';
+      } else if (ua.includes('Linux')) {
+        return 'Linux';
+      } else {
+        return extended ? 'Windows' : 'Win32';
+      }
+    };
+
+    // Source in C++: https://source.chromium.org/chromium/chromium/src/+/master:components/embedder_support/user_agent_utils.cc;l=55-100
+    const _getBrands = () => {
+      const seed = uaVersion.split('.')[0]; // the major version number of Chrome
+
+      const order = [
+        [0, 1, 2],
+        [0, 2, 1],
+        [1, 0, 2],
+        [1, 2, 0],
+        [2, 0, 1],
+        [2, 1, 0],
+      ][seed % 6];
+      const escapedChars = [' ', ' ', ';'];
+
+      const greaseyBrand = `${escapedChars[order[0]]}Not${
+        escapedChars[order[1]]
+      }A${escapedChars[order[2]]}Brand`;
+
+      const greasedBrandVersionList = [];
+      greasedBrandVersionList[order[0]] = {
+        brand: greaseyBrand,
+        version: '99',
+      };
+      greasedBrandVersionList[order[1]] = {
+        brand: 'Chromium',
+        version: seed,
+      };
+      greasedBrandVersionList[order[2]] = {
+        brand: 'Google Chrome',
+        version: seed,
+      };
+
+      return greasedBrandVersionList;
+    };
+
+    // Return OS version
+    const _getPlatformVersion = () => {
+      if (ua.includes('Mac OS X ')) {
+        return ua.match(/Mac OS X ([^)]+)/)[1];
+      } else if (ua.includes('Android ')) {
+        return ua.match(/Android ([^;]+)/)[1];
+      } else if (ua.includes('Windows ')) {
+        return ua.match(/Windows .*?([\d|.]+);?/)[1];
+      } else {
+        return '';
+      }
+    };
+
+    // Get architecture, this seems to be empty on mobile and x86 on desktop
+    const _getPlatformArch = () => (_getMobile() ? '' : 'x86');
+
+    // Return the Android model, empty on desktop
+    const _getPlatformModel = () =>
+      _getMobile() ? ua.match(/Android.*?;\s([^)]+)/)[1] : '';
+
+    const _getMobile = () => ua.includes('Android');
+
+    const override = {
+      userAgent: ua,
+      platform: _getPlatform(),
+      userAgentMetadata: {
+        brands: _getBrands(),
+        fullVersion: uaVersion,
+        platform: _getPlatform(true),
+        platformVersion: _getPlatformVersion(),
+        architecture: _getPlatformArch(),
+        model: _getPlatformModel(),
+        mobile: _getMobile(),
+      },
+    };
+
+    // In case of headless, override the acceptLanguage in CDP.
+    // This is not preferred, as it messed up the header order.
+    // On headful, we set the user preference language setting instead.
+    if (this._headless) {
+      override.acceptLanguage = this.opts.locale || 'en-US,en';
+    }
+
+    this.debug('onPageCreated - Will set these user agent options', {
+      override,
+      opts: this.opts,
+    });
+
+    const client =
+      typeof page._client === 'function' ? page._client() : page._client;
+    client.send('Network.setUserAgentOverride', override);
+  }
+
+  async beforeLaunch(options) {
+    // Check if launched headless
+    this._headless = options.headless;
+  }
+
+  async beforeConnect() {
+    // Treat browsers using connect() as headless browsers
+    this._headless = true;
+  }
+
+  get data() {
+    return [
+      {
+        name: 'userPreferences',
+        value: {
+          intl: { accept_languages: this.opts.locale || 'en-US,en' },
+        },
+      },
+    ];
+  }
+}
+
+const defaultExport = opts => new Plugin(opts);
+module.exports = defaultExport;

package/src/evasions/user-agent-override/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/webgl.vendor/README.md

@@ -0,0 +1,20 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/index.js#L17-L55)
+
+- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
+  - `opts.vendor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The vendor string to use (default: `Intel Inc.`)
+  - `opts.renderer` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The renderer string (default: `Intel Iris OpenGL Engine`)
+
+**Extends: PuppeteerExtraPlugin**
+
+Fix WebGL Vendor/Renderer being set to Google in headless mode
+
+Example data (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"}
+
+---

package/src/evasions/webgl.vendor/index.js

@@ -0,0 +1,57 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+import withUtils from '../_utils/withUtils.js';
+
+/**
+ * Fix WebGL Vendor/Renderer being set to Google in headless mode
+ *
+ * Example data (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"}
+ *
+ * @param {Object} [opts] - Options
+ * @param {string} [opts.vendor] - The vendor string to use (default: `Intel Inc.`)
+ * @param {string} [opts.renderer] - The renderer string (default: `Intel Iris OpenGL Engine`)
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/webgl.vendor';
+  }
+
+  /* global WebGLRenderingContext WebGL2RenderingContext */
+  async onPageCreated(page) {
+    await withUtils(page).evaluateOnNewDocument((utils, opts) => {
+      const getParameterProxyHandler = {
+        apply: (target, ctx, args) => {
+          const param = (args || [])[0];
+          const result = utils.cache.Reflect.apply(target, ctx, args);
+          // UNMASKED_VENDOR_WEBGL
+          if (param === 37445) {
+            return opts.vendor || 'Intel Inc.'; // default in headless: Google Inc.
+          }
+          // UNMASKED_RENDERER_WEBGL
+          if (param === 37446) {
+            return opts.renderer || 'Intel Iris OpenGL Engine'; // default in headless: Google SwiftShader
+          }
+          return result;
+        },
+      };
+
+      // There's more than one WebGL rendering context
+      // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility
+      // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter)
+      const addProxy = (obj, propName) => {
+        utils.replaceWithProxy(obj, propName, getParameterProxyHandler);
+      };
+      // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing:
+      addProxy(WebGLRenderingContext.prototype, 'getParameter');
+      addProxy(WebGL2RenderingContext.prototype, 'getParameter');
+    }, this.opts);
+  }
+}
+
+export default function (pluginConfig) {
+  return new Plugin(pluginConfig);
+}

package/src/evasions/webgl.vendor/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/window.outerdimensions/README.md

@@ -0,0 +1,17 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/index.js#L9-L40)
+
+- `opts` (optional, default `{}`)
+
+**Extends: PuppeteerExtraPlugin**
+
+Fix missing window.outerWidth/window.outerHeight in headless mode
+Will also set the viewport to match window size, unless specified by user
+
+---