@zorilla/puppeteer-extra-plugin-stealth 1.0.0

package/src/evasions/navigator.hardwareConcurrency/index.js

@@ -0,0 +1,45 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+import withUtils from '../_utils/withUtils.js';
+
+/**
+ * Set the hardwareConcurrency to 4 (optionally configurable with `hardwareConcurrency`)
+ *
+ * @see https://arh.antoinevastel.com/reports/stats/osName_hardwareConcurrency_report.html
+ *
+ * @param {Object} [opts] - Options
+ * @param {number} [opts.hardwareConcurrency] - The value to use in `navigator.hardwareConcurrency` (default: `4`)
+ */
+
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/navigator.hardwareConcurrency';
+  }
+
+  get defaults() {
+    return {
+      hardwareConcurrency: 4,
+    };
+  }
+
+  async onPageCreated(page) {
+    await withUtils(page).evaluateOnNewDocument(
+      (utils, { opts }) => {
+        utils.replaceGetterWithProxy(
+          Object.getPrototypeOf(navigator),
+          'hardwareConcurrency',
+          utils.makeHandler().getterValue(opts.hardwareConcurrency)
+        );
+      },
+      {
+        opts: this.opts,
+      }
+    );
+  }
+}
+
+module.exports = pluginConfig => new Plugin(pluginConfig);

package/src/evasions/navigator.hardwareConcurrency/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/navigator.languages/README.md

@@ -0,0 +1,17 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js#L11-L28)
+
+- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
+  - `opts.languages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>?** The languages to use (default: `['en-US', 'en']`)
+
+**Extends: PuppeteerExtraPlugin**
+
+Pass the Languages Test. Allows setting custom languages.
+
+---

package/src/evasions/navigator.languages/index.js

@@ -0,0 +1,44 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+import withUtils from '../_utils/withUtils.js';
+
+/**
+ * Pass the Languages Test. Allows setting custom languages.
+ *
+ * @param {Object} [opts] - Options
+ * @param {Array<string>} [opts.languages] - The languages to use (default: `['en-US', 'en']`)
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/navigator.languages';
+  }
+
+  get defaults() {
+    return {
+      languages: [], // Empty default, otherwise this would be merged with user defined array override
+    };
+  }
+
+  async onPageCreated(page) {
+    await withUtils(page).evaluateOnNewDocument(
+      (utils, { opts }) => {
+        const languages = opts.languages.length
+          ? opts.languages
+          : ['en-US', 'en'];
+        utils.replaceGetterWithProxy(
+          Object.getPrototypeOf(navigator),
+          'languages',
+          utils.makeHandler().getterValue(Object.freeze([...languages]))
+        );
+      },
+      {
+        opts: this.opts,
+      }
+    );
+  }
+}
+
+module.exports = pluginConfig => new Plugin(pluginConfig);

package/src/evasions/navigator.languages/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/navigator.permissions/README.md

@@ -0,0 +1,16 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js#L12-L45)
+
+- `opts` (optional, default `{}`)
+
+**Extends: PuppeteerExtraPlugin**
+
+Pass the Permissions Test.
+
+---

package/src/evasions/navigator.permissions/index.js

@@ -0,0 +1,66 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+import withUtils from '../_utils/withUtils.js';
+
+/**
+ * Fix `Notification.permission` behaving weirdly in headless mode
+ *
+ * @see https://bugs.chromium.org/p/chromium/issues/detail?id=1052332
+ */
+
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/navigator.permissions';
+  }
+
+  /* global Notification Permissions PermissionStatus */
+  async onPageCreated(page) {
+    await withUtils(page).evaluateOnNewDocument((utils, _opts) => {
+      const isSecure = document.location.protocol.startsWith('https');
+
+      // In headful on secure origins the permission should be "default", not "denied"
+      if (isSecure) {
+        utils.replaceGetterWithProxy(Notification, 'permission', {
+          apply() {
+            return 'default';
+          },
+        });
+      }
+
+      // Another weird behavior:
+      // On insecure origins in headful the state is "denied",
+      // whereas in headless it's "prompt"
+      if (!isSecure) {
+        const handler = {
+          apply(_target, _ctx, args) {
+            const param = (args || [])[0];
+
+            const isNotifications =
+              param?.name && param.name === 'notifications';
+            if (!isNotifications) {
+              return utils.cache.Reflect.apply(...arguments);
+            }
+
+            return Promise.resolve(
+              Object.setPrototypeOf(
+                {
+                  state: 'denied',
+                  onchange: null,
+                },
+                PermissionStatus.prototype
+              )
+            );
+          },
+        };
+        // Note: Don't use `Object.getPrototypeOf` here
+        utils.replaceWithProxy(Permissions.prototype, 'query', handler);
+      }
+    }, this.opts);
+  }
+}
+
+module.exports = pluginConfig => new Plugin(pluginConfig);

package/src/evasions/navigator.permissions/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/navigator.plugins/README.md

@@ -0,0 +1,24 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js#L26-L88)
+
+- `opts` (optional, default `{}`)
+
+**Extends: PuppeteerExtraPlugin**
+
+In headless mode `navigator.mimeTypes` and `navigator.plugins` are empty.
+This plugin emulates both of these with functional mocks to match regular headful Chrome.
+
+Note: mimeTypes and plugins cross-reference each other, so it makes sense to do them at the same time.
+
+- **See: <https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes>**
+- **See: <https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray>**
+- **See: <https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins>**
+- **See: <https://developer.mozilla.org/en-US/docs/Web/API/PluginArray>**
+
+---

package/src/evasions/navigator.plugins/data.json

@@ -0,0 +1,48 @@
+{
+  "mimeTypes": [
+    {
+      "type": "application/pdf",
+      "suffixes": "pdf",
+      "description": "Portable Document Format",
+      "__pluginName": "PDF Viewer"
+    },
+    {
+      "type": "text/pdf",
+      "suffixes": "pdf",
+      "description": "Portable Document Format",
+      "__pluginName": "PDF Viewer"
+    }
+  ],
+  "plugins": [
+    {
+      "name": "PDF Viewer",
+      "filename": "internal-pdf-viewer",
+      "description": "Portable Document Format",
+      "__mimeTypes": ["application/pdf", "text/pdf"]
+    },
+    {
+      "name": "Chrome PDF Viewer",
+      "filename": "mhjfbmdgcfjbbpaeojofohoefgiehjai",
+      "description": "Portable Document Format",
+      "__mimeTypes": ["application/pdf", "text/pdf"]
+    },
+    {
+      "name": "Chromium PDF Viewer",
+      "filename": "chromium-pdf-viewer",
+      "description": "Portable Document Format",
+      "__mimeTypes": ["application/pdf", "text/pdf"]
+    },
+    {
+      "name": "Microsoft Edge PDF Viewer",
+      "filename": "edge-pdf-viewer",
+      "description": "Portable Document Format",
+      "__mimeTypes": ["application/pdf", "text/pdf"]
+    },
+    {
+      "name": "WebKit built-in PDF",
+      "filename": "webkit-pdf-plugin",
+      "description": "Portable Document Format",
+      "__mimeTypes": ["application/pdf", "text/pdf"]
+    }
+  ]
+}

package/src/evasions/navigator.plugins/functionMocks.js

@@ -0,0 +1,47 @@
+/**
+ * `navigator.{plugins,mimeTypes}` share similar custom functions to look up properties
+ *
+ * Note: This is meant to be run in the context of the page.
+ */
+export const generateFunctionMocks =
+  utils => (proto, itemMainProp, dataArray) => ({
+    /** Returns the MimeType object with the specified index. */
+    item: utils.createProxy(proto.item, {
+      apply(_target, _ctx, args) {
+        if (!args.length) {
+          throw new TypeError(
+            `Failed to execute 'item' on '${
+              proto[Symbol.toStringTag]
+            }': 1 argument required, but only 0 present.`
+          );
+        }
+        // Special behavior alert:
+        // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup
+        // - If anything else than an integer (including as string) is provided it will return the first entry
+        const isInteger = args[0] && Number.isInteger(Number(args[0])); // Cast potential string to number first, then check for integer
+        // Note: Vanilla never returns `undefined`
+        return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null;
+      },
+    }),
+    /** Returns the MimeType object with the specified name. */
+    namedItem: utils.createProxy(proto.namedItem, {
+      apply(_target, _ctx, args) {
+        if (!args.length) {
+          throw new TypeError(
+            `Failed to execute 'namedItem' on '${
+              proto[Symbol.toStringTag]
+            }': 1 argument required, but only 0 present.`
+          );
+        }
+        return dataArray.find(mt => mt[itemMainProp] === args[0]) || null; // Not `undefined`!
+      },
+    }),
+    /** Does nothing and shall return nothing */
+    refresh: proto.refresh
+      ? utils.createProxy(proto.refresh, {
+          apply(_target, _ctx, _args) {
+            return undefined;
+          },
+        })
+      : undefined,
+  });

package/src/evasions/navigator.plugins/index.js

@@ -0,0 +1,98 @@
+import { PuppeteerExtraPlugin } from '@zorilla/puppeteer-extra-plugin';
+
+import utils from '../_utils/index.js';
+
+import withUtils from '../_utils/withUtils.js';
+import data from './data.json' with { type: 'json' };
+import { generateFunctionMocks } from './functionMocks.js';
+import { generateMagicArray } from './magicArray.js';
+import { generateMimeTypeArray } from './mimeTypes.js';
+import { generatePluginArray } from './plugins.js';
+
+/**
+ * In headless mode `navigator.mimeTypes` and `navigator.plugins` are empty.
+ * This plugin emulates both of these with functional mocks to match regular headful Chrome.
+ *
+ * Note: mimeTypes and plugins cross-reference each other, so it makes sense to do them at the same time.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/PluginArray
+ */
+class Plugin extends PuppeteerExtraPlugin {
+  constructor(opts = {}) {
+    super(opts);
+  }
+
+  get name() {
+    return 'stealth/evasions/navigator.plugins';
+  }
+
+  async onPageCreated(page) {
+    await withUtils(page).evaluateOnNewDocument(
+      (utils, { fns, data }) => {
+        fns = utils.materializeFns(fns);
+
+        // That means we're running headful
+        const hasPlugins = 'plugins' in navigator && navigator.plugins.length;
+        if (hasPlugins) {
+          return; // nothing to do here
+        }
+
+        const mimeTypes = fns.generateMimeTypeArray(utils, fns)(data.mimeTypes);
+        const plugins = fns.generatePluginArray(utils, fns)(data.plugins);
+
+        // Plugin and MimeType cross-reference each other, let's do that now
+        // Note: We're looping through `data.plugins` here, not the generated `plugins`
+        for (const pluginData of data.plugins) {
+          pluginData.__mimeTypes.forEach((type, index) => {
+            plugins[pluginData.name][index] = mimeTypes[type];
+
+            Object.defineProperty(plugins[pluginData.name], type, {
+              value: mimeTypes[type],
+              writable: false,
+              enumerable: false, // Not enumerable
+              configurable: true,
+            });
+            Object.defineProperty(mimeTypes[type], 'enabledPlugin', {
+              value:
+                type === 'application/x-pnacl'
+                  ? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks
+                  : new Proxy(plugins[pluginData.name], {}), // Prevent circular references
+              writable: false,
+              enumerable: false, // Important: `JSON.stringify(navigator.plugins)`
+              configurable: true,
+            });
+          });
+        }
+
+        const patchNavigator = (name, value) =>
+          utils.replaceProperty(Object.getPrototypeOf(navigator), name, {
+            get() {
+              return value;
+            },
+          });
+
+        patchNavigator('mimeTypes', mimeTypes);
+        patchNavigator('plugins', plugins);
+
+        // All done
+      },
+      {
+        // We pass some functions to evaluate to structure the code more nicely
+        fns: utils.stringifyFns({
+          generateMimeTypeArray,
+          generatePluginArray,
+          generateMagicArray,
+          generateFunctionMocks,
+        }),
+        data,
+      }
+    );
+  }
+}
+
+export default function (pluginConfig) {
+  return new Plugin(pluginConfig);
+}

package/src/evasions/navigator.plugins/magicArray.js

@@ -0,0 +1,145 @@
+/* global MimeType MimeTypeArray Plugin PluginArray  */
+
+/**
+ * Generate a convincing and functional MimeType or Plugin array from scratch.
+ * They're so similar that it makes sense to use a single generator here.
+ *
+ * Note: This is meant to be run in the context of the page.
+ */
+export const generateMagicArray =
+  (utils, fns) =>
+  (
+    dataArray = [],
+    proto = MimeTypeArray.prototype,
+    itemProto = MimeType.prototype,
+    itemMainProp = 'type'
+  ) => {
+    // Quick helper to set props with the same descriptors vanilla is using
+    const defineProp = (obj, prop, value) =>
+      Object.defineProperty(obj, prop, {
+        value,
+        writable: false,
+        enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)`
+        configurable: true,
+      });
+
+    // Loop over our fake data and construct items
+    const makeItem = data => {
+      const item = {};
+      for (const prop of Object.keys(data)) {
+        if (prop.startsWith('__')) {
+          continue;
+        }
+        defineProp(item, prop, data[prop]);
+      }
+      return patchItem(item, data);
+    };
+
+    const patchItem = (item, data) => {
+      let descriptor = Object.getOwnPropertyDescriptors(item);
+
+      // Special case: Plugins have a magic length property which is not enumerable
+      // e.g. `navigator.plugins[i].length` should always be the length of the assigned mimeTypes
+      if (itemProto === Plugin.prototype) {
+        descriptor = {
+          ...descriptor,
+          length: {
+            value: data.__mimeTypes.length,
+            writable: false,
+            enumerable: false,
+            configurable: true, // Important to be able to use the ownKeys trap in a Proxy to strip `length`
+          },
+        };
+      }
+
+      // We need to spoof a specific `MimeType` or `Plugin` object
+      const obj = Object.create(itemProto, descriptor);
+
+      // Virtually all property keys are not enumerable in vanilla
+      const blacklist = [...Object.keys(data), 'length', 'enabledPlugin'];
+      return new Proxy(obj, {
+        ownKeys(target) {
+          return Reflect.ownKeys(target).filter(k => !blacklist.includes(k));
+        },
+        getOwnPropertyDescriptor(target, prop) {
+          if (blacklist.includes(prop)) {
+            return undefined;
+          }
+          return Reflect.getOwnPropertyDescriptor(target, prop);
+        },
+      });
+    };
+
+    const magicArray = [];
+
+    // Loop through our fake data and use that to create convincing entities
+    dataArray.forEach(data => {
+      magicArray.push(makeItem(data));
+    });
+
+    // Add direct property access  based on types (e.g. `obj['application/pdf']`) afterwards
+    magicArray.forEach(entry => {
+      defineProp(magicArray, entry[itemMainProp], entry);
+    });
+
+    // This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)`
+    const magicArrayObj = Object.create(proto, {
+      ...Object.getOwnPropertyDescriptors(magicArray),
+
+      // There's one ugly quirk we unfortunately need to take care of:
+      // The `MimeTypeArray` prototype has an enumerable `length` property,
+      // but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`.
+      // To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap.
+      length: {
+        value: magicArray.length,
+        writable: false,
+        enumerable: false,
+        configurable: true, // Important to be able to use the ownKeys trap in a Proxy to strip `length`
+      },
+    });
+
+    // Generate our functional function mocks :-)
+    const functionMocks = fns.generateFunctionMocks(utils)(
+      proto,
+      itemMainProp,
+      magicArray
+    );
+
+    // We need to overlay our custom object with a JS Proxy
+    const magicArrayObjProxy = new Proxy(magicArrayObj, {
+      get(_target, key = '') {
+        // Redirect function calls to our custom proxied versions mocking the vanilla behavior
+        if (key === 'item') {
+          return functionMocks.item;
+        }
+        if (key === 'namedItem') {
+          return functionMocks.namedItem;
+        }
+        if (proto === PluginArray.prototype && key === 'refresh') {
+          return functionMocks.refresh;
+        }
+        // Everything else can pass through as normal
+        return utils.cache.Reflect.get(...arguments);
+      },
+      ownKeys(_target) {
+        // There are a couple of quirks where the original property demonstrates "magical" behavior that makes no sense
+        // This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length`
+        // My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly
+        // For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing
+        // Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing
+        const keys = [];
+        const typeProps = magicArray.map(mt => mt[itemMainProp]);
+        typeProps.forEach((_, i) => keys.push(`${i}`));
+        typeProps.forEach(propName => keys.push(propName));
+        return keys;
+      },
+      getOwnPropertyDescriptor(target, prop) {
+        if (prop === 'length') {
+          return undefined;
+        }
+        return Reflect.getOwnPropertyDescriptor(target, prop);
+      },
+    });
+
+    return magicArrayObjProxy;
+  };

package/src/evasions/navigator.plugins/mimeTypes.js

@@ -0,0 +1,18 @@
+/* global MimeType MimeTypeArray */
+
+/**
+ * Generate a convincing and functional MimeTypeArray (with mime types) from scratch.
+ *
+ * Note: This is meant to be run in the context of the page.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray
+ */
+export const generateMimeTypeArray = (utils, fns) => mimeTypesData => {
+  return fns.generateMagicArray(utils, fns)(
+    mimeTypesData,
+    MimeTypeArray.prototype,
+    MimeType.prototype,
+    'type'
+  );
+};

package/src/evasions/navigator.plugins/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "main": "index.js"
+}

package/src/evasions/navigator.plugins/plugins.js

@@ -0,0 +1,18 @@
+/* global Plugin PluginArray */
+
+/**
+ * Generate a convincing and functional PluginArray (with plugins) from scratch.
+ *
+ * Note: This is meant to be run in the context of the page.
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins
+ * @see https://developer.mozilla.org/en-US/docs/Web/API/PluginArray
+ */
+export const generatePluginArray = (utils, fns) => pluginsData => {
+  return fns.generateMagicArray(utils, fns)(
+    pluginsData,
+    PluginArray.prototype,
+    Plugin.prototype,
+    'name'
+  );
+};

package/src/evasions/navigator.vendor/README.md

@@ -0,0 +1,36 @@
+## API
+
+
+#### Table of Contents
+
+- [class: Plugin](#class-plugin)
+
+### class: [Plugin](https://github.com/zorillajs/zorilla/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.js#L28-L55)
+
+- `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`)
+  - `opts.vendor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The vendor to use in `navigator.vendor` (default: `Google Inc.`)
+
+**Extends: PuppeteerExtraPlugin**
+
+By default puppeteer will have a fixed `navigator.vendor` property.
+
+This plugin makes it possible to change this property.
+
+Example:
+
+```javascript
+const puppeteer = require('@zorilla/puppeteer-extra')
+
+const StealthPlugin = require('@zorilla/puppeteer-extra-plugin-stealth')
+const stealth = StealthPlugin()
+// Remove this specific stealth plugin from the default set
+stealth.enabledEvasions.delete('navigator.vendor')
+puppeteer.use(stealth)
+
+// Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such
+const NavigatorVendorPlugin = require('@zorilla/puppeteer-extra-plugin-stealth/evasions/navigator.vendor')
+const nvp = NavigatorVendorPlugin({ vendor: 'Apple Computer, Inc.' }) // Custom vendor
+puppeteer.use(nvp)
+```
+
+---