header-generator 1.1.2 → 2.0.0-beta.0

package/README.md

@@ -5,6 +5,7 @@ NodeJs package for generating browser-like headers.
 
 - [Installation](#installation)
 - [Usage](#usage)
+- [Presets](#presets)
 - [Result example](#result-example)
 - [API Reference](#api-reference)
 
@@ -15,7 +16,7 @@ Run the `npm install header-generator` command. No further setup is needed after
 ## Usage
 To use the generator, you need to create an instance of the `HeaderGenerator` class which is exported from this package. Constructor of this class accepts a `HeaderGeneratorOptions` object, which can be used to globally specify what kind of headers you are looking for: 
 ```js
-const HeaderGenerator = require('header-generator');
+const { HeaderGenerator } = require('header-generator');
 let headerGenerator = new HeaderGenerator({
         browsers: [
             {name: "firefox", minVersion: 80},
@@ -40,6 +41,15 @@ let headers = headersGenerator.getHeaders({
 });
 ```
 This method always generates a random realistic set of headers, excluding the request dependant headers, which need to be filled in afterwards. Since the generation is randomized, multiple calls to this method with the same parameters can generate multiple different outputs.
+
+## Presets
+Presets are setting templates for common use cases. It saves time writing the same configuration over and over.
+```js
+const { HeaderGenerator, PRESETS } = require('header-generator');
+let headerGenerator = new HeaderGenerator(PRESETS.MODERN_WINDOWS_CHROME);
+```
+
+This preset will fill the configuration for the latest five versions of chrome for windows desktops. Checkout the available presets list here @TODO: LINK.
 ## Result example
 A result that can be generated for the usage example above:
 ```json
@@ -134,6 +144,7 @@ Returns a new object that contains ordered headers.
 | Param | Type | Description |
 | --- | --- | --- |
 | browsers | <code>Array.&lt;(BrowserSpecification\|string)&gt;</code> | List of BrowserSpecifications to generate the headers for,  or one of `chrome`, `firefox` and `safari`. |
+| browserListQuery | <code>string</code> | Browser generation query based on the real world data.  For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).  If `browserListQuery` is passed the `browsers` array is ignored. |
 | operatingSystems | <code>Array.&lt;string&gt;</code> | List of operating systems to generate the headers for.  The options are `windows`, `macos`, `linux`, `android` and `ios`. |
 | devices | <code>Array.&lt;string&gt;</code> | List of devices to generate the headers for. Options are `desktop` and `mobile`. |
 | locales | <code>Array.&lt;string&gt;</code> | List of at most 10 languages to include in the  [Accept-Language](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header  in the language format accepted by that header, for example `en`, `en-US` or `de`. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "header-generator",
-  "version": "1.1.2",
+  "version": "2.0.0-beta.0",
   "description": "NodeJs package for generating browser-like headers.",
   "author": {
     "name": "Apify",
@@ -14,6 +14,7 @@
     "src"
   ],
   "dependencies": {
+    "browserslist": "^4.19.1",
     "generative-bayesian-network": "0.1.0-beta.1",
     "ow": "^0.23.0"
   },

package/src/constants.js

@@ -0,0 +1,5 @@
+exports.SUPPORTED_BROWSERS = [
+    'chrome',
+    'firefox',
+    'safari',
+];

package/src/header-generator.js

@@ -11,7 +11,8 @@ const headerNetworkDefinition = require('./data_files/header-network-definition.
 const inputNetworkDefinition = require('./data_files/input-network-definition.json');
 const headersOrder = require('./data_files/headers-order.json');
 const uniqueBrowserStrings = require('./data_files/browser-helper-file.json');
-const { getBrowser, getUserAgent } = require('./utils');
+const { getBrowser, getUserAgent, getBrowsersFromQuery } = require('./utils');
+const { SUPPORTED_BROWSERS } = require('./constants');
 
 const uniqueBrowsers = [];
 for (const browserString of uniqueBrowserStrings) {
@@ -104,6 +105,7 @@ const headerGeneratorOptionsShape = {
     devices: ow.optional.array.ofType(ow.string),
     locales: ow.optional.array.ofType(ow.string),
     httpVersion: ow.optional.string,
+    browserListQuery: ow.optional.string,
 };
 
 /**
@@ -130,10 +132,14 @@ function getOrderFromUserAgent(headers) {
  * @param {string} httpVersion - Http version to be used to generate headers (the headers differ depending on the version).
  *  Either 1 or 2. If none specified the httpVersion specified in `HeaderGeneratorOptions` is used.
  */
+
 /**
  * @typedef HeaderGeneratorOptions
  * @param {Array<BrowserSpecification|string>} browsers - List of BrowserSpecifications to generate the headers for,
  *  or one of `chrome`, `firefox` and `safari`.
+ * @param {string} browserListQuery - Browser generation query based on the real world data.
+ *  For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).
+ *  If `browserListQuery` is passed the `browsers` array is ignored.
  * @param {Array<string>} operatingSystems - List of operating systems to generate the headers for.
  *  The options are `windows`, `macos`, `linux`, `android` and `ios`.
  * @param {Array<string>} devices - List of devices to generate the headers for. Options are `desktop` and `mobile`.
@@ -153,30 +159,23 @@ class HeaderGenerator {
      */
     constructor(options = {}) {
         ow(options, 'HeaderGeneratorOptions', ow.object.exactShape(headerGeneratorOptionsShape));
-        this.defaultOptions = JSON.parse(JSON.stringify(options));
         // Use a default setup when the necessary values are not provided
-        if (!this.defaultOptions.locales) {
-            this.defaultOptions.locales = ['en-US'];
-        }
-        if (!this.defaultOptions.httpVersion) {
-            this.defaultOptions.httpVersion = '2';
-        }
-        if (!this.defaultOptions.browsers) {
-            this.defaultOptions.browsers = [
-                { name: 'chrome' },
-                { name: 'firefox' },
-                { name: 'safari' },
-            ];
-        }
-        if (!this.defaultOptions.operatingSystems) {
-            this.defaultOptions.operatingSystems = [
-                'windows',
-                'macos',
-                'linux',
-                'android',
-                'ios',
-            ];
-        }
+        const {
+            browsers = SUPPORTED_BROWSERS,
+            operatingSystems = ['windows', 'macos', 'linux', 'android', 'ios'],
+            devices = ['desktop'],
+            locales = ['en-US'],
+            httpVersion = '2',
+            browserListQuery,
+        } = options;
+        this.browserListQuery = browserListQuery;
+        this.globalOptions = {
+            browsers: this._prepareBrowsersConfig(browsers, browserListQuery, httpVersion),
+            operatingSystems,
+            devices,
+            locales,
+            httpVersion,
+        };
 
         this.inputGeneratorNetwork = new BayesianNetwork(inputNetworkDefinition);
         this.headerGeneratorNetwork = new BayesianNetwork(headerNetworkDefinition);
@@ -190,17 +189,8 @@ class HeaderGenerator {
      */
     getHeaders(options = {}, requestDependentHeaders = {}) {
         ow(options, 'HeaderGeneratorOptions', ow.object.exactShape(headerGeneratorOptionsShape));
-        const headerOptions = JSON.parse(JSON.stringify({ ...this.defaultOptions, ...options }));
-        headerOptions.browsers = headerOptions.browsers.map((browserObject) => {
-            if (typeof browserObject === 'string') {
-                browserObject = { name: browserObject };
-            }
-
-            if (!browserObject.httpVersion) {
-                browserObject.httpVersion = headerOptions.httpVersion;
-            }
-            return browserObject;
-        });
+        const headerOptions = JSON.parse(JSON.stringify({ ...this.globalOptions, ...options }));
+        headerOptions.browsers = this._prepareBrowsersConfig(headerOptions.browsers, headerOptions.browserListQuery, headerOptions.httpVersion);
 
         const possibleAttributeValues = {};
 
@@ -248,7 +238,7 @@ class HeaderGenerator {
         let highLevelLocales = [];
         for (const locale of headerOptions.locales) {
             if (!locale.includes('-')) {
-                highLevelLocales.push();
+                highLevelLocales.push(locale);
             }
         }
 
@@ -279,7 +269,7 @@ class HeaderGenerator {
 
         let acceptLanguageFieldValue = localesInAddingOrder[0];
         for (let x = 1; x < localesInAddingOrder.length; x++) {
-            acceptLanguageFieldValue += `,${localesInAddingOrder[x]};${1 - x * 0.1}`;
+            acceptLanguageFieldValue += `,${localesInAddingOrder[x]};q=${1 - x * 0.1}`;
         }
 
         generatedSample[acceptLanguageFieldName] = acceptLanguageFieldValue;
@@ -329,6 +319,23 @@ class HeaderGenerator {
 
         return orderedSample;
     }
+
+    _prepareBrowsersConfig(browsers, browserListQuery, httpVersion) {
+        let finalBrowsers = browsers;
+
+        if (browserListQuery) {
+            finalBrowsers = getBrowsersFromQuery(browserListQuery);
+        }
+
+        return finalBrowsers.map((browser) => {
+            if (typeof browser === 'string') {
+                return { name: browser, httpVersion };
+            }
+
+            browser.httpVersion = httpVersion;
+            return browser;
+        });
+    }
 }
 
 module.exports = HeaderGenerator;

package/src/main.js

@@ -1,3 +1,7 @@
 const HeaderGenerator = require('./header-generator');
+const PRESETS = require('./presets');
 
-module.exports = HeaderGenerator;
+module.exports = {
+    HeaderGenerator,
+    PRESETS,
+};

package/src/presets.js

@@ -0,0 +1,62 @@
+const MODERN_DESKTOP = {
+    browserListQuery: 'last 5 versions',
+};
+
+const MODERN_MOBILE = {
+    ...MODERN_DESKTOP,
+    devices: ['mobile'],
+};
+
+exports.MODERN_DESKTOP = MODERN_DESKTOP;
+exports.MODERN_MOBILE = MODERN_MOBILE;
+
+exports.MODERN_LINUX = {
+    ...MODERN_DESKTOP,
+    operatingSystems: ['linux'],
+};
+
+exports.MODERN_LINUX_FIREFOX = {
+    browserListQuery: 'last 5 firefox versions',
+    operatingSystems: ['linux'],
+};
+
+exports.MODERN_LINUX_CHROME = {
+    browserListQuery: 'last 5 chrome versions',
+    operatingSystems: ['linux'],
+};
+
+exports.MODERN_WINDOWS = {
+    ...MODERN_DESKTOP,
+    operatingSystems: ['windows'],
+};
+
+exports.MODERN_WINDOWS_FIREFOX = {
+    browserListQuery: 'last 5 firefox versions',
+    operatingSystems: ['windows'],
+};
+
+exports.MODERN_WINDOWS_CHROME = {
+    browserListQuery: 'last 5 chrome versions',
+    operatingSystems: ['windows'],
+};
+
+exports.MODERN_MACOS = {
+    ...MODERN_DESKTOP,
+    operatingSystems: ['macos'],
+};
+
+exports.MODERN_MACOS_FIREFOX = {
+    browserListQuery: 'last 5 firefox versions',
+    operatingSystems: ['macos'],
+};
+
+exports.MODERN_MACOS_CHROME = {
+    browserListQuery: 'last 5 chrome versions',
+    operatingSystems: ['macos'],
+};
+
+exports.MODERN_ANDROID = {
+    ...MODERN_MOBILE,
+    operatingSystems: ['android'],
+
+};

package/src/utils.js

@@ -1,3 +1,6 @@
+const browsersList = require('browserslist');
+const { SUPPORTED_BROWSERS } = require('./constants');
+
 const getUserAgent = (headers) => {
     let userAgent;
     for (const [header, value] of Object.entries(headers)) {
@@ -27,7 +30,58 @@ const getBrowser = (userAgent) => {
     return browser;
 };
 
+const getBrowsersWithVersions = (browserList) => {
+    const browsersWithVersions = {};
+
+    for (const browserDefinition of browserList) {
+        const [browser, versionString] = browserDefinition.split(' ');
+        const version = parseInt(versionString, 10);
+        if (!SUPPORTED_BROWSERS.includes(browser)) {
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+
+        if (browsersWithVersions[browser]) {
+            browsersWithVersions[browser].push(version);
+        } else {
+            browsersWithVersions[browser] = [version];
+        }
+    }
+    return browsersWithVersions;
+};
+const getOptimizedVersionDistribution = (browsersWithVersions) => {
+    const finalOptimizedBrowsers = [];
+
+    Object.entries(browsersWithVersions).forEach(([browser, versions]) => {
+        const sortedVersions = versions.sort((a, b) => a - b);
+        let lowestVersionSoFar = sortedVersions[0];
+
+        sortedVersions.forEach((version, index) => {
+            const nextVersion = sortedVersions[index + 1];
+            const isLast = index === sortedVersions.length - 1;
+            const isNextVersionGap = nextVersion - version > 1;
+
+            if (isNextVersionGap || isLast) {
+                finalOptimizedBrowsers.push({
+                    name: browser,
+                    minVersion: lowestVersionSoFar,
+                    maxVersion: version,
+                });
+                lowestVersionSoFar = nextVersion;
+            }
+        });
+    });
+    return finalOptimizedBrowsers;
+};
+
+const getBrowsersFromQuery = (browserListQuery) => {
+    const browserList = browsersList(browserListQuery);
+    const browsersWithVersions = getBrowsersWithVersions(browserList);
+    return getOptimizedVersionDistribution(browsersWithVersions);
+};
+
 module.exports = {
     getUserAgent,
     getBrowser,
+    getBrowsersFromQuery,
 };