header-generator 1.1.3 → 1.2.0-beta.0

package/README.md

@@ -134,6 +134,7 @@ Returns a new object that contains ordered headers.
 | Param | Type | Description |
 | --- | --- | --- |
 | browsers | <code>Array.&lt;(BrowserSpecification\|string)&gt;</code> | List of BrowserSpecifications to generate the headers for,  or one of `chrome`, `firefox` and `safari`. |
+| browserListQuery | <code>string</code> | Browser generation query based on the real world data.  For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).  If `browserListQuery` is passed the `browsers` array is ignored. |
 | operatingSystems | <code>Array.&lt;string&gt;</code> | List of operating systems to generate the headers for.  The options are `windows`, `macos`, `linux`, `android` and `ios`. |
 | devices | <code>Array.&lt;string&gt;</code> | List of devices to generate the headers for. Options are `desktop` and `mobile`. |
 | locales | <code>Array.&lt;string&gt;</code> | List of at most 10 languages to include in the  [Accept-Language](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header  in the language format accepted by that header, for example `en`, `en-US` or `de`. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "header-generator",
-  "version": "1.1.3",
+  "version": "1.2.0-beta.0",
   "description": "NodeJs package for generating browser-like headers.",
   "author": {
     "name": "Apify",
@@ -14,6 +14,7 @@
     "src"
   ],
   "dependencies": {
+    "browserslist": "^4.19.1",
     "generative-bayesian-network": "0.1.0-beta.1",
     "ow": "^0.23.0"
   },

package/src/constants.js

@@ -0,0 +1,5 @@
+exports.SUPPORTED_BROWSERS = [
+    'chrome',
+    'firefox',
+    'safari',
+];

package/src/header-generator.js

@@ -11,7 +11,8 @@ const headerNetworkDefinition = require('./data_files/header-network-definition.
 const inputNetworkDefinition = require('./data_files/input-network-definition.json');
 const headersOrder = require('./data_files/headers-order.json');
 const uniqueBrowserStrings = require('./data_files/browser-helper-file.json');
-const { getBrowser, getUserAgent } = require('./utils');
+const { getBrowser, getUserAgent, getBrowsersFromQuery } = require('./utils');
+const { SUPPORTED_BROWSERS } = require('./constants');
 
 const uniqueBrowsers = [];
 for (const browserString of uniqueBrowserStrings) {
@@ -104,6 +105,7 @@ const headerGeneratorOptionsShape = {
     devices: ow.optional.array.ofType(ow.string),
     locales: ow.optional.array.ofType(ow.string),
     httpVersion: ow.optional.string,
+    browserListQuery: ow.optional.string,
 };
 
 /**
@@ -130,10 +132,14 @@ function getOrderFromUserAgent(headers) {
  * @param {string} httpVersion - Http version to be used to generate headers (the headers differ depending on the version).
  *  Either 1 or 2. If none specified the httpVersion specified in `HeaderGeneratorOptions` is used.
  */
+
 /**
  * @typedef HeaderGeneratorOptions
  * @param {Array<BrowserSpecification|string>} browsers - List of BrowserSpecifications to generate the headers for,
  *  or one of `chrome`, `firefox` and `safari`.
+ * @param {string} browserListQuery - Browser generation query based on the real world data.
+ *  For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).
+ *  If `browserListQuery` is passed the `browsers` array is ignored.
  * @param {Array<string>} operatingSystems - List of operating systems to generate the headers for.
  *  The options are `windows`, `macos`, `linux`, `android` and `ios`.
  * @param {Array<string>} devices - List of devices to generate the headers for. Options are `desktop` and `mobile`.
@@ -153,30 +159,23 @@ class HeaderGenerator {
      */
     constructor(options = {}) {
         ow(options, 'HeaderGeneratorOptions', ow.object.exactShape(headerGeneratorOptionsShape));
-        this.defaultOptions = JSON.parse(JSON.stringify(options));
         // Use a default setup when the necessary values are not provided
-        if (!this.defaultOptions.locales) {
-            this.defaultOptions.locales = ['en-US'];
-        }
-        if (!this.defaultOptions.httpVersion) {
-            this.defaultOptions.httpVersion = '2';
-        }
-        if (!this.defaultOptions.browsers) {
-            this.defaultOptions.browsers = [
-                { name: 'chrome' },
-                { name: 'firefox' },
-                { name: 'safari' },
-            ];
-        }
-        if (!this.defaultOptions.operatingSystems) {
-            this.defaultOptions.operatingSystems = [
-                'windows',
-                'macos',
-                'linux',
-                'android',
-                'ios',
-            ];
-        }
+        const {
+            browsers = SUPPORTED_BROWSERS,
+            operatingSystems = ['windows', 'macos', 'linux', 'android', 'ios'],
+            devices = ['desktop'],
+            locales = ['en-US'],
+            httpVersion = '2',
+            browserListQuery,
+        } = options;
+        this.browserListQuery = browserListQuery;
+        this.globalOptions = {
+            browsers: this._prepareBrowsersConfig(browsers, browserListQuery, httpVersion),
+            operatingSystems,
+            devices,
+            locales,
+            httpVersion,
+        };
 
         this.inputGeneratorNetwork = new BayesianNetwork(inputNetworkDefinition);
         this.headerGeneratorNetwork = new BayesianNetwork(headerNetworkDefinition);
@@ -190,17 +189,8 @@ class HeaderGenerator {
      */
     getHeaders(options = {}, requestDependentHeaders = {}) {
         ow(options, 'HeaderGeneratorOptions', ow.object.exactShape(headerGeneratorOptionsShape));
-        const headerOptions = JSON.parse(JSON.stringify({ ...this.defaultOptions, ...options }));
-        headerOptions.browsers = headerOptions.browsers.map((browserObject) => {
-            if (typeof browserObject === 'string') {
-                browserObject = { name: browserObject };
-            }
-
-            if (!browserObject.httpVersion) {
-                browserObject.httpVersion = headerOptions.httpVersion;
-            }
-            return browserObject;
-        });
+        const headerOptions = JSON.parse(JSON.stringify({ ...this.globalOptions, ...options }));
+        headerOptions.browsers = this._prepareBrowsersConfig(headerOptions.browsers, headerOptions.browserListQuery, headerOptions.httpVersion);
 
         const possibleAttributeValues = {};
 
@@ -329,6 +319,23 @@ class HeaderGenerator {
 
         return orderedSample;
     }
+
+    _prepareBrowsersConfig(browsers, browserListQuery, httpVersion) {
+        let finalBrowsers = browsers;
+
+        if (browserListQuery) {
+            finalBrowsers = getBrowsersFromQuery(browserListQuery);
+        }
+
+        return finalBrowsers.map((browser) => {
+            if (typeof browser === 'string') {
+                return { name: browser, httpVersion };
+            }
+
+            browser.httpVersion = httpVersion;
+            return browser;
+        });
+    }
 }
 
 module.exports = HeaderGenerator;

package/src/utils.js

@@ -1,3 +1,6 @@
+const browsersList = require('browserslist');
+const { SUPPORTED_BROWSERS } = require('./constants');
+
 const getUserAgent = (headers) => {
     let userAgent;
     for (const [header, value] of Object.entries(headers)) {
@@ -27,7 +30,57 @@ const getBrowser = (userAgent) => {
     return browser;
 };
 
+const getBrowsersWithVersions = (browserList) => {
+    const browsersWithVersions = {};
+
+    for (const browserDefinition of browserList) {
+        const [browser, version] = browserDefinition.split(' ');
+        if (!SUPPORTED_BROWSERS.includes(browser)) {
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+
+        if (browsersWithVersions[browser]) {
+            browsersWithVersions[browser].push(version);
+        } else {
+            browsersWithVersions[browser] = [version];
+        }
+    }
+    return browsersWithVersions;
+};
+const getOptimizedVersionDistribution = (browsersWithVersions) => {
+    const finalOptimizedBrowsers = [];
+
+    Object.entries(browsersWithVersions).forEach(([browser, versions]) => {
+        const sortedVersions = versions.sort((a, b) => a - b);
+        let lowestVersionSoFar = sortedVersions[0];
+
+        sortedVersions.forEach((version, index) => {
+            const nextVersion = sortedVersions[index + 1];
+            const isLast = index === sortedVersions.length - 1;
+            const isNextVersionGap = nextVersion - version > 1;
+
+            if (isNextVersionGap || isLast) {
+                finalOptimizedBrowsers.push({
+                    name: browser,
+                    minVersion: lowestVersionSoFar,
+                    maxVersion: version,
+                });
+                lowestVersionSoFar = nextVersion;
+            }
+        });
+    });
+    return finalOptimizedBrowsers;
+};
+
+const getBrowsersFromQuery = (browserListQuery) => {
+    const browserList = browsersList(browserListQuery);
+    const browsersWithVersions = getBrowsersWithVersions(browserList);
+    return getOptimizedVersionDistribution(browsersWithVersions);
+};
+
 module.exports = {
     getUserAgent,
     getBrowser,
+    getBrowsersFromQuery,
 };