es6-crawler-detect 3.1.0 → 3.2.0

package/src/lib/crawler/exclusions.js

@@ -1,15 +1,62 @@
-'use strict';
-
-const Provider = require("./provider");
-
-class Exclusions extends Provider
-{
-	constructor()
-	{
-		super();
-
-		this.data = ["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"];
-	}
-}
-
-module.exports = Exclusions;
+'use strict';
+
+const Provider = require('./provider');
+
+class Exclusions extends Provider {
+  constructor() {
+    super();
+
+    this.data = [
+      'Safari.[\\d\\.]*',
+      'Firefox.[\\d\\.]*',
+      ' Chrome.[\\d\\.]*',
+      'Chromium.[\\d\\.]*',
+      'MSIE.[\\d\\.]',
+      'Opera\\/[\\d\\.]*',
+      'Mozilla.[\\d\\.]*',
+      'AppleWebKit.[\\d\\.]*',
+      'Trident.[\\d\\.]*',
+      'Windows NT.[\\d\\.]*',
+      'Android [\\d\\.]*',
+      'Macintosh.',
+      'Ubuntu',
+      'Linux',
+      '[ ]Intel',
+      'Mac OS X [\\d_]*',
+      '(like )?Gecko(.[\\d\\.]*)?',
+      'KHTML,',
+      'CriOS.[\\d\\.]*',
+      'CPU iPhone OS ([0-9_])* like Mac OS X',
+      'CPU OS ([0-9_])* like Mac OS X',
+      'iPod',
+      'compatible',
+      'x86_..',
+      'i686',
+      'x64',
+      'X11',
+      'rv:[\\d\\.]*',
+      'Version.[\\d\\.]*',
+      'WOW64',
+      'Win64',
+      'Dalvik.[\\d\\.]*',
+      ' \\.NET CLR [\\d\\.]*',
+      'Presto.[\\d\\.]*',
+      'Media Center PC',
+      'BlackBerry',
+      'Build',
+      'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
+      'Opera',
+      ' \\.NET[\\d\\.]*',
+      'cubot',
+      '; M bot',
+      '; CRONO',
+      '; B bot',
+      '; IDbot',
+      '; ID bot',
+      '; POWER BOT',
+      ';',
+    ];
+  }
+}
+
+module.exports = Exclusions;

package/src/lib/crawler/headers.js

@@ -1,15 +1,24 @@
-'use strict';
-
-const Provider = require("./provider");
-
-class Headers extends Provider
-{
-	constructor()
-	{
-		super();
-
-		this.data = ["HTTP_USER_AGENT","HTTP_X_OPERAMINI_PHONE_UA","HTTP_X_DEVICE_USER_AGENT","HTTP_X_ORIGINAL_USER_AGENT","HTTP_X_SKYFIRE_PHONE","HTTP_X_BOLT_PHONE_UA","HTTP_DEVICE_STOCK_UA","HTTP_X_UCBROWSER_DEVICE_UA","HTTP_FROM","HTTP_X_SCANNER"];
-	}
-}
-
-module.exports = Headers;
+'use strict';
+
+const Provider = require('./provider');
+
+class Headers extends Provider {
+  constructor() {
+    super();
+
+    this.data = [
+      'USER-AGENT',
+      'X-OPERAMINI-PHONE-UA',
+      'X-DEVICE-USER-AGENT',
+      'X-ORIGINAL-USER-AGENT',
+      'X-SKYFIRE-PHONE',
+      'X-BOLT-PHONE-UA',
+      'DEVICE-STOCK-UA',
+      'X-UCBROWSER-DEVICE-UA',
+      'FROM',
+      'X-SCANNER',
+    ];
+  }
+}
+
+module.exports = Headers;

package/src/lib/crawler/provider.js

@@ -1,13 +1,11 @@
-'use strict';
-
-class Provider
-{
-	constructor(){}
-
-	getAll()
-	{
-		return this.data;
-	}
-}
-
-module.exports = Provider;
+'use strict';
+
+class Provider {
+  constructor() {}
+
+  getAll() {
+    return this.data;
+  }
+}
+
+module.exports = Provider;

package/src/lib/crawler.js

@@ -1,139 +1,135 @@
-'use strict';
-
-const Crawlers   = require('./crawler/crawlers');
-const Exclusions = require('./crawler/exclusions');
-const Headers    = require('./crawler/headers');
-
-class Crawler
-{
-	constructor(request, headers, userAgent)
-	{
-		/**
-		 * Init classes
-		 */
-		this._init();
-
-		/**
-		 * This request must be an object
-		 */
-		this.request = typeof request === "object" ? request : {};
-
-		// The regex-list must not be used with g-flag!
-		// See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
-		this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
-
-		// The exclusions should be used with g-flag in order to remove each value.
-		this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'g');
-
-		/**
-		 * Set http headers
-		 */
-		this.setHttpHeaders(headers);
-
-		/**
-		 * Set userAgent
-		 */
-		this.userAgent = this.setUserAgent(userAgent);
-	}
-
-	/**
-	 * Init Classes Instances
-	 */
-	_init()
-	{
-		this.crawlers   = new Crawlers();
-		this.headers    = new Headers();
-		this.exclusions = new Exclusions();
-	}
-
-	compileRegex(patterns, flags) {
-		return new RegExp(patterns.join('|').trim(), flags);
-	}
-
-	/**
-	 * Set HTTP headers.
-	 */
-	setHttpHeaders(headers)
-	{
-		// Use the Request headers if httpHeaders is not defined
-        if ( typeof headers !== "object" || Object.keys(headers).length == 0 )
-        {
-        	headers = Object.keys(this.request).length ? this.request.headers : {};
-        }
-
-		// Clear existing headers.
-		this.httpHeaders = [];
-
-        // Only save HTTP headers. In NodeJS land, that means
-        // Only Request vars that start with HTTP-.
-		for ( const key in headers )
-		{
-			if ( key.substring(0, 5) === "http-" )
-			{
-				this.httpHeaders[key] = headers[key];
-			}
-		}
-	}
-
-	/**
-	 * Set user agent
-	 */
-	setUserAgent(userAgent)
-	{
-		if ( typeof userAgent == "undefined" || userAgent == null || ! userAgent.length )
-		{
-			for ( const header of this.getUaHttpHeaders() )
-			{
-				if ( Object.keys(this.httpHeaders).indexOf(header.toLowerCase()) === 0 )
-				{
-					userAgent += this.httpHeaders[header] + ' ';
-				}
-			}
-		}
-
-		return userAgent;
-	}
-
-	/**
-	 * Get user agent headers
-	 */
-	getUaHttpHeaders()
-	{
-		return this.headers.getAll();
-	}
-
-	/**
-	 * Check user agent string against the regex.
-	 */
-	isCrawler(userAgent = undefined)
-	{
-		var agent = (typeof userAgent == "undefined" || userAgent == null ? this.userAgent : userAgent);
-
-		// test on compiled regx
-		agent = agent.replace(this.compiledExclusions, '');
-
-		if ( agent.trim().length == 0 )
-		{
-			return false;
-		}
-
-		var matches = this.compiledRegexList.exec(agent.trim());
-
-		if ( matches )
-		{
-			this.matches = matches;
-		}
-
-		return matches !== null ? (matches.length ? true : false) : false;
-	}
-
-	/**
-	 * Return the matches.
-	 */
-	getMatches()
-	{
-		return this.matches !== undefined ? (this.matches.length ? this.matches[0] : null) : {};
-	}
-}
-
-module.exports = Crawler;
+'use strict';
+
+const Crawlers = require('./crawler/crawlers');
+const Exclusions = require('./crawler/exclusions');
+const Headers = require('./crawler/headers');
+
+class Crawler {
+  constructor(request, headers, userAgent) {
+    /**
+     * Init classes
+     */
+    this._init();
+
+    /**
+     * This request must be an object
+     */
+    this.request = typeof request === 'object' ? request : {};
+
+    // The regex-list must not be used with g-flag!
+    // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
+    this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
+
+    // The exclusions should be used with g-flag in order to remove each value.
+    this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'g');
+
+    /**
+     * Set http headers
+     */
+    this.setHttpHeaders(headers);
+
+    /**
+     * Set userAgent
+     */
+    this.userAgent = this.setUserAgent(userAgent);
+  }
+
+  /**
+   * Init Classes Instances
+   */
+  _init() {
+    this.crawlers = new Crawlers();
+    this.headers = new Headers();
+    this.exclusions = new Exclusions();
+  }
+
+  compileRegex(patterns, flags) {
+    return new RegExp(patterns.join('|').trim(), flags);
+  }
+
+  /**
+   * Set HTTP headers.
+   */
+  setHttpHeaders(headers) {
+    // Use the Request headers if httpHeaders is not defined
+    if (typeof headers === 'undefined' || Object.keys(headers).length === 0) {
+      headers = Object.keys(this.request).length ? this.request.headers : {};
+    }
+
+    // Clear existing headers.
+    this.httpHeaders = [];
+
+    // Only save HTTP headers.
+    for (const key in headers) {
+      this.httpHeaders[key] = headers[key];
+    }
+  }
+
+  /**
+   * Set user agent
+   */
+  setUserAgent(userAgent) {
+    if (
+      typeof userAgent === 'undefined' ||
+      userAgent === null ||
+      !userAgent.length
+    ) {
+      for (const header of this.getUaHttpHeaders()) {
+        if (Object.keys(this.httpHeaders).indexOf(header.toLowerCase()) >= 0) {
+          userAgent += this.httpHeaders[header] + ' ';
+        }
+      }
+    }
+
+    return userAgent;
+  }
+
+  /**
+   * Get user agent headers
+   */
+  getUaHttpHeaders() {
+    return this.headers.getAll();
+  }
+
+  /**
+   * Check user agent string against the regex.
+   */
+  isCrawler(userAgent = undefined) {
+    if (Buffer.byteLength(userAgent || '', 'utf8') > 4096) {
+      return false;
+    }
+
+    var agent =
+      typeof userAgent === 'undefined' || userAgent === null
+        ? this.userAgent
+        : userAgent;
+
+    // test on compiled regx
+    agent = agent.replace(this.compiledExclusions, '');
+
+    if (agent.trim().length === 0) {
+      return false;
+    }
+
+    var matches = this.compiledRegexList.exec(agent.trim());
+
+    if (matches) {
+      this.matches = matches;
+    }
+
+    return matches !== null ? (matches.length ? true : false) : false;
+  }
+
+  /**
+   * Return the matches.
+   */
+  getMatches() {
+    return this.matches !== undefined
+      ? this.matches.length
+        ? this.matches[0]
+        : null
+      : {};
+  }
+}
+
+module.exports = Crawler;

package/test/lib/crawler.test.js

@@ -1,35 +1,52 @@
-var assert = require("assert");
-
-const Crawler = require("../../src/lib/crawler");
-
-describe("crawler", () => {
-    var crawler = new Crawler();
-
-    it("will identify crawlers correctly on subsequent calls", () => {
-        assert.strictEqual(crawler.isCrawler("Zombie.js"), true);
-        assert.strictEqual(crawler.isCrawler("Zombie.js"), true, "crawler was not identified on subsequent call");
-    });
-
-    it("will identify telegram bot", () => {
-        assert.strictEqual(crawler.isCrawler("TelegramBot (like TwitterBot)"), true);
-    });
-
-    describe("regex-compilation", () => {
-        it("will join list of patterns with pipes", () => {
-            assert.strictEqual(crawler.compileRegex(["some", "patterns"]).source, "some|patterns");
-            assert.strictEqual(crawler.compileRegex(["single"]).source, "single");
-            assert.strictEqual(crawler.compileRegex(["  remove-whitespaces "]).source, "remove-whitespaces");
-        });
-
-        it("will accept regex-flags for compilation", () => {
-            var patterns = ['some', 'patterns'];
-            assert.strictEqual(crawler.compileRegex(patterns, "g").flags, "g");
-            assert.strictEqual(crawler.compileRegex(patterns, "i").flags, "i");
-        });
-
-        it("should be case insensitive", ()=>{
-            assert.strictEqual(crawler.isCrawler("Facebot\1.0"), true);
-            assert.strictEqual(crawler.getMatches("Facebot\1.0"), "Facebot", "Crawler was not able to indentify crawler correctly");
-        })
-    });
-});
+var assert = require('assert');
+
+const Crawler = require('../../src/lib/crawler');
+
+describe('crawler', () => {
+  var crawler = new Crawler();
+
+  it('will identify crawlers correctly on subsequent calls', () => {
+    assert.strictEqual(crawler.isCrawler('Zombie.js'), true);
+    assert.strictEqual(
+      crawler.isCrawler('Zombie.js'),
+      true,
+      'crawler was not identified on subsequent call'
+    );
+  });
+
+  it('will identify telegram bot', () => {
+    assert.strictEqual(
+      crawler.isCrawler('TelegramBot (like TwitterBot)'),
+      true
+    );
+  });
+
+  describe('regex-compilation', () => {
+    it('will join list of patterns with pipes', () => {
+      assert.strictEqual(
+        crawler.compileRegex(['some', 'patterns']).source,
+        'some|patterns'
+      );
+      assert.strictEqual(crawler.compileRegex(['single']).source, 'single');
+      assert.strictEqual(
+        crawler.compileRegex(['  remove-whitespaces ']).source,
+        'remove-whitespaces'
+      );
+    });
+
+    it('will accept regex-flags for compilation', () => {
+      var patterns = ['some', 'patterns'];
+      assert.strictEqual(crawler.compileRegex(patterns, 'g').flags, 'g');
+      assert.strictEqual(crawler.compileRegex(patterns, 'i').flags, 'i');
+    });
+
+    it('should be case insensitive', () => {
+      assert.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
+      assert.strictEqual(
+        crawler.getMatches('Facebot\\1.0'),
+        'Facebot',
+        'Crawler was not able to indentify crawler correctly'
+      );
+    });
+  });
+});

package/webpack.common.js

@@ -1,20 +1,20 @@
-const path = require('path')
-
-module.exports = {
-  context: path.resolve(__dirname, 'src'),
-  entry: './index.js',
-  output: {
-    path: path.resolve(__dirname, 'dist'),
-    filename: '[name].bundle.js',
-    library: 'Crawler'
-  },
-  module: {
-    rules: [
-      {
-        test: /\.js$/,
-        use: 'babel-loader',
-        exclude: /node_modules/
-      }
-    ]
-  }
-}
+const path = require('path');
+
+module.exports = {
+  context: path.resolve(__dirname, 'src'),
+  entry: './index.js',
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: '[name].bundle.js',
+    library: 'Crawler',
+  },
+  module: {
+    rules: [
+      {
+        test: /\.js$/,
+        use: 'babel-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+};

package/webpack.dev.js

@@ -1,7 +1,7 @@
-const merge = require('webpack-merge')
-const common = require('./webpack.common')
-
-module.exports = merge(common, {
-  mode: 'development',
-  devtool: 'inline-source-map'
-})
+const { merge } = require('webpack-merge');
+const common = require('./webpack.common');
+
+module.exports = merge(common, {
+  mode: 'development',
+  devtool: 'inline-source-map',
+});

package/webpack.prod.js

@@ -1,15 +1,14 @@
-const merge = require('webpack-merge')
-const TerserPlugin = require('terser-webpack-plugin')
-const common = require('./webpack.common')
-
-module.exports = merge(common, {
-  mode: 'production',
-  optimization: {
-    minimizer: [
-      new TerserPlugin({
-        cache: true,
-        parallel: true
-      })
-    ]
-  }
-})
+const { merge } = require('webpack-merge');
+const TerserPlugin = require('terser-webpack-plugin');
+const common = require('./webpack.common');
+
+module.exports = merge(common, {
+  mode: 'production',
+  optimization: {
+    minimizer: [
+      new TerserPlugin({
+        parallel: true,
+      }),
+    ],
+  },
+});