es6-crawler-detect 4.0.0-beta.1 → 4.0.0-beta.2

package/src/lib/crawler/exclusions.js

@@ -1,58 +0,0 @@
-'use strict';
-Object.defineProperty(exports, '__esModule', { value: true });
-exports.Exclusions = void 0;
-class Exclusions {
-  getAll() {
-    return [
-      'Safari.[\\d\\.]*',
-      'Firefox.[\\d\\.]*',
-      ' Chrome.[\\d\\.]*',
-      'Chromium.[\\d\\.]*',
-      'MSIE.[\\d\\.]',
-      'Opera\\/[\\d\\.]*',
-      'Mozilla.[\\d\\.]*',
-      'AppleWebKit.[\\d\\.]*',
-      'Trident.[\\d\\.]*',
-      'Windows NT.[\\d\\.]*',
-      'Android [\\d\\.]*',
-      'Macintosh.',
-      'Ubuntu',
-      'Linux',
-      '[ ]Intel',
-      'Mac OS X [\\d_]*',
-      '(like )?Gecko(.[\\d\\.]*)?',
-      'KHTML,',
-      'CriOS.[\\d\\.]*',
-      'CPU iPhone OS ([0-9_])* like Mac OS X',
-      'CPU OS ([0-9_])* like Mac OS X',
-      'iPod',
-      'compatible',
-      'x86_..',
-      'i686',
-      'x64',
-      'X11',
-      'rv:[\\d\\.]*',
-      'Version.[\\d\\.]*',
-      'WOW64',
-      'Win64',
-      'Dalvik.[\\d\\.]*',
-      ' \\.NET CLR [\\d\\.]*',
-      'Presto.[\\d\\.]*',
-      'Media Center PC',
-      'BlackBerry',
-      'Build',
-      'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
-      'Opera',
-      ' \\.NET[\\d\\.]*',
-      'cubot',
-      '; M bot',
-      '; CRONO',
-      '; B bot',
-      '; IDbot',
-      '; ID bot',
-      '; POWER BOT',
-      'OCTOPUS-CORE',
-    ];
-  }
-}
-exports.Exclusions = Exclusions;

package/src/lib/crawler/headers.js

@@ -1,20 +0,0 @@
-'use strict';
-Object.defineProperty(exports, '__esModule', { value: true });
-exports.Headers = void 0;
-class Headers {
-  getAll() {
-    return [
-      'USER-AGENT',
-      'X-OPERAMINI-PHONE-UA',
-      'X-DEVICE-USER-AGENT',
-      'X-ORIGINAL-USER-AGENT',
-      'X-SKYFIRE-PHONE',
-      'X-BOLT-PHONE-UA',
-      'DEVICE-STOCK-UA',
-      'X-UCBROWSER-DEVICE-UA',
-      'FROM',
-      'X-SCANNER',
-    ];
-  }
-}
-exports.Headers = Headers;

package/src/lib/crawler/provider.js

@@ -1,2 +0,0 @@
-'use strict';
-Object.defineProperty(exports, '__esModule', { value: true });

package/src/lib/crawler.js

@@ -1,98 +0,0 @@
-'use strict';
-Object.defineProperty(exports, '__esModule', { value: true });
-exports.Crawler = void 0;
-const crawlers_1 = require('./crawler/crawlers');
-const exclusions_1 = require('./crawler/exclusions');
-const headers_1 = require('./crawler/headers');
-class Crawler {
-  crawlers;
-  headers;
-  exclusions;
-  request;
-  compiledRegexList;
-  compiledExclusions;
-  httpHeaders;
-  userAgent;
-  matches;
-  constructor(request, headers, userAgent) {
-    this.crawlers = new crawlers_1.Crawlers();
-    this.headers = new headers_1.Headers();
-    this.exclusions = new exclusions_1.Exclusions();
-    this.request = request ?? {};
-    // The regex-list must not be used with g-flag!
-    // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
-    this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
-    // The exclusions should be used with g-flag in order to remove each value.
-    this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
-    this.httpHeaders = this.setHttpHeaders(headers);
-    this.userAgent = this.setUserAgent(userAgent);
-  }
-  compileRegex(patterns, flags) {
-    return new RegExp(patterns.join('|'), flags);
-  }
-  setHttpHeaders(headers) {
-    // Use the Request headers if httpHeaders is not defined
-    if (!headers || Object.keys(headers).length === 0) {
-      if (Object.keys(this.request).length) {
-        if (this.request.headers) {
-          return this.request.headers;
-        }
-      }
-      return '';
-    }
-    // Save the headers.
-    return headers;
-  }
-  setUserAgent(userAgent) {
-    if (!userAgent?.length) {
-      userAgent = '';
-      for (const header of this.getUaHttpHeaders()) {
-        if (
-          typeof this.httpHeaders === 'object' &&
-          !Array.isArray(this.httpHeaders)
-        ) {
-          if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) {
-            const headerValue = this.httpHeaders[header.toLowerCase()];
-            if (typeof headerValue === 'string') {
-              const separator = userAgent.length > 0 ? ' ' : '';
-              userAgent += separator + headerValue;
-            } else if (Array.isArray(headerValue)) {
-              const separator = userAgent.length > 0 ? ' ' : '';
-              userAgent += separator + headerValue.join(' ');
-            }
-          }
-        }
-      }
-    }
-    return userAgent;
-  }
-  getUaHttpHeaders() {
-    return this.headers.getAll();
-  }
-  getMatches() {
-    if (this.matches !== undefined) {
-      if (this.matches?.length) {
-        return this.matches[0];
-      }
-      return null;
-    }
-    return {};
-  }
-  isCrawler(userAgent) {
-    if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) {
-      return false;
-    }
-    let agent = userAgent ?? this.userAgent;
-    // test on compiled regx
-    agent = agent.replace(this.compiledExclusions, '');
-    if (agent.trim().length === 0) {
-      return false;
-    }
-    const matches = this.compiledRegexList.exec(agent);
-    if (matches) {
-      this.matches = matches;
-    }
-    return matches !== null && matches.length > 0;
-  }
-}
-exports.Crawler = Crawler;

package/src/types.js

@@ -1,2 +0,0 @@
-'use strict';
-Object.defineProperty(exports, '__esModule', { value: true });

package/test/lib/crawler.spec.js

@@ -1,149 +0,0 @@
-'use strict';
-var __createBinding =
-  (this && this.__createBinding) ||
-  (Object.create
-    ? function (o, m, k, k2) {
-        if (k2 === undefined) k2 = k;
-        var desc = Object.getOwnPropertyDescriptor(m, k);
-        if (
-          !desc ||
-          ('get' in desc ? !m.__esModule : desc.writable || desc.configurable)
-        ) {
-          desc = {
-            enumerable: true,
-            get: function () {
-              return m[k];
-            },
-          };
-        }
-        Object.defineProperty(o, k2, desc);
-      }
-    : function (o, m, k, k2) {
-        if (k2 === undefined) k2 = k;
-        o[k2] = m[k];
-      });
-var __setModuleDefault =
-  (this && this.__setModuleDefault) ||
-  (Object.create
-    ? function (o, v) {
-        Object.defineProperty(o, 'default', { enumerable: true, value: v });
-      }
-    : function (o, v) {
-        o['default'] = v;
-      });
-var __importStar =
-  (this && this.__importStar) ||
-  function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null)
-      for (var k in mod)
-        if (k !== 'default' && Object.prototype.hasOwnProperty.call(mod, k))
-          __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-  };
-var __importDefault =
-  (this && this.__importDefault) ||
-  function (mod) {
-    return mod && mod.__esModule ? mod : { default: mod };
-  };
-Object.defineProperty(exports, '__esModule', { value: true });
-const fs = __importStar(require('fs'));
-const readline = __importStar(require('readline'));
-const assert_1 = __importDefault(require('assert'));
-const src_1 = require('../../src');
-let crawler = new src_1.Crawler();
-describe('regex-compilation', () => {
-  it('will join list of patterns with pipes', () => {
-    assert_1.default.strictEqual(
-      crawler.compileRegex(['some', 'patterns']).source,
-      'some|patterns'
-    );
-    assert_1.default.strictEqual(
-      crawler.compileRegex(['single']).source,
-      'single'
-    );
-  });
-  it('keeps the whitespace', () => {
-    assert_1.default.strictEqual(
-      crawler.compileRegex(['  keep-whitespaces ']).source,
-      '  keep-whitespaces '
-    );
-  });
-  it('will accept regex-flags for compilation', () => {
-    const patterns = ['some', 'patterns'];
-    assert_1.default.strictEqual(
-      crawler.compileRegex(patterns, 'g').flags,
-      'g'
-    );
-    assert_1.default.strictEqual(
-      crawler.compileRegex(patterns, 'i').flags,
-      'i'
-    );
-  });
-  it('should be case insensitive', () => {
-    assert_1.default.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
-    assert_1.default.strictEqual(
-      crawler.getMatches(),
-      'Facebot',
-      'Crawler was not able to identify crawler correctly'
-    );
-  });
-});
-describe('crawler-identification', () => {
-  it('should be able to identify crawlers', async () => {
-    const rl = readline.createInterface({
-      input: fs.createReadStream('./test/lib/database/crawlers.txt'),
-      crlfDelay: Infinity,
-    });
-    for await (const line of rl) {
-      assert_1.default.strictEqual(
-        crawler.isCrawler(line),
-        true,
-        `${line} is not a crawler`
-      );
-    }
-    rl.close();
-  });
-  it('should be able to identify devices', async () => {
-    const rl = readline.createInterface({
-      input: fs.createReadStream('./test/lib/database/devices.txt'),
-      crlfDelay: Infinity,
-    });
-    for await (const line of rl) {
-      assert_1.default.strictEqual(
-        crawler.isCrawler(line),
-        false,
-        `${line} is not a device`
-      );
-    }
-    rl.close();
-  });
-  it('should identify the crawler from given headers', async () => {
-    crawler = new src_1.Crawler(undefined, {
-      host: '127.0.0.1:3000',
-      'user-agent': 'curl/7.73.0',
-      accept: '*/*',
-    });
-    assert_1.default.strictEqual(crawler.isCrawler(), true);
-  });
-  it('should identify the crawler from request headers', async () => {
-    crawler = new src_1.Crawler({
-      headers: { 'user-agent': 'curl/7.73.0', accept: '*/*' },
-    });
-    assert_1.default.strictEqual(crawler.isCrawler(), true);
-  });
-  it('should identify the crawler from request headers with exact pattern', async () => {
-    crawler = new src_1.Crawler({
-      headers: { 'user-agent': 'b0t', accept: '*/*' },
-    });
-    assert_1.default.strictEqual(crawler.isCrawler(), true);
-  });
-  it('should not throw an exception on empty request header', async () => {
-    crawler = new src_1.Crawler({
-      headers: { accept: '*/*' },
-    });
-    assert_1.default.doesNotThrow(() => crawler.isCrawler());
-  });
-});

package/webpack.common.js

@@ -1,20 +0,0 @@
-const path = require('path');
-
-module.exports = {
-  context: path.resolve(__dirname, 'src'),
-  entry: './index.js',
-  output: {
-    path: path.resolve(__dirname, 'dist'),
-    filename: '[name].bundle.js',
-    library: 'Crawler',
-  },
-  module: {
-    rules: [
-      {
-        test: /\.js$/,
-        use: 'babel-loader',
-        exclude: /node_modules/,
-      },
-    ],
-  },
-};

package/webpack.dev.js

@@ -1,7 +0,0 @@
-const { merge } = require('webpack-merge');
-const common = require('./webpack.common');
-
-module.exports = merge(common, {
-  mode: 'development',
-  devtool: 'inline-source-map',
-});

package/webpack.prod.js

@@ -1,14 +0,0 @@
-const { merge } = require('webpack-merge');
-const TerserPlugin = require('terser-webpack-plugin');
-const common = require('./webpack.common');
-
-module.exports = merge(common, {
-  mode: 'production',
-  optimization: {
-    minimizer: [
-      new TerserPlugin({
-        parallel: true,
-      }),
-    ],
-  },
-});