es6-crawler-detect 3.3.0 → 4.0.0-beta.1

package/test/lib/crawler.spec.js

@@ -0,0 +1,149 @@
+'use strict';
+var __createBinding =
+  (this && this.__createBinding) ||
+  (Object.create
+    ? function (o, m, k, k2) {
+        if (k2 === undefined) k2 = k;
+        var desc = Object.getOwnPropertyDescriptor(m, k);
+        if (
+          !desc ||
+          ('get' in desc ? !m.__esModule : desc.writable || desc.configurable)
+        ) {
+          desc = {
+            enumerable: true,
+            get: function () {
+              return m[k];
+            },
+          };
+        }
+        Object.defineProperty(o, k2, desc);
+      }
+    : function (o, m, k, k2) {
+        if (k2 === undefined) k2 = k;
+        o[k2] = m[k];
+      });
+var __setModuleDefault =
+  (this && this.__setModuleDefault) ||
+  (Object.create
+    ? function (o, v) {
+        Object.defineProperty(o, 'default', { enumerable: true, value: v });
+      }
+    : function (o, v) {
+        o['default'] = v;
+      });
+var __importStar =
+  (this && this.__importStar) ||
+  function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null)
+      for (var k in mod)
+        if (k !== 'default' && Object.prototype.hasOwnProperty.call(mod, k))
+          __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+  };
+var __importDefault =
+  (this && this.__importDefault) ||
+  function (mod) {
+    return mod && mod.__esModule ? mod : { default: mod };
+  };
+Object.defineProperty(exports, '__esModule', { value: true });
+const fs = __importStar(require('fs'));
+const readline = __importStar(require('readline'));
+const assert_1 = __importDefault(require('assert'));
+const src_1 = require('../../src');
+let crawler = new src_1.Crawler();
+describe('regex-compilation', () => {
+  it('will join list of patterns with pipes', () => {
+    assert_1.default.strictEqual(
+      crawler.compileRegex(['some', 'patterns']).source,
+      'some|patterns'
+    );
+    assert_1.default.strictEqual(
+      crawler.compileRegex(['single']).source,
+      'single'
+    );
+  });
+  it('keeps the whitespace', () => {
+    assert_1.default.strictEqual(
+      crawler.compileRegex(['  keep-whitespaces ']).source,
+      '  keep-whitespaces '
+    );
+  });
+  it('will accept regex-flags for compilation', () => {
+    const patterns = ['some', 'patterns'];
+    assert_1.default.strictEqual(
+      crawler.compileRegex(patterns, 'g').flags,
+      'g'
+    );
+    assert_1.default.strictEqual(
+      crawler.compileRegex(patterns, 'i').flags,
+      'i'
+    );
+  });
+  it('should be case insensitive', () => {
+    assert_1.default.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
+    assert_1.default.strictEqual(
+      crawler.getMatches(),
+      'Facebot',
+      'Crawler was not able to identify crawler correctly'
+    );
+  });
+});
+describe('crawler-identification', () => {
+  it('should be able to identify crawlers', async () => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream('./test/lib/database/crawlers.txt'),
+      crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+      assert_1.default.strictEqual(
+        crawler.isCrawler(line),
+        true,
+        `${line} is not a crawler`
+      );
+    }
+    rl.close();
+  });
+  it('should be able to identify devices', async () => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream('./test/lib/database/devices.txt'),
+      crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+      assert_1.default.strictEqual(
+        crawler.isCrawler(line),
+        false,
+        `${line} is not a device`
+      );
+    }
+    rl.close();
+  });
+  it('should identify the crawler from given headers', async () => {
+    crawler = new src_1.Crawler(undefined, {
+      host: '127.0.0.1:3000',
+      'user-agent': 'curl/7.73.0',
+      accept: '*/*',
+    });
+    assert_1.default.strictEqual(crawler.isCrawler(), true);
+  });
+  it('should identify the crawler from request headers', async () => {
+    crawler = new src_1.Crawler({
+      headers: { 'user-agent': 'curl/7.73.0', accept: '*/*' },
+    });
+    assert_1.default.strictEqual(crawler.isCrawler(), true);
+  });
+  it('should identify the crawler from request headers with exact pattern', async () => {
+    crawler = new src_1.Crawler({
+      headers: { 'user-agent': 'b0t', accept: '*/*' },
+    });
+    assert_1.default.strictEqual(crawler.isCrawler(), true);
+  });
+  it('should not throw an exception on empty request header', async () => {
+    crawler = new src_1.Crawler({
+      headers: { accept: '*/*' },
+    });
+    assert_1.default.doesNotThrow(() => crawler.isCrawler());
+  });
+});

package/test/lib/crawler.spec.ts

@@ -0,0 +1,116 @@
+import assert from 'assert';
+import { Request } from 'express-serve-static-core';
+import * as fs from 'fs';
+import * as readline from 'readline';
+
+import { Crawler } from '../../src';
+
+declare module 'express-serve-static-core' {
+  interface Request {
+    Crawler: Crawler;
+  }
+}
+
+let crawler = new Crawler();
+
+describe('regex-compilation', () => {
+  it('will join list of patterns with pipes', () => {
+    assert.strictEqual(
+      crawler.compileRegex(['some', 'patterns']).source,
+      'some|patterns'
+    );
+    assert.strictEqual(crawler.compileRegex(['single']).source, 'single');
+  });
+
+  it('keeps the whitespace', () => {
+    assert.strictEqual(
+      crawler.compileRegex(['  keep-whitespaces ']).source,
+      '  keep-whitespaces '
+    );
+  });
+
+  it('will accept regex-flags for compilation', () => {
+    const patterns = ['some', 'patterns'];
+    assert.strictEqual(crawler.compileRegex(patterns, 'g').flags, 'g');
+    assert.strictEqual(crawler.compileRegex(patterns, 'i').flags, 'i');
+  });
+
+  it('should be case insensitive', () => {
+    assert.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
+    assert.strictEqual(
+      crawler.getMatches(),
+      'Facebot',
+      'Crawler was not able to identify crawler correctly'
+    );
+  });
+});
+
+describe('crawler-identification', () => {
+  it('should be able to identify crawlers', async () => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream('./test/lib/database/crawlers.txt'),
+      crlfDelay: Infinity,
+    });
+
+    for await (const line of rl) {
+      assert.strictEqual(
+        crawler.isCrawler(line),
+        true,
+        `${line} is not a crawler`
+      );
+    }
+
+    rl.close();
+  });
+
+  it('should be able to identify devices', async () => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream('./test/lib/database/devices.txt'),
+      crlfDelay: Infinity,
+    });
+
+    for await (const line of rl) {
+      assert.strictEqual(
+        crawler.isCrawler(line),
+        false,
+        `${line} is not a device`
+      );
+    }
+
+    rl.close();
+  });
+
+  it('should identify the crawler from given headers', async () => {
+    crawler = new Crawler(undefined, {
+      host: '127.0.0.1:3000',
+      'user-agent': 'curl/7.73.0',
+      accept: '*/*',
+    });
+
+    assert.strictEqual(crawler.isCrawler(), true);
+  });
+
+  it('should identify the crawler from request headers', async () => {
+    crawler = new Crawler({
+      headers: { 'user-agent': 'curl/7.73.0', accept: '*/*' },
+    } as Request);
+
+    assert.strictEqual(crawler.isCrawler(), true);
+  });
+
+  it('should identify the crawler from request headers with exact pattern', async () => {
+    crawler = new Crawler({
+      headers: { 'user-agent': 'b0t', accept: '*/*' },
+    } as Request);
+
+    assert.strictEqual(crawler.isCrawler(), true);
+  });
+
+  it('should not throw an exception on empty request header', async () => {
+    crawler = new Crawler({
+      headers: { accept: '*/*' },
+    } as Request);
+
+    assert.doesNotThrow(() => crawler.isCrawler());
+  });
+});

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+    "$schema": "https://json.schemastore.org/tsconfig",
+    "display": "Node 20",
+    "_version": "20.1.0",
+  
+    "compilerOptions": {
+      "lib": ["es2023"],
+      "module": "node16",
+      "target": "es2022",
+  
+      "strict": true,
+      "esModuleInterop": true,
+      "skipLibCheck": true,
+      "moduleResolution": "node16"
+    }
+}

package/webpack.common.js

@@ -1,20 +1,20 @@
-const path = require('path');
-
-module.exports = {
-  context: path.resolve(__dirname, 'src'),
-  entry: './index.js',
-  output: {
-    path: path.resolve(__dirname, 'dist'),
-    filename: '[name].bundle.js',
-    library: 'Crawler',
-  },
-  module: {
-    rules: [
-      {
-        test: /\.js$/,
-        use: 'babel-loader',
-        exclude: /node_modules/,
-      },
-    ],
-  },
-};
+const path = require('path');
+
+module.exports = {
+  context: path.resolve(__dirname, 'src'),
+  entry: './index.js',
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: '[name].bundle.js',
+    library: 'Crawler',
+  },
+  module: {
+    rules: [
+      {
+        test: /\.js$/,
+        use: 'babel-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+};

package/webpack.dev.js

@@ -1,7 +1,7 @@
-const { merge } = require('webpack-merge');
-const common = require('./webpack.common');
-
-module.exports = merge(common, {
-  mode: 'development',
-  devtool: 'inline-source-map',
-});
+const { merge } = require('webpack-merge');
+const common = require('./webpack.common');
+
+module.exports = merge(common, {
+  mode: 'development',
+  devtool: 'inline-source-map',
+});

package/webpack.prod.js

@@ -1,14 +1,14 @@
-const { merge } = require('webpack-merge');
-const TerserPlugin = require('terser-webpack-plugin');
-const common = require('./webpack.common');
-
-module.exports = merge(common, {
-  mode: 'production',
-  optimization: {
-    minimizer: [
-      new TerserPlugin({
-        parallel: true,
-      }),
-    ],
-  },
-});
+const { merge } = require('webpack-merge');
+const TerserPlugin = require('terser-webpack-plugin');
+const common = require('./webpack.common');
+
+module.exports = merge(common, {
+  mode: 'production',
+  optimization: {
+    minimizer: [
+      new TerserPlugin({
+        parallel: true,
+      }),
+    ],
+  },
+});

package/.eslintignore

@@ -1,6 +0,0 @@
-!.*
-coverage/
-junit/
-dist/
-mobiles/
-**/node_modules/*

package/.eslintrc.json

@@ -1,30 +0,0 @@
-{
-  "extends": ["eslint:recommended", "plugin:prettier/recommended"],
-  "env": {
-    "browser": false,
-    "commonjs": true,
-    "es6": true,
-    "node": true,
-    "mocha": true
-  },
-  "rules": {
-    "max-len": [
-      "warn",
-      {
-        "code": 80,
-        "tabWidth": 2,
-        "comments": 80,
-        "ignoreComments": false,
-        "ignoreTrailingComments": true,
-        "ignoreUrls": true,
-        "ignoreStrings": true,
-        "ignoreTemplateLiterals": true,
-        "ignoreRegExpLiterals": true
-      }
-    ]
-  },
-  "parser": "@babel/eslint-parser",
-  "parserOptions": {
-    "requireConfigFile": false
-  }
-}

package/example/node/node_modules/.bin/mime

@@ -1,15 +0,0 @@
-#!/bin/sh
-basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
-
-case `uname` in
-    *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
-esac
-
-if [ -x "$basedir/node" ]; then
-  "$basedir/node"  "$basedir/../mime/cli.js" "$@"
-  ret=$?
-else 
-  node  "$basedir/../mime/cli.js" "$@"
-  ret=$?
-fi
-exit $ret

package/example/node/node_modules/.bin/mime.cmd

@@ -1,7 +0,0 @@
-@IF EXIST "%~dp0\node.exe" (
-  "%~dp0\node.exe"  "%~dp0\..\mime\cli.js" %*
-) ELSE (
-  @SETLOCAL
-  @SET PATHEXT=%PATHEXT:;.JS;=;%
-  node  "%~dp0\..\mime\cli.js" %*
-)

package/example/node/node_modules/.yarn-integrity

@@ -1,67 +0,0 @@
-{
-  "systemParams": "win32-x64-93",
-  "modulesFolders": [
-    "node_modules"
-  ],
-  "flags": [],
-  "linkedModules": [],
-  "topLevelPatterns": [
-    "express@^4.17.2"
-  ],
-  "lockfileEntries": {
-    "accepts@~1.3.7": "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd",
-    "array-flatten@1.1.1": "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2",
-    "body-parser@1.19.1": "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.1.tgz#1499abbaa9274af3ecc9f6f10396c995943e31d4",
-    "bytes@3.1.1": "https://registry.yarnpkg.com/bytes/-/bytes-3.1.1.tgz#3f018291cb4cbad9accb6e6970bca9c8889e879a",
-    "content-disposition@0.5.4": "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe",
-    "content-type@~1.0.4": "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b",
-    "cookie-signature@1.0.6": "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c",
-    "cookie@0.4.1": "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1",
-    "debug@2.6.9": "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f",
-    "depd@~1.1.2": "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9",
-    "destroy@~1.0.4": "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80",
-    "ee-first@1.1.1": "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d",
-    "encodeurl@~1.0.2": "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59",
-    "escape-html@~1.0.3": "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988",
-    "etag@~1.8.1": "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887",
-    "express@^4.17.2": "https://registry.yarnpkg.com/express/-/express-4.17.2.tgz#c18369f265297319beed4e5558753cc8c1364cb3",
-    "finalhandler@~1.1.2": "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d",
-    "forwarded@0.2.0": "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811",
-    "fresh@0.5.2": "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7",
-    "http-errors@1.8.1": "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c",
-    "iconv-lite@0.4.24": "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b",
-    "inherits@2.0.4": "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c",
-    "ipaddr.js@1.9.1": "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3",
-    "media-typer@0.3.0": "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748",
-    "merge-descriptors@1.0.1": "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61",
-    "methods@~1.1.2": "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee",
-    "mime-db@1.51.0": "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c",
-    "mime-types@~2.1.24": "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24",
-    "mime@1.6.0": "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1",
-    "ms@2.0.0": "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8",
-    "ms@2.1.3": "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2",
-    "negotiator@0.6.2": "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb",
-    "on-finished@~2.3.0": "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947",
-    "parseurl@~1.3.3": "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4",
-    "path-to-regexp@0.1.7": "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c",
-    "proxy-addr@~2.0.7": "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025",
-    "qs@6.9.6": "https://registry.yarnpkg.com/qs/-/qs-6.9.6.tgz#26ed3c8243a431b2924aca84cc90471f35d5a0ee",
-    "range-parser@~1.2.1": "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031",
-    "raw-body@2.4.2": "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.2.tgz#baf3e9c21eebced59dd6533ac872b71f7b61cb32",
-    "safe-buffer@5.2.1": "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6",
-    "safer-buffer@>= 2.1.2 < 3": "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a",
-    "send@0.17.2": "https://registry.yarnpkg.com/send/-/send-0.17.2.tgz#926622f76601c41808012c8bf1688fe3906f7820",
-    "serve-static@1.14.2": "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.2.tgz#722d6294b1d62626d41b43a013ece4598d292bfa",
-    "setprototypeof@1.2.0": "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424",
-    "statuses@>= 1.5.0 < 2": "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c",
-    "statuses@~1.5.0": "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c",
-    "toidentifier@1.0.1": "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35",
-    "type-is@~1.6.18": "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131",
-    "unpipe@1.0.0": "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec",
-    "unpipe@~1.0.0": "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec",
-    "utils-merge@1.0.1": "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713",
-    "vary@~1.1.2": "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
-  },
-  "files": [],
-  "artifacts": {}
-}