es6-crawler-detect 3.2.0 → 4.0.0-beta.1

package/src/lib/crawler/exclusions.js

@@ -1,12 +1,9 @@
 'use strict';
-
-const Provider = require('./provider');
-
-class Exclusions extends Provider {
-  constructor() {
-    super();
-
-    this.data = [
+Object.defineProperty(exports, '__esModule', { value: true });
+exports.Exclusions = void 0;
+class Exclusions {
+  getAll() {
+    return [
       'Safari.[\\d\\.]*',
       'Firefox.[\\d\\.]*',
       ' Chrome.[\\d\\.]*',
@@ -54,9 +51,8 @@ class Exclusions extends Provider {
       '; IDbot',
       '; ID bot',
       '; POWER BOT',
-      ';',
+      'OCTOPUS-CORE',
     ];
   }
 }
-
-module.exports = Exclusions;
+exports.Exclusions = Exclusions;

package/src/lib/crawler/exclusions.ts

@@ -0,0 +1,56 @@
+import { Provider } from './provider';
+
+export class Exclusions implements Provider {
+  getAll(): string[] {
+    return [
+      'Safari.[\\d\\.]*',
+      'Firefox.[\\d\\.]*',
+      ' Chrome.[\\d\\.]*',
+      'Chromium.[\\d\\.]*',
+      'MSIE.[\\d\\.]',
+      'Opera\\/[\\d\\.]*',
+      'Mozilla.[\\d\\.]*',
+      'AppleWebKit.[\\d\\.]*',
+      'Trident.[\\d\\.]*',
+      'Windows NT.[\\d\\.]*',
+      'Android [\\d\\.]*',
+      'Macintosh.',
+      'Ubuntu',
+      'Linux',
+      '[ ]Intel',
+      'Mac OS X [\\d_]*',
+      '(like )?Gecko(.[\\d\\.]*)?',
+      'KHTML,',
+      'CriOS.[\\d\\.]*',
+      'CPU iPhone OS ([0-9_])* like Mac OS X',
+      'CPU OS ([0-9_])* like Mac OS X',
+      'iPod',
+      'compatible',
+      'x86_..',
+      'i686',
+      'x64',
+      'X11',
+      'rv:[\\d\\.]*',
+      'Version.[\\d\\.]*',
+      'WOW64',
+      'Win64',
+      'Dalvik.[\\d\\.]*',
+      ' \\.NET CLR [\\d\\.]*',
+      'Presto.[\\d\\.]*',
+      'Media Center PC',
+      'BlackBerry',
+      'Build',
+      'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
+      'Opera',
+      ' \\.NET[\\d\\.]*',
+      'cubot',
+      '; M bot',
+      '; CRONO',
+      '; B bot',
+      '; IDbot',
+      '; ID bot',
+      '; POWER BOT',
+      'OCTOPUS-CORE',
+    ];
+  }
+}

package/src/lib/crawler/headers.js

@@ -1,12 +1,9 @@
 'use strict';
-
-const Provider = require('./provider');
-
-class Headers extends Provider {
-  constructor() {
-    super();
-
-    this.data = [
+Object.defineProperty(exports, '__esModule', { value: true });
+exports.Headers = void 0;
+class Headers {
+  getAll() {
+    return [
       'USER-AGENT',
       'X-OPERAMINI-PHONE-UA',
       'X-DEVICE-USER-AGENT',
@@ -20,5 +17,4 @@ class Headers extends Provider {
     ];
   }
 }
-
-module.exports = Headers;
+exports.Headers = Headers;

package/src/lib/crawler/headers.ts

@@ -0,0 +1,18 @@
+import { Provider } from './provider';
+
+export class Headers implements Provider {
+  getAll(): string[] {
+    return [
+      'USER-AGENT',
+      'X-OPERAMINI-PHONE-UA',
+      'X-DEVICE-USER-AGENT',
+      'X-ORIGINAL-USER-AGENT',
+      'X-SKYFIRE-PHONE',
+      'X-BOLT-PHONE-UA',
+      'DEVICE-STOCK-UA',
+      'X-UCBROWSER-DEVICE-UA',
+      'FROM',
+      'X-SCANNER',
+    ];
+  }
+}

package/src/lib/crawler/provider.js

@@ -1,11 +1,2 @@
 'use strict';
-
-class Provider {
-  constructor() {}
-
-  getAll() {
-    return this.data;
-  }
-}
-
-module.exports = Provider;
+Object.defineProperty(exports, '__esModule', { value: true });

package/src/lib/crawler/provider.ts

@@ -0,0 +1,3 @@
+export interface Provider {
+  getAll(): string[];
+}

package/src/lib/crawler.js

@@ -1,135 +1,98 @@
 'use strict';
-
-const Crawlers = require('./crawler/crawlers');
-const Exclusions = require('./crawler/exclusions');
-const Headers = require('./crawler/headers');
-
+Object.defineProperty(exports, '__esModule', { value: true });
+exports.Crawler = void 0;
+const crawlers_1 = require('./crawler/crawlers');
+const exclusions_1 = require('./crawler/exclusions');
+const headers_1 = require('./crawler/headers');
 class Crawler {
+  crawlers;
+  headers;
+  exclusions;
+  request;
+  compiledRegexList;
+  compiledExclusions;
+  httpHeaders;
+  userAgent;
+  matches;
   constructor(request, headers, userAgent) {
-    /**
-     * Init classes
-     */
-    this._init();
-
-    /**
-     * This request must be an object
-     */
-    this.request = typeof request === 'object' ? request : {};
-
+    this.crawlers = new crawlers_1.Crawlers();
+    this.headers = new headers_1.Headers();
+    this.exclusions = new exclusions_1.Exclusions();
+    this.request = request ?? {};
     // The regex-list must not be used with g-flag!
     // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
     this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
-
     // The exclusions should be used with g-flag in order to remove each value.
-    this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'g');
-
-    /**
-     * Set http headers
-     */
-    this.setHttpHeaders(headers);
-
-    /**
-     * Set userAgent
-     */
+    this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
+    this.httpHeaders = this.setHttpHeaders(headers);
     this.userAgent = this.setUserAgent(userAgent);
   }
-
-  /**
-   * Init Classes Instances
-   */
-  _init() {
-    this.crawlers = new Crawlers();
-    this.headers = new Headers();
-    this.exclusions = new Exclusions();
-  }
-
   compileRegex(patterns, flags) {
-    return new RegExp(patterns.join('|').trim(), flags);
+    return new RegExp(patterns.join('|'), flags);
   }
-
-  /**
-   * Set HTTP headers.
-   */
   setHttpHeaders(headers) {
     // Use the Request headers if httpHeaders is not defined
-    if (typeof headers === 'undefined' || Object.keys(headers).length === 0) {
-      headers = Object.keys(this.request).length ? this.request.headers : {};
-    }
-
-    // Clear existing headers.
-    this.httpHeaders = [];
-
-    // Only save HTTP headers.
-    for (const key in headers) {
-      this.httpHeaders[key] = headers[key];
+    if (!headers || Object.keys(headers).length === 0) {
+      if (Object.keys(this.request).length) {
+        if (this.request.headers) {
+          return this.request.headers;
+        }
+      }
+      return '';
     }
+    // Save the headers.
+    return headers;
   }
-
-  /**
-   * Set user agent
-   */
   setUserAgent(userAgent) {
-    if (
-      typeof userAgent === 'undefined' ||
-      userAgent === null ||
-      !userAgent.length
-    ) {
+    if (!userAgent?.length) {
+      userAgent = '';
       for (const header of this.getUaHttpHeaders()) {
-        if (Object.keys(this.httpHeaders).indexOf(header.toLowerCase()) >= 0) {
-          userAgent += this.httpHeaders[header] + ' ';
+        if (
+          typeof this.httpHeaders === 'object' &&
+          !Array.isArray(this.httpHeaders)
+        ) {
+          if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) {
+            const headerValue = this.httpHeaders[header.toLowerCase()];
+            if (typeof headerValue === 'string') {
+              const separator = userAgent.length > 0 ? ' ' : '';
+              userAgent += separator + headerValue;
+            } else if (Array.isArray(headerValue)) {
+              const separator = userAgent.length > 0 ? ' ' : '';
+              userAgent += separator + headerValue.join(' ');
+            }
+          }
         }
       }
     }
-
     return userAgent;
   }
-
-  /**
-   * Get user agent headers
-   */
   getUaHttpHeaders() {
     return this.headers.getAll();
   }
-
-  /**
-   * Check user agent string against the regex.
-   */
-  isCrawler(userAgent = undefined) {
-    if (Buffer.byteLength(userAgent || '', 'utf8') > 4096) {
+  getMatches() {
+    if (this.matches !== undefined) {
+      if (this.matches?.length) {
+        return this.matches[0];
+      }
+      return null;
+    }
+    return {};
+  }
+  isCrawler(userAgent) {
+    if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) {
       return false;
     }
-
-    var agent =
-      typeof userAgent === 'undefined' || userAgent === null
-        ? this.userAgent
-        : userAgent;
-
+    let agent = userAgent ?? this.userAgent;
     // test on compiled regx
     agent = agent.replace(this.compiledExclusions, '');
-
     if (agent.trim().length === 0) {
       return false;
     }
-
-    var matches = this.compiledRegexList.exec(agent.trim());
-
+    const matches = this.compiledRegexList.exec(agent);
     if (matches) {
       this.matches = matches;
     }
-
-    return matches !== null ? (matches.length ? true : false) : false;
-  }
-
-  /**
-   * Return the matches.
-   */
-  getMatches() {
-    return this.matches !== undefined
-      ? this.matches.length
-        ? this.matches[0]
-        : null
-      : {};
+    return matches !== null && matches.length > 0;
   }
 }
-
-module.exports = Crawler;
+exports.Crawler = Crawler;

package/src/lib/crawler.ts

@@ -0,0 +1,128 @@
+import { Request } from 'express-serve-static-core';
+import { IncomingHttpHeaders } from 'http2';
+
+import { Crawlers } from './crawler/crawlers';
+import { Exclusions } from './crawler/exclusions';
+import { Headers } from './crawler/headers';
+
+export class Crawler {
+  private crawlers: Crawlers;
+  private headers: Headers;
+  private exclusions: Exclusions;
+
+  private request: Request | NodeJS.Dict<string | string[]>;
+  private compiledRegexList: RegExp;
+  private compiledExclusions: RegExp;
+  private httpHeaders: string | string[] | IncomingHttpHeaders;
+  private userAgent: string;
+  private matches?: RegExpExecArray | null;
+
+  constructor(
+    request?: Request,
+    headers?: IncomingHttpHeaders,
+    userAgent?: string
+  ) {
+    this.crawlers = new Crawlers();
+    this.headers = new Headers();
+    this.exclusions = new Exclusions();
+
+    this.request = request ?? {};
+
+    // The regex-list must not be used with g-flag!
+    // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
+    this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
+
+    // The exclusions should be used with g-flag in order to remove each value.
+    this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
+
+    this.httpHeaders = this.setHttpHeaders(headers);
+    this.userAgent = this.setUserAgent(userAgent);
+  }
+
+  public compileRegex(patterns: string[], flags?: string): RegExp {
+    return new RegExp(patterns.join('|'), flags);
+  }
+
+  private setHttpHeaders(
+    headers?: string | string[] | IncomingHttpHeaders
+  ): string | string[] | IncomingHttpHeaders {
+    // Use the Request headers if httpHeaders is not defined
+    if (!headers || Object.keys(headers).length === 0) {
+      if (Object.keys(this.request).length) {
+        if (this.request.headers) {
+          return this.request.headers;
+        }
+      }
+
+      return '';
+    }
+
+    // Save the headers.
+    return headers;
+  }
+
+  private setUserAgent(userAgent?: string): string {
+    if (!userAgent?.length) {
+      userAgent = '';
+      for (const header of this.getUaHttpHeaders()) {
+        if (
+          typeof this.httpHeaders === 'object' &&
+          !Array.isArray(this.httpHeaders)
+        ) {
+          if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) {
+            const headerValue = this.httpHeaders[header.toLowerCase()];
+
+            if (typeof headerValue === 'string') {
+              const separator = userAgent.length > 0 ? ' ' : '';
+              userAgent += separator + headerValue;
+            } else if (Array.isArray(headerValue)) {
+              const separator = userAgent.length > 0 ? ' ' : '';
+              userAgent += separator + headerValue.join(' ');
+            }
+          }
+        }
+      }
+    }
+
+    return userAgent;
+  }
+
+  private getUaHttpHeaders() {
+    return this.headers.getAll();
+  }
+
+  public getMatches(): string | null | object {
+    if (this.matches !== undefined) {
+      if (this.matches?.length) {
+        return this.matches[0];
+      }
+
+      return null;
+    }
+
+    return {};
+  }
+
+  public isCrawler(userAgent?: string): boolean {
+    if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) {
+      return false;
+    }
+
+    let agent = userAgent ?? this.userAgent;
+
+    // test on compiled regx
+    agent = agent.replace(this.compiledExclusions, '');
+
+    if (agent.trim().length === 0) {
+      return false;
+    }
+
+    const matches = this.compiledRegexList.exec(agent);
+
+    if (matches) {
+      this.matches = matches;
+    }
+
+    return matches !== null && matches.length > 0;
+  }
+}

package/src/types.js

@@ -0,0 +1,2 @@
+'use strict';
+Object.defineProperty(exports, '__esModule', { value: true });

package/src/types.ts

@@ -0,0 +1,15 @@
+import { NextFunction, Request, Response } from 'express-serve-static-core';
+
+import { Crawler } from './lib/crawler';
+
+export type Middleware = (
+  request: Request,
+  response: Response,
+  next: NextFunction
+) => Promise<Response>;
+
+declare module 'express-serve-static-core' {
+  interface Request {
+    Crawler: Crawler;
+  }
+}

package/test/lib/crawler.spec.js

@@ -0,0 +1,149 @@
+'use strict';
+var __createBinding =
+  (this && this.__createBinding) ||
+  (Object.create
+    ? function (o, m, k, k2) {
+        if (k2 === undefined) k2 = k;
+        var desc = Object.getOwnPropertyDescriptor(m, k);
+        if (
+          !desc ||
+          ('get' in desc ? !m.__esModule : desc.writable || desc.configurable)
+        ) {
+          desc = {
+            enumerable: true,
+            get: function () {
+              return m[k];
+            },
+          };
+        }
+        Object.defineProperty(o, k2, desc);
+      }
+    : function (o, m, k, k2) {
+        if (k2 === undefined) k2 = k;
+        o[k2] = m[k];
+      });
+var __setModuleDefault =
+  (this && this.__setModuleDefault) ||
+  (Object.create
+    ? function (o, v) {
+        Object.defineProperty(o, 'default', { enumerable: true, value: v });
+      }
+    : function (o, v) {
+        o['default'] = v;
+      });
+var __importStar =
+  (this && this.__importStar) ||
+  function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null)
+      for (var k in mod)
+        if (k !== 'default' && Object.prototype.hasOwnProperty.call(mod, k))
+          __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+  };
+var __importDefault =
+  (this && this.__importDefault) ||
+  function (mod) {
+    return mod && mod.__esModule ? mod : { default: mod };
+  };
+Object.defineProperty(exports, '__esModule', { value: true });
+const fs = __importStar(require('fs'));
+const readline = __importStar(require('readline'));
+const assert_1 = __importDefault(require('assert'));
+const src_1 = require('../../src');
+let crawler = new src_1.Crawler();
+describe('regex-compilation', () => {
+  it('will join list of patterns with pipes', () => {
+    assert_1.default.strictEqual(
+      crawler.compileRegex(['some', 'patterns']).source,
+      'some|patterns'
+    );
+    assert_1.default.strictEqual(
+      crawler.compileRegex(['single']).source,
+      'single'
+    );
+  });
+  it('keeps the whitespace', () => {
+    assert_1.default.strictEqual(
+      crawler.compileRegex(['  keep-whitespaces ']).source,
+      '  keep-whitespaces '
+    );
+  });
+  it('will accept regex-flags for compilation', () => {
+    const patterns = ['some', 'patterns'];
+    assert_1.default.strictEqual(
+      crawler.compileRegex(patterns, 'g').flags,
+      'g'
+    );
+    assert_1.default.strictEqual(
+      crawler.compileRegex(patterns, 'i').flags,
+      'i'
+    );
+  });
+  it('should be case insensitive', () => {
+    assert_1.default.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
+    assert_1.default.strictEqual(
+      crawler.getMatches(),
+      'Facebot',
+      'Crawler was not able to identify crawler correctly'
+    );
+  });
+});
+describe('crawler-identification', () => {
+  it('should be able to identify crawlers', async () => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream('./test/lib/database/crawlers.txt'),
+      crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+      assert_1.default.strictEqual(
+        crawler.isCrawler(line),
+        true,
+        `${line} is not a crawler`
+      );
+    }
+    rl.close();
+  });
+  it('should be able to identify devices', async () => {
+    const rl = readline.createInterface({
+      input: fs.createReadStream('./test/lib/database/devices.txt'),
+      crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+      assert_1.default.strictEqual(
+        crawler.isCrawler(line),
+        false,
+        `${line} is not a device`
+      );
+    }
+    rl.close();
+  });
+  it('should identify the crawler from given headers', async () => {
+    crawler = new src_1.Crawler(undefined, {
+      host: '127.0.0.1:3000',
+      'user-agent': 'curl/7.73.0',
+      accept: '*/*',
+    });
+    assert_1.default.strictEqual(crawler.isCrawler(), true);
+  });
+  it('should identify the crawler from request headers', async () => {
+    crawler = new src_1.Crawler({
+      headers: { 'user-agent': 'curl/7.73.0', accept: '*/*' },
+    });
+    assert_1.default.strictEqual(crawler.isCrawler(), true);
+  });
+  it('should identify the crawler from request headers with exact pattern', async () => {
+    crawler = new src_1.Crawler({
+      headers: { 'user-agent': 'b0t', accept: '*/*' },
+    });
+    assert_1.default.strictEqual(crawler.isCrawler(), true);
+  });
+  it('should not throw an exception on empty request header', async () => {
+    crawler = new src_1.Crawler({
+      headers: { accept: '*/*' },
+    });
+    assert_1.default.doesNotThrow(() => crawler.isCrawler());
+  });
+});