es6-crawler-detect 3.3.0 → 4.0.0-beta.2

package/example/node/node_modules/vary/package.json

@@ -1,43 +0,0 @@
-{
-  "name": "vary",
-  "description": "Manipulate the HTTP Vary header",
-  "version": "1.1.2",
-  "author": "Douglas Christopher Wilson <doug@somethingdoug.com>",
-  "license": "MIT",
-  "keywords": [
-    "http",
-    "res",
-    "vary"
-  ],
-  "repository": "jshttp/vary",
-  "devDependencies": {
-    "beautify-benchmark": "0.2.4",
-    "benchmark": "2.1.4",
-    "eslint": "3.19.0",
-    "eslint-config-standard": "10.2.1",
-    "eslint-plugin-import": "2.7.0",
-    "eslint-plugin-markdown": "1.0.0-beta.6",
-    "eslint-plugin-node": "5.1.1",
-    "eslint-plugin-promise": "3.5.0",
-    "eslint-plugin-standard": "3.0.1",
-    "istanbul": "0.4.5",
-    "mocha": "2.5.3",
-    "supertest": "1.1.0"
-  },
-  "files": [
-    "HISTORY.md",
-    "LICENSE",
-    "README.md",
-    "index.js"
-  ],
-  "engines": {
-    "node": ">= 0.8"
-  },
-  "scripts": {
-    "bench": "node benchmark/index.js",
-    "lint": "eslint --plugin markdown --ext js,md .",
-    "test": "mocha --reporter spec --bail --check-leaks test/",
-    "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- --reporter dot --check-leaks test/",
-    "test-travis": "istanbul cover node_modules/mocha/bin/_mocha --report lcovonly -- --reporter spec --check-leaks test/"
-  }
-}

package/example/node/server.js

@@ -1,38 +0,0 @@
-'use strict';
-
-const path = require('path');
-const express = require('express');
-const { middleware, Crawler } = require('../../src/index');
-
-const app = express();
-const port = 3000;
-
-app.use(
-  middleware(() => {
-    console.log('Testing the callback\n');
-  })
-);
-
-app.use('/dist', express.static(path.join(__dirname + '/dist')));
-
-app.get('/', function (req, res) {
-  res.sendFile(path.join(__dirname + '/index.html'));
-});
-
-app.get('/crawler', function async(request, response) {
-  // or check a user agent string
-  request.Crawler.isCrawler(
-    'TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)'
-  );
-
-  // Output the name of the bot that matched (if any)
-  response.send(request.Crawler.getMatches());
-});
-
-app.get('/curl', function async(request, response) {
-  var CrawlerDetector = new Crawler(request);
-  CrawlerDetector.isCrawler(); // true
-  response.send(CrawlerDetector.getMatches());
-});
-
-app.listen(port, () => console.log(`Example app listening on port ${port}!`));

package/src/index.js

@@ -1,16 +0,0 @@
-const Crawler = require('./lib/crawler');
-
-module.exports = {
-  Crawler,
-  middleware(cb) {
-    return (req, res, next) => {
-      // If there is a cb, execute it
-      if (typeof cb === 'function') {
-        cb.call(req, res);
-      }
-      // Initiate
-      req.Crawler = new Crawler(req);
-      next();
-    };
-  },
-};

package/src/lib/crawler/provider.js

@@ -1,11 +0,0 @@
-'use strict';
-
-class Provider {
-  constructor() {}
-
-  getAll() {
-    return this.data;
-  }
-}
-
-module.exports = Provider;

package/src/lib/crawler.js

@@ -1,130 +0,0 @@
-'use strict';
-
-const Crawlers = require('./crawler/crawlers');
-const Exclusions = require('./crawler/exclusions');
-const Headers = require('./crawler/headers');
-
-class Crawler {
-  constructor(request, headers, userAgent) {
-    /**
-     * Init classes
-     */
-    this._init();
-
-    /**
-     * This request must be an object
-     */
-    this.request = typeof request === 'object' ? request : {};
-
-    // The regex-list must not be used with g-flag!
-    // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
-    this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
-
-    // The exclusions should be used with g-flag in order to remove each value.
-    this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
-
-    /**
-     * Set http headers
-     */
-    this.setHttpHeaders(headers);
-
-    /**
-     * Set userAgent
-     */
-    this.userAgent = this.setUserAgent(userAgent);
-  }
-
-  /**
-   * Init Classes Instances
-   */
-  _init() {
-    this.crawlers = new Crawlers();
-    this.headers = new Headers();
-    this.exclusions = new Exclusions();
-  }
-
-  compileRegex(patterns, flags) {
-    return new RegExp(patterns.join('|'), flags);
-  }
-
-  /**
-   * Set HTTP headers.
-   */
-  setHttpHeaders(headers) {
-    // Use the Request headers if httpHeaders is not defined
-    if (typeof headers === 'undefined' || Object.keys(headers).length === 0) {
-      headers = Object.keys(this.request).length ? this.request.headers : {};
-    }
-
-    // Save the headers.
-    this.httpHeaders = headers;
-  }
-
-  /**
-   * Set user agent
-   */
-  setUserAgent(userAgent) {
-    if (
-      typeof userAgent === 'undefined' ||
-      userAgent === null ||
-      !userAgent.length
-    ) {
-      for (const header of this.getUaHttpHeaders()) {
-        if (Object.keys(this.httpHeaders).indexOf(header.toLowerCase()) >= 0) {
-          userAgent += this.httpHeaders[header.toLowerCase()] + ' ';
-        }
-      }
-    }
-
-    return userAgent;
-  }
-
-  /**
-   * Get user agent headers
-   */
-  getUaHttpHeaders() {
-    return this.headers.getAll();
-  }
-
-  /**
-   * Check user agent string against the regex.
-   */
-  isCrawler(userAgent = undefined) {
-    if (Buffer.byteLength(userAgent || '', 'utf8') > 4096) {
-      return false;
-    }
-
-    var agent =
-      typeof userAgent === 'undefined' || userAgent === null
-        ? this.userAgent
-        : userAgent;
-
-    // test on compiled regx
-    agent = agent.replace(this.compiledExclusions, '');
-
-    if (agent.trim().length === 0) {
-      return false;
-    }
-
-    var matches = this.compiledRegexList.exec(agent);
-
-    if (matches) {
-      this.matches = matches;
-    }
-
-    return matches !== null ? (matches.length ? true : false) : false;
-  }
-
-  /**
-   * Return the matches.
-   */
-  getMatches() {
-    return this.matches !== undefined
-      ? this.matches.length
-        ? this.matches[0]
-        : null
-      : {};
-  }
-}
-
-module.exports = Crawler;

package/test/lib/crawler.test.js

@@ -1,94 +0,0 @@
-const readline = require('readline');
-const fs = require('fs');
-const assert = require('assert');
-const Crawler = require('../../src/lib/crawler');
-
-describe('crawler', () => {
-  var crawler = new Crawler();
-
-  describe('regex-compilation', () => {
-    it('will join list of patterns with pipes', () => {
-      assert.strictEqual(
-        crawler.compileRegex(['some', 'patterns']).source,
-        'some|patterns'
-      );
-      assert.strictEqual(crawler.compileRegex(['single']).source, 'single');
-    });
-
-    it('keeps the whitespace', () => {
-      assert.strictEqual(
-        crawler.compileRegex(['  keep-whitespaces ']).source,
-        '  keep-whitespaces '
-      );
-    });
-
-    it('will accept regex-flags for compilation', () => {
-      var patterns = ['some', 'patterns'];
-      assert.strictEqual(crawler.compileRegex(patterns, 'g').flags, 'g');
-      assert.strictEqual(crawler.compileRegex(patterns, 'i').flags, 'i');
-    });
-
-    it('should be case insensitive', () => {
-      assert.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
-      assert.strictEqual(
-        crawler.getMatches('Facebot\\1.0'),
-        'Facebot',
-        'Crawler was not able to indentify crawler correctly'
-      );
-    });
-  });
-
-  describe('crawler-identification', () => {
-    it('should be able to identify crawlers', async () => {
-      const rl = readline.createInterface({
-        input: fs.createReadStream('./test/lib/database/crawlers.txt'),
-        crlfDelay: Infinity,
-      });
-
-      for await (const line of rl) {
-        assert.strictEqual(
-          crawler.isCrawler(line),
-          true,
-          `${line} is not a crawler`
-        );
-      }
-
-      rl.close();
-    });
-
-    it('should be able to identify devices', async () => {
-      const rl = readline.createInterface({
-        input: fs.createReadStream('./test/lib/database/devices.txt'),
-        crlfDelay: Infinity,
-      });
-
-      for await (const line of rl) {
-        assert.strictEqual(
-          crawler.isCrawler(line),
-          false,
-          `${line} is not a device`
-        );
-      }
-
-      rl.close();
-    });
-
-    it('should identify the crawler from a given headers', async () => {
-      crawler = new Crawler(null, {
-        host: '127.0.0.1:3000',
-        'user-agent': 'curl/7.73.0',
-        accept: '*/*',
-      });
-
-      assert.strictEqual(crawler.isCrawler(), true);
-    });
-
-    it('should identify the crawler from request headers', async () => {
-      crawler = new Crawler({
-        headers: { 'user-agent': 'curl/7.73.0', accept: '*/*' },
-      });
-
-      assert.strictEqual(crawler.isCrawler(), true);
-    });
-  });
-});

package/webpack.common.js

@@ -1,20 +0,0 @@
-const path = require('path');
-
-module.exports = {
-  context: path.resolve(__dirname, 'src'),
-  entry: './index.js',
-  output: {
-    path: path.resolve(__dirname, 'dist'),
-    filename: '[name].bundle.js',
-    library: 'Crawler',
-  },
-  module: {
-    rules: [
-      {
-        test: /\.js$/,
-        use: 'babel-loader',
-        exclude: /node_modules/,
-      },
-    ],
-  },
-};

package/webpack.dev.js

@@ -1,7 +0,0 @@
-const { merge } = require('webpack-merge');
-const common = require('./webpack.common');
-
-module.exports = merge(common, {
-  mode: 'development',
-  devtool: 'inline-source-map',
-});

package/webpack.prod.js

@@ -1,14 +0,0 @@
-const { merge } = require('webpack-merge');
-const TerserPlugin = require('terser-webpack-plugin');
-const common = require('./webpack.common');
-
-module.exports = merge(common, {
-  mode: 'production',
-  optimization: {
-    minimizer: [
-      new TerserPlugin({
-        parallel: true,
-      }),
-    ],
-  },
-});