@hyperwatch/hyperwatch 3.9.3 → 4.0.0

package/README.md

@@ -12,7 +12,7 @@ Hyperwatch is built on a real-time stream processor handling logs from inputs of
 
 ## Install
 
-Make sure you have Node.js version >= 11.14.
+Make sure you have Node.js version >= 14.
 
 We recommend using [nvm](https://github.com/creationix/nvm): `nvm install && nvm use`.
 

package/docs/input.md

@@ -152,10 +152,10 @@ Hyperwatch is distributed with two Nginx formats.
 - The default `combined` format:
   `'$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent"'`
 
-- the `access_watch_combined` format that extracts important HTTP headers from the request:
+- the `hyperwatch_combined` format that extracts important HTTP headers from the request:
   `'$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host"'`.
 
-If possible, we recommend you to use the `access_watch_combined` format to take full advantage of the Hyperwatch data augmentation in the pipeline.
+If possible, we recommend you to use the `hyperwatch_combined` format to take full advantage of the Hyperwatch data augmentation in the pipeline.
 
 ### Apache
 
@@ -176,10 +176,10 @@ Hyperwatch is distributed with two Apache formats.
 - The default `combined` format:
   `'%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'`
 
-- the `access_watch_combined` format that extracts important HTTP headers from the request:
+- the `hyperwatch_combined` format that extracts important HTTP headers from the request:
   `'%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" "%{Accept}i" "%{Accept-Charset}i" "%{Accept-Encoding}i" "%{Accept-Language}i" "%{Connection}i" "%{Dnt}i" "%{From}i" "%{Host}i"'`.
 
-If possible, we recommend you to use the `access_watch_combined` format to take full advantage of the Hyperwatch data augmentation in the pipeline.
+If possible, we recommend you to use the `hyperwatch_combined` format to take full advantage of the Hyperwatch data augmentation in the pipeline.
 
 ### Custom
 
@@ -225,8 +225,8 @@ npm start config/custom
 For more detailed log processing, it is recommended to use the _Hyperwatch combined_ log format:
 
 ```
-log_format access_watch_combined '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host" "$http_x_forwarded_for"'
-access_log /logs/access.log access_watch_combined;
+log_format hyperwatch_combined '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host"'
+access_log /logs/access.log hyperwatch_combined;
 ```
 
 With the following configuration for Hyperwatch:
@@ -234,7 +234,7 @@ With the following configuration for Hyperwatch:
 ```
 const defaultInput = input.file.create({
   path: '/logs/access.log',
-  parse: format.nginx.parser({format: format.nginx.formats.accessWatchCombined})
+  parse: format.nginx.parser({format: format.nginx.formats.hyperwatch_combined})
 })
 ```
 
@@ -243,8 +243,8 @@ const defaultInput = input.file.create({
 If behind a proxy, you might want to also report the `HTTP_X_FORWARDED_FOR` header to allow Hyperwatch to properly detect the client IP address.
 
 ```
-log_format access_watch_combined_with_x_forwarded_for '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host" "$http_x_forwarded_for"'
-access_log /logs/access.log access_watch_combined_with_x_forwarded_for;
+log_format hyperwatch_combined_with_x_forwarded_for '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host" "$http_x_forwarded_for"'
+access_log /logs/access.log hyperwatch_combined_with_x_forwarded_for;
 ```
 
 With the following configuration for Hyperwatch:

package/docs/tutorials/apache_input.md

@@ -51,7 +51,7 @@ module.exports = function (hyperwatch) {
   const syslogApacheHyperwatchCombinedInput = input.syslog.create({
     port: 1518,
     parse: format.apache.parser({
-      format: format.apache.formats.hyperwatchCombined,
+      format: format.apache.formats.hyperwatch_combined,
     }),
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
-  "version": "3.9.3",
+  "version": "4.0.0",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",
@@ -34,14 +34,14 @@
     "chalk": "^4.1.0",
     "country-code-emoji": "^2.3.0",
     "debug": "^4.3.4",
-    "dnsbl": "^3.2.0",
+    "dnsbl": "^4.0.0",
     "express": "^4.18.2",
     "express-ws": "^5.0.2",
     "geoip-lite": "^1.4.6",
     "immutable": "^4.1.0",
     "ip-cidr": "^3.0.10",
     "lodash": "^4.17.21",
-    "lru-cache": "^6.0.0",
+    "lru-cache": "^10.0.1",
     "micro-strptime": "^0.2.3",
     "proxy-addr": "^2.0.7",
     "rc": "^1.2.8",
@@ -58,11 +58,12 @@
     "husky": "^4.3.5",
     "lint-staged": "^10.5.3",
     "mocha": "^10.0.0",
-    "prettier": "^2.7.1",
+    "node-fetch": "^2.0.0",
+    "prettier": "^3.0.3",
     "prettier-package-json": "^2.7.0"
   },
   "engines": {
-    "node": ">=12.0"
+    "node": ">=14.0"
   },
   "depcheck": {
     "ignores": [

package/scripts/cloudfront-ips.js

@@ -0,0 +1,56 @@
+/* eslint-disable-next-line node/no-unpublished-require */
+const fetch = require('node-fetch');
+
+async function main() {
+  const response = await fetch(
+    'https://ip-ranges.amazonaws.com/ip-ranges.json'
+  );
+
+  const ipRanges = await response.json();
+
+  console.log(
+    ipRanges.prefixes
+      .filter((p) => p.service === 'CLOUDFRONT')
+      .map((el) => el.ip_prefix)
+      .sort((a, b) => {
+        const aArray = a.split('.').map(Number);
+        const bArray = b.split('.').map(Number);
+        if (aArray[0] === bArray[0]) {
+          if (aArray[1] === bArray[1]) {
+            if (aArray[2] === bArray[2]) {
+              return aArray[3] > bArray[3] ? 1 : -1;
+            }
+            return aArray[2] > bArray[2] ? 1 : -1;
+          }
+          return aArray[1] > bArray[1] ? 1 : -1;
+        }
+        return aArray[0] > bArray[0] ? 1 : -1;
+      })
+      .map((el) => `"${el}"`)
+      .join(',\n')
+  );
+
+  console.log(
+    ipRanges.ipv6_prefixes
+      .filter((p) => p.service === 'CLOUDFRONT')
+      .map((el) => el.ipv6_prefix)
+      .sort((a, b) => {
+        const aArray = a.split(':').map(Number);
+        const bArray = b.split(':').map(Number);
+        if (aArray[0] === bArray[0]) {
+          if (aArray[1] === bArray[1]) {
+            if (aArray[2] === bArray[2]) {
+              return aArray[3] > bArray[3] ? 1 : -1;
+            }
+            return aArray[2] > bArray[2] ? 1 : -1;
+          }
+          return aArray[1] > bArray[1] ? 1 : -1;
+        }
+        return aArray[0] > bArray[0] ? 1 : -1;
+      })
+      .map((el) => `"${el}"`)
+      .join(',\n')
+  );
+}
+
+main();

package/src/format/apache.js

@@ -94,7 +94,7 @@ function parser({ format }) {
 
 const formats = {
   combined: '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
-  hyperwatchCombined:
+  hyperwatch_combined:
     '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" "%{Accept}i" "%{Accept-Charset}i" "%{Accept-Encoding}i" "%{Accept-Language}i" "%{Connection}i" "%{Dnt}i" "%{From}i" "%{Host}i"',
 };
 

package/src/format/nginx.js

@@ -95,9 +95,7 @@ function parser({ format = formats.combined } = {}) {
 const formats = {
   combined:
     '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent"',
-  accessWatch:
-    '"$time_iso8601" "$remote_addr" "$http_host" "$request" $status "$http_user_agent" "$http_accept" "$http_accept_language" "$http_accept_charset" "$http_accept_encoding" "$http_from" "$http_dnt" "$http_connection" "$http_referer"',
-  hyperwatchCombined:
+  hyperwatch_combined:
     '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host"',
 };
 

package/src/input/vercel.js

@@ -0,0 +1,81 @@
+const crypto = require('crypto');
+
+const { fromJS } = require('immutable');
+
+const app = require('../app/api');
+
+/**
+ * Create a log from Vercel JSON log drain format
+ */
+function parseVercel(request) {
+  return fromJS({
+    request: {
+      time: new Date(request.proxy.timestamp).toISOString(),
+      address: request.proxy.clientIp,
+      scheme: request.proxy.scheme,
+      method: request.proxy.method,
+      url: request.proxy.path,
+      headers: {
+        host: request.proxy.host,
+        'user-agent': request.proxy.userAgent[0],
+      },
+      captured_headers: ['host', 'user-agent'],
+    },
+    response: {
+      status: request.proxy.statusCode,
+    },
+  });
+}
+
+function verifySignature(req, secret) {
+  if (!req.headers['x-vercel-signature'] || !secret) {
+    return true;
+  }
+
+  return (
+    req.headers['x-vercel-signature'] ===
+    crypto.createHmac('sha1', secret).update(req.rawBody).digest('hex')
+  );
+}
+
+function create({
+  name = 'HTTP input (Vercel JSON log drain format)',
+  path = '/input/vercel',
+  parse = parseVercel,
+  verify,
+  secret,
+}) {
+  return {
+    name: name,
+    start: ({ success, reject, status }) => {
+      app.post(path, (req, res) => {
+        // Send back verify header (only needed to setup the drain)
+        if (verify) {
+          res.set('x-vercel-verify', verify);
+        }
+        // Verify signature
+        if (!verifySignature(req, secret)) {
+          res.statusCode = 403;
+          res.send("Signature didn't match");
+          return;
+        }
+        // No extra validation before sending the response to the client
+        res.send('Ok');
+        // Processing the message(s)
+        const messages = Array.isArray(req.body) ? req.body : [req.body];
+        messages.forEach((message) => {
+          try {
+            success(parse(message));
+          } catch (err) {
+            reject(err);
+          }
+        });
+      });
+      status(null, `Listening on http://__HOST__${path}`);
+    },
+  };
+}
+
+module.exports = {
+  create: create,
+};

package/src/lib/cache.js

@@ -1,14 +1,14 @@
-const lruCache = require('lru-cache');
+const { LRUCache } = require('lru-cache');
 
 const lruProvider = (options) => {
-  const lruInstance = new lruCache(options);
+  const lruInstance = new LRUCache(options);
   return {
-    clear: async () => lruInstance.reset(),
-    del: async (key) => lruInstance.del(key),
+    clear: async () => lruInstance.clear(),
+    del: async (key) => lruInstance.delete(key),
     get: async (key) => lruInstance.get(key),
     has: async (key) => lruInstance.has(key),
     set: async (key, value, ttlInSeconds) =>
-      lruInstance.set(key, value, ttlInSeconds ? ttlInSeconds * 1000 : null),
+      lruInstance.set(key, value, { ttl: ttlInSeconds * 1000 }),
   };
 };
 
@@ -30,4 +30,12 @@ async function set(key, value, ttlInSeconds) {
   return cacheProvider.set(key, value, ttlInSeconds);
 }
 
-module.exports = { has, get, set, setProvider };
+async function del(key) {
+  return cacheProvider.del(key);
+}
+
+async function clear(key) {
+  return cacheProvider.clear(key);
+}
+
+module.exports = { has, get, set, del, clear, setProvider };

package/src/modules/identity.js

@@ -186,6 +186,19 @@ function augment(log) {
       return hostname && hostname.endsWith('.dataforseo.com')
         ? log.set('identity', 'DataForSeo')
         : log;
+    case 'Qwantify':
+    case 'Qwantify-dev':
+      return hostname && hostname.endsWith('.qwant.com')
+        ? log.set('identity', 'Qwant')
+        : log;
+    case 'InfoTigerBot':
+      return hostname && hostname.endsWith('.infotiger.com')
+        ? log.set('identity', 'InfoTiger')
+        : log;
+    case 'Amazonbot':
+      return hostname && hostname.endsWith('.crawl.amazonbot.amazon')
+        ? log.set('identity', 'Amazonbot')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -253,10 +266,6 @@ function augment(log) {
         hostname.endsWith('.eu-central-1.compute.amazonaws.com')
         ? log.set('identity', 'Wise')
         : log;
-    case 'Amazonbot':
-      return hostname && hostname.endsWith('.compute-1.amazonaws.com')
-        ? log.set('identity', 'Amazonbot')
-        : log;
 
     // GCE
     case 'VelenPublicWebCrawler':

package/src/plugins/proxy.js

@@ -3,52 +3,169 @@ const proxyaddr = require('proxy-addr');
 
 const ipRanges = {
   cloudflare: [
+    // Source: https://www.cloudflare.com/ips/
+    '173.245.48.0/20',
     '103.21.244.0/22',
     '103.22.200.0/22',
     '103.31.4.0/22',
-    '104.16.0.0/12',
-    '108.162.192.0/18',
-    '131.0.72.0/22',
     '141.101.64.0/18',
-    '162.158.0.0/15',
-    '172.64.0.0/13',
-    '173.245.48.0/20',
-    '188.114.96.0/20',
+    '108.162.192.0/18',
     '190.93.240.0/20',
+    '188.114.96.0/20',
     '197.234.240.0/22',
     '198.41.128.0/17',
-    '199.27.128.0/21',
+    '162.158.0.0/15',
+    '104.16.0.0/13',
+    '104.24.0.0/14',
+    '172.64.0.0/13',
+    '131.0.72.0/22',
     '2400:cb00::/32',
-    '2405:8100::/32',
-    '2405:b500::/32',
     '2606:4700::/32',
     '2803:f800::/32',
+    '2405:b500::/32',
+    '2405:8100::/32',
+    '2a06:98c0::/29',
+    '2c0f:f248::/32',
   ],
   cloudfront: [
+    // Source: scripts/cloudfront-ips.js
+    '3.10.17.128/25',
+    '3.11.53.0/24',
+    '3.35.130.128/25',
+    '3.101.158.0/23',
+    '3.128.93.0/24',
+    '3.134.215.0/24',
+    '3.160.0.0/14',
+    '3.231.2.0/25',
+    '3.234.232.224/27',
+    '3.236.48.0/23',
+    '3.236.169.192/26',
     '13.32.0.0/15',
+    '13.35.0.0/16',
+    '13.48.32.0/24',
     '13.54.63.128/26',
+    '13.59.250.0/26',
+    '13.113.196.64/26',
+    '13.113.203.0/24',
+    '13.124.199.0/24',
+    '13.210.67.128/26',
+    '13.224.0.0/14',
+    '13.228.69.0/24',
+    '13.233.177.192/26',
+    '13.249.0.0/16',
+    '15.158.0.0/16',
+    '15.188.184.0/24',
+    '15.207.13.128/25',
+    '15.207.213.128/25',
+    '18.64.0.0/14',
+    '18.68.0.0/16',
+    '18.154.0.0/15',
+    '18.160.0.0/15',
+    '18.164.0.0/15',
+    '18.172.0.0/15',
+    '18.192.142.0/23',
+    '18.200.212.0/23',
+    '18.216.170.128/25',
+    '18.229.220.192/26',
+    '18.238.0.0/15',
+    '18.244.0.0/15',
     '34.195.252.0/24',
+    '34.216.51.0/25',
+    '34.223.12.224/27',
+    '34.223.80.192/26',
+    '34.226.14.0/24',
+    '35.158.136.0/24',
     '35.162.63.192/26',
     '35.167.191.128/26',
+    '36.103.232.0/25',
+    '36.103.232.128/26',
+    '44.227.178.0/24',
+    '44.234.90.252/30',
+    '44.234.108.128/25',
     '52.15.127.128/26',
     '52.46.0.0/18',
+    '52.47.139.0/24',
     '52.52.191.128/26',
     '52.56.127.0/25',
     '52.57.254.0/24',
     '52.66.194.128/26',
     '52.78.247.128/26',
+    '52.82.128.0/19',
     '52.84.0.0/15',
+    '52.124.128.0/17',
     '52.199.127.192/26',
     '52.212.248.0/26',
     '52.220.191.0/26',
     '52.222.128.0/17',
     '54.182.0.0/16',
     '54.192.0.0/16',
-    '54.230.0.0/16',
+    '54.230.0.0/17',
+    '54.230.128.0/18',
+    '54.230.200.0/21',
+    '54.230.208.0/20',
+    '54.230.224.0/19',
     '54.233.255.128/26',
     '54.239.128.0/18',
     '54.239.192.0/19',
     '54.240.128.0/18',
+    '58.254.138.128/26',
+    '58.254.138.0/25',
+    '64.252.64.0/18',
+    '64.252.128.0/18',
+    '65.8.0.0/16',
+    '65.9.0.0/17',
+    '65.9.128.0/18',
+    '70.132.0.0/18',
+    '71.152.0.0/17',
+    '99.79.169.0/24',
+    '99.84.0.0/16',
+    '99.86.0.0/16',
+    '108.138.0.0/15',
+    '108.156.0.0/14',
+    '111.13.171.192/26',
+    '111.13.171.128/26',
+    '111.13.185.64/27',
+    '111.13.185.32/27',
+    '116.129.226.0/25',
+    '116.129.226.128/26',
+    '118.193.97.128/25',
+    '118.193.97.64/26',
+    '119.147.182.128/26',
+    '119.147.182.0/25',
+    '120.52.12.64/26',
+    '120.52.22.96/27',
+    '120.52.39.128/27',
+    '120.52.153.192/26',
+    '120.232.236.128/26',
+    '120.232.236.0/25',
+    '120.253.240.192/26',
+    '120.253.241.160/27',
+    '120.253.245.192/27',
+    '120.253.245.128/26',
+    '130.176.0.0/17',
+    '130.176.128.0/18',
+    '130.176.192.0/19',
+    '130.176.224.0/20',
+    '143.204.0.0/16',
+    '144.220.0.0/16',
+    '180.163.57.0/25',
+    '180.163.57.128/26',
+    '204.246.164.0/22',
+    '204.246.168.0/22',
+    '204.246.172.0/24',
+    '204.246.173.0/24',
+    '204.246.174.0/23',
+    '204.246.176.0/20',
+    '205.251.200.0/21',
+    '205.251.208.0/20',
+    '205.251.249.0/24',
+    '205.251.250.0/23',
+    '205.251.252.0/23',
+    '205.251.254.0/24',
+    '216.137.32.0/19',
+    '223.71.11.0/27',
+    '223.71.71.96/27',
+    '223.71.71.128/25',
     '204.246.164.0/22',
     '204.246.168.0/22',
     '204.246.174.0/23',
@@ -59,9 +176,43 @@ const ipRanges = {
     '205.251.252.0/23',
     '205.251.254.0/24',
     '216.137.32.0/19',
+    '2400:7fc0:500::/40',
+    '2404:c2c0:500::/40',
+    '2409:8c00:2421:400::/56',
+    '2409:8c00:2421:300::/56',
+    '2600:9000:eee::/48',
+    '2600:9000:f580::/41',
+    '2600:9000:5310::/44',
+    '2600:9000:f520::/44',
+    '2600:9000:f534::/46',
+    '2600:9000:5308::/45',
+    '2600:9000:fff::/48',
+    '2600:9000:1000::/36',
+    '2600:9000:2000::/36',
+    '2600:9000:4000::/36',
+    '2600:9000:5320::/43',
+    '2600:9000:5340::/42',
+    '2600:9000:5380::/41',
+    '2600:9000:f538::/45',
+    '2600:9000:f400::/40',
+    '2600:9000:f800::/37',
+    '2600:9000:ddd::/48',
+    '2600:9000:f500::/43',
+    '2600:9000:f000::/38',
+    '2600:9000:f540::/42',
+    '2600:9000:f600::/39',
+    '2600:9000:3000::/36',
+  ],
+  sucuri: [
+    // Source: https://docs.sucuri.net/website-firewall/sucuri-firewall-troubleshooting-guide/
+    '192.88.134.0/23',
+    '185.93.228.0/22',
+    '66.248.200.0/22',
+    '2a02:fe80::/29',
+    '208.109.0.0/22',
   ],
-  cloudproxy: ['192.88.134.0/23', '185.93.228.0/22', '66.248.200.0/22'],
-  incapsula: [
+  imperva: [
+    // Source: https://docs.imperva.com/howto/c85245b7
     '199.83.128.0/21',
     '198.143.32.0/19',
     '149.126.72.0/21',