@hyperwatch/hyperwatch 3.9.4 → 4.1.0

package/README.md

@@ -12,7 +12,7 @@ Hyperwatch is built on a real-time stream processor handling logs from inputs of
 
 ## Install
 
-Make sure you have Node.js version >= 11.14.
+Make sure you have Node.js version >= 14.
 
 We recommend using [nvm](https://github.com/creationix/nvm): `nvm install && nvm use`.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
-  "version": "3.9.4",
+  "version": "4.1.0",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",
@@ -20,49 +20,50 @@
     "depcheck": "depcheck",
     "lint": "eslint '**/*.js'",
     "lint:fix": "eslint '**/*.js' --fix",
-    "start": "node start",
-    "test": "mocha --recursive",
     "prettier": "prettier \"**/*.@(js|json|md|yml)\"",
     "prettier:check": "npm run prettier -- --list-different",
-    "prettier:write": "npm run prettier -- --write"
+    "prettier:write": "npm run prettier -- --write",
+    "start": "node start",
+    "test": "mocha --recursive"
   },
   "dependencies": {
-    "@hyperwatch/useragent": "^3.9.1",
+    "@hyperwatch/useragent": "^3.9.3",
     "accept-language-parser": "^1.5.0",
-    "ajv": "^8.11.0",
+    "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",
     "chalk": "^4.1.0",
     "country-code-emoji": "^2.3.0",
     "debug": "^4.3.4",
-    "dnsbl": "^4.0.0",
+    "dnsbl": "^4.0.1",
     "express": "^4.18.2",
     "express-ws": "^5.0.2",
-    "geoip-lite": "^1.4.6",
-    "immutable": "^4.1.0",
-    "ip-cidr": "^3.0.10",
+    "geoip-lite": "^1.4.8",
+    "immutable": "^4.3.4",
+    "ip-cidr": "^3.1.0",
     "lodash": "^4.17.21",
-    "lru-cache": "^6.0.0",
+    "lru-cache": "^10.0.1",
     "micro-strptime": "^0.2.3",
     "proxy-addr": "^2.0.7",
     "rc": "^1.2.8",
     "syslog-parse": "^1.3.1",
-    "tail": "^2.2.4",
-    "uuid": "^9.0.0",
-    "ws": "^8.9.0"
+    "tail": "^2.2.6",
+    "uuid": "^9.0.1",
+    "ws": "^8.14.2"
   },
   "devDependencies": {
-    "depcheck": "^1.4.3",
-    "eslint": "^8.25.0",
-    "eslint-plugin-import": "^2.26.0",
+    "depcheck": "^1.4.7",
+    "eslint": "^8.53.0",
+    "eslint-plugin-import": "^2.29.0",
     "eslint-plugin-node": "^11.1.0",
     "husky": "^4.3.5",
     "lint-staged": "^10.5.3",
-    "mocha": "^10.0.0",
-    "prettier": "^2.7.1",
-    "prettier-package-json": "^2.7.0"
+    "mocha": "^10.2.0",
+    "node-fetch": "^2.0.0",
+    "prettier": "^3.3.3",
+    "prettier-package-json": "^2.8.0"
   },
   "engines": {
-    "node": ">=12.0"
+    "node": ">=14.0"
   },
   "depcheck": {
     "ignores": [

package/scripts/cloudfront-ips.js

@@ -0,0 +1,56 @@
+/* eslint-disable-next-line node/no-unpublished-require */
+const fetch = require('node-fetch');
+
+async function main() {
+  const response = await fetch(
+    'https://ip-ranges.amazonaws.com/ip-ranges.json'
+  );
+
+  const ipRanges = await response.json();
+
+  console.log(
+    ipRanges.prefixes
+      .filter((p) => p.service === 'CLOUDFRONT')
+      .map((el) => el.ip_prefix)
+      .sort((a, b) => {
+        const aArray = a.split('.').map(Number);
+        const bArray = b.split('.').map(Number);
+        if (aArray[0] === bArray[0]) {
+          if (aArray[1] === bArray[1]) {
+            if (aArray[2] === bArray[2]) {
+              return aArray[3] > bArray[3] ? 1 : -1;
+            }
+            return aArray[2] > bArray[2] ? 1 : -1;
+          }
+          return aArray[1] > bArray[1] ? 1 : -1;
+        }
+        return aArray[0] > bArray[0] ? 1 : -1;
+      })
+      .map((el) => `"${el}"`)
+      .join(',\n')
+  );
+
+  console.log(
+    ipRanges.ipv6_prefixes
+      .filter((p) => p.service === 'CLOUDFRONT')
+      .map((el) => el.ipv6_prefix)
+      .sort((a, b) => {
+        const aArray = a.split(':').map(Number);
+        const bArray = b.split(':').map(Number);
+        if (aArray[0] === bArray[0]) {
+          if (aArray[1] === bArray[1]) {
+            if (aArray[2] === bArray[2]) {
+              return aArray[3] > bArray[3] ? 1 : -1;
+            }
+            return aArray[2] > bArray[2] ? 1 : -1;
+          }
+          return aArray[1] > bArray[1] ? 1 : -1;
+        }
+        return aArray[0] > bArray[0] ? 1 : -1;
+      })
+      .map((el) => `"${el}"`)
+      .join(',\n')
+  );
+}
+
+main();

package/src/lib/cache.js

@@ -1,14 +1,14 @@
-const lruCache = require('lru-cache');
+const { LRUCache } = require('lru-cache');
 
 const lruProvider = (options) => {
-  const lruInstance = new lruCache(options);
+  const lruInstance = new LRUCache(options);
   return {
-    clear: async () => lruInstance.reset(),
-    del: async (key) => lruInstance.del(key),
+    clear: async () => lruInstance.clear(),
+    del: async (key) => lruInstance.delete(key),
     get: async (key) => lruInstance.get(key),
     has: async (key) => lruInstance.has(key),
     set: async (key, value, ttlInSeconds) =>
-      lruInstance.set(key, value, ttlInSeconds ? ttlInSeconds * 1000 : null),
+      lruInstance.set(key, value, { ttl: ttlInSeconds * 1000 }),
   };
 };
 
@@ -30,4 +30,12 @@ async function set(key, value, ttlInSeconds) {
   return cacheProvider.set(key, value, ttlInSeconds);
 }
 
-module.exports = { has, get, set, setProvider };
+async function del(key) {
+  return cacheProvider.del(key);
+}
+
+async function clear(key) {
+  return cacheProvider.clear(key);
+}
+
+module.exports = { has, get, set, del, clear, setProvider };

package/src/lib/formatter.js

@@ -34,8 +34,8 @@ const executionTime = (log, output) => {
   return log.get('executionTime') <= 100
     ? colorize('green', `${log.get('executionTime')}ms`, output)
     : log.get('executionTime') >= 1000
-    ? colorize('red', `${log.get('executionTime')}ms`, output)
-    : colorize('yellow', `${log.get('executionTime')}ms`, output);
+      ? colorize('red', `${log.get('executionTime')}ms`, output)
+      : colorize('yellow', `${log.get('executionTime')}ms`, output);
 };
 
 const identity = (log) => log.getIn(['identity'], '');

package/src/modules/identity.js

@@ -195,6 +195,10 @@ function augment(log) {
       return hostname && hostname.endsWith('.infotiger.com')
         ? log.set('identity', 'InfoTiger')
         : log;
+    case 'Amazonbot':
+      return hostname && hostname.endsWith('.crawl.amazonbot.amazon')
+        ? log.set('identity', 'Amazonbot')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -262,9 +266,9 @@ function augment(log) {
         hostname.endsWith('.eu-central-1.compute.amazonaws.com')
         ? log.set('identity', 'Wise')
         : log;
-    case 'Amazonbot':
-      return hostname && hostname.endsWith('.compute-1.amazonaws.com')
-        ? log.set('identity', 'Amazonbot')
+    case 'ClaudeBot':
+      return hostname && hostname.endsWith('.us-east-2.compute.amazonaws.com')
+        ? log.set('identity', 'Claude')
         : log;
 
     // GCE
@@ -308,6 +312,9 @@ function augment(log) {
     if (hostname.endsWith('.us.archive.org')) {
       return log.set('identity', 'Archive.org');
     }
+    if (hostname.endsWith('.qwant.com')) {
+      return log.set('identity', 'Qwant');
+    }
   }
 
   // Signature

package/src/plugins/proxy.js

@@ -3,52 +3,169 @@ const proxyaddr = require('proxy-addr');
 
 const ipRanges = {
   cloudflare: [
+    // Source: https://www.cloudflare.com/ips/
+    '173.245.48.0/20',
     '103.21.244.0/22',
     '103.22.200.0/22',
     '103.31.4.0/22',
-    '104.16.0.0/12',
-    '108.162.192.0/18',
-    '131.0.72.0/22',
     '141.101.64.0/18',
-    '162.158.0.0/15',
-    '172.64.0.0/13',
-    '173.245.48.0/20',
-    '188.114.96.0/20',
+    '108.162.192.0/18',
     '190.93.240.0/20',
+    '188.114.96.0/20',
     '197.234.240.0/22',
     '198.41.128.0/17',
-    '199.27.128.0/21',
+    '162.158.0.0/15',
+    '104.16.0.0/13',
+    '104.24.0.0/14',
+    '172.64.0.0/13',
+    '131.0.72.0/22',
     '2400:cb00::/32',
-    '2405:8100::/32',
-    '2405:b500::/32',
     '2606:4700::/32',
     '2803:f800::/32',
+    '2405:b500::/32',
+    '2405:8100::/32',
+    '2a06:98c0::/29',
+    '2c0f:f248::/32',
   ],
   cloudfront: [
+    // Source: scripts/cloudfront-ips.js
+    '3.10.17.128/25',
+    '3.11.53.0/24',
+    '3.35.130.128/25',
+    '3.101.158.0/23',
+    '3.128.93.0/24',
+    '3.134.215.0/24',
+    '3.160.0.0/14',
+    '3.231.2.0/25',
+    '3.234.232.224/27',
+    '3.236.48.0/23',
+    '3.236.169.192/26',
     '13.32.0.0/15',
+    '13.35.0.0/16',
+    '13.48.32.0/24',
     '13.54.63.128/26',
+    '13.59.250.0/26',
+    '13.113.196.64/26',
+    '13.113.203.0/24',
+    '13.124.199.0/24',
+    '13.210.67.128/26',
+    '13.224.0.0/14',
+    '13.228.69.0/24',
+    '13.233.177.192/26',
+    '13.249.0.0/16',
+    '15.158.0.0/16',
+    '15.188.184.0/24',
+    '15.207.13.128/25',
+    '15.207.213.128/25',
+    '18.64.0.0/14',
+    '18.68.0.0/16',
+    '18.154.0.0/15',
+    '18.160.0.0/15',
+    '18.164.0.0/15',
+    '18.172.0.0/15',
+    '18.192.142.0/23',
+    '18.200.212.0/23',
+    '18.216.170.128/25',
+    '18.229.220.192/26',
+    '18.238.0.0/15',
+    '18.244.0.0/15',
     '34.195.252.0/24',
+    '34.216.51.0/25',
+    '34.223.12.224/27',
+    '34.223.80.192/26',
+    '34.226.14.0/24',
+    '35.158.136.0/24',
     '35.162.63.192/26',
     '35.167.191.128/26',
+    '36.103.232.0/25',
+    '36.103.232.128/26',
+    '44.227.178.0/24',
+    '44.234.90.252/30',
+    '44.234.108.128/25',
     '52.15.127.128/26',
     '52.46.0.0/18',
+    '52.47.139.0/24',
     '52.52.191.128/26',
     '52.56.127.0/25',
     '52.57.254.0/24',
     '52.66.194.128/26',
     '52.78.247.128/26',
+    '52.82.128.0/19',
     '52.84.0.0/15',
+    '52.124.128.0/17',
     '52.199.127.192/26',
     '52.212.248.0/26',
     '52.220.191.0/26',
     '52.222.128.0/17',
     '54.182.0.0/16',
     '54.192.0.0/16',
-    '54.230.0.0/16',
+    '54.230.0.0/17',
+    '54.230.128.0/18',
+    '54.230.200.0/21',
+    '54.230.208.0/20',
+    '54.230.224.0/19',
     '54.233.255.128/26',
     '54.239.128.0/18',
     '54.239.192.0/19',
     '54.240.128.0/18',
+    '58.254.138.128/26',
+    '58.254.138.0/25',
+    '64.252.64.0/18',
+    '64.252.128.0/18',
+    '65.8.0.0/16',
+    '65.9.0.0/17',
+    '65.9.128.0/18',
+    '70.132.0.0/18',
+    '71.152.0.0/17',
+    '99.79.169.0/24',
+    '99.84.0.0/16',
+    '99.86.0.0/16',
+    '108.138.0.0/15',
+    '108.156.0.0/14',
+    '111.13.171.192/26',
+    '111.13.171.128/26',
+    '111.13.185.64/27',
+    '111.13.185.32/27',
+    '116.129.226.0/25',
+    '116.129.226.128/26',
+    '118.193.97.128/25',
+    '118.193.97.64/26',
+    '119.147.182.128/26',
+    '119.147.182.0/25',
+    '120.52.12.64/26',
+    '120.52.22.96/27',
+    '120.52.39.128/27',
+    '120.52.153.192/26',
+    '120.232.236.128/26',
+    '120.232.236.0/25',
+    '120.253.240.192/26',
+    '120.253.241.160/27',
+    '120.253.245.192/27',
+    '120.253.245.128/26',
+    '130.176.0.0/17',
+    '130.176.128.0/18',
+    '130.176.192.0/19',
+    '130.176.224.0/20',
+    '143.204.0.0/16',
+    '144.220.0.0/16',
+    '180.163.57.0/25',
+    '180.163.57.128/26',
+    '204.246.164.0/22',
+    '204.246.168.0/22',
+    '204.246.172.0/24',
+    '204.246.173.0/24',
+    '204.246.174.0/23',
+    '204.246.176.0/20',
+    '205.251.200.0/21',
+    '205.251.208.0/20',
+    '205.251.249.0/24',
+    '205.251.250.0/23',
+    '205.251.252.0/23',
+    '205.251.254.0/24',
+    '216.137.32.0/19',
+    '223.71.11.0/27',
+    '223.71.71.96/27',
+    '223.71.71.128/25',
     '204.246.164.0/22',
     '204.246.168.0/22',
     '204.246.174.0/23',
@@ -59,9 +176,43 @@ const ipRanges = {
     '205.251.252.0/23',
     '205.251.254.0/24',
     '216.137.32.0/19',
+    '2400:7fc0:500::/40',
+    '2404:c2c0:500::/40',
+    '2409:8c00:2421:400::/56',
+    '2409:8c00:2421:300::/56',
+    '2600:9000:eee::/48',
+    '2600:9000:f580::/41',
+    '2600:9000:5310::/44',
+    '2600:9000:f520::/44',
+    '2600:9000:f534::/46',
+    '2600:9000:5308::/45',
+    '2600:9000:fff::/48',
+    '2600:9000:1000::/36',
+    '2600:9000:2000::/36',
+    '2600:9000:4000::/36',
+    '2600:9000:5320::/43',
+    '2600:9000:5340::/42',
+    '2600:9000:5380::/41',
+    '2600:9000:f538::/45',
+    '2600:9000:f400::/40',
+    '2600:9000:f800::/37',
+    '2600:9000:ddd::/48',
+    '2600:9000:f500::/43',
+    '2600:9000:f000::/38',
+    '2600:9000:f540::/42',
+    '2600:9000:f600::/39',
+    '2600:9000:3000::/36',
+  ],
+  sucuri: [
+    // Source: https://docs.sucuri.net/website-firewall/sucuri-firewall-troubleshooting-guide/
+    '192.88.134.0/23',
+    '185.93.228.0/22',
+    '66.248.200.0/22',
+    '2a02:fe80::/29',
+    '208.109.0.0/22',
   ],
-  cloudproxy: ['192.88.134.0/23', '185.93.228.0/22', '66.248.200.0/22'],
-  incapsula: [
+  imperva: [
+    // Source: https://docs.imperva.com/howto/c85245b7
     '199.83.128.0/21',
     '198.143.32.0/19',
     '149.126.72.0/21',