@hyperwatch/hyperwatch 4.2.0 → 4.3.1

package/README.md

@@ -12,7 +12,7 @@ Hyperwatch is built on a real-time stream processor handling logs from inputs of
 
 ## Install
 
-Make sure you have Node.js version >= 14.
+Make sure you have Node.js version >= 20.
 
 We recommend using [nvm](https://github.com/creationix/nvm): `nvm install && nvm use`.
 

package/config/apache_syslog_example.js

@@ -0,0 +1,29 @@
+module.exports = function (hyperwatch) {
+  const { pipeline, input, format, logger } = hyperwatch;
+
+  /* Init Hyperwatch first (will load modules) */
+
+  hyperwatch.init();
+
+  /* Input configuration */
+  /* =================== */
+
+  /* Syslog input  */
+  /* ------------- */
+
+  const syslogApacheHyperwatchCombinedInput = input.syslog.create({
+    port: 1518,
+    parse: format.apache.parser({
+      format: format.apache.formats.hyperwatchCombined,
+    }),
+  });
+
+  pipeline.registerInput(syslogApacheHyperwatchCombinedInput);
+
+  /* Pipeline configuration */
+  /* ====================== */
+
+  pipeline
+    .getNode('main')
+    .map((log) => console.log(logger.defaultFormatter.format(log, 'console')));
+};

package/config/default.js

@@ -76,7 +76,7 @@ if (require.main === module) {
   try {
     hyperwatch = require('../hyperwatch');
   } catch (e) {
-    // eslint-disable-next-line node/no-missing-require
+    // eslint-disable-next-line n/no-missing-require
     hyperwatch = require('@hyperwatch/hyperwatch');
   }
 

package/config/example.js

@@ -165,7 +165,7 @@ if (require.main === module) {
   try {
     hyperwatch = require('../hyperwatch');
   } catch (e) {
-    // eslint-disable-next-line node/no-missing-require
+    // eslint-disable-next-line n/no-missing-require
     hyperwatch = require('@hyperwatch/hyperwatch');
   }
 

package/config/express_websocket_example.js

@@ -0,0 +1,28 @@
+module.exports = function (hyperwatch) {
+  const { pipeline, input, logger } = hyperwatch;
+
+  /* Init Hyperwatch first (will load modules) */
+
+  hyperwatch.init();
+
+  /* Input configuration */
+  /* =================== */
+
+  /* Syslog input  */
+  /* ------------- */
+
+  const webSocketServerInput = input.websocket.create({
+    name: 'WebSocket server (JSON standard format)',
+    type: 'server',
+    path: '/input/log',
+  });
+
+  pipeline.registerInput(webSocketServerInput);
+
+  /* Pipeline configuration */
+  /* ====================== */
+
+  pipeline
+    .getNode('main')
+    .map((log) => console.log(logger.defaultFormatter.format(log, 'console')));
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
-  "version": "4.2.0",
+  "version": "4.3.1",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",
@@ -10,16 +10,25 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/hyperwatch/hyperwatch.git"
+    "url": "git+https://github.com/hyperwatch/hyperwatch.git"
   },
   "main": "hyperwatch.js",
   "bin": {
-    "hyperwatch": "./bin/hyperwatch"
+    "hyperwatch": "bin/hyperwatch"
   },
+  "files": [
+    "bin",
+    "config",
+    "docs",
+    "scripts",
+    "src",
+    "start.js"
+  ],
   "scripts": {
     "depcheck": "depcheck",
-    "lint": "eslint '**/*.js'",
-    "lint:fix": "eslint '**/*.js' --fix",
+    "lint": "eslint",
+    "lint:fix": "eslint --fix",
+    "prepare": "husky",
     "prettier": "prettier \"**/*.@(js|json|md|yml)\"",
     "prettier:check": "npm run prettier -- --list-different",
     "prettier:write": "npm run prettier -- --write",
@@ -29,52 +38,48 @@
   "dependencies": {
     "@hyperwatch/useragent": "^3.9.3",
     "accept-language-parser": "^1.5.0",
-    "ajv": "^8.12.0",
-    "ajv-formats": "^2.1.1",
-    "chalk": "^4.1.0",
+    "ajv": "^8.20.0",
+    "ajv-formats": "^3.0.1",
+    "chalk": "^4.1.2",
     "country-code-emoji": "^2.3.0",
-    "debug": "^4.3.4",
-    "dnsbl": "^4.0.1",
-    "express": "^4.18.2",
+    "csv-stringify": "^6.8.3",
+    "debug": "^4.4.3",
+    "dnsbl": "^4.0.3",
+    "express": "^4.22.2",
     "express-ws": "^5.0.2",
-    "geoip-lite": "^1.4.8",
-    "immutable": "^4.3.4",
-    "ip-cidr": "^3.1.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^10.0.1",
-    "micro-strptime": "^0.2.3",
+    "geoip-lite": "^1.4.10",
+    "immutable": "^5.1.9",
+    "ip-cidr": "^4.0.2",
+    "lodash": "^4.18.1",
+    "lru-cache": "^11.5.2",
+    "micro-strptime": "^1.0.0",
     "proxy-addr": "^2.0.7",
     "rc": "^1.2.8",
     "syslog-parse": "^1.3.1",
     "tail": "^2.2.6",
-    "uuid": "^9.0.1",
-    "ws": "^8.14.2"
+    "ws": "^8.21.3"
   },
   "devDependencies": {
+    "@eslint/js": "^9.39.5",
     "depcheck": "^1.4.7",
-    "eslint": "^8.53.0",
-    "eslint-plugin-import": "^2.29.0",
-    "eslint-plugin-node": "^11.1.0",
-    "husky": "^4.3.5",
-    "lint-staged": "^10.5.3",
-    "mocha": "^10.2.0",
-    "node-fetch": "^2.0.0",
-    "prettier": "^3.3.3",
+    "eslint": "^9.39.5",
+    "eslint-plugin-import": "^2.32.0",
+    "eslint-plugin-n": "^17.24.0",
+    "globals": "^17.11.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^16.4.0",
+    "mocha": "^11.8.0",
+    "prettier": "^3.9.6",
     "prettier-package-json": "^2.8.0"
   },
   "engines": {
-    "node": ">=14.0"
+    "node": ">=20"
   },
   "depcheck": {
     "ignores": [
       "prettier-package-json"
     ]
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
-  },
   "lint-staged": {
     "*.{js,json,md,yml}": [
       "prettier --write"

package/scripts/fetch-cloudflare-ips.js

@@ -0,0 +1,27 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+async function fetchLines(url) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to fetch ${url}: ${res.status}`);
+  }
+  const text = await res.text();
+  return text.trim().split('\n').filter(Boolean);
+}
+
+async function main() {
+  const ipv4 = await fetchLines('https://www.cloudflare.com/ips-v4/');
+  const ipv6 = await fetchLines('https://www.cloudflare.com/ips-v6/');
+  const cidrs = [...ipv4, ...ipv6];
+
+  const filePath = path.join(dataDir, 'cloudflare-ips.json');
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(
+    `cloudflare-ips: ${cidrs.length} CIDRs (${ipv4.length} IPv4, ${ipv6.length} IPv6)`
+  );
+}
+
+main().catch(console.error);

package/scripts/fetch-cloudfront-ips.js

@@ -0,0 +1,29 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+async function main() {
+  const response = await fetch(
+    'https://ip-ranges.amazonaws.com/ip-ranges.json'
+  );
+  const ipRanges = await response.json();
+
+  const ipv4 = ipRanges.prefixes
+    .filter((p) => p.service === 'CLOUDFRONT')
+    .map((p) => p.ip_prefix);
+
+  const ipv6 = ipRanges.ipv6_prefixes
+    .filter((p) => p.service === 'CLOUDFRONT')
+    .map((p) => p.ipv6_prefix);
+
+  const cidrs = [...ipv4, ...ipv6];
+
+  const filePath = path.join(dataDir, 'cloudfront-ips.json');
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(
+    `cloudfront-ips: ${cidrs.length} CIDRs (${ipv4.length} IPv4, ${ipv6.length} IPv6)`
+  );
+}
+
+main().catch(console.error);

package/scripts/fetch-openai-ips.js

@@ -0,0 +1,34 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+const sources = [
+  { name: 'chatgpt-user-ips', url: 'https://openai.com/chatgpt-user.json' },
+  { name: 'gptbot-ips', url: 'https://openai.com/gptbot.json' },
+  { name: 'openai-searchbot-ips', url: 'https://openai.com/searchbot.json' },
+];
+
+async function fetchAndStore({ name, url }) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    console.error(`Failed to fetch ${name}: ${res.status} ${res.statusText}`);
+    return;
+  }
+  const data = await res.json();
+  const cidrs = data.prefixes.map((p) => {
+    const cidr = p.ipv4Prefix || p.ip_prefix;
+    return cidr.includes('/') ? cidr : `${cidr}/32`;
+  });
+  const filePath = path.join(dataDir, `${name}.json`);
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(`${name}: ${cidrs.length} CIDRs`);
+}
+
+async function main() {
+  for (const source of sources) {
+    await fetchAndStore(source);
+  }
+}
+
+main().catch(console.error);

package/src/app/api.js

@@ -1,10 +1,12 @@
+const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
 
+const { stringify } = require('csv-stringify/sync');
 const express = require('express');
-const uuid = require('uuid');
 
 const monitoring = require('../lib/monitoring');
+const persistence = require('../lib/persistence');
 const { formatTable } = require('../lib/util');
 const stylesheet = require('../stylesheet');
 
@@ -47,7 +49,7 @@ app.streamToHttp = (
   updateMonitoringStatus();
 
   app.get(endpoint, (req, res) => {
-    const requestId = uuid.v4();
+    const requestId = crypto.randomUUID();
     requests[requestId] = [req, res];
     updateMonitoringStatus();
 
@@ -87,20 +89,33 @@ body { display: flex; flex-direction: column-reverse; }
 };
 
 app.registerAggregator = (name, aggregator) => {
-  app.get(`/${name}.:format(json)?`, (req, res) => {
+  persistence.register(name, aggregator);
+  app.get(`/${name}.:format(json|csv)?`, (req, res) => {
     const raw = req.query.raw ? true : false;
     const format = req.params.format || (raw ? 'json' : null);
     const limit = req.query.limit || 100;
-    const sort = req.query.sort || '15m';
+    const sort = req.query.sort || 'count15m';
 
     const data = aggregator.getData({
       sort,
       limit,
-      format,
+      format: format === 'csv' ? 'json' : format,
       raw,
     });
 
-    if (format === 'json') {
+    if (format === 'csv') {
+      const rows = data.toJS();
+      const columns = Object.keys(rows[0] || {}).filter(
+        (k) => k !== 'activity'
+      );
+      const csv = stringify(rows, { header: true, columns });
+      res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+      res.setHeader(
+        'Content-Disposition',
+        `attachment; filename="${name}.csv"`
+      );
+      res.send(csv);
+    } else if (format === 'json') {
       res.send(data);
     } else {
       res.setHeader('Content-Type', 'text/html; charset=utf-8');

package/src/app/websocket.js

@@ -1,7 +1,9 @@
+const crypto = require('crypto');
+
 const express = require('express');
 const expressWs = require('express-ws');
-const uuid = require('uuid');
 
+const constants = require('../constants');
 const monitoring = require('../lib/monitoring');
 
 const app = express();
@@ -36,18 +38,27 @@ app.streamToWebsocket = (
   updateMonitoringStatus();
 
   app.ws(endpoint, (client, req) => {
-    const clientId = req.query.clientId || uuid.v4();
+    const clientId = req.query.clientId || crypto.randomUUID();
     if (clients[clientId]) {
       console.log(`Client '${clientId}' is already connected. Terminating.`);
       client.terminate();
       return;
     }
     clients[clientId] = client;
+    client.isAlive = true;
     updateMonitoringStatus();
+    client.on('pong', () => {
+      client.isAlive = true;
+    });
     client.on('close', () => {
+      console.log(`Client '${clientId}' closed.`);
       delete clients[clientId];
       updateMonitoringStatus();
     });
+    client.on('error', (error) => {
+      console.log(`Client '${clientId}' error.`);
+      console.log(error);
+    });
   });
 
   stream.map((log) => {
@@ -60,6 +71,21 @@ app.streamToWebsocket = (
       }
     });
   });
+
+  // Heartbeat: detect and clean up stale connections
+  setInterval(() => {
+    Object.entries(clients).forEach(([clientId, client]) => {
+      if (!client.isAlive) {
+        console.log(`Client '${clientId}' stale. Terminating.`);
+        client.terminate();
+        delete clients[clientId];
+        updateMonitoringStatus();
+        return;
+      }
+      client.isAlive = false;
+      client.ping();
+    });
+  }, constants.heartbeatInterval || 30000);
 };
 
 module.exports = app;

package/src/constants.js

@@ -47,11 +47,20 @@ const constants = {
       active: false,
       priority: 620,
     },
+    history: {
+      active: false,
+      priority: 700,
+    },
     sparkline: {
       active: false,
       priority: 800,
     },
   },
+  persistence: {
+    enabled: false,
+    path: null,
+    namespace: null,
+  },
 };
 
 module.exports = rc('hyperwatch', constants);