@hyperwatch/hyperwatch 4.2.0 → 5.0.0

package/docs/tutorials/apache_input.md

@@ -10,10 +10,10 @@ Let's start.
 
 On the same server where Apache is running, or on a server that is reachable by it, install the Hyperwatch processor.
 
-As a prerequirement, you'll need Node.js >= 7. Use nvm if you're in trouble.
+As a prerequisite, you'll need Node.js >= 24. We recommend [nvm](https://github.com/nvm-sh/nvm).
 
 ```bash
-nvm install node
+nvm install 24
 ```
 
 #### Install from npm
@@ -24,7 +24,7 @@ npm install -g @hyperwatch/hyperwatch
 
 #### Install from Git
 
-Alternatively, for developemnt purpose, you can use Git and clone the public repository:
+Alternatively, for development purpose, you can use Git and clone the public repository:
 
 ```bash
 git clone https://github.com/hyperwatch/hyperwatch.git
@@ -34,13 +34,13 @@ npm install
 
 ### Configure Hyperwatch
 
-In our suggested configuration, Hyperwatch will be listening for access logs in the the `access_watch_combined` format on port `1518`.
+In our suggested configuration, Hyperwatch will be listening for access logs in the `hyperwatch_combined` format on port `1518`.
 
-We always recommand using the `access_watch_combined` format, which is logging more detailed information and allows for a much better analysis than the regular `combined` format.
+We always recommend using the `hyperwatch_combined` format, which is logging more detailed information and allows for a much better analysis than the regular `combined` format.
 
-To get more familiar, you can inspect default and example configurations in [`config/default.js`](<(../../config/default.js)>) and [`config/example.js`](../../config/example.js) file.
+To get more familiar, you can inspect default and example configurations in [`config/default.js`](../../config/default.js) and [`config/example.js`](../../config/example.js) file.
 
-Now, you can create your own configuration in `apache_syslog_example.js`:
+Now, you can create your own configuration in `apache_syslog_example.js` (a complete version is available in [`config/apache_syslog_example.js`](../../config/apache_syslog_example.js)):
 
 ```javascript
 module.exports = function (hyperwatch) {
@@ -61,7 +61,7 @@ module.exports = function (hyperwatch) {
 
 ### Configure Apache
 
-First, if you're following our recommendation and opted for the `hyperwatch_combined` format, you need to define it in the Apache configuration. This will not replace the standard log format, just create an additional one.
+First, if you're following our recommendation and opted for the `hyperwatch_combined` format, you need to define it in the Apache configuration. This will not replace the standard log format, just create an additional one.
 
 ```
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Accept}i\" \"%{Accept-Charset}i\" \"%{Accept-Encoding}i\" \"%{Accept-Language}i\" \"%{Connection}i\" \"%{Dnt}i\" \"%{From}i\" \"%{Host}i\"" hyperwatch_combined
@@ -79,13 +79,13 @@ Note: This is known to be working on _Ubuntu 16.04, 18.04_ with _logger 2.27.1,
 
 In this example, there are 3 important things:
 
-1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
-   If it's on a different server, replace `localhost` by the proper private or public IP address.
-2. We configured Hyperwatch to listen for syslog messages in the `hyperwatch_combined` format on port `1518`.
+1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
+   If it's on a different server, replace `localhost` by the proper private or public IP address.
+2. We configured Hyperwatch to listen for syslog messages in the `hyperwatch_combined` format on port `1518`.
    We're properly passing that port in the configuration
-3. Finally, we're asking Apache to use the `hyperwatch_combined` log format we previously configured.
+3. Finally, we're asking Apache to use the `hyperwatch_combined` log format we previously configured.
 
-Don't forget to reload Aapche with the updated configuration. On Ubuntu, it would be:
+Don't forget to reload Apache with the updated configuration. On Ubuntu, it would be:
 
 ```bash
 service apache2 reload
@@ -101,4 +101,6 @@ hyperwatch apache_syslog_example.js
 
 ### Browse the interface
 
-Now, you can point your browser to the IP/port where Hyperwatch is running. If you see data flowing, congrats you made it!
+Now, you can point your browser to the `/status` page on the IP/port where Hyperwatch is running (e.g. `http://localhost:3000/status`). If you see traffic going through your input, congrats you made it!
+
+To watch the logs live at `/logs/main` and explore aggregations such as `/addresses` or `/identities`, activate the corresponding modules. See [Global Configuration](../configuration.md#modules).

package/docs/tutorials/express_input.md

@@ -10,10 +10,10 @@ Let's start!
 
 On the same server where the Node/Express application is running, or on a server that is reachable by it, install the Hyperwatch processor.
 
-As a prerequirement, you'll need Node.js >= 7. Use nvm if you're in trouble.
+As a prerequisite, you'll need Node.js >= 24. We recommend [nvm](https://github.com/nvm-sh/nvm).
 
 ```bash
-nvm install node
+nvm install 24
 ```
 
 #### Install from npm
@@ -24,7 +24,7 @@ npm install -g @hyperwatch/hyperwatch
 
 #### Install from Git
 
-Alternatively, for developement purpose, you can use Git and clone the public repository:
+Alternatively, for development purpose, you can use Git and clone the public repository:
 
 ```bash
 git clone https://github.com/hyperwatch/hyperwatch.git
@@ -38,7 +38,7 @@ In our suggested configuration, Hyperwatch will be listening for access logs usi
 
 All communications between your Node/Express application and Hyperwatch will be happening in clear, please only use that setup on your internal network. If on the public internet, we're advising to use the Websocket Secure protocol (wss) which is straightforward but out of the scope of this tutorial.
 
-Now, you can create your own configuration in `express_websocket_example.js`:
+Now, you can create your own configuration in `express_websocket_example.js` (a complete version is available in [`config/express_websocket_example.js`](../../config/express_websocket_example.js)):
 
 ```javascript
 module.exports = function (hyperwatch) {
@@ -77,10 +77,10 @@ app.use(hyperwatchExpressLogger('websocket', 'ws://localhost:3000/input/log'));
 
 In this example, there are 3 important things:
 
-1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
-   If it's on a different server, replace `localhost` by the proper private or public IP address.
+1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
+   If it's on a different server, replace `localhost` by the proper private or public IP address.
 2. Replace the port (here `3000`) by the relevant one, it should be the main port where Hyperwatch is running.
-3. Finally, the path `/input/log` should match the one configured on Hyperwatch side, If you're following this tutorial from start to begin, nothing to change!
+3. Finally, the path `/input/log` should match the one configured on Hyperwatch side, If you're following this tutorial from start to end, nothing to change!
 
 Now, that you added and configured the Hyperwatch middleware, you can deploy and restart your application.
 
@@ -96,4 +96,6 @@ hyperwatch express_websocket_example.js
 
 ### Browse the interface
 
-Now, you can point your browser to the IP/port where Hyperwatch is running. If you see data flowing, congrats you made it!
+Now, you can point your browser to the `/status` page on the IP/port where Hyperwatch is running (e.g. `http://localhost:3000/status`). If you see traffic going through your input, congrats you made it!
+
+To watch the logs live at `/logs/main` and explore aggregations such as `/addresses` or `/identities`, activate the corresponding modules. See [Global Configuration](../configuration.md#modules).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
-  "version": "4.2.0",
+  "version": "5.0.0",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",
@@ -10,16 +10,25 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/hyperwatch/hyperwatch.git"
+    "url": "git+https://github.com/hyperwatch/hyperwatch.git"
   },
   "main": "hyperwatch.js",
   "bin": {
-    "hyperwatch": "./bin/hyperwatch"
+    "hyperwatch": "bin/hyperwatch"
   },
+  "files": [
+    "bin",
+    "config",
+    "docs",
+    "scripts",
+    "src",
+    "start.js"
+  ],
   "scripts": {
     "depcheck": "depcheck",
-    "lint": "eslint '**/*.js'",
-    "lint:fix": "eslint '**/*.js' --fix",
+    "lint": "eslint",
+    "lint:fix": "eslint --fix",
+    "prepare": "husky",
     "prettier": "prettier \"**/*.@(js|json|md|yml)\"",
     "prettier:check": "npm run prettier -- --list-different",
     "prettier:write": "npm run prettier -- --write",
@@ -29,52 +38,50 @@
   "dependencies": {
     "@hyperwatch/useragent": "^3.9.3",
     "accept-language-parser": "^1.5.0",
-    "ajv": "^8.12.0",
-    "ajv-formats": "^2.1.1",
-    "chalk": "^4.1.0",
+    "ajv": "^8.20.0",
+    "ajv-formats": "^3.0.1",
+    "chalk": "^6.0.0",
     "country-code-emoji": "^2.3.0",
-    "debug": "^4.3.4",
-    "dnsbl": "^4.0.1",
-    "express": "^4.18.2",
-    "express-ws": "^5.0.2",
-    "geoip-lite": "^1.4.8",
-    "immutable": "^4.3.4",
-    "ip-cidr": "^3.1.0",
-    "lodash": "^4.17.21",
-    "lru-cache": "^10.0.1",
-    "micro-strptime": "^0.2.3",
+    "csv-stringify": "^6.8.3",
+    "debug": "^4.4.3",
+    "dnsbl": "^5.1.1",
+    "express": "^5.2.1",
+    "geoip-lite": "^2.0.3",
+    "immutable": "^5.1.9",
+    "ip-cidr": "^4.0.2",
+    "lodash": "^4.18.1",
+    "lru-cache": "^11.5.2",
+    "micro-strptime": "^1.0.0",
     "proxy-addr": "^2.0.7",
     "rc": "^1.2.8",
-    "syslog-parse": "^1.3.1",
+    "syslog-parse": "^2.0.0",
     "tail": "^2.2.6",
-    "uuid": "^9.0.1",
-    "ws": "^8.14.2"
+    "ws": "^8.21.3"
   },
   "devDependencies": {
+    "@eslint/js": "^10.0.1",
     "depcheck": "^1.4.7",
-    "eslint": "^8.53.0",
-    "eslint-plugin-import": "^2.29.0",
-    "eslint-plugin-node": "^11.1.0",
-    "husky": "^4.3.5",
-    "lint-staged": "^10.5.3",
-    "mocha": "^10.2.0",
-    "node-fetch": "^2.0.0",
-    "prettier": "^3.3.3",
+    "eslint": "^10.9.1",
+    "eslint-plugin-import-x": "^4.17.1",
+    "eslint-plugin-n": "^18.3.0",
+    "globals": "^17.11.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^17.4.1",
+    "mocha": "^12.0.0",
+    "prettier": "^3.9.6",
     "prettier-package-json": "^2.8.0"
   },
   "engines": {
-    "node": ">=14.0"
+    "node": ">=24"
+  },
+  "allowScripts": {
+    "unrs-resolver": false
   },
   "depcheck": {
     "ignores": [
       "prettier-package-json"
     ]
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
-  },
   "lint-staged": {
     "*.{js,json,md,yml}": [
       "prettier --write"

package/scripts/fetch-anthropic-ips.js

@@ -0,0 +1,60 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+// Anthropic publishes a single list covering all Claude crawlers
+// (ClaudeBot, Claude-User, Claude-SearchBot).
+const sources = [
+  { name: 'claude-bot-ips', url: 'https://claude.com/crawling/bots.json' },
+];
+
+function extractCidrs(data) {
+  const prefixes = Array.isArray(data) ? data : data.prefixes;
+  if (!Array.isArray(prefixes)) {
+    throw new Error('unexpected payload: no prefixes array');
+  }
+  return prefixes
+    .map((prefix) => {
+      if (typeof prefix === 'string') {
+        return prefix;
+      }
+      return prefix.ipv4Prefix || prefix.ipv6Prefix || prefix.ip_prefix;
+    })
+    .filter(Boolean)
+    .map((cidr) => {
+      if (cidr.includes('/')) {
+        return cidr;
+      }
+      return cidr.includes(':') ? `${cidr}/128` : `${cidr}/32`;
+    });
+}
+
+async function fetchAndStore({ name, url }) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    console.error(`Failed to fetch ${name}: ${res.status} ${res.statusText}`);
+    process.exitCode = 1;
+    return;
+  }
+  const cidrs = extractCidrs(await res.json());
+  if (cidrs.length === 0) {
+    console.error(`Failed to fetch ${name}: empty list, keeping current file`);
+    process.exitCode = 1;
+    return;
+  }
+  const filePath = path.join(dataDir, `${name}.json`);
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(`${name}: ${cidrs.length} CIDRs`);
+}
+
+async function main() {
+  for (const source of sources) {
+    await fetchAndStore(source);
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

package/scripts/fetch-cloudflare-ips.js

@@ -0,0 +1,27 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+async function fetchLines(url) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to fetch ${url}: ${res.status}`);
+  }
+  const text = await res.text();
+  return text.trim().split('\n').filter(Boolean);
+}
+
+async function main() {
+  const ipv4 = await fetchLines('https://www.cloudflare.com/ips-v4/');
+  const ipv6 = await fetchLines('https://www.cloudflare.com/ips-v6/');
+  const cidrs = [...ipv4, ...ipv6];
+
+  const filePath = path.join(dataDir, 'cloudflare-ips.json');
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(
+    `cloudflare-ips: ${cidrs.length} CIDRs (${ipv4.length} IPv4, ${ipv6.length} IPv6)`
+  );
+}
+
+main().catch(console.error);

package/scripts/fetch-cloudfront-ips.js

@@ -0,0 +1,29 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+async function main() {
+  const response = await fetch(
+    'https://ip-ranges.amazonaws.com/ip-ranges.json'
+  );
+  const ipRanges = await response.json();
+
+  const ipv4 = ipRanges.prefixes
+    .filter((p) => p.service === 'CLOUDFRONT')
+    .map((p) => p.ip_prefix);
+
+  const ipv6 = ipRanges.ipv6_prefixes
+    .filter((p) => p.service === 'CLOUDFRONT')
+    .map((p) => p.ipv6_prefix);
+
+  const cidrs = [...ipv4, ...ipv6];
+
+  const filePath = path.join(dataDir, 'cloudfront-ips.json');
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(
+    `cloudfront-ips: ${cidrs.length} CIDRs (${ipv4.length} IPv4, ${ipv6.length} IPv6)`
+  );
+}
+
+main().catch(console.error);

package/scripts/fetch-openai-ips.js

@@ -0,0 +1,34 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+const sources = [
+  { name: 'chatgpt-user-ips', url: 'https://openai.com/chatgpt-user.json' },
+  { name: 'gptbot-ips', url: 'https://openai.com/gptbot.json' },
+  { name: 'openai-searchbot-ips', url: 'https://openai.com/searchbot.json' },
+];
+
+async function fetchAndStore({ name, url }) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    console.error(`Failed to fetch ${name}: ${res.status} ${res.statusText}`);
+    return;
+  }
+  const data = await res.json();
+  const cidrs = data.prefixes.map((p) => {
+    const cidr = p.ipv4Prefix || p.ip_prefix;
+    return cidr.includes('/') ? cidr : `${cidr}/32`;
+  });
+  const filePath = path.join(dataDir, `${name}.json`);
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(`${name}: ${cidrs.length} CIDRs`);
+}
+
+async function main() {
+  for (const source of sources) {
+    await fetchAndStore(source);
+  }
+}
+
+main().catch(console.error);

package/src/app/api.js

@@ -1,19 +1,137 @@
+const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
 
+const { stringify } = require('csv-stringify/sync');
 const express = require('express');
-const uuid = require('uuid');
 
 const monitoring = require('../lib/monitoring');
+const persistence = require('../lib/persistence');
+const pipeline = require('../lib/pipeline');
 const { formatTable } = require('../lib/util');
 const stylesheet = require('../stylesheet');
 
+const wsServer = require('./ws-server');
+
 const script = fs.readFileSync(path.join(__dirname, '..', 'script.js'));
 
 const app = express();
 
+// WebSocket upgrades dispatched by mount() when Hyperwatch is embedded
+app.use(wsServer.middleware);
+
 app.use(express.json());
 
+function renderHtmlTree(node) {
+  const label = [];
+  if (node.name) {
+    label.push(`<strong>${node.name}</strong>`);
+  }
+  if (node.op) {
+    label.push(`<span class="op">[${node.op}]</span>`);
+  }
+  if (node.module) {
+    label.push(`<span class="module">(${node.module})</span>`);
+  }
+  if (node.fnName) {
+    label.push(`<span class="fn">${node.fnName}</span>`);
+  }
+  if (node.label) {
+    label.push(`<span class="label">${node.label}</span>`);
+  }
+
+  let html = `<li>${label.join(' ')}`;
+  if (node.children && node.children.length > 0) {
+    html += '<ul>';
+    for (const child of node.children) {
+      html += renderHtmlTree(child);
+    }
+    html += '</ul>';
+  }
+  html += '</li>';
+  return html;
+}
+
+function renderHtmlInputs(inputs) {
+  if (!inputs || inputs.length === 0) {
+    return '';
+  }
+  let html = '<ul>';
+  for (const input of inputs) {
+    html += `<li><strong>${input.name}</strong> <span class="op">[input]</span>`;
+    if (input.status) {
+      html += ` <span class="module">(${input.status})</span>`;
+    }
+    html += ` accepted: ${input.accepted}, rejected: ${input.rejected}`;
+    if (input.tree) {
+      html += `<ul>${renderHtmlTree(input.tree)}</ul>`;
+    }
+    html += '</li>';
+  }
+  html += '</ul>';
+  return html;
+}
+
+app.get('/nodes{.:format}', (req, res) => {
+  const nodes = Object.keys(pipeline.nodes);
+  const format = req.params.format;
+  const view = req.query.view;
+
+  if (format && !['csv', 'json'].includes(format)) {
+    res.sendStatus(404);
+    return;
+  }
+
+  if (format === 'csv') {
+    const csv = stringify(
+      nodes.map((name) => ({ name })),
+      {
+        header: true,
+        columns: ['name'],
+      }
+    );
+    res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+    res.setHeader('Content-Disposition', 'attachment; filename="nodes.csv"');
+    res.send(csv);
+  } else if (format === 'json') {
+    if (view === 'tree') {
+      res.json(pipeline.getTree());
+    } else {
+      res.json(nodes);
+    }
+  } else {
+    res.setHeader('Content-Type', 'text/html; charset=utf-8');
+    if (view === 'tree') {
+      const tree = pipeline.getTree();
+      res.send(
+        `<!DOCTYPE html>
+<html>
+<head>
+<style>${stylesheet}
+.op { color: #666; }
+.module { color: #0a0; }
+.fn { color: #00a; }
+.label { color: #a50; }
+ul { list-style: none; padding-left: 1.5em; }
+</style>
+</head>
+<body>${renderHtmlInputs(tree.inputs)}<ul>${renderHtmlTree(tree)}</ul></body>
+</html>`
+      );
+    } else {
+      res.send(
+        `<!DOCTYPE html>
+<html>
+<head>
+<style>${stylesheet}</style>
+</head>
+<body>${formatTable(nodes.map((name) => ({ name })))}</body>
+</html>`
+      );
+    }
+  }
+});
+
 app.streamToHttp = (
   endpoint,
   stream,
@@ -47,7 +165,7 @@ app.streamToHttp = (
   updateMonitoringStatus();
 
   app.get(endpoint, (req, res) => {
-    const requestId = uuid.v4();
+    const requestId = crypto.randomUUID();
     requests[requestId] = [req, res];
     updateMonitoringStatus();
 
@@ -83,24 +201,42 @@ body { display: flex; flex-direction: column-reverse; }
         res.write(`<div>${line}</div>\n`);
       }
     });
-  });
+  }, `http:${endpoint}`);
 };
 
 app.registerAggregator = (name, aggregator) => {
-  app.get(`/${name}.:format(json)?`, (req, res) => {
+  persistence.register(name, aggregator);
+  app.get(`/${name}{.:format}`, (req, res) => {
     const raw = req.query.raw ? true : false;
     const format = req.params.format || (raw ? 'json' : null);
     const limit = req.query.limit || 100;
-    const sort = req.query.sort || '15m';
+    const sort = req.query.sort || 'count15m';
+
+    if (format && !['csv', 'json'].includes(format)) {
+      res.sendStatus(404);
+      return;
+    }
 
     const data = aggregator.getData({
       sort,
       limit,
-      format,
+      format: format === 'csv' ? 'json' : format,
       raw,
     });
 
-    if (format === 'json') {
+    if (format === 'csv') {
+      const rows = data.toJS();
+      const columns = Object.keys(rows[0] || {}).filter(
+        (k) => k !== 'activity'
+      );
+      const csv = stringify(rows, { header: true, columns });
+      res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+      res.setHeader(
+        'Content-Disposition',
+        `attachment; filename="${name}.csv"`
+      );
+      res.send(csv);
+    } else if (format === 'json') {
       res.send(data);
     } else {
       res.setHeader('Content-Type', 'text/html; charset=utf-8');
@@ -117,7 +253,12 @@ app.registerAggregator = (name, aggregator) => {
     }
   });
 
-  app.get(`/${name}/:identifier.:format(json)?`, (req, res) => {
+  app.delete(`/${name}`, (req, res) => {
+    aggregator.reset();
+    res.send({ success: true });
+  });
+
+  app.get(`/${name}/:identifier{.json}`, (req, res) => {
     const entry = aggregator.get(req.params.identifier);
     if (!entry) {
       res.status(404).send('Not Found');

package/src/app/index.js

@@ -1,18 +1,22 @@
 const http = require('http');
 
 const express = require('express');
-const expressWs = require('express-ws');
 
 const constants = require('../constants');
 
 const api = require('./api');
+const mount = require('./mount');
 const websocket = require('./websocket');
+const wsServer = require('./ws-server');
 
 const app = express();
 const httpServer = http.createServer(app);
-expressWs(app, httpServer);
 
-app.use(api, websocket);
+httpServer.on('upgrade', (request, socket, head) => {
+  wsServer.handleUpgrade(request, socket, head);
+});
+
+app.use(api);
 
 function start() {
   const port = process.env.PORT || constants.port;
@@ -25,4 +29,4 @@ function stop() {
   httpServer.close();
 }
 
-module.exports = { api, start, stop, websocket };
+module.exports = { api, mount, start, stop, websocket };