@hyperwatch/hyperwatch 4.3.1 → 5.0.1

package/docs/tutorials/express_input.md

@@ -10,10 +10,10 @@ Let's start!
 
 On the same server where the Node/Express application is running, or on a server that is reachable by it, install the Hyperwatch processor.
 
-As a prerequirement, you'll need Node.js >= 7. Use nvm if you're in trouble.
+As a prerequisite, you'll need Node.js >= 24. We recommend [nvm](https://github.com/nvm-sh/nvm).
 
 ```bash
-nvm install node
+nvm install 24
 ```
 
 #### Install from npm
@@ -24,7 +24,7 @@ npm install -g @hyperwatch/hyperwatch
 
 #### Install from Git
 
-Alternatively, for developement purpose, you can use Git and clone the public repository:
+Alternatively, for development purpose, you can use Git and clone the public repository:
 
 ```bash
 git clone https://github.com/hyperwatch/hyperwatch.git
@@ -38,7 +38,7 @@ In our suggested configuration, Hyperwatch will be listening for access logs usi
 
 All communications between your Node/Express application and Hyperwatch will be happening in clear, please only use that setup on your internal network. If on the public internet, we're advising to use the Websocket Secure protocol (wss) which is straightforward but out of the scope of this tutorial.
 
-Now, you can create your own configuration in `express_websocket_example.js`:
+Now, you can create your own configuration in `express_websocket_example.js` (a complete version is available in [`config/express_websocket_example.js`](../../config/express_websocket_example.js)):
 
 ```javascript
 module.exports = function (hyperwatch) {
@@ -77,10 +77,10 @@ app.use(hyperwatchExpressLogger('websocket', 'ws://localhost:3000/input/log'));
 
 In this example, there are 3 important things:
 
-1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
-   If it's on a different server, replace `localhost` by the proper private or public IP address.
+1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
+   If it's on a different server, replace `localhost` by the proper private or public IP address.
 2. Replace the port (here `3000`) by the relevant one, it should be the main port where Hyperwatch is running.
-3. Finally, the path `/input/log` should match the one configured on Hyperwatch side, If you're following this tutorial from start to begin, nothing to change!
+3. Finally, the path `/input/log` should match the one configured on Hyperwatch side, If you're following this tutorial from start to end, nothing to change!
 
 Now, that you added and configured the Hyperwatch middleware, you can deploy and restart your application.
 
@@ -96,4 +96,6 @@ hyperwatch express_websocket_example.js
 
 ### Browse the interface
 
-Now, you can point your browser to the IP/port where Hyperwatch is running. If you see data flowing, congrats you made it!
+Now, you can point your browser to the `/status` page on the IP/port where Hyperwatch is running (e.g. `http://localhost:3000/status`). If you see traffic going through your input, congrats you made it!
+
+To watch the logs live at `/logs/main` and explore aggregations such as `/addresses` or `/identities`, activate the corresponding modules. See [Global Configuration](../configuration.md#modules).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
-  "version": "4.3.1",
+  "version": "5.0.1",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",
@@ -40,14 +40,13 @@
     "accept-language-parser": "^1.5.0",
     "ajv": "^8.20.0",
     "ajv-formats": "^3.0.1",
-    "chalk": "^4.1.2",
+    "chalk": "^6.0.0",
     "country-code-emoji": "^2.3.0",
     "csv-stringify": "^6.8.3",
     "debug": "^4.4.3",
-    "dnsbl": "^4.0.3",
-    "express": "^4.22.2",
-    "express-ws": "^5.0.2",
-    "geoip-lite": "^1.4.10",
+    "dnsbl": "^5.1.1",
+    "express": "^5.2.1",
+    "geoip-lite": "^2.0.3",
     "immutable": "^5.1.9",
     "ip-cidr": "^4.0.2",
     "lodash": "^4.18.1",
@@ -55,25 +54,28 @@
     "micro-strptime": "^1.0.0",
     "proxy-addr": "^2.0.7",
     "rc": "^1.2.8",
-    "syslog-parse": "^1.3.1",
+    "syslog-parse": "^2.0.0",
     "tail": "^2.2.6",
     "ws": "^8.21.3"
   },
   "devDependencies": {
-    "@eslint/js": "^9.39.5",
+    "@eslint/js": "^10.0.1",
     "depcheck": "^1.4.7",
-    "eslint": "^9.39.5",
-    "eslint-plugin-import": "^2.32.0",
-    "eslint-plugin-n": "^17.24.0",
+    "eslint": "^10.9.1",
+    "eslint-plugin-import-x": "^4.17.1",
+    "eslint-plugin-n": "^18.3.0",
     "globals": "^17.11.0",
     "husky": "^9.1.7",
-    "lint-staged": "^16.4.0",
-    "mocha": "^11.8.0",
+    "lint-staged": "^17.4.1",
+    "mocha": "^12.0.0",
     "prettier": "^3.9.6",
     "prettier-package-json": "^2.8.0"
   },
   "engines": {
-    "node": ">=20"
+    "node": ">=24"
+  },
+  "allowScripts": {
+    "unrs-resolver": false
   },
   "depcheck": {
     "ignores": [

package/scripts/fetch-anthropic-ips.js

@@ -0,0 +1,60 @@
+const fs = require('fs');
+const path = require('path');
+
+const dataDir = path.join(__dirname, '..', 'src', 'data');
+
+// Anthropic publishes a single list covering all Claude crawlers
+// (ClaudeBot, Claude-User, Claude-SearchBot).
+const sources = [
+  { name: 'claude-bot-ips', url: 'https://claude.com/crawling/bots.json' },
+];
+
+function extractCidrs(data) {
+  const prefixes = Array.isArray(data) ? data : data.prefixes;
+  if (!Array.isArray(prefixes)) {
+    throw new Error('unexpected payload: no prefixes array');
+  }
+  return prefixes
+    .map((prefix) => {
+      if (typeof prefix === 'string') {
+        return prefix;
+      }
+      return prefix.ipv4Prefix || prefix.ipv6Prefix || prefix.ip_prefix;
+    })
+    .filter(Boolean)
+    .map((cidr) => {
+      if (cidr.includes('/')) {
+        return cidr;
+      }
+      return cidr.includes(':') ? `${cidr}/128` : `${cidr}/32`;
+    });
+}
+
+async function fetchAndStore({ name, url }) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    console.error(`Failed to fetch ${name}: ${res.status} ${res.statusText}`);
+    process.exitCode = 1;
+    return;
+  }
+  const cidrs = extractCidrs(await res.json());
+  if (cidrs.length === 0) {
+    console.error(`Failed to fetch ${name}: empty list, keeping current file`);
+    process.exitCode = 1;
+    return;
+  }
+  const filePath = path.join(dataDir, `${name}.json`);
+  fs.writeFileSync(filePath, `${JSON.stringify(cidrs, null, 2)}\n`);
+  console.log(`${name}: ${cidrs.length} CIDRs`);
+}
+
+async function main() {
+  for (const source of sources) {
+    await fetchAndStore(source);
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

package/src/app/api.js

@@ -7,15 +7,131 @@ const express = require('express');
 
 const monitoring = require('../lib/monitoring');
 const persistence = require('../lib/persistence');
+const pipeline = require('../lib/pipeline');
 const { formatTable } = require('../lib/util');
 const stylesheet = require('../stylesheet');
 
+const wsServer = require('./ws-server');
+
 const script = fs.readFileSync(path.join(__dirname, '..', 'script.js'));
 
 const app = express();
 
+// WebSocket upgrades dispatched by mount() when Hyperwatch is embedded
+app.use(wsServer.middleware);
+
 app.use(express.json());
 
+function renderHtmlTree(node) {
+  const label = [];
+  if (node.name) {
+    label.push(`<strong>${node.name}</strong>`);
+  }
+  if (node.op) {
+    label.push(`<span class="op">[${node.op}]</span>`);
+  }
+  if (node.module) {
+    label.push(`<span class="module">(${node.module})</span>`);
+  }
+  if (node.fnName) {
+    label.push(`<span class="fn">${node.fnName}</span>`);
+  }
+  if (node.label) {
+    label.push(`<span class="label">${node.label}</span>`);
+  }
+
+  let html = `<li>${label.join(' ')}`;
+  if (node.children && node.children.length > 0) {
+    html += '<ul>';
+    for (const child of node.children) {
+      html += renderHtmlTree(child);
+    }
+    html += '</ul>';
+  }
+  html += '</li>';
+  return html;
+}
+
+function renderHtmlInputs(inputs) {
+  if (!inputs || inputs.length === 0) {
+    return '';
+  }
+  let html = '<ul>';
+  for (const input of inputs) {
+    html += `<li><strong>${input.name}</strong> <span class="op">[input]</span>`;
+    if (input.status) {
+      html += ` <span class="module">(${input.status})</span>`;
+    }
+    html += ` accepted: ${input.accepted}, rejected: ${input.rejected}`;
+    if (input.tree) {
+      html += `<ul>${renderHtmlTree(input.tree)}</ul>`;
+    }
+    html += '</li>';
+  }
+  html += '</ul>';
+  return html;
+}
+
+app.get('/nodes{.:format}', (req, res) => {
+  const nodes = Object.keys(pipeline.nodes);
+  const format = req.params.format;
+  const view = req.query.view;
+
+  if (format && !['csv', 'json'].includes(format)) {
+    res.sendStatus(404);
+    return;
+  }
+
+  if (format === 'csv') {
+    const csv = stringify(
+      nodes.map((name) => ({ name })),
+      {
+        header: true,
+        columns: ['name'],
+      }
+    );
+    res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+    res.setHeader('Content-Disposition', 'attachment; filename="nodes.csv"');
+    res.send(csv);
+  } else if (format === 'json') {
+    if (view === 'tree') {
+      res.json(pipeline.getTree());
+    } else {
+      res.json(nodes);
+    }
+  } else {
+    res.setHeader('Content-Type', 'text/html; charset=utf-8');
+    if (view === 'tree') {
+      const tree = pipeline.getTree();
+      res.send(
+        `<!DOCTYPE html>
+<html>
+<head>
+<style>${stylesheet}
+.op { color: #666; }
+.module { color: #0a0; }
+.fn { color: #00a; }
+.label { color: #a50; }
+ul { list-style: none; padding-left: 1.5em; }
+</style>
+</head>
+<body>${renderHtmlInputs(tree.inputs)}<ul>${renderHtmlTree(tree)}</ul></body>
+</html>`
+      );
+    } else {
+      res.send(
+        `<!DOCTYPE html>
+<html>
+<head>
+<style>${stylesheet}</style>
+</head>
+<body>${formatTable(nodes.map((name) => ({ name })))}</body>
+</html>`
+      );
+    }
+  }
+});
+
 app.streamToHttp = (
   endpoint,
   stream,
@@ -85,17 +201,22 @@ body { display: flex; flex-direction: column-reverse; }
         res.write(`<div>${line}</div>\n`);
       }
     });
-  });
+  }, `http:${endpoint}`);
 };
 
 app.registerAggregator = (name, aggregator) => {
   persistence.register(name, aggregator);
-  app.get(`/${name}.:format(json|csv)?`, (req, res) => {
+  app.get(`/${name}{.:format}`, (req, res) => {
     const raw = req.query.raw ? true : false;
     const format = req.params.format || (raw ? 'json' : null);
     const limit = req.query.limit || 100;
     const sort = req.query.sort || 'count15m';
 
+    if (format && !['csv', 'json'].includes(format)) {
+      res.sendStatus(404);
+      return;
+    }
+
     const data = aggregator.getData({
       sort,
       limit,
@@ -132,7 +253,12 @@ app.registerAggregator = (name, aggregator) => {
     }
   });
 
-  app.get(`/${name}/:identifier.:format(json)?`, (req, res) => {
+  app.delete(`/${name}`, (req, res) => {
+    aggregator.reset();
+    res.send({ success: true });
+  });
+
+  app.get(`/${name}/:identifier{.json}`, (req, res) => {
     const entry = aggregator.get(req.params.identifier);
     if (!entry) {
       res.status(404).send('Not Found');

package/src/app/index.js

@@ -1,18 +1,22 @@
 const http = require('http');
 
 const express = require('express');
-const expressWs = require('express-ws');
 
 const constants = require('../constants');
 
 const api = require('./api');
+const mount = require('./mount');
 const websocket = require('./websocket');
+const wsServer = require('./ws-server');
 
 const app = express();
 const httpServer = http.createServer(app);
-expressWs(app, httpServer);
 
-app.use(api, websocket);
+httpServer.on('upgrade', (request, socket, head) => {
+  wsServer.handleUpgrade(request, socket, head);
+});
+
+app.use(api);
 
 function start() {
   const port = process.env.PORT || constants.port;
@@ -25,4 +29,4 @@ function stop() {
   httpServer.close();
 }
 
-module.exports = { api, start, stop, websocket };
+module.exports = { api, mount, start, stop, websocket };

package/src/app/mount.js

@@ -0,0 +1,115 @@
+const api = require('./api');
+const wsServer = require('./ws-server');
+
+// Servers where Hyperwatch handles WebSocket upgrades, with their listener
+const upgradeListeners = new WeakMap();
+
+// Paths where Hyperwatch is mounted, per app. Express can't remove routes, so
+// a path stays mounted for the life of the app.
+const mountedPaths = new WeakMap();
+
+// A path as the app routes it: case-insensitive unless the app says otherwise
+function routingKey(app, path) {
+  return app.enabled('case sensitive routing') ? path : path.toLowerCase();
+}
+
+function validate(app, { server, path, middleware, fallback }) {
+  if (typeof app !== 'function' || typeof app.use !== 'function') {
+    throw new TypeError('mount() expects an Express app');
+  }
+  if (!server || typeof server.on !== 'function') {
+    throw new TypeError(
+      'mount() expects the HTTP server of the app: { server }'
+    );
+  }
+  if (typeof path !== 'string' || !path.startsWith('/') || path.endsWith('/')) {
+    throw new TypeError(
+      `Invalid mount path "${path}": it must start with "/" and not end with "/"`
+    );
+  }
+  const middlewares = middleware === undefined ? [] : [].concat(middleware);
+  if (!middlewares.every((fn) => typeof fn === 'function')) {
+    throw new TypeError(
+      'middleware must be a function or an array of functions'
+    );
+  }
+  if (fallback !== undefined && typeof fallback !== 'function') {
+    throw new TypeError('fallback must be a function');
+  }
+  const paths = mountedPaths.get(app);
+  if (paths && paths.has(routingKey(app, path))) {
+    throw new Error(
+      `Hyperwatch is already mounted on this app at ${path}: its routes can't be removed, so mounting again can't change its options (e.g. middleware)`
+    );
+  }
+  if (upgradeListeners.has(server)) {
+    throw new Error('Hyperwatch is already mounted on this server');
+  }
+  return middlewares;
+}
+
+// Whether a request is under the mount path, matching case like Express
+function isUnderPath(target, path, caseSensitive) {
+  const pathname = caseSensitive
+    ? target.pathname
+    : target.pathname.toLowerCase();
+  const mountPath = caseSensitive ? path : path.toLowerCase();
+  return pathname === mountPath || pathname.startsWith(`${mountPath}/`);
+}
+
+/**
+ * Mount Hyperwatch in an Express app, under an explicit path:
+ *
+ *   hyperwatch.app.mount(app, { server, path: '/_hyperwatch', middleware: auth });
+ *
+ * - Registers `app.use(path, ...middleware, router)` where it's called, so the
+ *   app's middleware order is kept.
+ * - Adds one 'upgrade' listener to `server`. WebSocket upgrades under `path`
+ *   go through the app like HTTP requests, so `middleware` applies to both.
+ *   Other upgrades go to `fallback` when given, or are left to the server's
+ *   other listeners. Malformed targets get 400.
+ * - Never creates a server nor listens.
+ *
+ * Mounting again on the same server, or on the same app at the same path
+ * (even after detachUpgrades()), throws before registering anything.
+ * Returns `{ path, detachUpgrades }`: detachUpgrades() removes the upgrade
+ * listener and releases the server. Express can't remove routes, so the
+ * HTTP routes stay mounted on the app, with their original middleware.
+ */
+function mount(app, options = {}) {
+  const middlewares = validate(app, options);
+  const { server, path, fallback } = options;
+
+  app.use(path, ...middlewares, api);
+  if (!mountedPaths.has(app)) {
+    mountedPaths.set(app, new Set());
+  }
+  mountedPaths.get(app).add(routingKey(app, path));
+
+  const listener = (req, socket, head) => {
+    const target = wsServer.parseTarget(req.url);
+    // Malformed targets are rejected before reaching the app or fallback
+    if (!target) {
+      return wsServer.reject(socket, 400, 'Bad Request');
+    }
+    if (isUnderPath(target, path, app.enabled('case sensitive routing'))) {
+      wsServer.dispatch(app, req, socket, head);
+    } else if (fallback) {
+      fallback(req, socket, head);
+    }
+  };
+  server.on('upgrade', listener);
+  upgradeListeners.set(server, listener);
+
+  return {
+    path,
+    detachUpgrades() {
+      if (upgradeListeners.get(server) === listener) {
+        server.off('upgrade', listener);
+        upgradeListeners.delete(server);
+      }
+    },
+  };
+}
+
+module.exports = mount;

package/src/app/websocket.js

@@ -1,15 +1,18 @@
 const crypto = require('crypto');
 
-const express = require('express');
-const expressWs = require('express-ws');
-
 const constants = require('../constants');
 const monitoring = require('../lib/monitoring');
 
-const app = express();
-expressWs(app);
+const wsServer = require('./ws-server');
+
+/**
+ * Express middleware completing the Hyperwatch WebSocket upgrades dispatched
+ * by hyperwatch.app.mount(), which already includes it. Kept for apps that
+ * mounted it explicitly: on its own, it doesn't handle any upgrade.
+ */
+const websocket = (req, res, next) => wsServer.middleware(req, res, next);
 
-app.streamToWebsocket = (
+websocket.streamToWebsocket = (
   endpoint,
   stream,
   { name = `WebSocket: ${endpoint}`, monitoringEnabled = false } = {}
@@ -37,7 +40,7 @@ app.streamToWebsocket = (
   };
   updateMonitoringStatus();
 
-  app.ws(endpoint, (client, req) => {
+  wsServer.ws(endpoint, (client, req) => {
     const clientId = req.query.clientId || crypto.randomUUID();
     if (clients[clientId]) {
       console.log(`Client '${clientId}' is already connected. Terminating.`);
@@ -70,7 +73,7 @@ app.streamToWebsocket = (
         client.send(JSON.stringify(log));
       }
     });
-  });
+  }, `ws:${endpoint}`);
 
   // Heartbeat: detect and clean up stale connections
   setInterval(() => {
@@ -88,4 +91,4 @@ app.streamToWebsocket = (
   }, constants.heartbeatInterval || 30000);
 };
 
-module.exports = app;
+module.exports = websocket;

package/src/app/ws-server.js

@@ -0,0 +1,123 @@
+const http = require('http');
+
+const { WebSocketServer } = require('ws');
+
+const wss = new WebSocketServer({ noServer: true });
+
+const routes = new Map();
+
+// Marks an upgrade request dispatched through an Express app
+const upgradeKey = Symbol('hyperwatch.upgrade');
+
+function ws(path, handler) {
+  routes.set(path, handler);
+}
+
+/**
+ * Parse the target of an upgrade request without throwing. WebSocket
+ * upgrades use the origin form ("/path?query"): anything else is malformed.
+ * `pathname` is kept as sent, like Express routing does; `urlPathname` is
+ * normalized by the URL parser.
+ */
+function parseTarget(url) {
+  if (typeof url !== 'string' || !url.startsWith('/')) {
+    return null;
+  }
+  try {
+    const { pathname: urlPathname, searchParams } = new URL(
+      url,
+      'http://localhost'
+    );
+    return {
+      pathname: url.split(/[?#]/)[0],
+      urlPathname,
+      query: Object.fromEntries(searchParams),
+    };
+  } catch (err) {
+    return null;
+  }
+}
+
+function reject(socket, status, message) {
+  socket.end(`HTTP/1.1 ${status} ${message}\r\nConnection: close\r\n\r\n`);
+}
+
+// Find the handler of a route, matching case like Express routing does
+function findRoute(pathname, caseSensitive) {
+  if (caseSensitive) {
+    return routes.get(pathname);
+  }
+  const lowerCase = pathname.toLowerCase();
+  for (const [route, handler] of routes) {
+    if (route.toLowerCase() === lowerCase) {
+      return handler;
+    }
+  }
+}
+
+function handleUpgrade(request, socket, head) {
+  const target = parseTarget(request.url);
+  if (!target) {
+    return reject(socket, 400, 'Bad Request');
+  }
+  request.query = target.query;
+
+  // Standalone mode: exact match, stream names are case-sensitive
+  const handler = routes.get(target.urlPathname);
+  if (handler) {
+    wss.handleUpgrade(request, socket, head, (client) => {
+      handler(client, request);
+    });
+  } else {
+    socket.destroy();
+  }
+}
+
+/**
+ * Express middleware completing the WebSocket upgrades sent by dispatch().
+ * The route is matched on the path relative to where the middleware is
+ * mounted, so `app.use('/_hyperwatch', middleware)` serves `/_hyperwatch/logs/raw`.
+ */
+function middleware(req, res, next) {
+  const upgrade = req[upgradeKey];
+  if (!upgrade) {
+    return next();
+  }
+  const handler = findRoute(
+    req.path,
+    req.app.enabled('case sensitive routing')
+  );
+  if (!handler) {
+    // The mount path is Hyperwatch's: unknown routes end here
+    return res.status(404).end();
+  }
+  req[upgradeKey] = null;
+  res.detachSocket(upgrade.socket);
+  wss.handleUpgrade(req, upgrade.socket, upgrade.head, (client) => {
+    handler(client, req);
+  });
+}
+
+/**
+ * Send a WebSocket upgrade through an Express app, like an HTTP request: it
+ * goes through the app's middlewares (e.g. authentication) and reaches the
+ * WebSocket middleware where it's mounted. The response is bound to the
+ * socket, so a middleware can reject the upgrade with an HTTP response, and
+ * Express answers errors and unknown routes as usual (e.g. 401, 503, 404).
+ */
+function dispatch(app, req, socket, head) {
+  req[upgradeKey] = { socket, head };
+  const res = new http.ServerResponse(req);
+  res.assignSocket(socket);
+  res.on('finish', () => socket.end());
+  app(req, res);
+}
+
+module.exports = {
+  ws,
+  handleUpgrade,
+  middleware,
+  dispatch,
+  parseTarget,
+  reject,
+};