@hyperwatch/hyperwatch 5.0.0 → 5.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
-  "version": "5.0.0",
+  "version": "5.0.1",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",

package/src/index.js

@@ -45,7 +45,12 @@ function start() {
 }
 
 async function stop() {
-  await pipeline.stop();
+  try {
+    await pipeline.stop();
+  } catch (err) {
+    console.error('Error stopping the pipeline:', err.message);
+  }
+  // Persist even if stopping the inputs failed
   try {
     if (constants.persistence.enabled) {
       persistence.dump(getPersistenceDir());

package/src/input/websocket.js

@@ -19,12 +19,20 @@ function create({
   heartbeatInterval = 30000,
 }) {
   let client;
+  let keepAlive;
+  let reconnectTimer;
+  // Each connection belongs to a generation. stop() and every new connection
+  // move to the next one, so events from an obsolete socket (a late close
+  // after a restart, its heartbeat) can't reconnect or touch the current one.
+  let generation = 0;
 
   let reconnectAttempts = 0;
 
   const setupWebSocketClient = ({ status, success, reject }) => {
+    const current = ++generation;
+    const isCurrent = () => current === generation;
     let isAlive;
-    let keepAlive;
+    let heartbeat;
 
     if (username && password) {
       options.headers = options.headers || {};
@@ -33,32 +41,37 @@ function create({
       ).toString('base64')}`;
     }
 
-    client = new WebSocket(address, [], options);
+    const socket = new WebSocket(address, [], options);
+    client = socket;
     status(null, `Waiting for connection to ${address}`);
 
-    client.on('open', () => {
+    socket.on('open', () => {
+      if (!isCurrent()) {
+        return;
+      }
       isAlive = true;
       reconnectAttempts = 0;
       status(null, `Listening to ${address}`);
 
       // Heartbeat: detect stale connections
-      keepAlive = setInterval(() => {
+      heartbeat = setInterval(() => {
         if (isAlive === false) {
-          client.terminate();
-          clearInterval(keepAlive);
+          socket.terminate();
+          clearInterval(heartbeat);
         } else {
           try {
-            client.ping();
+            socket.ping();
           } catch (err) {
             status(err, 'Websocket error');
           }
           isAlive = false;
         }
       }, heartbeatInterval);
+      keepAlive = heartbeat;
     });
 
-    client.on('message', (message) => {
-      if (sample !== 1 && Math.random() > sample) {
+    socket.on('message', (message) => {
+      if (!isCurrent() || (sample !== 1 && Math.random() > sample)) {
         return;
       }
       try {
@@ -68,19 +81,22 @@ function create({
       }
     });
 
-    client.on('error', (err) => {
-      status(err, 'Websocket error');
+    socket.on('error', (err) => {
+      if (isCurrent()) {
+        status(err, 'Websocket error');
+      }
     });
 
-    client.on('pong', () => {
+    socket.on('pong', () => {
       isAlive = true;
     });
 
-    client.on('close', () => {
-      status(null, 'Websocket connection has been closed');
-      if (keepAlive) {
-        clearInterval(keepAlive);
+    socket.on('close', () => {
+      clearInterval(heartbeat);
+      if (!isCurrent()) {
+        return;
       }
+      status(null, 'Websocket connection has been closed');
       if (reconnectOnClose) {
         reconnectAttempts++;
         const delay = Math.min(
@@ -91,7 +107,7 @@ function create({
           null,
           `Reconnecting Websocket in ${delay / 1000}s (attempt ${reconnectAttempts})`
         );
-        setTimeout(() => {
+        reconnectTimer = setTimeout(() => {
           setupWebSocketClient({ status, success, reject });
         }, delay);
       }
@@ -117,6 +133,7 @@ function create({
   return {
     name: `${name} ${type}`,
     start: ({ success, reject, status, log }) => {
+      reconnectAttempts = 0;
       if (type === 'client') {
         setupWebSocketClient({ status, success, reject });
       } else if (type === 'server') {
@@ -126,9 +143,28 @@ function create({
         log(new Error(errMsg), 'error');
       }
     },
+    // Never throws: a failing input must not prevent the others from stopping,
+    // nor Hyperwatch from persisting its data on shutdown.
     stop: () => {
-      if (client) {
-        client.close();
+      // Obsoletes the current connection: its close won't reconnect
+      generation++;
+      clearTimeout(reconnectTimer);
+      clearInterval(keepAlive);
+      if (!client) {
+        return;
+      }
+      try {
+        if (client.readyState === WebSocket.CONNECTING) {
+          // close() throws while the connection is not established yet
+          client.terminate();
+        } else if (client.readyState === WebSocket.OPEN) {
+          client.close();
+        }
+      } catch (err) {
+        console.error(
+          `${name}: error while closing the Websocket:`,
+          err.message
+        );
       }
     },
   };

package/src/lib/pipeline.js

@@ -311,10 +311,25 @@ class Pipeline extends Builder {
     });
   }
 
-  stop() {
-    return Promise.all(
-      this.inputs.filter((input) => input.stop).map((input) => input.stop())
+  // Stops every input, even if some of them throw or reject, then rejects
+  // with their errors if any failed
+  async stop() {
+    const results = await Promise.allSettled(
+      this.inputs
+        .filter((input) => input.stop)
+        .map((input) => Promise.resolve().then(() => input.stop()))
     );
+    const errors = results
+      .filter((result) => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(
+        errors,
+        `${errors.length} input(s) failed to stop: ${errors
+          .map((err) => err.message)
+          .join('; ')}`
+      );
+    }
   }
 
   registerNode(name, node) {

package/src/modules/hostname.js

@@ -1,4 +1,5 @@
 const dns = require('dns').promises;
+const net = require('net');
 
 const debug = require('debug')('hyperwatch:hostname');
 
@@ -13,6 +14,12 @@ function ignoreError() {
   return null;
 }
 
+// Compare addresses in canonical form: DNS and logs may write the same IPv6
+// address differently (zero compression, case)
+function canonical(ip) {
+  return net.isIPv6(ip) ? new URL(`http://[${ip}]`).hostname : ip;
+}
+
 function isValid(hostname) {
   return (
     hostname &&
@@ -40,11 +47,12 @@ async function lookup(ip, { fast = false } = {}) {
     if (isValid(reverse)) {
       entry.value = reverse;
       debug(`Resolve ${reverse} ...`);
-      const reverseIps = await dns.resolve(reverse).catch(ignoreError);
+      const reverseIps = await (
+        net.isIPv6(ip) ? dns.resolve6(reverse) : dns.resolve4(reverse)
+      ).catch(ignoreError);
       if (reverseIps) {
-        const reverseIp = reverseIps[0];
-        debug(`Resolve ${reverse}: ${reverseIp}`);
-        if (reverseIp === ip) {
+        debug(`Resolve ${reverse}: ${reverseIps.join(', ')}`);
+        if (reverseIps.map(canonical).includes(canonical(ip))) {
           entry.verified = true;
         }
       } else {

package/src/modules/identity.js

@@ -22,6 +22,11 @@ const claudeBotCidrs = claudeBotIps.map((cidr) => new IPCIDR(cidr));
 function augment(log) {
   const family = log.getIn(['agent', 'family']);
   const hostname = log.getIn(['address', 'hostname']);
+  // Reverse DNS confirmed by a forward lookup back to the same address. The
+  // older identities below still use the unconfirmed hostname.
+  const verifiedHostname = log.getIn(['hostname', 'verified'])
+    ? hostname
+    : undefined;
   const address =
     log.getIn(['address', 'value']) || log.getIn(['request', 'address']);
   const signature = log.getIn(['signature', 'id']);
@@ -62,8 +67,23 @@ function augment(log) {
       return hostname && hostname.endsWith('.yandex.com')
         ? log.set('identity', 'Yandex')
         : log;
+    // https://www.semrush.com/bot/ (SemrushBot-BA and SemrushBot-SI parse
+    // as SemrushBot)
     case 'SemrushBot':
-      if (hostname && hostname.endsWith('.semrush.com')) {
+    case 'SemrushBot-SWA':
+    case 'SemrushBot-OCOB':
+    case 'SemrushBot-FT':
+    case 'SemrushBot-ESI':
+    case 'SiteAuditBot':
+    case 'SplitSignalBot':
+    case 'RyteBot':
+      // Some SemrushBot addresses have the generic PTR bot.semrush.com, which
+      // doesn't resolve back to them: accept Semrush's own range too
+      // (85.208.98.0/24, announced by AS209366)
+      if (
+        (verifiedHostname && verifiedHostname.endsWith('.semrush.com')) ||
+        (address && new IPCIDR('85.208.98.0/24').contains(address))
+      ) {
         return log.set('identity', 'Semrush');
       }
       break;
@@ -187,9 +207,14 @@ function augment(log) {
         : log;
     case 'Reflectionbot':
       // https://reflection.ai/bot
-      return hostname && hostname.endsWith('.reflection.ai')
+      return verifiedHostname && verifiedHostname.endsWith('.reflection.ai')
         ? log.set('identity', 'Reflection')
         : log;
+    case 'SEOkicks':
+      // https://www.seokicks.de/robot.html
+      return verifiedHostname && verifiedHostname.endsWith('.seokicks.de')
+        ? log.set('identity', 'SEOkicks')
+        : log;
     case 'bnf.fr bot':
       return hostname && hostname.endsWith('.bnf.fr')
         ? log.set('identity', 'BnF.fr')
@@ -305,6 +330,11 @@ function augment(log) {
       return address && new IPCIDR('203.133.160.0/19').contains(address)
         ? log.set('identity', family)
         : log;
+    case 'LinkupBot':
+      // https://www.linkup.so/linkupbot-ips.txt
+      return address && new IPCIDR('35.198.113.100/32').contains(address)
+        ? log.set('identity', 'Linkup')
+        : log;
     case 'OAI-SearchBot':
       return openaiSearchbotIps.some((cidr) =>
         new IPCIDR(cidr).contains(address)
@@ -463,6 +493,10 @@ function augment(log) {
     if (hostname.endsWith('.qwant.com')) {
       return log.set('identity', 'Qwant');
     }
+    // Semrush crawlers not named above
+    if (verifiedHostname && verifiedHostname.endsWith('.semrush.com')) {
+      return log.set('identity', 'Semrush');
+    }
   }
 
   // Signature