@hyperwatch/hyperwatch 4.3.1 → 5.0.1

package/src/input/websocket.js

@@ -1,7 +1,7 @@
 const { fromJS } = require('immutable');
 const WebSocket = require('ws');
 
-const app = require('../app/websocket');
+const wsServer = require('../app/ws-server');
 
 const defaultParse = (s) => fromJS(JSON.parse(s));
 
@@ -19,12 +19,20 @@ function create({
   heartbeatInterval = 30000,
 }) {
   let client;
+  let keepAlive;
+  let reconnectTimer;
+  // Each connection belongs to a generation. stop() and every new connection
+  // move to the next one, so events from an obsolete socket (a late close
+  // after a restart, its heartbeat) can't reconnect or touch the current one.
+  let generation = 0;
 
   let reconnectAttempts = 0;
 
   const setupWebSocketClient = ({ status, success, reject }) => {
+    const current = ++generation;
+    const isCurrent = () => current === generation;
     let isAlive;
-    let keepAlive;
+    let heartbeat;
 
     if (username && password) {
       options.headers = options.headers || {};
@@ -33,32 +41,37 @@ function create({
       ).toString('base64')}`;
     }
 
-    client = new WebSocket(address, [], options);
+    const socket = new WebSocket(address, [], options);
+    client = socket;
     status(null, `Waiting for connection to ${address}`);
 
-    client.on('open', () => {
+    socket.on('open', () => {
+      if (!isCurrent()) {
+        return;
+      }
       isAlive = true;
       reconnectAttempts = 0;
       status(null, `Listening to ${address}`);
 
       // Heartbeat: detect stale connections
-      keepAlive = setInterval(() => {
+      heartbeat = setInterval(() => {
         if (isAlive === false) {
-          client.terminate();
-          clearInterval(keepAlive);
+          socket.terminate();
+          clearInterval(heartbeat);
         } else {
           try {
-            client.ping();
+            socket.ping();
           } catch (err) {
             status(err, 'Websocket error');
           }
           isAlive = false;
         }
       }, heartbeatInterval);
+      keepAlive = heartbeat;
     });
 
-    client.on('message', (message) => {
-      if (sample !== 1 && Math.random() > sample) {
+    socket.on('message', (message) => {
+      if (!isCurrent() || (sample !== 1 && Math.random() > sample)) {
         return;
       }
       try {
@@ -68,19 +81,22 @@ function create({
       }
     });
 
-    client.on('error', (err) => {
-      status(err, 'Websocket error');
+    socket.on('error', (err) => {
+      if (isCurrent()) {
+        status(err, 'Websocket error');
+      }
     });
 
-    client.on('pong', () => {
+    socket.on('pong', () => {
       isAlive = true;
     });
 
-    client.on('close', () => {
-      status(null, 'Websocket connection has been closed');
-      if (keepAlive) {
-        clearInterval(keepAlive);
+    socket.on('close', () => {
+      clearInterval(heartbeat);
+      if (!isCurrent()) {
+        return;
       }
+      status(null, 'Websocket connection has been closed');
       if (reconnectOnClose) {
         reconnectAttempts++;
         const delay = Math.min(
@@ -91,7 +107,7 @@ function create({
           null,
           `Reconnecting Websocket in ${delay / 1000}s (attempt ${reconnectAttempts})`
         );
-        setTimeout(() => {
+        reconnectTimer = setTimeout(() => {
           setupWebSocketClient({ status, success, reject });
         }, delay);
       }
@@ -99,7 +115,7 @@ function create({
   };
 
   const setupWebSocketServer = ({ status, success, reject }) => {
-    app.ws(path, (ws) => {
+    wsServer.ws(path, (ws) => {
       ws.on('message', (message) => {
         if (sample !== 1 && Math.random() > sample) {
           return;
@@ -117,6 +133,7 @@ function create({
   return {
     name: `${name} ${type}`,
     start: ({ success, reject, status, log }) => {
+      reconnectAttempts = 0;
       if (type === 'client') {
         setupWebSocketClient({ status, success, reject });
       } else if (type === 'server') {
@@ -126,9 +143,28 @@ function create({
         log(new Error(errMsg), 'error');
       }
     },
+    // Never throws: a failing input must not prevent the others from stopping,
+    // nor Hyperwatch from persisting its data on shutdown.
     stop: () => {
-      if (client) {
-        client.close();
+      // Obsoletes the current connection: its close won't reconnect
+      generation++;
+      clearTimeout(reconnectTimer);
+      clearInterval(keepAlive);
+      if (!client) {
+        return;
+      }
+      try {
+        if (client.readyState === WebSocket.CONNECTING) {
+          // close() throws while the connection is not established yet
+          client.terminate();
+        } else if (client.readyState === WebSocket.OPEN) {
+          client.close();
+        }
+      } catch (err) {
+        console.error(
+          `${name}: error while closing the Websocket:`,
+          err.message
+        );
       }
     },
   };

package/src/lib/aggregator.js

@@ -1,4 +1,4 @@
-const { Map, Set, fromJS, is } = require('immutable');
+const { List, Map, Set, fromJS, is } = require('immutable');
 
 const { Formatter, address, identity } = require('../lib/formatter');
 const { Speed } = require('../lib/speed');
@@ -9,17 +9,35 @@ const {
   md5,
 } = require('../lib/util');
 
+const lastSeen = (entry) => {
+  const ts =
+    entry.getIn(['speed', 'per_minute']) &&
+    entry.getIn(['speed', 'per_minute']).latest;
+  return ts ? new Date(ts * 1000).toISOString() : '';
+};
+
+const statusCount = (key) => (entry) =>
+  entry.getIn(['speed', key]) ? aggregateCount(entry, key) : 0;
+
 const defaultFormatter = new Formatter();
 defaultFormatter.setFormats([
   ['identity', identity],
 
-  ['address', address],
+  ['address', (entry) => entry.getIn(['address', 'value']) || ''],
+  ['hostname', address],
 
   ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
   ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
 
+  ['2xx15m', statusCount('2xx_per_minute')],
+  ['2xx24h', statusCount('2xx_per_hour')],
+  ['4xx15m', statusCount('4xx_per_minute')],
+  ['4xx24h', statusCount('4xx_per_hour')],
+
   ['execTime15m', (entry) => formatDuration(aggregateSum(entry, 'per_minute'))],
   ['execTime24h', (entry) => formatDuration(aggregateSum(entry, 'per_hour'))],
+
+  ['lastSeen', lastSeen],
 ]);
 
 const defaultEnricher = (entry, log) => {
@@ -52,6 +70,10 @@ const defaultIdentifier = (log) => {
 const defaultSorters = {
   count15m: (entry) => aggregateCount(entry, 'per_minute'),
   count24h: (entry) => aggregateCount(entry, 'per_hour'),
+  '2xx15m': statusCount('2xx_per_minute'),
+  '2xx24h': statusCount('2xx_per_hour'),
+  '4xx15m': statusCount('4xx_per_minute'),
+  '4xx24h': statusCount('4xx_per_hour'),
   latest: (entry) => entry.getIn(['speed', 'per_minute']).latest,
   execTime15m: (entry) => aggregateSum(entry, 'per_minute'),
   execTime24h: (entry) => aggregateSum(entry, 'per_hour'),
@@ -60,10 +82,14 @@ const defaultSorters = {
 class Aggregator {
   constructor() {
     this.entries = new Map();
-    this.formatter = defaultFormatter;
+    // Snapshot the default formats: modules extend defaultFormatter during
+    // init, aggregators are created during start, and per-aggregator
+    // insertFormat calls must not leak into other aggregators.
+    this.formatter = defaultFormatter.clone();
     this.enricher = defaultEnricher;
     this.identifier = defaultIdentifier;
-    this.sorters = defaultSorters;
+    this.sorters = { ...defaultSorters };
+    this.entryGc = null;
     this.gcSize = 1000;
 
     setInterval(() => this.gc(), 60 * 1000).unref();
@@ -81,6 +107,13 @@ class Aggregator {
     return this;
   }
 
+  // Per-entry cleanup run on every gc pass, regardless of aggregator size
+  setEntryGc(fn) {
+    this.entryGc = fn;
+
+    return this;
+  }
+
   setIdentifier(fn) {
     this.identifier = fn;
 
@@ -93,6 +126,8 @@ class Aggregator {
     const rawExecTime = Number(log.get('executionTime'));
     const executionTime = Number.isFinite(rawExecTime) ? rawExecTime : 0;
 
+    const status = log.getIn(['response', 'status']);
+
     if (!this.entries.has(id)) {
       this.entries = this.entries
         .setIn([id, 'id'], id)
@@ -104,7 +139,11 @@ class Aggregator {
         .setIn(
           [id, 'speed', 'per_hour'],
           new Speed(3600, 24).hit(undefined, executionTime)
-        );
+        )
+        .setIn([id, 'speed', '2xx_per_minute'], new Speed(60, 15))
+        .setIn([id, 'speed', '2xx_per_hour'], new Speed(3600, 24))
+        .setIn([id, 'speed', '4xx_per_minute'], new Speed(60, 15))
+        .setIn([id, 'speed', '4xx_per_hour'], new Speed(3600, 24));
     } else {
       this.entries = this.entries
         .updateIn([id, 'speed', 'per_minute'], (speed) =>
@@ -115,6 +154,16 @@ class Aggregator {
         );
     }
 
+    if (status >= 200 && status < 300) {
+      this.entries = this.entries
+        .updateIn([id, 'speed', '2xx_per_minute'], (speed) => speed.hit())
+        .updateIn([id, 'speed', '2xx_per_hour'], (speed) => speed.hit());
+    } else if (status >= 400 && status < 500) {
+      this.entries = this.entries
+        .updateIn([id, 'speed', '4xx_per_minute'], (speed) => speed.hit())
+        .updateIn([id, 'speed', '4xx_per_hour'], (speed) => speed.hit());
+    }
+
     this.entries = this.entries.updateIn([id], (entry) =>
       this.enricher(entry, log)
     );
@@ -131,10 +180,8 @@ class Aggregator {
       sort = 'count15m';
     }
 
-    const rawData = this.entries
-      .map(this.sorters[sort])
-      .sort()
-      .reverse()
+    const sorted = this.entries.map(this.sorters[sort]).sort().reverse();
+    const rawData = sorted
       .slice(0, limit || 100)
       .keySeq()
       .map((id) => this.entries.get(id));
@@ -146,7 +193,15 @@ class Aggregator {
       : rawData.map((entry) => this.formatter.formatObject(entry, output));
   }
 
+  reset() {
+    this.entries = new Map();
+  }
+
   gc() {
+    if (this.entryGc) {
+      this.entries = this.entries.map(this.entryGc);
+    }
+
     if (this.entries.size < this.gcSize) {
       return;
     }
@@ -169,10 +224,10 @@ class Aggregator {
     return this.entries
       .map((entry) => {
         const plain = entry.toJS();
-        plain.speed = {
-          per_minute: entry.getIn(['speed', 'per_minute']).toJSON(),
-          per_hour: entry.getIn(['speed', 'per_hour']).toJSON(),
-        };
+        plain.speed = entry
+          .get('speed')
+          .map((speed) => speed.toJSON())
+          .toObject();
         return plain;
       })
       .valueSeq()
@@ -183,21 +238,59 @@ class Aggregator {
     for (const item of data) {
       const { speed, ...rest } = item;
       // fromJS deep-converts everything to Immutable structures.
-      // Signature headers must stay as a plain object (used with Object.entries),
-      // and addresses must be a Set, not a List — fix both after conversion.
+      // Signature headers must stay as a plain object (used with Object.entries).
       let entry = fromJS(rest);
       if (entry.hasIn(['signature', 'headers'])) {
         entry = entry.setIn(['signature', 'headers'], rest.signature.headers);
       }
-      if (entry.has('addresses')) {
-        entry = entry.update('addresses', (list) => Set(list));
+      // addresses/signatures are Map<id, lastSeen>. Dumps from before that
+      // stored plain arrays, which the enrichers can't update — discard them
+      // and let the counts rebuild from live traffic.
+      for (const key of ['addresses', 'signatures']) {
+        if (List.isList(entry.get(key))) {
+          entry = entry.delete(key);
+        }
       }
       entry = entry
         .setIn(['speed', 'per_minute'], Speed.fromJSON(speed.per_minute))
-        .setIn(['speed', 'per_hour'], Speed.fromJSON(speed.per_hour));
+        .setIn(['speed', 'per_hour'], Speed.fromJSON(speed.per_hour))
+        .setIn(
+          ['speed', '2xx_per_minute'],
+          speed['2xx_per_minute']
+            ? Speed.fromJSON(speed['2xx_per_minute'])
+            : new Speed(60, 15)
+        )
+        .setIn(
+          ['speed', '2xx_per_hour'],
+          speed['2xx_per_hour']
+            ? Speed.fromJSON(speed['2xx_per_hour'])
+            : new Speed(3600, 24)
+        )
+        .setIn(
+          ['speed', '4xx_per_minute'],
+          speed['4xx_per_minute']
+            ? Speed.fromJSON(speed['4xx_per_minute'])
+            : new Speed(60, 15)
+        )
+        .setIn(
+          ['speed', '4xx_per_hour'],
+          speed['4xx_per_hour']
+            ? Speed.fromJSON(speed['4xx_per_hour'])
+            : new Speed(3600, 24)
+        );
       this.entries = this.entries.set(rest.id, entry);
     }
+    // Drop members that went stale while the process was down
+    if (this.entryGc) {
+      this.entries = this.entries.map(this.entryGc);
+    }
   }
 }
 
-module.exports = { Aggregator, defaultFormatter, defaultEnricher };
+module.exports = {
+  Aggregator,
+  defaultFormatter,
+  defaultEnricher,
+  lastSeen,
+  statusCount,
+};

package/src/lib/formatter.js

@@ -1,4 +1,4 @@
-const chalk = require('chalk');
+const { default: chalk } = require('chalk');
 
 const colorize = (name, value, output) => {
   if (output === 'console') {
@@ -65,6 +65,15 @@ class Formatter {
     };
   }
 
+  clone() {
+    const formatter = new Formatter();
+    formatter.formats = [...this.formats];
+    formatter.colors = { ...this.colors };
+    formatter.output = this.output;
+
+    return formatter;
+  }
+
   setOutput(output) {
     this.output = output;
 

package/src/lib/log-buffer.js

@@ -0,0 +1,45 @@
+/**
+ * Keep the latest `capacity` logs of a pipeline node, newest first.
+ * dump() / load() let the persistence module save it across restarts.
+ */
+const { fromJS } = require('immutable');
+
+class LogBuffer {
+  constructor(capacity = 1000) {
+    this.capacity = capacity;
+    this.buffer = new Array(capacity);
+    this.pointer = 0;
+    this.size = 0;
+  }
+
+  push(item) {
+    this.buffer[this.pointer] = item;
+    this.pointer = (this.pointer + 1) % this.capacity;
+    if (this.size < this.capacity) {
+      this.size++;
+    }
+  }
+
+  toArray() {
+    if (this.size < this.capacity) {
+      return this.buffer.slice(0, this.size).reverse();
+    }
+    return [
+      ...this.buffer.slice(this.pointer),
+      ...this.buffer.slice(0, this.pointer),
+    ].reverse();
+  }
+
+  // Oldest first, never more than capacity
+  dump() {
+    return this.toArray()
+      .reverse()
+      .map((log) => log.toJS());
+  }
+
+  load(data) {
+    data.slice(-this.capacity).forEach((item) => this.push(fromJS(item)));
+  }
+}
+
+module.exports = LogBuffer;

package/src/lib/persistence.js

@@ -9,6 +9,16 @@ const aggregators = Object.create(null);
 
 const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
 
+// Encode any string into a name accepted by register(). Characters outside
+// [A-Za-z0-9.-], including '_' itself, become _<hex code point>_, so distinct
+// inputs never map to the same name.
+function safeName(name) {
+  return String(name).replace(
+    /[^A-Za-z0-9.-]/gu,
+    (c) => `_${c.codePointAt(0).toString(16)}_`
+  );
+}
+
 function register(name, aggregator) {
   if (!SAFE_NAME.test(name)) {
     throw new Error(`Invalid aggregator name for persistence: "${name}"`);
@@ -69,4 +79,4 @@ function load(dir) {
   }
 }
 
-module.exports = { register, dump, load };
+module.exports = { register, dump, load, safeName };