@hyperwatch/hyperwatch 4.2.0 → 5.0.0

package/src/input/websocket.js

@@ -1,7 +1,7 @@
 const { fromJS } = require('immutable');
 const WebSocket = require('ws');
 
-const app = require('../app/websocket');
+const wsServer = require('../app/ws-server');
 
 const defaultParse = (s) => fromJS(JSON.parse(s));
 
@@ -16,11 +16,15 @@ function create({
   parse = defaultParse,
   sample = 1,
   reconnectOnClose = false,
+  heartbeatInterval = 30000,
 }) {
   let client;
 
+  let reconnectAttempts = 0;
+
   const setupWebSocketClient = ({ status, success, reject }) => {
     let isAlive;
+    let keepAlive;
 
     if (username && password) {
       options.headers = options.headers || {};
@@ -34,7 +38,23 @@ function create({
 
     client.on('open', () => {
       isAlive = true;
+      reconnectAttempts = 0;
       status(null, `Listening to ${address}`);
+
+      // Heartbeat: detect stale connections
+      keepAlive = setInterval(() => {
+        if (isAlive === false) {
+          client.terminate();
+          clearInterval(keepAlive);
+        } else {
+          try {
+            client.ping();
+          } catch (err) {
+            status(err, 'Websocket error');
+          }
+          isAlive = false;
+        }
+      }, heartbeatInterval);
     });
 
     client.on('message', (message) => {
@@ -52,38 +72,34 @@ function create({
       status(err, 'Websocket error');
     });
 
-    const keepAlive = setInterval(() => {
-      if (isAlive === false) {
-        client.terminate();
-        clearInterval(keepAlive);
-      } else {
-        try {
-          client.ping();
-        } catch (err) {
-          status(err, 'Websocket error');
-        }
-        isAlive = false;
-      }
-    }, 10 * 1000);
-
     client.on('pong', () => {
       isAlive = true;
     });
 
     client.on('close', () => {
       status(null, 'Websocket connection has been closed');
+      if (keepAlive) {
+        clearInterval(keepAlive);
+      }
       if (reconnectOnClose) {
+        reconnectAttempts++;
+        const delay = Math.min(
+          10 * 1000 * Math.pow(2, reconnectAttempts - 1),
+          5 * 60 * 1000
+        );
+        status(
+          null,
+          `Reconnecting Websocket in ${delay / 1000}s (attempt ${reconnectAttempts})`
+        );
         setTimeout(() => {
-          status(null, 'Reconnecting Websocket');
           setupWebSocketClient({ status, success, reject });
-        }, 10 * 1000);
+        }, delay);
       }
-      clearInterval(keepAlive);
     });
   };
 
   const setupWebSocketServer = ({ status, success, reject }) => {
-    app.ws(path, (ws) => {
+    wsServer.ws(path, (ws) => {
       ws.on('message', (message) => {
         if (sample !== 1 && Math.random() > sample) {
           return;

package/src/lib/aggregator.js

@@ -1,17 +1,43 @@
-const { Map, Set, is } = require('immutable');
+const { List, Map, Set, fromJS, is } = require('immutable');
 
 const { Formatter, address, identity } = require('../lib/formatter');
 const { Speed } = require('../lib/speed');
-const { aggregateSpeed, md5 } = require('../lib/util');
+const {
+  aggregateCount,
+  aggregateSum,
+  formatDuration,
+  md5,
+} = require('../lib/util');
+
+const lastSeen = (entry) => {
+  const ts =
+    entry.getIn(['speed', 'per_minute']) &&
+    entry.getIn(['speed', 'per_minute']).latest;
+  return ts ? new Date(ts * 1000).toISOString() : '';
+};
+
+const statusCount = (key) => (entry) =>
+  entry.getIn(['speed', key]) ? aggregateCount(entry, key) : 0;
 
 const defaultFormatter = new Formatter();
 defaultFormatter.setFormats([
   ['identity', identity],
 
-  ['address', address],
+  ['address', (entry) => entry.getIn(['address', 'value']) || ''],
+  ['hostname', address],
+
+  ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
+  ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
 
-  ['15m', (entry) => aggregateSpeed(entry, 'per_minute')],
-  ['24h', (entry) => aggregateSpeed(entry, 'per_hour')],
+  ['2xx15m', statusCount('2xx_per_minute')],
+  ['2xx24h', statusCount('2xx_per_hour')],
+  ['4xx15m', statusCount('4xx_per_minute')],
+  ['4xx24h', statusCount('4xx_per_hour')],
+
+  ['execTime15m', (entry) => formatDuration(aggregateSum(entry, 'per_minute'))],
+  ['execTime24h', (entry) => formatDuration(aggregateSum(entry, 'per_hour'))],
+
+  ['lastSeen', lastSeen],
 ]);
 
 const defaultEnricher = (entry, log) => {
@@ -42,21 +68,31 @@ const defaultIdentifier = (log) => {
 };
 
 const defaultSorters = {
-  '15m': (entry) => aggregateSpeed(entry, 'per_minute'),
-  '24h': (entry) => aggregateSpeed(entry, 'per_hour'),
+  count15m: (entry) => aggregateCount(entry, 'per_minute'),
+  count24h: (entry) => aggregateCount(entry, 'per_hour'),
+  '2xx15m': statusCount('2xx_per_minute'),
+  '2xx24h': statusCount('2xx_per_hour'),
+  '4xx15m': statusCount('4xx_per_minute'),
+  '4xx24h': statusCount('4xx_per_hour'),
   latest: (entry) => entry.getIn(['speed', 'per_minute']).latest,
+  execTime15m: (entry) => aggregateSum(entry, 'per_minute'),
+  execTime24h: (entry) => aggregateSum(entry, 'per_hour'),
 };
 
 class Aggregator {
   constructor() {
     this.entries = new Map();
-    this.formatter = defaultFormatter;
+    // Snapshot the default formats: modules extend defaultFormatter during
+    // init, aggregators are created during start, and per-aggregator
+    // insertFormat calls must not leak into other aggregators.
+    this.formatter = defaultFormatter.clone();
     this.enricher = defaultEnricher;
     this.identifier = defaultIdentifier;
-    this.sorters = defaultSorters;
+    this.sorters = { ...defaultSorters };
+    this.entryGc = null;
     this.gcSize = 1000;
 
-    setInterval(() => this.gc(), 60 * 1000);
+    setInterval(() => this.gc(), 60 * 1000).unref();
   }
 
   setFormatter(fn) {
@@ -71,6 +107,13 @@ class Aggregator {
     return this;
   }
 
+  // Per-entry cleanup run on every gc pass, regardless of aggregator size
+  setEntryGc(fn) {
+    this.entryGc = fn;
+
+    return this;
+  }
+
   setIdentifier(fn) {
     this.identifier = fn;
 
@@ -80,17 +123,45 @@ class Aggregator {
   processLog(log) {
     const identifier = this.identifier(log);
     const id = md5(identifier);
+    const rawExecTime = Number(log.get('executionTime'));
+    const executionTime = Number.isFinite(rawExecTime) ? rawExecTime : 0;
+
+    const status = log.getIn(['response', 'status']);
 
     if (!this.entries.has(id)) {
       this.entries = this.entries
         .setIn([id, 'id'], id)
         .setIn([id, 'identifier'], identifier)
-        .setIn([id, 'speed', 'per_minute'], new Speed(60, 15).hit())
-        .setIn([id, 'speed', 'per_hour'], new Speed(3600, 24).hit());
+        .setIn(
+          [id, 'speed', 'per_minute'],
+          new Speed(60, 15).hit(undefined, executionTime)
+        )
+        .setIn(
+          [id, 'speed', 'per_hour'],
+          new Speed(3600, 24).hit(undefined, executionTime)
+        )
+        .setIn([id, 'speed', '2xx_per_minute'], new Speed(60, 15))
+        .setIn([id, 'speed', '2xx_per_hour'], new Speed(3600, 24))
+        .setIn([id, 'speed', '4xx_per_minute'], new Speed(60, 15))
+        .setIn([id, 'speed', '4xx_per_hour'], new Speed(3600, 24));
     } else {
       this.entries = this.entries
-        .updateIn([id, 'speed', 'per_minute'], (speed) => speed.hit())
-        .updateIn([id, 'speed', 'per_hour'], (speed) => speed.hit());
+        .updateIn([id, 'speed', 'per_minute'], (speed) =>
+          speed.hit(undefined, executionTime)
+        )
+        .updateIn([id, 'speed', 'per_hour'], (speed) =>
+          speed.hit(undefined, executionTime)
+        );
+    }
+
+    if (status >= 200 && status < 300) {
+      this.entries = this.entries
+        .updateIn([id, 'speed', '2xx_per_minute'], (speed) => speed.hit())
+        .updateIn([id, 'speed', '2xx_per_hour'], (speed) => speed.hit());
+    } else if (status >= 400 && status < 500) {
+      this.entries = this.entries
+        .updateIn([id, 'speed', '4xx_per_minute'], (speed) => speed.hit())
+        .updateIn([id, 'speed', '4xx_per_hour'], (speed) => speed.hit());
     }
 
     this.entries = this.entries.updateIn([id], (entry) =>
@@ -106,13 +177,11 @@ class Aggregator {
 
   getData({ raw, sort, format, limit }) {
     if (!sort || !this.sorters[sort]) {
-      sort = '15m';
+      sort = 'count15m';
     }
 
-    const rawData = this.entries
-      .map(this.sorters[sort])
-      .sort()
-      .reverse()
+    const sorted = this.entries.map(this.sorters[sort]).sort().reverse();
+    const rawData = sorted
       .slice(0, limit || 100)
       .keySeq()
       .map((id) => this.entries.get(id));
@@ -124,7 +193,15 @@ class Aggregator {
       : rawData.map((entry) => this.formatter.formatObject(entry, output));
   }
 
+  reset() {
+    this.entries = new Map();
+  }
+
   gc() {
+    if (this.entryGc) {
+      this.entries = this.entries.map(this.entryGc);
+    }
+
     if (this.entries.size < this.gcSize) {
       return;
     }
@@ -142,6 +219,78 @@ class Aggregator {
 
     this.entries = this.entries.filter((value, key) => keepList.has(key));
   }
+
+  dump() {
+    return this.entries
+      .map((entry) => {
+        const plain = entry.toJS();
+        plain.speed = entry
+          .get('speed')
+          .map((speed) => speed.toJSON())
+          .toObject();
+        return plain;
+      })
+      .valueSeq()
+      .toArray();
+  }
+
+  load(data) {
+    for (const item of data) {
+      const { speed, ...rest } = item;
+      // fromJS deep-converts everything to Immutable structures.
+      // Signature headers must stay as a plain object (used with Object.entries).
+      let entry = fromJS(rest);
+      if (entry.hasIn(['signature', 'headers'])) {
+        entry = entry.setIn(['signature', 'headers'], rest.signature.headers);
+      }
+      // addresses/signatures are Map<id, lastSeen>. Dumps from before that
+      // stored plain arrays, which the enrichers can't update — discard them
+      // and let the counts rebuild from live traffic.
+      for (const key of ['addresses', 'signatures']) {
+        if (List.isList(entry.get(key))) {
+          entry = entry.delete(key);
+        }
+      }
+      entry = entry
+        .setIn(['speed', 'per_minute'], Speed.fromJSON(speed.per_minute))
+        .setIn(['speed', 'per_hour'], Speed.fromJSON(speed.per_hour))
+        .setIn(
+          ['speed', '2xx_per_minute'],
+          speed['2xx_per_minute']
+            ? Speed.fromJSON(speed['2xx_per_minute'])
+            : new Speed(60, 15)
+        )
+        .setIn(
+          ['speed', '2xx_per_hour'],
+          speed['2xx_per_hour']
+            ? Speed.fromJSON(speed['2xx_per_hour'])
+            : new Speed(3600, 24)
+        )
+        .setIn(
+          ['speed', '4xx_per_minute'],
+          speed['4xx_per_minute']
+            ? Speed.fromJSON(speed['4xx_per_minute'])
+            : new Speed(60, 15)
+        )
+        .setIn(
+          ['speed', '4xx_per_hour'],
+          speed['4xx_per_hour']
+            ? Speed.fromJSON(speed['4xx_per_hour'])
+            : new Speed(3600, 24)
+        );
+      this.entries = this.entries.set(rest.id, entry);
+    }
+    // Drop members that went stale while the process was down
+    if (this.entryGc) {
+      this.entries = this.entries.map(this.entryGc);
+    }
+  }
 }
 
-module.exports = { Aggregator, defaultFormatter, defaultEnricher };
+module.exports = {
+  Aggregator,
+  defaultFormatter,
+  defaultEnricher,
+  lastSeen,
+  statusCount,
+};

package/src/lib/formatter.js

@@ -1,4 +1,4 @@
-const chalk = require('chalk');
+const { default: chalk } = require('chalk');
 
 const colorize = (name, value, output) => {
   if (output === 'console') {
@@ -65,6 +65,15 @@ class Formatter {
     };
   }
 
+  clone() {
+    const formatter = new Formatter();
+    formatter.formats = [...this.formats];
+    formatter.colors = { ...this.colors };
+    formatter.output = this.output;
+
+    return formatter;
+  }
+
   setOutput(output) {
     this.output = output;
 

package/src/lib/index.js

@@ -2,6 +2,7 @@ const aggregator = require('./aggregator');
 const cache = require('./cache');
 const formatter = require('./formatter');
 const logger = require('./logger');
+const persistence = require('./persistence');
 const pipeline = require('./pipeline');
 const util = require('./util');
 
@@ -10,6 +11,7 @@ module.exports = {
   cache,
   formatter,
   logger,
+  persistence,
   pipeline,
   util,
 };

package/src/lib/log-buffer.js

@@ -0,0 +1,45 @@
+/**
+ * Keep the latest `capacity` logs of a pipeline node, newest first.
+ * dump() / load() let the persistence module save it across restarts.
+ */
+const { fromJS } = require('immutable');
+
+class LogBuffer {
+  constructor(capacity = 1000) {
+    this.capacity = capacity;
+    this.buffer = new Array(capacity);
+    this.pointer = 0;
+    this.size = 0;
+  }
+
+  push(item) {
+    this.buffer[this.pointer] = item;
+    this.pointer = (this.pointer + 1) % this.capacity;
+    if (this.size < this.capacity) {
+      this.size++;
+    }
+  }
+
+  toArray() {
+    if (this.size < this.capacity) {
+      return this.buffer.slice(0, this.size).reverse();
+    }
+    return [
+      ...this.buffer.slice(this.pointer),
+      ...this.buffer.slice(0, this.pointer),
+    ].reverse();
+  }
+
+  // Oldest first, never more than capacity
+  dump() {
+    return this.toArray()
+      .reverse()
+      .map((log) => log.toJS());
+  }
+
+  load(data) {
+    data.slice(-this.capacity).forEach((item) => this.push(fromJS(item)));
+  }
+}
+
+module.exports = LogBuffer;

package/src/lib/persistence.js

@@ -0,0 +1,82 @@
+const fs = require('fs');
+const path = require('path');
+
+const debug = require('debug');
+
+const debugPersistence = debug('hyperwatch:persistence');
+
+const aggregators = Object.create(null);
+
+const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
+
+// Encode any string into a name accepted by register(). Characters outside
+// [A-Za-z0-9.-], including '_' itself, become _<hex code point>_, so distinct
+// inputs never map to the same name.
+function safeName(name) {
+  return String(name).replace(
+    /[^A-Za-z0-9.-]/gu,
+    (c) => `_${c.codePointAt(0).toString(16)}_`
+  );
+}
+
+function register(name, aggregator) {
+  if (!SAFE_NAME.test(name)) {
+    throw new Error(`Invalid aggregator name for persistence: "${name}"`);
+  }
+  aggregators[name] = aggregator;
+}
+
+function dump(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+  for (const [name, aggregator] of Object.entries(aggregators)) {
+    const data = aggregator.dump();
+    const target = path.join(dir, `${name}.json`);
+    const tmp = path.join(dir, `${name}.json.tmp`);
+    fs.writeFileSync(tmp, JSON.stringify(data));
+    fs.renameSync(tmp, target);
+  }
+  debugPersistence(
+    `Dumped ${Object.keys(aggregators).length} aggregator(s) to ${dir}`
+  );
+}
+
+function load(dir) {
+  if (!fs.existsSync(dir)) {
+    return;
+  }
+  let files;
+  try {
+    files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
+  } catch (err) {
+    debugPersistence(
+      `Cannot read persistence directory ${dir}: ${err.message}`
+    );
+    return;
+  }
+  let loaded = 0;
+  for (const file of files) {
+    const name = path.basename(file, '.json');
+    if (aggregators[name]) {
+      try {
+        const data = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8'));
+        if (!Array.isArray(data)) {
+          debugPersistence(
+            `Skipping ${file}: expected array, got ${typeof data}`
+          );
+          continue;
+        }
+        aggregators[name].load(data);
+        loaded += data.length;
+      } catch (err) {
+        debugPersistence(`Skipping ${file}: ${err.message}`);
+      }
+    }
+  }
+  if (loaded) {
+    debugPersistence(`Loaded ${loaded} entries from ${dir}`);
+  }
+}
+
+module.exports = { register, dump, load, safeName };