@hyperwatch/hyperwatch 4.3.1 → 5.0.0

package/src/lib/aggregator.js

@@ -1,4 +1,4 @@
-const { Map, Set, fromJS, is } = require('immutable');
+const { List, Map, Set, fromJS, is } = require('immutable');
 
 const { Formatter, address, identity } = require('../lib/formatter');
 const { Speed } = require('../lib/speed');
@@ -9,17 +9,35 @@ const {
   md5,
 } = require('../lib/util');
 
+const lastSeen = (entry) => {
+  const ts =
+    entry.getIn(['speed', 'per_minute']) &&
+    entry.getIn(['speed', 'per_minute']).latest;
+  return ts ? new Date(ts * 1000).toISOString() : '';
+};
+
+const statusCount = (key) => (entry) =>
+  entry.getIn(['speed', key]) ? aggregateCount(entry, key) : 0;
+
 const defaultFormatter = new Formatter();
 defaultFormatter.setFormats([
   ['identity', identity],
 
-  ['address', address],
+  ['address', (entry) => entry.getIn(['address', 'value']) || ''],
+  ['hostname', address],
 
   ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
   ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
 
+  ['2xx15m', statusCount('2xx_per_minute')],
+  ['2xx24h', statusCount('2xx_per_hour')],
+  ['4xx15m', statusCount('4xx_per_minute')],
+  ['4xx24h', statusCount('4xx_per_hour')],
+
   ['execTime15m', (entry) => formatDuration(aggregateSum(entry, 'per_minute'))],
   ['execTime24h', (entry) => formatDuration(aggregateSum(entry, 'per_hour'))],
+
+  ['lastSeen', lastSeen],
 ]);
 
 const defaultEnricher = (entry, log) => {
@@ -52,6 +70,10 @@ const defaultIdentifier = (log) => {
 const defaultSorters = {
   count15m: (entry) => aggregateCount(entry, 'per_minute'),
   count24h: (entry) => aggregateCount(entry, 'per_hour'),
+  '2xx15m': statusCount('2xx_per_minute'),
+  '2xx24h': statusCount('2xx_per_hour'),
+  '4xx15m': statusCount('4xx_per_minute'),
+  '4xx24h': statusCount('4xx_per_hour'),
   latest: (entry) => entry.getIn(['speed', 'per_minute']).latest,
   execTime15m: (entry) => aggregateSum(entry, 'per_minute'),
   execTime24h: (entry) => aggregateSum(entry, 'per_hour'),
@@ -60,10 +82,14 @@ const defaultSorters = {
 class Aggregator {
   constructor() {
     this.entries = new Map();
-    this.formatter = defaultFormatter;
+    // Snapshot the default formats: modules extend defaultFormatter during
+    // init, aggregators are created during start, and per-aggregator
+    // insertFormat calls must not leak into other aggregators.
+    this.formatter = defaultFormatter.clone();
     this.enricher = defaultEnricher;
     this.identifier = defaultIdentifier;
-    this.sorters = defaultSorters;
+    this.sorters = { ...defaultSorters };
+    this.entryGc = null;
     this.gcSize = 1000;
 
     setInterval(() => this.gc(), 60 * 1000).unref();
@@ -81,6 +107,13 @@ class Aggregator {
     return this;
   }
 
+  // Per-entry cleanup run on every gc pass, regardless of aggregator size
+  setEntryGc(fn) {
+    this.entryGc = fn;
+
+    return this;
+  }
+
   setIdentifier(fn) {
     this.identifier = fn;
 
@@ -93,6 +126,8 @@ class Aggregator {
     const rawExecTime = Number(log.get('executionTime'));
     const executionTime = Number.isFinite(rawExecTime) ? rawExecTime : 0;
 
+    const status = log.getIn(['response', 'status']);
+
     if (!this.entries.has(id)) {
       this.entries = this.entries
         .setIn([id, 'id'], id)
@@ -104,7 +139,11 @@ class Aggregator {
         .setIn(
           [id, 'speed', 'per_hour'],
           new Speed(3600, 24).hit(undefined, executionTime)
-        );
+        )
+        .setIn([id, 'speed', '2xx_per_minute'], new Speed(60, 15))
+        .setIn([id, 'speed', '2xx_per_hour'], new Speed(3600, 24))
+        .setIn([id, 'speed', '4xx_per_minute'], new Speed(60, 15))
+        .setIn([id, 'speed', '4xx_per_hour'], new Speed(3600, 24));
     } else {
       this.entries = this.entries
         .updateIn([id, 'speed', 'per_minute'], (speed) =>
@@ -115,6 +154,16 @@ class Aggregator {
         );
     }
 
+    if (status >= 200 && status < 300) {
+      this.entries = this.entries
+        .updateIn([id, 'speed', '2xx_per_minute'], (speed) => speed.hit())
+        .updateIn([id, 'speed', '2xx_per_hour'], (speed) => speed.hit());
+    } else if (status >= 400 && status < 500) {
+      this.entries = this.entries
+        .updateIn([id, 'speed', '4xx_per_minute'], (speed) => speed.hit())
+        .updateIn([id, 'speed', '4xx_per_hour'], (speed) => speed.hit());
+    }
+
     this.entries = this.entries.updateIn([id], (entry) =>
       this.enricher(entry, log)
     );
@@ -131,10 +180,8 @@ class Aggregator {
       sort = 'count15m';
     }
 
-    const rawData = this.entries
-      .map(this.sorters[sort])
-      .sort()
-      .reverse()
+    const sorted = this.entries.map(this.sorters[sort]).sort().reverse();
+    const rawData = sorted
       .slice(0, limit || 100)
       .keySeq()
       .map((id) => this.entries.get(id));
@@ -146,7 +193,15 @@ class Aggregator {
       : rawData.map((entry) => this.formatter.formatObject(entry, output));
   }
 
+  reset() {
+    this.entries = new Map();
+  }
+
   gc() {
+    if (this.entryGc) {
+      this.entries = this.entries.map(this.entryGc);
+    }
+
     if (this.entries.size < this.gcSize) {
       return;
     }
@@ -169,10 +224,10 @@ class Aggregator {
     return this.entries
       .map((entry) => {
         const plain = entry.toJS();
-        plain.speed = {
-          per_minute: entry.getIn(['speed', 'per_minute']).toJSON(),
-          per_hour: entry.getIn(['speed', 'per_hour']).toJSON(),
-        };
+        plain.speed = entry
+          .get('speed')
+          .map((speed) => speed.toJSON())
+          .toObject();
         return plain;
       })
       .valueSeq()
@@ -183,21 +238,59 @@ class Aggregator {
     for (const item of data) {
       const { speed, ...rest } = item;
       // fromJS deep-converts everything to Immutable structures.
-      // Signature headers must stay as a plain object (used with Object.entries),
-      // and addresses must be a Set, not a List — fix both after conversion.
+      // Signature headers must stay as a plain object (used with Object.entries).
       let entry = fromJS(rest);
       if (entry.hasIn(['signature', 'headers'])) {
         entry = entry.setIn(['signature', 'headers'], rest.signature.headers);
       }
-      if (entry.has('addresses')) {
-        entry = entry.update('addresses', (list) => Set(list));
+      // addresses/signatures are Map<id, lastSeen>. Dumps from before that
+      // stored plain arrays, which the enrichers can't update — discard them
+      // and let the counts rebuild from live traffic.
+      for (const key of ['addresses', 'signatures']) {
+        if (List.isList(entry.get(key))) {
+          entry = entry.delete(key);
+        }
       }
       entry = entry
         .setIn(['speed', 'per_minute'], Speed.fromJSON(speed.per_minute))
-        .setIn(['speed', 'per_hour'], Speed.fromJSON(speed.per_hour));
+        .setIn(['speed', 'per_hour'], Speed.fromJSON(speed.per_hour))
+        .setIn(
+          ['speed', '2xx_per_minute'],
+          speed['2xx_per_minute']
+            ? Speed.fromJSON(speed['2xx_per_minute'])
+            : new Speed(60, 15)
+        )
+        .setIn(
+          ['speed', '2xx_per_hour'],
+          speed['2xx_per_hour']
+            ? Speed.fromJSON(speed['2xx_per_hour'])
+            : new Speed(3600, 24)
+        )
+        .setIn(
+          ['speed', '4xx_per_minute'],
+          speed['4xx_per_minute']
+            ? Speed.fromJSON(speed['4xx_per_minute'])
+            : new Speed(60, 15)
+        )
+        .setIn(
+          ['speed', '4xx_per_hour'],
+          speed['4xx_per_hour']
+            ? Speed.fromJSON(speed['4xx_per_hour'])
+            : new Speed(3600, 24)
+        );
       this.entries = this.entries.set(rest.id, entry);
     }
+    // Drop members that went stale while the process was down
+    if (this.entryGc) {
+      this.entries = this.entries.map(this.entryGc);
+    }
   }
 }
 
-module.exports = { Aggregator, defaultFormatter, defaultEnricher };
+module.exports = {
+  Aggregator,
+  defaultFormatter,
+  defaultEnricher,
+  lastSeen,
+  statusCount,
+};

package/src/lib/formatter.js

@@ -1,4 +1,4 @@
-const chalk = require('chalk');
+const { default: chalk } = require('chalk');
 
 const colorize = (name, value, output) => {
   if (output === 'console') {
@@ -65,6 +65,15 @@ class Formatter {
     };
   }
 
+  clone() {
+    const formatter = new Formatter();
+    formatter.formats = [...this.formats];
+    formatter.colors = { ...this.colors };
+    formatter.output = this.output;
+
+    return formatter;
+  }
+
   setOutput(output) {
     this.output = output;
 

package/src/lib/log-buffer.js

@@ -0,0 +1,45 @@
+/**
+ * Keep the latest `capacity` logs of a pipeline node, newest first.
+ * dump() / load() let the persistence module save it across restarts.
+ */
+const { fromJS } = require('immutable');
+
+class LogBuffer {
+  constructor(capacity = 1000) {
+    this.capacity = capacity;
+    this.buffer = new Array(capacity);
+    this.pointer = 0;
+    this.size = 0;
+  }
+
+  push(item) {
+    this.buffer[this.pointer] = item;
+    this.pointer = (this.pointer + 1) % this.capacity;
+    if (this.size < this.capacity) {
+      this.size++;
+    }
+  }
+
+  toArray() {
+    if (this.size < this.capacity) {
+      return this.buffer.slice(0, this.size).reverse();
+    }
+    return [
+      ...this.buffer.slice(this.pointer),
+      ...this.buffer.slice(0, this.pointer),
+    ].reverse();
+  }
+
+  // Oldest first, never more than capacity
+  dump() {
+    return this.toArray()
+      .reverse()
+      .map((log) => log.toJS());
+  }
+
+  load(data) {
+    data.slice(-this.capacity).forEach((item) => this.push(fromJS(item)));
+  }
+}
+
+module.exports = LogBuffer;

package/src/lib/persistence.js

@@ -9,6 +9,16 @@ const aggregators = Object.create(null);
 
 const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
 
+// Encode any string into a name accepted by register(). Characters outside
+// [A-Za-z0-9.-], including '_' itself, become _<hex code point>_, so distinct
+// inputs never map to the same name.
+function safeName(name) {
+  return String(name).replace(
+    /[^A-Za-z0-9.-]/gu,
+    (c) => `_${c.codePointAt(0).toString(16)}_`
+  );
+}
+
 function register(name, aggregator) {
   if (!SAFE_NAME.test(name)) {
     throw new Error(`Invalid aggregator name for persistence: "${name}"`);
@@ -69,4 +79,4 @@ function load(dir) {
   }
 }
 
-module.exports = { register, dump, load };
+module.exports = { register, dump, load, safeName };

package/src/lib/pipeline.js

@@ -128,35 +128,79 @@ class Builder {
     this.children = [];
     this.name = null;
     this.pipeline = pipeline;
+    this.parent = null;
+    this.op = null;
+    this.module = null;
+    this.fnName = null;
+    this._label = null;
+    // Lightweight per-node counter for throughput tracking
+    this.counter = 0;
+    // Fixed one-second buckets keep rate tracking bounded and O(1) per event.
+    this.rateBuckets = Array.from({ length: 10 }, () => ({
+      second: null,
+      count: 0,
+    }));
   }
 
-  add(xf) {
+  add(xf, { op = null, fnName = null, label = null } = {}) {
     const child = new Builder(xf, this.pipeline);
+    child.parent = this;
+    child.op = op;
+    child.module = this.pipeline ? this.pipeline.currentModule : null;
+    child.fnName = fnName || null;
+    child._label = label || null;
     this.children.push(child);
     return child;
   }
 
-  map(f) {
-    return this.add(map(f));
+  map(f, label) {
+    return this.add(map(f), { op: 'map', fnName: f.name, label });
   }
 
-  filter(pred) {
-    return this.add(filter(pred));
+  filter(pred, label) {
+    return this.add(filter(pred), { op: 'filter', fnName: pred.name, label });
   }
 
-  split(pred) {
-    return [this.add(filter(pred)), this.add(filter(complement(pred)))];
+  split(pred, labels) {
+    return [
+      this.add(filter(pred), {
+        op: 'split-true',
+        fnName: pred.name,
+        label: labels && labels[0],
+      }),
+      this.add(filter(complement(pred)), {
+        op: 'split-false',
+        fnName: pred.name,
+        label: labels && labels[1],
+      }),
+    ];
   }
 
-  by(f) {
-    return this.add(by(f));
+  by(f, label) {
+    return this.add(by(f), { op: 'by', fnName: f.name, label });
   }
 
   create() {
+    const self = this;
+    const countXf = (stream) => (event) => {
+      self.counter++;
+      const second = Math.floor(Date.now() / 1000);
+      const bucket = self.rateBuckets[second % self.rateBuckets.length];
+      if (bucket.second !== second) {
+        bucket.second = second;
+        bucket.count = 0;
+      }
+      bucket.count++;
+      forward(stream, event);
+    };
     if (this.children.length === 0) {
-      return this.xf;
+      return comp([this.xf, countXf]);
     }
-    return comp([this.xf, multiplex(this.children.map((b) => b.create()))]);
+    return comp([
+      this.xf,
+      countXf,
+      multiplex(this.children.map((b) => b.create())),
+    ]);
   }
 
   // Helpers
@@ -194,6 +238,9 @@ class Pipeline extends Builder {
       main: this,
     };
     this.pipeline = this;
+    this.name = 'raw';
+    this.currentModule = null;
+    this.op = 'identity';
   }
 
   registerInput(input) {
@@ -204,6 +251,16 @@ class Pipeline extends Builder {
       type: 'input',
     });
     this.monitors.push(monitor);
+    // Create a tap node so this input gets its own /logs/ WebSocket endpoint
+    const inputIndex = this.inputs.length;
+    const nodeName = `input-${inputIndex}`;
+    const tap = new Builder(identity(), this);
+    tap.name = nodeName;
+    tap.op = 'input';
+    tap.module = null;
+    tap.fnName = null;
+    this.registerNode(nodeName, tap);
+    input._tap = tap;
   }
 
   start() {
@@ -213,6 +270,8 @@ class Pipeline extends Builder {
       const monitor = this.monitors.find(
         (monitor) => monitor.type === 'input' && monitor.name === input.name
       );
+      // Build the tap stream so it can forward logs to WebSocket clients
+      const tapStream = input._tap ? input._tap.create()(() => {}) : null;
       input.start({
         success: (log) => {
           const valid = validate(log.toJS());
@@ -224,6 +283,9 @@ class Pipeline extends Builder {
               data: log,
             });
             monitor.hit('accepted');
+            if (tapStream) {
+              forward(tapStream, event);
+            }
             forward(stream, event);
           } else {
             monitor.hit('rejected');
@@ -256,12 +318,61 @@ class Pipeline extends Builder {
   }
 
   registerNode(name, node) {
+    const previous = this.nodes[name];
+    if (previous && previous !== node && previous.name === name) {
+      previous.name = null;
+    }
     this.nodes[name] = node;
   }
 
   getNode(name) {
     return this.nodes[name];
   }
+
+  getTree(node = this) {
+    const second = Math.floor(Date.now() / 1000);
+    const windowStart = second - node.rateBuckets.length + 1;
+    const rateCount = node.rateBuckets.reduce(
+      (count, bucket) =>
+        bucket.second >= windowStart && bucket.second <= second
+          ? count + bucket.count
+          : count,
+      0
+    );
+    const obj = {
+      name: node.name || null,
+      op: node.op || null,
+      module: node.module || null,
+      fnName: node.fnName || null,
+      label: node._label || null,
+      count: node.counter,
+      rate: rateCount / node.rateBuckets.length,
+      children: node.children.map((child) => this.getTree(child)),
+    };
+    // Include inputs at the root level
+    if (node === this) {
+      obj.inputs = this.inputs.map((input) => {
+        const monitor = this.monitors.find(
+          (m) => m.type === 'input' && m.name === input.name
+        );
+        const accepted = monitor
+          ? monitor.speeds.accepted.per_minute.compute()
+          : null;
+        const rejected = monitor
+          ? monitor.speeds.rejected.per_minute.compute()
+          : null;
+        return {
+          name: input.name,
+          node: input._tap ? input._tap.name : null,
+          tree: input._tap ? this.getTree(input._tap) : null,
+          status: monitor ? monitor.status : null,
+          accepted: accepted ? accepted.reduce((a, b) => a + b, 0) : 0,
+          rejected: rejected ? rejected.reduce((a, b) => a + b, 0) : 0,
+        };
+      });
+    }
+    return obj;
+  }
 }
 
 module.exports = new Pipeline();

package/src/lib/recent-map.js

@@ -0,0 +1,23 @@
+/**
+ * Track distinct keys with the time they were last seen, bounded to a
+ * rolling window. Backs the addressCount15m/24h and signatureCount15m/24h
+ * counters.
+ */
+const { Map } = require('immutable');
+
+const { now } = require('./util');
+
+// Rolling window in seconds
+const WINDOW = 24 * 3600;
+
+// Record `key` as seen at `time` in an Immutable Map<key, unix seconds>
+const touch = (map = Map(), key, time = now()) => map.set(key, time);
+
+// Drop keys not seen in the last WINDOW seconds
+const prune = (map, time = now()) => map.filter((seen) => seen > time - WINDOW);
+
+// Number of keys seen in the last `window` seconds
+const countRecent = (map, window = WINDOW, time = now()) =>
+  map ? map.count((seen) => seen > time - window) : 0;
+
+module.exports = { WINDOW, touch, prune, countRecent };

package/src/modules/address.js

@@ -1,9 +1,18 @@
+const { is } = require('immutable');
+
 const api = require('../app/api');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
+const { touch, prune, countRecent } = require('../lib/recent-map');
 
 const identifier = (log) => log.getIn(['address', 'value']);
 
+const signatureCount15m = (entry) =>
+  countRecent(entry.get('signatures'), 15 * 60);
+const signatureCount24h = (entry) => countRecent(entry.get('signatures'));
+
+let aggregator;
+
 function fill(log) {
   if (!log.hasIn(['address', 'value'])) {
     log = log.setIn(['address', 'value'], log.getIn(['request', 'address']));
@@ -16,11 +25,56 @@ function init() {
 }
 
 function start() {
-  const aggregator = new Aggregator();
+  aggregator = new Aggregator();
 
   aggregator.setIdentifier(identifier);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  const enricher = (entry, log) => {
+    for (const field of [
+      'address',
+      'identity',
+      'cloudflare',
+      'dnsbl',
+      'geoip',
+      'hostname',
+      'agent',
+      'language',
+      'signature',
+    ]) {
+      if (log.has(field) && !is(log.get(field), entry.get(field))) {
+        entry = entry.set(field, log.get(field));
+      }
+    }
+
+    // Distinct signature IDs with last-seen time, pruned to 24h; backs both
+    // the 15m and 24h counts
+    const signatureId = log.getIn(['signature', 'id']);
+    if (signatureId) {
+      entry = entry.update('signatures', (map) => touch(map, signatureId));
+    }
+
+    return entry;
+  };
+
+  aggregator.setEnricher(enricher);
+
+  aggregator.setEntryGc((entry) =>
+    entry.has('signatures') ? entry.update('signatures', prune) : entry
+  );
+
+  aggregator.formatter.insertFormat('signatureCount15m', signatureCount15m, {
+    before: 'count15m',
+  });
+  aggregator.formatter.insertFormat('signatureCount24h', signatureCount24h, {
+    before: 'count15m',
+  });
+
+  aggregator.sorters.signatureCount15m = signatureCount15m;
+  aggregator.sorters.signatureCount24h = signatureCount24h;
+
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('addresses', aggregator);
 }
@@ -28,4 +82,7 @@ function start() {
 module.exports = {
   init,
   start,
+  get aggregator() {
+    return aggregator;
+  },
 };

package/src/modules/dnsbl.js

@@ -4,11 +4,19 @@ const aggregator = require('../lib/aggregator');
 const cache = require('../lib/cache');
 const pipeline = require('../lib/pipeline');
 
+// Indirection so tests can swap the DNS lookup without hitting the network.
+const defaultLookup = (ip, blacklist) => dnsbl.lookup(ip, blacklist);
+let lookup = defaultLookup;
+
+function setLookup(fn = defaultLookup) {
+  lookup = fn;
+}
+
 async function xblLookup(ip) {
   if (await cache.has(`xbl-${ip}`)) {
     return cache.get(`xbl-${ip}`);
   }
-  const result = await dnsbl.lookup(ip, 'xbl.spamhaus.org');
+  const result = await lookup(ip, 'xbl.spamhaus.org');
   cache.set(`xbl-${ip}`, result);
   return result;
 }
@@ -45,4 +53,6 @@ function init() {
 module.exports = {
   augment,
   init,
+  setLookup,
+  xblFormat,
 };