@hyperwatch/hyperwatch 4.2.0 → 5.0.0

package/src/lib/pipeline.js

@@ -128,35 +128,79 @@ class Builder {
     this.children = [];
     this.name = null;
     this.pipeline = pipeline;
+    this.parent = null;
+    this.op = null;
+    this.module = null;
+    this.fnName = null;
+    this._label = null;
+    // Lightweight per-node counter for throughput tracking
+    this.counter = 0;
+    // Fixed one-second buckets keep rate tracking bounded and O(1) per event.
+    this.rateBuckets = Array.from({ length: 10 }, () => ({
+      second: null,
+      count: 0,
+    }));
   }
 
-  add(xf) {
+  add(xf, { op = null, fnName = null, label = null } = {}) {
     const child = new Builder(xf, this.pipeline);
+    child.parent = this;
+    child.op = op;
+    child.module = this.pipeline ? this.pipeline.currentModule : null;
+    child.fnName = fnName || null;
+    child._label = label || null;
     this.children.push(child);
     return child;
   }
 
-  map(f) {
-    return this.add(map(f));
+  map(f, label) {
+    return this.add(map(f), { op: 'map', fnName: f.name, label });
   }
 
-  filter(pred) {
-    return this.add(filter(pred));
+  filter(pred, label) {
+    return this.add(filter(pred), { op: 'filter', fnName: pred.name, label });
   }
 
-  split(pred) {
-    return [this.add(filter(pred)), this.add(filter(complement(pred)))];
+  split(pred, labels) {
+    return [
+      this.add(filter(pred), {
+        op: 'split-true',
+        fnName: pred.name,
+        label: labels && labels[0],
+      }),
+      this.add(filter(complement(pred)), {
+        op: 'split-false',
+        fnName: pred.name,
+        label: labels && labels[1],
+      }),
+    ];
   }
 
-  by(f) {
-    return this.add(by(f));
+  by(f, label) {
+    return this.add(by(f), { op: 'by', fnName: f.name, label });
   }
 
   create() {
+    const self = this;
+    const countXf = (stream) => (event) => {
+      self.counter++;
+      const second = Math.floor(Date.now() / 1000);
+      const bucket = self.rateBuckets[second % self.rateBuckets.length];
+      if (bucket.second !== second) {
+        bucket.second = second;
+        bucket.count = 0;
+      }
+      bucket.count++;
+      forward(stream, event);
+    };
     if (this.children.length === 0) {
-      return this.xf;
+      return comp([this.xf, countXf]);
     }
-    return comp([this.xf, multiplex(this.children.map((b) => b.create()))]);
+    return comp([
+      this.xf,
+      countXf,
+      multiplex(this.children.map((b) => b.create())),
+    ]);
   }
 
   // Helpers
@@ -194,6 +238,9 @@ class Pipeline extends Builder {
       main: this,
     };
     this.pipeline = this;
+    this.name = 'raw';
+    this.currentModule = null;
+    this.op = 'identity';
   }
 
   registerInput(input) {
@@ -204,6 +251,16 @@ class Pipeline extends Builder {
       type: 'input',
     });
     this.monitors.push(monitor);
+    // Create a tap node so this input gets its own /logs/ WebSocket endpoint
+    const inputIndex = this.inputs.length;
+    const nodeName = `input-${inputIndex}`;
+    const tap = new Builder(identity(), this);
+    tap.name = nodeName;
+    tap.op = 'input';
+    tap.module = null;
+    tap.fnName = null;
+    this.registerNode(nodeName, tap);
+    input._tap = tap;
   }
 
   start() {
@@ -213,6 +270,8 @@ class Pipeline extends Builder {
       const monitor = this.monitors.find(
         (monitor) => monitor.type === 'input' && monitor.name === input.name
       );
+      // Build the tap stream so it can forward logs to WebSocket clients
+      const tapStream = input._tap ? input._tap.create()(() => {}) : null;
       input.start({
         success: (log) => {
           const valid = validate(log.toJS());
@@ -224,6 +283,9 @@ class Pipeline extends Builder {
               data: log,
             });
             monitor.hit('accepted');
+            if (tapStream) {
+              forward(tapStream, event);
+            }
             forward(stream, event);
           } else {
             monitor.hit('rejected');
@@ -256,12 +318,61 @@ class Pipeline extends Builder {
   }
 
   registerNode(name, node) {
+    const previous = this.nodes[name];
+    if (previous && previous !== node && previous.name === name) {
+      previous.name = null;
+    }
     this.nodes[name] = node;
   }
 
   getNode(name) {
     return this.nodes[name];
   }
+
+  getTree(node = this) {
+    const second = Math.floor(Date.now() / 1000);
+    const windowStart = second - node.rateBuckets.length + 1;
+    const rateCount = node.rateBuckets.reduce(
+      (count, bucket) =>
+        bucket.second >= windowStart && bucket.second <= second
+          ? count + bucket.count
+          : count,
+      0
+    );
+    const obj = {
+      name: node.name || null,
+      op: node.op || null,
+      module: node.module || null,
+      fnName: node.fnName || null,
+      label: node._label || null,
+      count: node.counter,
+      rate: rateCount / node.rateBuckets.length,
+      children: node.children.map((child) => this.getTree(child)),
+    };
+    // Include inputs at the root level
+    if (node === this) {
+      obj.inputs = this.inputs.map((input) => {
+        const monitor = this.monitors.find(
+          (m) => m.type === 'input' && m.name === input.name
+        );
+        const accepted = monitor
+          ? monitor.speeds.accepted.per_minute.compute()
+          : null;
+        const rejected = monitor
+          ? monitor.speeds.rejected.per_minute.compute()
+          : null;
+        return {
+          name: input.name,
+          node: input._tap ? input._tap.name : null,
+          tree: input._tap ? this.getTree(input._tap) : null,
+          status: monitor ? monitor.status : null,
+          accepted: accepted ? accepted.reduce((a, b) => a + b, 0) : 0,
+          rejected: rejected ? rejected.reduce((a, b) => a + b, 0) : 0,
+        };
+      });
+    }
+    return obj;
+  }
 }
 
 module.exports = new Pipeline();

package/src/lib/recent-map.js

@@ -0,0 +1,23 @@
+/**
+ * Track distinct keys with the time they were last seen, bounded to a
+ * rolling window. Backs the addressCount15m/24h and signatureCount15m/24h
+ * counters.
+ */
+const { Map } = require('immutable');
+
+const { now } = require('./util');
+
+// Rolling window in seconds
+const WINDOW = 24 * 3600;
+
+// Record `key` as seen at `time` in an Immutable Map<key, unix seconds>
+const touch = (map = Map(), key, time = now()) => map.set(key, time);
+
+// Drop keys not seen in the last WINDOW seconds
+const prune = (map, time = now()) => map.filter((seen) => seen > time - WINDOW);
+
+// Number of keys seen in the last `window` seconds
+const countRecent = (map, window = WINDOW, time = now()) =>
+  map ? map.count((seen) => seen > time - window) : 0;
+
+module.exports = { WINDOW, touch, prune, countRecent };

package/src/lib/speed.js

@@ -10,6 +10,7 @@ class Speed {
     this.windowSize = windowSize;
     this.size = size;
     this.counters = Map();
+    this.sums = Map();
     this.started = null;
     this.latest = null;
   }
@@ -18,23 +19,26 @@ class Speed {
   gc() {
     const cutoff = now() - this.size * this.windowSize;
     this.counters = this.counters.filter((c, t) => parseInt(t) > cutoff);
+    this.sums = this.sums.filter((c, t) => parseInt(t) > cutoff);
   }
 
-  hit(time = now()) {
+  hit(time = now(), value) {
     this.started = !this.started ? time : Math.min(this.started, time);
     this.latest = !this.latest ? time : Math.max(this.latest, time);
     const idx = time - (time % this.windowSize);
     this.counters = this.counters.update(`${idx}`, 0, (n) => n + 1);
+    if (value !== undefined) {
+      this.sums = this.sums.update(`${idx}`, 0, (n) => n + value);
+    }
     this.gc();
     return this;
   }
 
-  compute() {
+  compute(time = now()) {
     this.gc();
     if (!this.started) {
       return List();
     }
-    const time = now();
     return Range(0, this.size)
       .map((n) => {
         let t = time - n * this.windowSize;
@@ -45,6 +49,42 @@ class Speed {
       })
       .filter((v) => v !== undefined);
   }
+
+  computeSum(time = now()) {
+    this.gc();
+    if (!this.started) {
+      return List();
+    }
+    return Range(0, this.size)
+      .map((n) => {
+        let t = time - n * this.windowSize;
+        t = t - (t % this.windowSize);
+        if (t >= this.started - (this.started % this.windowSize)) {
+          return this.sums.get(`${t}`, 0);
+        }
+      })
+      .filter((v) => v !== undefined);
+  }
+
+  toJSON() {
+    return {
+      windowSize: this.windowSize,
+      size: this.size,
+      counters: this.counters.toObject(),
+      sums: this.sums.toObject(),
+      started: this.started,
+      latest: this.latest,
+    };
+  }
+
+  static fromJSON(data) {
+    const speed = new Speed(data.windowSize, data.size);
+    speed.counters = Map(data.counters);
+    speed.sums = Map(data.sums);
+    speed.started = data.started;
+    speed.latest = data.latest;
+    return speed;
+  }
 }
 
 module.exports = {

package/src/lib/util.js

@@ -34,12 +34,28 @@ exports.createLog = (req, res) => {
   });
 };
 
-exports.aggregateSpeed = (entry, key) =>
+exports.aggregateCount = (entry, key) =>
   entry
     .getIn(['speed', key])
     .compute()
     .reduce((p, c) => p + c, 0);
 
+exports.aggregateSum = (entry, key) =>
+  entry
+    .getIn(['speed', key])
+    .computeSum()
+    .reduce((p, c) => p + c, 0);
+
+exports.formatDuration = (ms) => {
+  const totalSeconds = ms / 1000;
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = Math.round(totalSeconds % 60);
+  if (minutes > 0) {
+    return seconds > 0 ? `${minutes}m${seconds}s` : `${minutes}m`;
+  }
+  return `${totalSeconds.toFixed(1)}s`;
+};
+
 exports.formatTable = (data) => {
   if (!data || data.length === 0) {
     return '';

package/src/modules/address.js

@@ -1,9 +1,18 @@
+const { is } = require('immutable');
+
 const api = require('../app/api');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
+const { touch, prune, countRecent } = require('../lib/recent-map');
 
 const identifier = (log) => log.getIn(['address', 'value']);
 
+const signatureCount15m = (entry) =>
+  countRecent(entry.get('signatures'), 15 * 60);
+const signatureCount24h = (entry) => countRecent(entry.get('signatures'));
+
+let aggregator;
+
 function fill(log) {
   if (!log.hasIn(['address', 'value'])) {
     log = log.setIn(['address', 'value'], log.getIn(['request', 'address']));
@@ -16,11 +25,56 @@ function init() {
 }
 
 function start() {
-  const aggregator = new Aggregator();
+  aggregator = new Aggregator();
 
   aggregator.setIdentifier(identifier);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  const enricher = (entry, log) => {
+    for (const field of [
+      'address',
+      'identity',
+      'cloudflare',
+      'dnsbl',
+      'geoip',
+      'hostname',
+      'agent',
+      'language',
+      'signature',
+    ]) {
+      if (log.has(field) && !is(log.get(field), entry.get(field))) {
+        entry = entry.set(field, log.get(field));
+      }
+    }
+
+    // Distinct signature IDs with last-seen time, pruned to 24h; backs both
+    // the 15m and 24h counts
+    const signatureId = log.getIn(['signature', 'id']);
+    if (signatureId) {
+      entry = entry.update('signatures', (map) => touch(map, signatureId));
+    }
+
+    return entry;
+  };
+
+  aggregator.setEnricher(enricher);
+
+  aggregator.setEntryGc((entry) =>
+    entry.has('signatures') ? entry.update('signatures', prune) : entry
+  );
+
+  aggregator.formatter.insertFormat('signatureCount15m', signatureCount15m, {
+    before: 'count15m',
+  });
+  aggregator.formatter.insertFormat('signatureCount24h', signatureCount24h, {
+    before: 'count15m',
+  });
+
+  aggregator.sorters.signatureCount15m = signatureCount15m;
+  aggregator.sorters.signatureCount24h = signatureCount24h;
+
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('addresses', aggregator);
 }
@@ -28,4 +82,7 @@ function start() {
 module.exports = {
   init,
   start,
+  get aggregator() {
+    return aggregator;
+  },
 };

package/src/modules/agent.js

@@ -70,7 +70,7 @@ function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('agent', agentFormat, {
-    before: '15m',
+    before: 'count15m',
     color: 'grey',
   });
   aggregator.defaultFormatter.insertFormat('os', osFormat, {

package/src/modules/dnsbl.js

@@ -4,11 +4,19 @@ const aggregator = require('../lib/aggregator');
 const cache = require('../lib/cache');
 const pipeline = require('../lib/pipeline');
 
+// Indirection so tests can swap the DNS lookup without hitting the network.
+const defaultLookup = (ip, blacklist) => dnsbl.lookup(ip, blacklist);
+let lookup = defaultLookup;
+
+function setLookup(fn = defaultLookup) {
+  lookup = fn;
+}
+
 async function xblLookup(ip) {
   if (await cache.has(`xbl-${ip}`)) {
     return cache.get(`xbl-${ip}`);
   }
-  const result = await dnsbl.lookup(ip, 'xbl.spamhaus.org');
+  const result = await lookup(ip, 'xbl.spamhaus.org');
   cache.set(`xbl-${ip}`, result);
   return result;
 }
@@ -45,4 +53,6 @@ function init() {
 module.exports = {
   augment,
   init,
+  setLookup,
+  xblFormat,
 };

package/src/modules/history.js

@@ -0,0 +1,62 @@
+const { api } = require('../app');
+const constants = require('../constants');
+const LogBuffer = require('../lib/log-buffer');
+const persistence = require('../lib/persistence');
+const pipeline = require('../lib/pipeline');
+
+function start() {
+  const capacity =
+    (constants.modules.history && constants.modules.history.capacity) || 1000;
+  const buffers = {};
+
+  function registerNodeHistory(name, node) {
+    const buffer = new LogBuffer(capacity);
+    buffers[name] = buffer;
+    persistence.register(`history-${persistence.safeName(name)}`, buffer);
+
+    node.map((log) => {
+      buffer.push(log);
+      return log;
+    });
+
+    api.get(`/history/${name}.json`, (req, res) => {
+      const { identity, signature, address } = req.query;
+      const limit = parseInt(req.query.limit, 10) || 100;
+
+      let logs = buffer.toArray();
+
+      if (identity) {
+        logs = logs.filter((log) => log.get('identity') === identity);
+      }
+      if (signature) {
+        logs = logs.filter(
+          (log) => log.getIn(['signature', 'id']) === signature
+        );
+      }
+      if (address) {
+        logs = logs.filter(
+          (log) => log.getIn(['address', 'value']) === address
+        );
+      }
+
+      logs = logs.slice(0, limit);
+
+      res.json(logs);
+    });
+  }
+
+  for (const [name, node] of Object.entries(pipeline.nodes)) {
+    registerNodeHistory(name, node);
+  }
+
+  // Auto-register future nodes
+  const originalRegisterNode = pipeline.registerNode.bind(pipeline);
+  pipeline.registerNode = function (name, node) {
+    originalRegisterNode(name, node);
+    if (!buffers[name]) {
+      registerNodeHistory(name, node);
+    }
+  };
+}
+
+module.exports = { start };