@hyperwatch/hyperwatch 4.3.1 → 5.0.1

package/src/lib/pipeline.js

@@ -128,35 +128,79 @@ class Builder {
     this.children = [];
     this.name = null;
     this.pipeline = pipeline;
+    this.parent = null;
+    this.op = null;
+    this.module = null;
+    this.fnName = null;
+    this._label = null;
+    // Lightweight per-node counter for throughput tracking
+    this.counter = 0;
+    // Fixed one-second buckets keep rate tracking bounded and O(1) per event.
+    this.rateBuckets = Array.from({ length: 10 }, () => ({
+      second: null,
+      count: 0,
+    }));
   }
 
-  add(xf) {
+  add(xf, { op = null, fnName = null, label = null } = {}) {
     const child = new Builder(xf, this.pipeline);
+    child.parent = this;
+    child.op = op;
+    child.module = this.pipeline ? this.pipeline.currentModule : null;
+    child.fnName = fnName || null;
+    child._label = label || null;
     this.children.push(child);
     return child;
   }
 
-  map(f) {
-    return this.add(map(f));
+  map(f, label) {
+    return this.add(map(f), { op: 'map', fnName: f.name, label });
   }
 
-  filter(pred) {
-    return this.add(filter(pred));
+  filter(pred, label) {
+    return this.add(filter(pred), { op: 'filter', fnName: pred.name, label });
   }
 
-  split(pred) {
-    return [this.add(filter(pred)), this.add(filter(complement(pred)))];
+  split(pred, labels) {
+    return [
+      this.add(filter(pred), {
+        op: 'split-true',
+        fnName: pred.name,
+        label: labels && labels[0],
+      }),
+      this.add(filter(complement(pred)), {
+        op: 'split-false',
+        fnName: pred.name,
+        label: labels && labels[1],
+      }),
+    ];
   }
 
-  by(f) {
-    return this.add(by(f));
+  by(f, label) {
+    return this.add(by(f), { op: 'by', fnName: f.name, label });
   }
 
   create() {
+    const self = this;
+    const countXf = (stream) => (event) => {
+      self.counter++;
+      const second = Math.floor(Date.now() / 1000);
+      const bucket = self.rateBuckets[second % self.rateBuckets.length];
+      if (bucket.second !== second) {
+        bucket.second = second;
+        bucket.count = 0;
+      }
+      bucket.count++;
+      forward(stream, event);
+    };
     if (this.children.length === 0) {
-      return this.xf;
+      return comp([this.xf, countXf]);
     }
-    return comp([this.xf, multiplex(this.children.map((b) => b.create()))]);
+    return comp([
+      this.xf,
+      countXf,
+      multiplex(this.children.map((b) => b.create())),
+    ]);
   }
 
   // Helpers
@@ -194,6 +238,9 @@ class Pipeline extends Builder {
       main: this,
     };
     this.pipeline = this;
+    this.name = 'raw';
+    this.currentModule = null;
+    this.op = 'identity';
   }
 
   registerInput(input) {
@@ -204,6 +251,16 @@ class Pipeline extends Builder {
       type: 'input',
     });
     this.monitors.push(monitor);
+    // Create a tap node so this input gets its own /logs/ WebSocket endpoint
+    const inputIndex = this.inputs.length;
+    const nodeName = `input-${inputIndex}`;
+    const tap = new Builder(identity(), this);
+    tap.name = nodeName;
+    tap.op = 'input';
+    tap.module = null;
+    tap.fnName = null;
+    this.registerNode(nodeName, tap);
+    input._tap = tap;
   }
 
   start() {
@@ -213,6 +270,8 @@ class Pipeline extends Builder {
       const monitor = this.monitors.find(
         (monitor) => monitor.type === 'input' && monitor.name === input.name
       );
+      // Build the tap stream so it can forward logs to WebSocket clients
+      const tapStream = input._tap ? input._tap.create()(() => {}) : null;
       input.start({
         success: (log) => {
           const valid = validate(log.toJS());
@@ -224,6 +283,9 @@ class Pipeline extends Builder {
               data: log,
             });
             monitor.hit('accepted');
+            if (tapStream) {
+              forward(tapStream, event);
+            }
             forward(stream, event);
           } else {
             monitor.hit('rejected');
@@ -249,19 +311,83 @@ class Pipeline extends Builder {
     });
   }
 
-  stop() {
-    return Promise.all(
-      this.inputs.filter((input) => input.stop).map((input) => input.stop())
+  // Stops every input, even if some of them throw or reject, then rejects
+  // with their errors if any failed
+  async stop() {
+    const results = await Promise.allSettled(
+      this.inputs
+        .filter((input) => input.stop)
+        .map((input) => Promise.resolve().then(() => input.stop()))
     );
+    const errors = results
+      .filter((result) => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(
+        errors,
+        `${errors.length} input(s) failed to stop: ${errors
+          .map((err) => err.message)
+          .join('; ')}`
+      );
+    }
   }
 
   registerNode(name, node) {
+    const previous = this.nodes[name];
+    if (previous && previous !== node && previous.name === name) {
+      previous.name = null;
+    }
     this.nodes[name] = node;
   }
 
   getNode(name) {
     return this.nodes[name];
   }
+
+  getTree(node = this) {
+    const second = Math.floor(Date.now() / 1000);
+    const windowStart = second - node.rateBuckets.length + 1;
+    const rateCount = node.rateBuckets.reduce(
+      (count, bucket) =>
+        bucket.second >= windowStart && bucket.second <= second
+          ? count + bucket.count
+          : count,
+      0
+    );
+    const obj = {
+      name: node.name || null,
+      op: node.op || null,
+      module: node.module || null,
+      fnName: node.fnName || null,
+      label: node._label || null,
+      count: node.counter,
+      rate: rateCount / node.rateBuckets.length,
+      children: node.children.map((child) => this.getTree(child)),
+    };
+    // Include inputs at the root level
+    if (node === this) {
+      obj.inputs = this.inputs.map((input) => {
+        const monitor = this.monitors.find(
+          (m) => m.type === 'input' && m.name === input.name
+        );
+        const accepted = monitor
+          ? monitor.speeds.accepted.per_minute.compute()
+          : null;
+        const rejected = monitor
+          ? monitor.speeds.rejected.per_minute.compute()
+          : null;
+        return {
+          name: input.name,
+          node: input._tap ? input._tap.name : null,
+          tree: input._tap ? this.getTree(input._tap) : null,
+          status: monitor ? monitor.status : null,
+          accepted: accepted ? accepted.reduce((a, b) => a + b, 0) : 0,
+          rejected: rejected ? rejected.reduce((a, b) => a + b, 0) : 0,
+        };
+      });
+    }
+    return obj;
+  }
 }
 
 module.exports = new Pipeline();

package/src/lib/recent-map.js

@@ -0,0 +1,23 @@
+/**
+ * Track distinct keys with the time they were last seen, bounded to a
+ * rolling window. Backs the addressCount15m/24h and signatureCount15m/24h
+ * counters.
+ */
+const { Map } = require('immutable');
+
+const { now } = require('./util');
+
+// Rolling window in seconds
+const WINDOW = 24 * 3600;
+
+// Record `key` as seen at `time` in an Immutable Map<key, unix seconds>
+const touch = (map = Map(), key, time = now()) => map.set(key, time);
+
+// Drop keys not seen in the last WINDOW seconds
+const prune = (map, time = now()) => map.filter((seen) => seen > time - WINDOW);
+
+// Number of keys seen in the last `window` seconds
+const countRecent = (map, window = WINDOW, time = now()) =>
+  map ? map.count((seen) => seen > time - window) : 0;
+
+module.exports = { WINDOW, touch, prune, countRecent };

package/src/modules/address.js

@@ -1,9 +1,18 @@
+const { is } = require('immutable');
+
 const api = require('../app/api');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
+const { touch, prune, countRecent } = require('../lib/recent-map');
 
 const identifier = (log) => log.getIn(['address', 'value']);
 
+const signatureCount15m = (entry) =>
+  countRecent(entry.get('signatures'), 15 * 60);
+const signatureCount24h = (entry) => countRecent(entry.get('signatures'));
+
+let aggregator;
+
 function fill(log) {
   if (!log.hasIn(['address', 'value'])) {
     log = log.setIn(['address', 'value'], log.getIn(['request', 'address']));
@@ -16,11 +25,56 @@ function init() {
 }
 
 function start() {
-  const aggregator = new Aggregator();
+  aggregator = new Aggregator();
 
   aggregator.setIdentifier(identifier);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  const enricher = (entry, log) => {
+    for (const field of [
+      'address',
+      'identity',
+      'cloudflare',
+      'dnsbl',
+      'geoip',
+      'hostname',
+      'agent',
+      'language',
+      'signature',
+    ]) {
+      if (log.has(field) && !is(log.get(field), entry.get(field))) {
+        entry = entry.set(field, log.get(field));
+      }
+    }
+
+    // Distinct signature IDs with last-seen time, pruned to 24h; backs both
+    // the 15m and 24h counts
+    const signatureId = log.getIn(['signature', 'id']);
+    if (signatureId) {
+      entry = entry.update('signatures', (map) => touch(map, signatureId));
+    }
+
+    return entry;
+  };
+
+  aggregator.setEnricher(enricher);
+
+  aggregator.setEntryGc((entry) =>
+    entry.has('signatures') ? entry.update('signatures', prune) : entry
+  );
+
+  aggregator.formatter.insertFormat('signatureCount15m', signatureCount15m, {
+    before: 'count15m',
+  });
+  aggregator.formatter.insertFormat('signatureCount24h', signatureCount24h, {
+    before: 'count15m',
+  });
+
+  aggregator.sorters.signatureCount15m = signatureCount15m;
+  aggregator.sorters.signatureCount24h = signatureCount24h;
+
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('addresses', aggregator);
 }
@@ -28,4 +82,7 @@ function start() {
 module.exports = {
   init,
   start,
+  get aggregator() {
+    return aggregator;
+  },
 };

package/src/modules/dnsbl.js

@@ -4,11 +4,19 @@ const aggregator = require('../lib/aggregator');
 const cache = require('../lib/cache');
 const pipeline = require('../lib/pipeline');
 
+// Indirection so tests can swap the DNS lookup without hitting the network.
+const defaultLookup = (ip, blacklist) => dnsbl.lookup(ip, blacklist);
+let lookup = defaultLookup;
+
+function setLookup(fn = defaultLookup) {
+  lookup = fn;
+}
+
 async function xblLookup(ip) {
   if (await cache.has(`xbl-${ip}`)) {
     return cache.get(`xbl-${ip}`);
   }
-  const result = await dnsbl.lookup(ip, 'xbl.spamhaus.org');
+  const result = await lookup(ip, 'xbl.spamhaus.org');
   cache.set(`xbl-${ip}`, result);
   return result;
 }
@@ -45,4 +53,6 @@ function init() {
 module.exports = {
   augment,
   init,
+  setLookup,
+  xblFormat,
 };

package/src/modules/history.js

@@ -1,42 +1,18 @@
 const { api } = require('../app');
 const constants = require('../constants');
+const LogBuffer = require('../lib/log-buffer');
+const persistence = require('../lib/persistence');
 const pipeline = require('../lib/pipeline');
 
-class CircularBuffer {
-  constructor(capacity = 1000) {
-    this.capacity = capacity;
-    this.buffer = new Array(capacity);
-    this.pointer = 0;
-    this.size = 0;
-  }
-
-  push(item) {
-    this.buffer[this.pointer] = item;
-    this.pointer = (this.pointer + 1) % this.capacity;
-    if (this.size < this.capacity) {
-      this.size++;
-    }
-  }
-
-  toArray() {
-    if (this.size < this.capacity) {
-      return this.buffer.slice(0, this.size).reverse();
-    }
-    return [
-      ...this.buffer.slice(this.pointer),
-      ...this.buffer.slice(0, this.pointer),
-    ].reverse();
-  }
-}
-
 function start() {
   const capacity =
     (constants.modules.history && constants.modules.history.capacity) || 1000;
   const buffers = {};
 
   function registerNodeHistory(name, node) {
-    const buffer = new CircularBuffer(capacity);
+    const buffer = new LogBuffer(capacity);
     buffers[name] = buffer;
+    persistence.register(`history-${persistence.safeName(name)}`, buffer);
 
     node.map((log) => {
       buffer.push(log);

package/src/modules/hostname.js

@@ -1,4 +1,5 @@
 const dns = require('dns').promises;
+const net = require('net');
 
 const debug = require('debug')('hyperwatch:hostname');
 
@@ -13,6 +14,12 @@ function ignoreError() {
   return null;
 }
 
+// Compare addresses in canonical form: DNS and logs may write the same IPv6
+// address differently (zero compression, case)
+function canonical(ip) {
+  return net.isIPv6(ip) ? new URL(`http://[${ip}]`).hostname : ip;
+}
+
 function isValid(hostname) {
   return (
     hostname &&
@@ -40,11 +47,12 @@ async function lookup(ip, { fast = false } = {}) {
     if (isValid(reverse)) {
       entry.value = reverse;
       debug(`Resolve ${reverse} ...`);
-      const reverseIps = await dns.resolve(reverse).catch(ignoreError);
+      const reverseIps = await (
+        net.isIPv6(ip) ? dns.resolve6(reverse) : dns.resolve4(reverse)
+      ).catch(ignoreError);
       if (reverseIps) {
-        const reverseIp = reverseIps[0];
-        debug(`Resolve ${reverse}: ${reverseIp}`);
-        if (reverseIp === ip) {
+        debug(`Resolve ${reverse}: ${reverseIps.join(', ')}`);
+        if (reverseIps.map(canonical).includes(canonical(ip))) {
           entry.verified = true;
         }
       } else {

package/src/modules/identity.js

@@ -3,21 +3,34 @@ const IPCIDR = require('ip-cidr').default;
 const api = require('../app/api');
 // Bot IP lists for identity verification
 // Run `node scripts/fetch-openai-ips.js` to update OpenAI lists
+// Run `node scripts/fetch-anthropic-ips.js` to update the Claude list
 const amazonSearchBotIps = require('../data/amazon-searchbot-ips.json');
 const amazonUserIps = require('../data/amazon-user-ips.json');
 const amazonBotIps = require('../data/amazonbot-ips.json');
 const chatgptUserIps = require('../data/chatgpt-user-ips.json');
+const claudeBotIps = require('../data/claude-bot-ips.json');
 const gptbotIps = require('../data/gptbot-ips.json');
 const openaiSearchbotIps = require('../data/openai-searchbot-ips.json');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
 
+// Anthropic publishes one list of ranges covering all Claude crawlers.
+// Reverse DNS is not usable here: Claude crawlers run on shared cloud
+// infrastructure, so their PTR records are not Anthropic-controlled.
+const claudeBotCidrs = claudeBotIps.map((cidr) => new IPCIDR(cidr));
+
 function augment(log) {
   const family = log.getIn(['agent', 'family']);
   const hostname = log.getIn(['address', 'hostname']);
+  // Reverse DNS confirmed by a forward lookup back to the same address. The
+  // older identities below still use the unconfirmed hostname.
+  const verifiedHostname = log.getIn(['hostname', 'verified'])
+    ? hostname
+    : undefined;
   const address =
     log.getIn(['address', 'value']) || log.getIn(['request', 'address']);
   const signature = log.getIn(['signature', 'id']);
+  const signatureAgent = log.getIn(['request', 'headers', 'signature-agent']);
 
   switch (family) {
     // Per hostname
@@ -54,8 +67,23 @@ function augment(log) {
       return hostname && hostname.endsWith('.yandex.com')
         ? log.set('identity', 'Yandex')
         : log;
+    // https://www.semrush.com/bot/ (SemrushBot-BA and SemrushBot-SI parse
+    // as SemrushBot)
     case 'SemrushBot':
-      if (hostname && hostname.endsWith('.semrush.com')) {
+    case 'SemrushBot-SWA':
+    case 'SemrushBot-OCOB':
+    case 'SemrushBot-FT':
+    case 'SemrushBot-ESI':
+    case 'SiteAuditBot':
+    case 'SplitSignalBot':
+    case 'RyteBot':
+      // Some SemrushBot addresses have the generic PTR bot.semrush.com, which
+      // doesn't resolve back to them: accept Semrush's own range too
+      // (85.208.98.0/24, announced by AS209366)
+      if (
+        (verifiedHostname && verifiedHostname.endsWith('.semrush.com')) ||
+        (address && new IPCIDR('85.208.98.0/24').contains(address))
+      ) {
         return log.set('identity', 'Semrush');
       }
       break;
@@ -177,6 +205,16 @@ function augment(log) {
       return hostname && hostname.endsWith('.babbar.eu')
         ? log.set('identity', 'Babbar')
         : log;
+    case 'Reflectionbot':
+      // https://reflection.ai/bot
+      return verifiedHostname && verifiedHostname.endsWith('.reflection.ai')
+        ? log.set('identity', 'Reflection')
+        : log;
+    case 'SEOkicks':
+      // https://www.seokicks.de/robot.html
+      return verifiedHostname && verifiedHostname.endsWith('.seokicks.de')
+        ? log.set('identity', 'SEOkicks')
+        : log;
     case 'bnf.fr bot':
       return hostname && hostname.endsWith('.bnf.fr')
         ? log.set('identity', 'BnF.fr')
@@ -237,6 +275,22 @@ function augment(log) {
       return hostname && hostname.endsWith('.blex.seranking.com')
         ? log.set('identity', 'SE Ranking')
         : log;
+    case 'SofyaBot':
+      return hostname && hostname.endsWith('.sofya.co')
+        ? log.set('identity', 'Sofya')
+        : log;
+    case 'YouBot':
+      // https://docs.you.com/youbot
+      return hostname && hostname.endsWith('.search.you.com')
+        ? log.set('identity', 'You.com')
+        : log;
+    case 'AIWebIndex':
+    case 'AIWebIndex-Agent':
+      // Lyrenth AI-readable web index, forward-confirmed rDNS under lyrenth.com
+      // https://lyrenth.com/bot
+      return hostname && hostname.endsWith('.lyrenth.com')
+        ? log.set('identity', 'Lyrenth')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -276,6 +330,11 @@ function augment(log) {
       return address && new IPCIDR('203.133.160.0/19').contains(address)
         ? log.set('identity', family)
         : log;
+    case 'LinkupBot':
+      // https://www.linkup.so/linkupbot-ips.txt
+      return address && new IPCIDR('35.198.113.100/32').contains(address)
+        ? log.set('identity', 'Linkup')
+        : log;
     case 'OAI-SearchBot':
       return openaiSearchbotIps.some((cidr) =>
         new IPCIDR(cidr).contains(address)
@@ -291,11 +350,18 @@ function augment(log) {
       return chatgptUserIps.some((cidr) => new IPCIDR(cidr).contains(address))
         ? log.set('identity', 'ChatGPT')
         : log;
+    case 'ClaudeBot':
+    case 'Claude-User':
+    case 'Claude-SearchBot':
+    case 'Claude-Web':
+    case 'anthropic-ai':
+      // https://claude.com/crawling/bots.json
+      return address && claudeBotCidrs.some((cidr) => cidr.contains(address))
+        ? log.set('identity', 'Claude')
+        : log;
     case 'meta-externalagent':
     case 'meta-webindexer':
-      return address &&
-        (new IPCIDR('2a03:2880::/29').contains(address) ||
-          new IPCIDR('2a06:98c0:3600::/48').contains(address))
+      return address && new IPCIDR('2a03:2880::/29').contains(address)
         ? log.set('identity', 'Meta')
         : log;
 
@@ -335,10 +401,6 @@ function augment(log) {
         hostname.endsWith('.eu-central-1.compute.amazonaws.com')
         ? log.set('identity', 'Wise')
         : log;
-    case 'ClaudeBot':
-      return hostname && hostname.endsWith('.us-east-2.compute.amazonaws.com')
-        ? log.set('identity', 'Claude')
-        : log;
     case 'PerplexityBot':
       return hostname && hostname.endsWith('.compute-1.amazonaws.com')
         ? log.set('identity', 'Perplexity')
@@ -365,6 +427,12 @@ function augment(log) {
       return hostname && hostname.endsWith('.googleusercontent.com')
         ? log.set('identity', 'Dolfe')
         : log;
+    case 'ShapBot':
+      // Parallel Web Systems crawler, runs on GCE
+      // https://docs.parallel.ai/resources/crawler
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Parallel')
+        : log;
 
     // Hetzner
     case 'Ubermetrics':
@@ -391,6 +459,15 @@ function augment(log) {
       return hostname && hostname.endsWith('.flipboard.com')
         ? log.set('identity', 'Flipboard')
         : log;
+
+    // Web Bot Auth
+    case 'ExaSearchBot':
+      // Exa search crawler. No published IP ranges or reverse DNS: requests
+      // are signed (RFC 9421) and carry a `Signature-Agent` header
+      // https://crawler.exa.ai/
+      return signatureAgent && signatureAgent.includes('https://crawler.exa.ai')
+        ? log.set('identity', 'Exa')
+        : log;
   }
 
   // Hostname only
@@ -416,6 +493,10 @@ function augment(log) {
     if (hostname.endsWith('.qwant.com')) {
       return log.set('identity', 'Qwant');
     }
+    // Semrush crawlers not named above
+    if (verifiedHostname && verifiedHostname.endsWith('.semrush.com')) {
+      return log.set('identity', 'Semrush');
+    }
   }
 
   // Signature
@@ -448,7 +529,9 @@ function start() {
 
   aggregator.setIdentifier(identifier);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('identities', aggregator);
 }

package/src/modules/index.js

@@ -1,6 +1,7 @@
 const debug = require('debug');
 
 const constants = require('../constants');
+const pipeline = require('../lib/pipeline');
 
 const debugModules = debug('hyperwatch:modules');
 
@@ -38,27 +39,35 @@ function get(module) {
   }
 }
 
-function activeModules() {
+function activeModulesWithKeys() {
   return Object.keys(constants.modules)
     .map((key) => Object.assign({ key }, constants.modules[key]))
     .sort((a, b) => a.priority - b.priority)
     .filter((m) => m.active === true)
-    .map((m) => get(m.key))
-    .filter((m) => m);
+    .map((m) => ({ key: m.key, module: get(m.key) }))
+    .filter((m) => m.module);
+}
+
+function activeModules() {
+  return activeModulesWithKeys().map((m) => m.module);
 }
 
 function init() {
-  for (const module of activeModules()) {
+  for (const { key, module } of activeModulesWithKeys()) {
     if (module && module.init) {
+      pipeline.currentModule = key;
       module.init();
+      pipeline.currentModule = null;
     }
   }
 }
 
 function start() {
-  for (const module of activeModules()) {
+  for (const { key, module } of activeModulesWithKeys()) {
     if (module && module.start) {
+      pipeline.currentModule = key;
       module.start();
+      pipeline.currentModule = null;
     }
   }
 }