@hyperwatch/hyperwatch 4.3.1 → 5.0.0

package/src/modules/history.js

@@ -1,42 +1,18 @@
 const { api } = require('../app');
 const constants = require('../constants');
+const LogBuffer = require('../lib/log-buffer');
+const persistence = require('../lib/persistence');
 const pipeline = require('../lib/pipeline');
 
-class CircularBuffer {
-  constructor(capacity = 1000) {
-    this.capacity = capacity;
-    this.buffer = new Array(capacity);
-    this.pointer = 0;
-    this.size = 0;
-  }
-
-  push(item) {
-    this.buffer[this.pointer] = item;
-    this.pointer = (this.pointer + 1) % this.capacity;
-    if (this.size < this.capacity) {
-      this.size++;
-    }
-  }
-
-  toArray() {
-    if (this.size < this.capacity) {
-      return this.buffer.slice(0, this.size).reverse();
-    }
-    return [
-      ...this.buffer.slice(this.pointer),
-      ...this.buffer.slice(0, this.pointer),
-    ].reverse();
-  }
-}
-
 function start() {
   const capacity =
     (constants.modules.history && constants.modules.history.capacity) || 1000;
   const buffers = {};
 
   function registerNodeHistory(name, node) {
-    const buffer = new CircularBuffer(capacity);
+    const buffer = new LogBuffer(capacity);
     buffers[name] = buffer;
+    persistence.register(`history-${persistence.safeName(name)}`, buffer);
 
     node.map((log) => {
       buffer.push(log);

package/src/modules/identity.js

@@ -3,21 +3,29 @@ const IPCIDR = require('ip-cidr').default;
 const api = require('../app/api');
 // Bot IP lists for identity verification
 // Run `node scripts/fetch-openai-ips.js` to update OpenAI lists
+// Run `node scripts/fetch-anthropic-ips.js` to update the Claude list
 const amazonSearchBotIps = require('../data/amazon-searchbot-ips.json');
 const amazonUserIps = require('../data/amazon-user-ips.json');
 const amazonBotIps = require('../data/amazonbot-ips.json');
 const chatgptUserIps = require('../data/chatgpt-user-ips.json');
+const claudeBotIps = require('../data/claude-bot-ips.json');
 const gptbotIps = require('../data/gptbot-ips.json');
 const openaiSearchbotIps = require('../data/openai-searchbot-ips.json');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
 
+// Anthropic publishes one list of ranges covering all Claude crawlers.
+// Reverse DNS is not usable here: Claude crawlers run on shared cloud
+// infrastructure, so their PTR records are not Anthropic-controlled.
+const claudeBotCidrs = claudeBotIps.map((cidr) => new IPCIDR(cidr));
+
 function augment(log) {
   const family = log.getIn(['agent', 'family']);
   const hostname = log.getIn(['address', 'hostname']);
   const address =
     log.getIn(['address', 'value']) || log.getIn(['request', 'address']);
   const signature = log.getIn(['signature', 'id']);
+  const signatureAgent = log.getIn(['request', 'headers', 'signature-agent']);
 
   switch (family) {
     // Per hostname
@@ -177,6 +185,11 @@ function augment(log) {
       return hostname && hostname.endsWith('.babbar.eu')
         ? log.set('identity', 'Babbar')
         : log;
+    case 'Reflectionbot':
+      // https://reflection.ai/bot
+      return hostname && hostname.endsWith('.reflection.ai')
+        ? log.set('identity', 'Reflection')
+        : log;
     case 'bnf.fr bot':
       return hostname && hostname.endsWith('.bnf.fr')
         ? log.set('identity', 'BnF.fr')
@@ -237,6 +250,22 @@ function augment(log) {
       return hostname && hostname.endsWith('.blex.seranking.com')
         ? log.set('identity', 'SE Ranking')
         : log;
+    case 'SofyaBot':
+      return hostname && hostname.endsWith('.sofya.co')
+        ? log.set('identity', 'Sofya')
+        : log;
+    case 'YouBot':
+      // https://docs.you.com/youbot
+      return hostname && hostname.endsWith('.search.you.com')
+        ? log.set('identity', 'You.com')
+        : log;
+    case 'AIWebIndex':
+    case 'AIWebIndex-Agent':
+      // Lyrenth AI-readable web index, forward-confirmed rDNS under lyrenth.com
+      // https://lyrenth.com/bot
+      return hostname && hostname.endsWith('.lyrenth.com')
+        ? log.set('identity', 'Lyrenth')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -291,11 +320,18 @@ function augment(log) {
       return chatgptUserIps.some((cidr) => new IPCIDR(cidr).contains(address))
         ? log.set('identity', 'ChatGPT')
         : log;
+    case 'ClaudeBot':
+    case 'Claude-User':
+    case 'Claude-SearchBot':
+    case 'Claude-Web':
+    case 'anthropic-ai':
+      // https://claude.com/crawling/bots.json
+      return address && claudeBotCidrs.some((cidr) => cidr.contains(address))
+        ? log.set('identity', 'Claude')
+        : log;
     case 'meta-externalagent':
     case 'meta-webindexer':
-      return address &&
-        (new IPCIDR('2a03:2880::/29').contains(address) ||
-          new IPCIDR('2a06:98c0:3600::/48').contains(address))
+      return address && new IPCIDR('2a03:2880::/29').contains(address)
         ? log.set('identity', 'Meta')
         : log;
 
@@ -335,10 +371,6 @@ function augment(log) {
         hostname.endsWith('.eu-central-1.compute.amazonaws.com')
         ? log.set('identity', 'Wise')
         : log;
-    case 'ClaudeBot':
-      return hostname && hostname.endsWith('.us-east-2.compute.amazonaws.com')
-        ? log.set('identity', 'Claude')
-        : log;
     case 'PerplexityBot':
       return hostname && hostname.endsWith('.compute-1.amazonaws.com')
         ? log.set('identity', 'Perplexity')
@@ -365,6 +397,12 @@ function augment(log) {
       return hostname && hostname.endsWith('.googleusercontent.com')
         ? log.set('identity', 'Dolfe')
         : log;
+    case 'ShapBot':
+      // Parallel Web Systems crawler, runs on GCE
+      // https://docs.parallel.ai/resources/crawler
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Parallel')
+        : log;
 
     // Hetzner
     case 'Ubermetrics':
@@ -391,6 +429,15 @@ function augment(log) {
       return hostname && hostname.endsWith('.flipboard.com')
         ? log.set('identity', 'Flipboard')
         : log;
+
+    // Web Bot Auth
+    case 'ExaSearchBot':
+      // Exa search crawler. No published IP ranges or reverse DNS: requests
+      // are signed (RFC 9421) and carry a `Signature-Agent` header
+      // https://crawler.exa.ai/
+      return signatureAgent && signatureAgent.includes('https://crawler.exa.ai')
+        ? log.set('identity', 'Exa')
+        : log;
   }
 
   // Hostname only
@@ -448,7 +495,9 @@ function start() {
 
   aggregator.setIdentifier(identifier);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('identities', aggregator);
 }

package/src/modules/index.js

@@ -1,6 +1,7 @@
 const debug = require('debug');
 
 const constants = require('../constants');
+const pipeline = require('../lib/pipeline');
 
 const debugModules = debug('hyperwatch:modules');
 
@@ -38,27 +39,35 @@ function get(module) {
   }
 }
 
-function activeModules() {
+function activeModulesWithKeys() {
   return Object.keys(constants.modules)
     .map((key) => Object.assign({ key }, constants.modules[key]))
     .sort((a, b) => a.priority - b.priority)
     .filter((m) => m.active === true)
-    .map((m) => get(m.key))
-    .filter((m) => m);
+    .map((m) => ({ key: m.key, module: get(m.key) }))
+    .filter((m) => m.module);
+}
+
+function activeModules() {
+  return activeModulesWithKeys().map((m) => m.module);
 }
 
 function init() {
-  for (const module of activeModules()) {
+  for (const { key, module } of activeModulesWithKeys()) {
     if (module && module.init) {
+      pipeline.currentModule = key;
       module.init();
+      pipeline.currentModule = null;
     }
   }
 }
 
 function start() {
-  for (const module of activeModules()) {
+  for (const { key, module } of activeModulesWithKeys()) {
     if (module && module.start) {
+      pipeline.currentModule = key;
       module.start();
+      pipeline.currentModule = null;
     }
   }
 }

package/src/modules/signature.js

@@ -1,9 +1,10 @@
-const { is, Set } = require('immutable');
+const { is } = require('immutable');
 
 const api = require('../app/api');
-const { Aggregator } = require('../lib/aggregator');
+const { Aggregator, lastSeen, statusCount } = require('../lib/aggregator');
 const { Formatter } = require('../lib/formatter');
 const pipeline = require('../lib/pipeline');
+const { touch, prune, countRecent } = require('../lib/recent-map');
 const {
   aggregateCount,
   aggregateSum,
@@ -40,6 +41,9 @@ function normalisedIdentityHeader(headers) {
   return obj;
 }
 
+const addressCount15m = (entry) => countRecent(entry.get('addresses'), 15 * 60);
+const addressCount24h = (entry) => countRecent(entry.get('addresses'));
+
 function computeSignature(headers) {
   const string = Object.keys(headers)
     .map((key) => [key, headers[key]].join(':'))
@@ -65,8 +69,10 @@ function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 }
 
+let _aggregator;
+
 function start() {
-  const aggregator = new Aggregator();
+  const aggregator = (_aggregator = new Aggregator());
 
   aggregator.setIdentifier((log) => log.getIn(['signature', 'id']));
 
@@ -75,15 +81,24 @@ function start() {
   signatureFormatter.setFormats([
     ['signature', (entry) => entry.getIn(['signature', 'id'])],
     ['identity', (entry) => entry.get('identity')],
-    ['addressCount', (entry) => entry.get('addresses').size],
+    ['addressCount15m', addressCount15m],
+    ['addressCount24h', addressCount24h],
     [
       'addresses',
       (entry) =>
-        entry
-          .get('addresses')
-          .map((address) => address.get('value'))
-          .slice(0, 10)
-          .join('<br>'),
+        entry.has('addresses')
+          ? entry.get('addresses').keySeq().slice(0, 10).join('<br>')
+          : '',
+    ],
+    [
+      'lastAddress',
+      (entry) => {
+        const addr = entry.get('lastAddress');
+        if (!addr) {
+          return '';
+        }
+        return addr.get('hostname') || addr.get('value') || '';
+      },
     ],
 
     [
@@ -96,9 +111,15 @@ function start() {
       },
     ],
 
+    ['lastSeen', lastSeen],
     ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
     ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
 
+    ['2xx15m', statusCount('2xx_per_minute')],
+    ['2xx24h', statusCount('2xx_per_hour')],
+    ['4xx15m', statusCount('4xx_per_minute')],
+    ['4xx24h', statusCount('4xx_per_hour')],
+
     [
       'execTime15m',
       (entry) => formatDuration(aggregateSum(entry, 'per_minute')),
@@ -121,10 +142,12 @@ function start() {
     }
 
     const address = log.get('address');
-    if (!entry.has('addresses')) {
-      entry = entry.set('addresses', new Set([address]));
-    } else if (!entry.get('addresses').has(address)) {
-      entry = entry.update('addresses', (set) => set.add(address));
+    entry = entry.set('lastAddress', address);
+    // Distinct IPs with last-seen time, pruned to 24h; backs both the 15m
+    // and 24h counts
+    const value = address && address.get('value');
+    if (value) {
+      entry = entry.update('addresses', (map) => touch(map, value));
     }
 
     return entry;
@@ -132,7 +155,16 @@ function start() {
 
   aggregator.setEnricher(enricher);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  aggregator.setEntryGc((entry) =>
+    entry.has('addresses') ? entry.update('addresses', prune) : entry
+  );
+
+  aggregator.sorters.addressCount15m = addressCount15m;
+  aggregator.sorters.addressCount24h = addressCount24h;
+
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('signatures', aggregator);
 }
@@ -140,4 +172,7 @@ function start() {
 module.exports = {
   init,
   start,
+  get aggregator() {
+    return _aggregator;
+  },
 };

package/src/modules/sparkline.js

@@ -1,6 +1,10 @@
 const aggregator = require('../lib/aggregator');
 
-const sparkline = (entry, key) => {
+const sparkline = (entry, key, output) => {
+  if (output === 'text') {
+    return;
+  }
+
   const id = entry.get('id');
 
   const points = entry
@@ -22,8 +26,8 @@ sparkline ('${id}', ${JSON.stringify(points)}, '#797979', 14, 5);
 };
 
 function init() {
-  aggregator.defaultFormatter.insertFormat('activity', (entry) =>
-    sparkline(entry, 'per_minute')
+  aggregator.defaultFormatter.insertFormat('activity', (entry, output) =>
+    sparkline(entry, 'per_minute', output)
   );
 }
 

package/src/modules/status.js

@@ -23,10 +23,15 @@ function mapper(entry, format) {
 }
 
 function start() {
-  api.get('/status(.:format(txt|json))?', (req, res) => {
+  api.get('/status{.:format}', (req, res) => {
     const raw = req.query.raw ? true : false;
     const format = req.params.format || (raw ? 'json' : null);
 
+    if (format && !['json', 'txt'].includes(format)) {
+      res.sendStatus(404);
+      return;
+    }
+
     let rawData = monitoring.getAllComputed();
 
     if (req.query.type) {