@hyperwatch/hyperwatch 4.2.0 → 5.0.0

package/src/modules/identity.js

@@ -1,15 +1,31 @@
-const IPCIDR = require('ip-cidr');
+const IPCIDR = require('ip-cidr').default;
 
 const api = require('../app/api');
+// Bot IP lists for identity verification
+// Run `node scripts/fetch-openai-ips.js` to update OpenAI lists
+// Run `node scripts/fetch-anthropic-ips.js` to update the Claude list
+const amazonSearchBotIps = require('../data/amazon-searchbot-ips.json');
+const amazonUserIps = require('../data/amazon-user-ips.json');
+const amazonBotIps = require('../data/amazonbot-ips.json');
+const chatgptUserIps = require('../data/chatgpt-user-ips.json');
+const claudeBotIps = require('../data/claude-bot-ips.json');
+const gptbotIps = require('../data/gptbot-ips.json');
+const openaiSearchbotIps = require('../data/openai-searchbot-ips.json');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
 
+// Anthropic publishes one list of ranges covering all Claude crawlers.
+// Reverse DNS is not usable here: Claude crawlers run on shared cloud
+// infrastructure, so their PTR records are not Anthropic-controlled.
+const claudeBotCidrs = claudeBotIps.map((cidr) => new IPCIDR(cidr));
+
 function augment(log) {
   const family = log.getIn(['agent', 'family']);
   const hostname = log.getIn(['address', 'hostname']);
   const address =
     log.getIn(['address', 'value']) || log.getIn(['request', 'address']);
   const signature = log.getIn(['signature', 'id']);
+  const signatureAgent = log.getIn(['request', 'headers', 'signature-agent']);
 
   switch (family) {
     // Per hostname
@@ -69,6 +85,7 @@ function augment(log) {
       return hostname && hostname.endsWith('.mojeek.com')
         ? log.set('identity', 'Mojeek')
         : log;
+    case 'AwarioBot':
     case 'BLEXBot':
       return hostname && hostname.endsWith('.webmeup.com')
         ? log.set('identity', 'WebMeUp')
@@ -89,13 +106,11 @@ function augment(log) {
       return hostname && hostname.endsWith('.naver.com')
         ? log.set('identity', 'Naver')
         : log;
-    case 'FacebookBot':
-      return hostname && hostname.endsWith('.fbsv.net')
-        ? log.set('identity', 'Facebook')
-        : log;
+
     case 'Stripe':
+    case 'Stripebot':
       return hostname && hostname.endsWith('.stripe.com')
-        ? log.set('identity', family)
+        ? log.set('identity', 'Stripe')
         : log;
     case 'UptimeRobot':
       return hostname && hostname.endsWith('.uptimerobot.com')
@@ -127,9 +142,17 @@ function augment(log) {
         ? log.set('identity', 'Sirportly')
         : log;
     case 'Bytespider':
-      return hostname && hostname.endsWith('.crawl.bytedance.com')
+      return hostname &&
+        (hostname.endsWith('.crawl.bytedance.com') ||
+          hostname.endsWith('.ap-southeast-1.compute.amazonaws.com'))
         ? log.set('identity', family)
         : log;
+    case 'TikTokSpider':
+      return hostname &&
+        (hostname.endsWith('.crawl.bytedance.com') ||
+          hostname.endsWith('.ap-southeast-1.compute.amazonaws.com'))
+        ? log.set('identity', 'TikTok')
+        : log;
     case 'Mail.RU Bot':
     case 'Mail.RU Bot Img':
     case 'Mail.RU Bot Fast':
@@ -162,6 +185,11 @@ function augment(log) {
       return hostname && hostname.endsWith('.babbar.eu')
         ? log.set('identity', 'Babbar')
         : log;
+    case 'Reflectionbot':
+      // https://reflection.ai/bot
+      return hostname && hostname.endsWith('.reflection.ai')
+        ? log.set('identity', 'Reflection')
+        : log;
     case 'bnf.fr bot':
       return hostname && hostname.endsWith('.bnf.fr')
         ? log.set('identity', 'BnF.fr')
@@ -182,6 +210,14 @@ function augment(log) {
       return hostname && hostname.endsWith('.neevabot.com')
         ? log.set('identity', 'Neevabot')
         : log;
+    case 'RootCrawl-Crawler':
+      return hostname && hostname.endsWith('.rootcrawl.org')
+        ? log.set('identity', 'RootCrawl')
+        : log;
+    case 'bl.uk ldfc bot':
+      return hostname && hostname.endsWith('.bl.uk')
+        ? log.set('identity', 'British Library')
+        : log;
     case 'DataForSeoBot':
       return hostname && hostname.endsWith('.dataforseo.com')
         ? log.set('identity', 'DataForSeo')
@@ -196,9 +232,40 @@ function augment(log) {
         ? log.set('identity', 'InfoTiger')
         : log;
     case 'Amazonbot':
-      return hostname && hostname.endsWith('.crawl.amazonbot.amazon')
+      return (hostname && hostname.endsWith('.crawl.amazonbot.amazon')) ||
+        amazonBotIps.some((cidr) => new IPCIDR(cidr).contains(address))
         ? log.set('identity', 'Amazonbot')
         : log;
+    case 'Amzn-SearchBot':
+      return (hostname && hostname.endsWith('.crawl.amazonbot.amazon')) ||
+        amazonSearchBotIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'Amazon SearchBot')
+        : log;
+    case 'Amzn-User':
+      return (hostname && hostname.endsWith('.crawl.amazonbot.amazon')) ||
+        amazonUserIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'Amazon User')
+        : log;
+    case 'SERankingBacklinksBot':
+      return hostname && hostname.endsWith('.blex.seranking.com')
+        ? log.set('identity', 'SE Ranking')
+        : log;
+    case 'SofyaBot':
+      return hostname && hostname.endsWith('.sofya.co')
+        ? log.set('identity', 'Sofya')
+        : log;
+    case 'YouBot':
+      // https://docs.you.com/youbot
+      return hostname && hostname.endsWith('.search.you.com')
+        ? log.set('identity', 'You.com')
+        : log;
+    case 'AIWebIndex':
+    case 'AIWebIndex-Agent':
+      // Lyrenth AI-readable web index, forward-confirmed rDNS under lyrenth.com
+      // https://lyrenth.com/bot
+      return hostname && hostname.endsWith('.lyrenth.com')
+        ? log.set('identity', 'Lyrenth')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -211,6 +278,11 @@ function augment(log) {
         (address && new IPCIDR('2a02:598::/32').contains(address))
         ? log.set('identity', 'Seznam')
         : log;
+    case 'FacebookBot':
+      return (hostname && hostname.endsWith('.fbsv.net')) ||
+        (address && new IPCIDR('2a03:2880::/29').contains(address))
+        ? log.set('identity', 'Facebook')
+        : log;
 
     // Per CIDR
     case 'github-camo':
@@ -221,6 +293,10 @@ function augment(log) {
       return address && new IPCIDR('216.244.64.0/19').contains(address)
         ? log.set('identity', 'Moz')
         : log;
+    case 'AliyunSecBot':
+      return address && new IPCIDR('8.217.0.0/16').contains(address)
+        ? log.set('identity', family)
+        : log;
     case '360Spider':
       return address && new IPCIDR('42.236.10.0/24').contains(address)
         ? log.set('identity', family)
@@ -230,26 +306,33 @@ function augment(log) {
         ? log.set('identity', family)
         : log;
     case 'OAI-SearchBot':
-      // https://openai.com/searchbot.json
-      return address &&
-        (new IPCIDR('20.42.10.176/28').contains(address) ||
-          new IPCIDR('172.203.190.128/28').contains(address) ||
-          new IPCIDR('51.8.102.0/24').contains(address))
+      return openaiSearchbotIps.some((cidr) =>
+        new IPCIDR(cidr).contains(address)
+      )
         ? log.set('identity', 'OpenAI SearchBot')
         : log;
     case 'GPTBot':
-      // https://openai.com/gptbot.json
-      return address &&
-        (new IPCIDR('52.230.152.0/24').contains(address) ||
-          new IPCIDR('52.233.106.0/24').contains(address) ||
-          new IPCIDR('20.171.206.0/24').contains(address) ||
-          new IPCIDR('20.171.207.0/24').contains(address) ||
-          new IPCIDR('4.227.36.0/25').contains(address))
+      return gptbotIps.some((cidr) => new IPCIDR(cidr).contains(address))
         ? log.set('identity', 'OpenAI GPTBot')
         : log;
+    case 'ChatGPT-User':
+      // https://openai.com/chatgpt-user.json
+      return chatgptUserIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'ChatGPT')
+        : log;
+    case 'ClaudeBot':
+    case 'Claude-User':
+    case 'Claude-SearchBot':
+    case 'Claude-Web':
+    case 'anthropic-ai':
+      // https://claude.com/crawling/bots.json
+      return address && claudeBotCidrs.some((cidr) => cidr.contains(address))
+        ? log.set('identity', 'Claude')
+        : log;
     case 'meta-externalagent':
-      return address && new IPCIDR('2a03:2880:f800::/48').contains(address)
-        ? log.set('identity', 'Meta External Agent')
+    case 'meta-webindexer':
+      return address && new IPCIDR('2a03:2880::/29').contains(address)
+        ? log.set('identity', 'Meta')
         : log;
 
     // EC2
@@ -288,16 +371,38 @@ function augment(log) {
         hostname.endsWith('.eu-central-1.compute.amazonaws.com')
         ? log.set('identity', 'Wise')
         : log;
-    case 'ClaudeBot':
-      return hostname && hostname.endsWith('.us-east-2.compute.amazonaws.com')
-        ? log.set('identity', 'Claude')
+    case 'PerplexityBot':
+      return hostname && hostname.endsWith('.compute-1.amazonaws.com')
+        ? log.set('identity', 'Perplexity')
         : log;
 
     // GCE
+    case 'Aranet-SearchBot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Aranet')
+        : log;
+    case 'Discordbot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Discord')
+        : log;
     case 'VelenPublicWebCrawler':
       return hostname && hostname.endsWith('.googleusercontent.com')
         ? log.set('identity', 'Velen')
         : log;
+    case 'SleepBot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'SleepBot')
+        : log;
+    case 'DolfeEngineCrawler':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Dolfe')
+        : log;
+    case 'ShapBot':
+      // Parallel Web Systems crawler, runs on GCE
+      // https://docs.parallel.ai/resources/crawler
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Parallel')
+        : log;
 
     // Hetzner
     case 'Ubermetrics':
@@ -312,6 +417,27 @@ function augment(log) {
       return hostname && hostname.endsWith('.clients.your-server.de')
         ? log.set('identity', 'MegaIndex.ru')
         : log;
+    case 'ev-crawler':
+      return hostname && hostname.endsWith('.headline.com')
+        ? log.set('identity', 'Headline')
+        : log;
+    case 'SentryUptimeBot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Sentry')
+        : log;
+    case 'FlipboardProxy':
+      return hostname && hostname.endsWith('.flipboard.com')
+        ? log.set('identity', 'Flipboard')
+        : log;
+
+    // Web Bot Auth
+    case 'ExaSearchBot':
+      // Exa search crawler. No published IP ranges or reverse DNS: requests
+      // are signed (RFC 9421) and carry a `Signature-Agent` header
+      // https://crawler.exa.ai/
+      return signatureAgent && signatureAgent.includes('https://crawler.exa.ai')
+        ? log.set('identity', 'Exa')
+        : log;
   }
 
   // Hostname only
@@ -369,7 +495,9 @@ function start() {
 
   aggregator.setIdentifier(identifier);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('identities', aggregator);
 }

package/src/modules/index.js

@@ -1,6 +1,7 @@
 const debug = require('debug');
 
 const constants = require('../constants');
+const pipeline = require('../lib/pipeline');
 
 const debugModules = debug('hyperwatch:modules');
 
@@ -16,6 +17,8 @@ function get(module) {
       return require('./dnsbl');
     case 'geoip':
       return require('./geoip');
+    case 'history':
+      return require('./history');
     case 'hostname':
       return require('./hostname');
     case 'identity':
@@ -36,27 +39,35 @@ function get(module) {
   }
 }
 
-function activeModules() {
+function activeModulesWithKeys() {
   return Object.keys(constants.modules)
     .map((key) => Object.assign({ key }, constants.modules[key]))
     .sort((a, b) => a.priority - b.priority)
     .filter((m) => m.active === true)
-    .map((m) => get(m.key))
-    .filter((m) => m);
+    .map((m) => ({ key: m.key, module: get(m.key) }))
+    .filter((m) => m.module);
+}
+
+function activeModules() {
+  return activeModulesWithKeys().map((m) => m.module);
 }
 
 function init() {
-  for (const module of activeModules()) {
+  for (const { key, module } of activeModulesWithKeys()) {
     if (module && module.init) {
+      pipeline.currentModule = key;
       module.init();
+      pipeline.currentModule = null;
     }
   }
 }
 
 function start() {
-  for (const module of activeModules()) {
+  for (const { key, module } of activeModulesWithKeys()) {
     if (module && module.start) {
+      pipeline.currentModule = key;
       module.start();
+      pipeline.currentModule = null;
     }
   }
 }

package/src/modules/language.js

@@ -33,7 +33,7 @@ function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('language', language, {
-    before: '15m',
+    before: 'count15m',
     color: 'grey',
   });
 }

package/src/modules/signature.js

@@ -1,10 +1,16 @@
-const { is, Set } = require('immutable');
+const { is } = require('immutable');
 
 const api = require('../app/api');
-const { Aggregator } = require('../lib/aggregator');
+const { Aggregator, lastSeen, statusCount } = require('../lib/aggregator');
 const { Formatter } = require('../lib/formatter');
 const pipeline = require('../lib/pipeline');
-const { aggregateSpeed, md5 } = require('../lib/util');
+const { touch, prune, countRecent } = require('../lib/recent-map');
+const {
+  aggregateCount,
+  aggregateSum,
+  formatDuration,
+  md5,
+} = require('../lib/util');
 
 const { agentFormat } = require('./agent');
 
@@ -35,6 +41,9 @@ function normalisedIdentityHeader(headers) {
   return obj;
 }
 
+const addressCount15m = (entry) => countRecent(entry.get('addresses'), 15 * 60);
+const addressCount24h = (entry) => countRecent(entry.get('addresses'));
+
 function computeSignature(headers) {
   const string = Object.keys(headers)
     .map((key) => [key, headers[key]].join(':'))
@@ -60,8 +69,10 @@ function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 }
 
+let _aggregator;
+
 function start() {
-  const aggregator = new Aggregator();
+  const aggregator = (_aggregator = new Aggregator());
 
   aggregator.setIdentifier((log) => log.getIn(['signature', 'id']));
 
@@ -70,15 +81,24 @@ function start() {
   signatureFormatter.setFormats([
     ['signature', (entry) => entry.getIn(['signature', 'id'])],
     ['identity', (entry) => entry.get('identity')],
-    ['addressCount', (entry) => entry.get('addresses').size],
+    ['addressCount15m', addressCount15m],
+    ['addressCount24h', addressCount24h],
     [
       'addresses',
       (entry) =>
-        entry
-          .get('addresses')
-          .map((address) => address.get('value'))
-          .slice(0, 10)
-          .join('<br>'),
+        entry.has('addresses')
+          ? entry.get('addresses').keySeq().slice(0, 10).join('<br>')
+          : '',
+    ],
+    [
+      'lastAddress',
+      (entry) => {
+        const addr = entry.get('lastAddress');
+        if (!addr) {
+          return '';
+        }
+        return addr.get('hostname') || addr.get('value') || '';
+      },
     ],
 
     [
@@ -91,12 +111,24 @@ function start() {
       },
     ],
 
-    ['15m', (entry) => aggregateSpeed(entry, 'per_minute')],
-    ['24h', (entry) => aggregateSpeed(entry, 'per_hour')],
+    ['lastSeen', lastSeen],
+    ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
+    ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
+
+    ['2xx15m', statusCount('2xx_per_minute')],
+    ['2xx24h', statusCount('2xx_per_hour')],
+    ['4xx15m', statusCount('4xx_per_minute')],
+    ['4xx24h', statusCount('4xx_per_hour')],
+
+    [
+      'execTime15m',
+      (entry) => formatDuration(aggregateSum(entry, 'per_minute')),
+    ],
+    ['execTime24h', (entry) => formatDuration(aggregateSum(entry, 'per_hour'))],
   ]);
 
   signatureFormatter.insertFormat('agent', agentFormat, {
-    before: '15m',
+    before: 'count15m',
     color: 'grey',
   });
 
@@ -110,10 +142,12 @@ function start() {
     }
 
     const address = log.get('address');
-    if (!entry.has('addresses')) {
-      entry = entry.set('addresses', new Set([address]));
-    } else if (!entry.get('addresses').has(address)) {
-      entry = entry.update('addresses', (set) => set.add(address));
+    entry = entry.set('lastAddress', address);
+    // Distinct IPs with last-seen time, pruned to 24h; backs both the 15m
+    // and 24h counts
+    const value = address && address.get('value');
+    if (value) {
+      entry = entry.update('addresses', (map) => touch(map, value));
     }
 
     return entry;
@@ -121,7 +155,16 @@ function start() {
 
   aggregator.setEnricher(enricher);
 
-  pipeline.getNode('main').map((log) => aggregator.processLog(log));
+  aggregator.setEntryGc((entry) =>
+    entry.has('addresses') ? entry.update('addresses', prune) : entry
+  );
+
+  aggregator.sorters.addressCount15m = addressCount15m;
+  aggregator.sorters.addressCount24h = addressCount24h;
+
+  pipeline
+    .getNode('main')
+    .map((log) => aggregator.processLog(log), 'aggregator');
 
   api.registerAggregator('signatures', aggregator);
 }
@@ -129,4 +172,7 @@ function start() {
 module.exports = {
   init,
   start,
+  get aggregator() {
+    return _aggregator;
+  },
 };

package/src/modules/sparkline.js

@@ -1,6 +1,10 @@
 const aggregator = require('../lib/aggregator');
 
-const sparkline = (entry, key) => {
+const sparkline = (entry, key, output) => {
+  if (output === 'text') {
+    return;
+  }
+
   const id = entry.get('id');
 
   const points = entry
@@ -22,8 +26,8 @@ sparkline ('${id}', ${JSON.stringify(points)}, '#797979', 14, 5);
 };
 
 function init() {
-  aggregator.defaultFormatter.insertFormat('activity', (entry) =>
-    sparkline(entry, 'per_minute')
+  aggregator.defaultFormatter.insertFormat('activity', (entry, output) =>
+    sparkline(entry, 'per_minute', output)
   );
 }
 

package/src/modules/status.js

@@ -3,30 +3,35 @@ const monitoring = require('../lib/monitoring');
 const { formatTable } = require('../lib/util');
 const stylesheet = require('../stylesheet');
 
-const aggregateSpeed = (entry, path) =>
+const aggregateCount = (entry, path) =>
   entry.hasIn(path) ? entry.getIn(path).reduce((p, c) => p + c, 0) : null;
 
 function mapper(entry, format) {
   return {
     name: entry.get('name'),
     type: entry.get('type'),
-    '15m':
-      aggregateSpeed(entry, ['speeds', 'processed', 'per_minute']) ||
-      aggregateSpeed(entry, ['speeds', 'accepted', 'per_minute']) ||
+    count15m:
+      aggregateCount(entry, ['speeds', 'processed', 'per_minute']) ||
+      aggregateCount(entry, ['speeds', 'accepted', 'per_minute']) ||
       (format !== 'json' ? '' : null),
-    '24h':
-      aggregateSpeed(entry, ['speeds', 'processed', 'per_hour']) ||
-      aggregateSpeed(entry, ['speeds', 'accepted', 'per_hour']) ||
+    count24h:
+      aggregateCount(entry, ['speeds', 'processed', 'per_hour']) ||
+      aggregateCount(entry, ['speeds', 'accepted', 'per_hour']) ||
       (format !== 'json' ? '' : null),
     status: entry.get('status'),
   };
 }
 
 function start() {
-  api.get('/status(.:format(txt|json))?', (req, res) => {
+  api.get('/status{.:format}', (req, res) => {
     const raw = req.query.raw ? true : false;
     const format = req.params.format || (raw ? 'json' : null);
 
+    if (format && !['json', 'txt'].includes(format)) {
+      res.sendStatus(404);
+      return;
+    }
+
     let rawData = monitoring.getAllComputed();
 
     if (req.query.type) {