@hyperwatch/hyperwatch 4.1.0 → 4.3.1

package/src/lib/aggregator.js

@@ -1,8 +1,13 @@
-const { Map, Set, is } = require('immutable');
+const { Map, Set, fromJS, is } = require('immutable');
 
 const { Formatter, address, identity } = require('../lib/formatter');
 const { Speed } = require('../lib/speed');
-const { aggregateSpeed, md5 } = require('../lib/util');
+const {
+  aggregateCount,
+  aggregateSum,
+  formatDuration,
+  md5,
+} = require('../lib/util');
 
 const defaultFormatter = new Formatter();
 defaultFormatter.setFormats([
@@ -10,8 +15,11 @@ defaultFormatter.setFormats([
 
   ['address', address],
 
-  ['15m', (entry) => aggregateSpeed(entry, 'per_minute')],
-  ['24h', (entry) => aggregateSpeed(entry, 'per_hour')],
+  ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
+  ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
+
+  ['execTime15m', (entry) => formatDuration(aggregateSum(entry, 'per_minute'))],
+  ['execTime24h', (entry) => formatDuration(aggregateSum(entry, 'per_hour'))],
 ]);
 
 const defaultEnricher = (entry, log) => {
@@ -42,9 +50,11 @@ const defaultIdentifier = (log) => {
 };
 
 const defaultSorters = {
-  '15m': (entry) => aggregateSpeed(entry, 'per_minute'),
-  '24h': (entry) => aggregateSpeed(entry, 'per_hour'),
+  count15m: (entry) => aggregateCount(entry, 'per_minute'),
+  count24h: (entry) => aggregateCount(entry, 'per_hour'),
   latest: (entry) => entry.getIn(['speed', 'per_minute']).latest,
+  execTime15m: (entry) => aggregateSum(entry, 'per_minute'),
+  execTime24h: (entry) => aggregateSum(entry, 'per_hour'),
 };
 
 class Aggregator {
@@ -56,7 +66,7 @@ class Aggregator {
     this.sorters = defaultSorters;
     this.gcSize = 1000;
 
-    setInterval(() => this.gc(), 60 * 1000);
+    setInterval(() => this.gc(), 60 * 1000).unref();
   }
 
   setFormatter(fn) {
@@ -80,17 +90,29 @@ class Aggregator {
   processLog(log) {
     const identifier = this.identifier(log);
     const id = md5(identifier);
+    const rawExecTime = Number(log.get('executionTime'));
+    const executionTime = Number.isFinite(rawExecTime) ? rawExecTime : 0;
 
     if (!this.entries.has(id)) {
       this.entries = this.entries
         .setIn([id, 'id'], id)
         .setIn([id, 'identifier'], identifier)
-        .setIn([id, 'speed', 'per_minute'], new Speed(60, 15).hit())
-        .setIn([id, 'speed', 'per_hour'], new Speed(3600, 24).hit());
+        .setIn(
+          [id, 'speed', 'per_minute'],
+          new Speed(60, 15).hit(undefined, executionTime)
+        )
+        .setIn(
+          [id, 'speed', 'per_hour'],
+          new Speed(3600, 24).hit(undefined, executionTime)
+        );
     } else {
       this.entries = this.entries
-        .updateIn([id, 'speed', 'per_minute'], (speed) => speed.hit())
-        .updateIn([id, 'speed', 'per_hour'], (speed) => speed.hit());
+        .updateIn([id, 'speed', 'per_minute'], (speed) =>
+          speed.hit(undefined, executionTime)
+        )
+        .updateIn([id, 'speed', 'per_hour'], (speed) =>
+          speed.hit(undefined, executionTime)
+        );
     }
 
     this.entries = this.entries.updateIn([id], (entry) =>
@@ -106,7 +128,7 @@ class Aggregator {
 
   getData({ raw, sort, format, limit }) {
     if (!sort || !this.sorters[sort]) {
-      sort = '15m';
+      sort = 'count15m';
     }
 
     const rawData = this.entries
@@ -142,6 +164,40 @@ class Aggregator {
 
     this.entries = this.entries.filter((value, key) => keepList.has(key));
   }
+
+  dump() {
+    return this.entries
+      .map((entry) => {
+        const plain = entry.toJS();
+        plain.speed = {
+          per_minute: entry.getIn(['speed', 'per_minute']).toJSON(),
+          per_hour: entry.getIn(['speed', 'per_hour']).toJSON(),
+        };
+        return plain;
+      })
+      .valueSeq()
+      .toArray();
+  }
+
+  load(data) {
+    for (const item of data) {
+      const { speed, ...rest } = item;
+      // fromJS deep-converts everything to Immutable structures.
+      // Signature headers must stay as a plain object (used with Object.entries),
+      // and addresses must be a Set, not a List — fix both after conversion.
+      let entry = fromJS(rest);
+      if (entry.hasIn(['signature', 'headers'])) {
+        entry = entry.setIn(['signature', 'headers'], rest.signature.headers);
+      }
+      if (entry.has('addresses')) {
+        entry = entry.update('addresses', (list) => Set(list));
+      }
+      entry = entry
+        .setIn(['speed', 'per_minute'], Speed.fromJSON(speed.per_minute))
+        .setIn(['speed', 'per_hour'], Speed.fromJSON(speed.per_hour));
+      this.entries = this.entries.set(rest.id, entry);
+    }
+  }
 }
 
 module.exports = { Aggregator, defaultFormatter, defaultEnricher };

package/src/lib/index.js

@@ -2,6 +2,7 @@ const aggregator = require('./aggregator');
 const cache = require('./cache');
 const formatter = require('./formatter');
 const logger = require('./logger');
+const persistence = require('./persistence');
 const pipeline = require('./pipeline');
 const util = require('./util');
 
@@ -10,6 +11,7 @@ module.exports = {
   cache,
   formatter,
   logger,
+  persistence,
   pipeline,
   util,
 };

package/src/lib/persistence.js

@@ -0,0 +1,72 @@
+const fs = require('fs');
+const path = require('path');
+
+const debug = require('debug');
+
+const debugPersistence = debug('hyperwatch:persistence');
+
+const aggregators = Object.create(null);
+
+const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
+
+function register(name, aggregator) {
+  if (!SAFE_NAME.test(name)) {
+    throw new Error(`Invalid aggregator name for persistence: "${name}"`);
+  }
+  aggregators[name] = aggregator;
+}
+
+function dump(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+  for (const [name, aggregator] of Object.entries(aggregators)) {
+    const data = aggregator.dump();
+    const target = path.join(dir, `${name}.json`);
+    const tmp = path.join(dir, `${name}.json.tmp`);
+    fs.writeFileSync(tmp, JSON.stringify(data));
+    fs.renameSync(tmp, target);
+  }
+  debugPersistence(
+    `Dumped ${Object.keys(aggregators).length} aggregator(s) to ${dir}`
+  );
+}
+
+function load(dir) {
+  if (!fs.existsSync(dir)) {
+    return;
+  }
+  let files;
+  try {
+    files = fs.readdirSync(dir).filter((f) => f.endsWith('.json'));
+  } catch (err) {
+    debugPersistence(
+      `Cannot read persistence directory ${dir}: ${err.message}`
+    );
+    return;
+  }
+  let loaded = 0;
+  for (const file of files) {
+    const name = path.basename(file, '.json');
+    if (aggregators[name]) {
+      try {
+        const data = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8'));
+        if (!Array.isArray(data)) {
+          debugPersistence(
+            `Skipping ${file}: expected array, got ${typeof data}`
+          );
+          continue;
+        }
+        aggregators[name].load(data);
+        loaded += data.length;
+      } catch (err) {
+        debugPersistence(`Skipping ${file}: ${err.message}`);
+      }
+    }
+  }
+  if (loaded) {
+    debugPersistence(`Loaded ${loaded} entries from ${dir}`);
+  }
+}
+
+module.exports = { register, dump, load };

package/src/lib/speed.js

@@ -10,6 +10,7 @@ class Speed {
     this.windowSize = windowSize;
     this.size = size;
     this.counters = Map();
+    this.sums = Map();
     this.started = null;
     this.latest = null;
   }
@@ -18,23 +19,26 @@ class Speed {
   gc() {
     const cutoff = now() - this.size * this.windowSize;
     this.counters = this.counters.filter((c, t) => parseInt(t) > cutoff);
+    this.sums = this.sums.filter((c, t) => parseInt(t) > cutoff);
   }
 
-  hit(time = now()) {
+  hit(time = now(), value) {
     this.started = !this.started ? time : Math.min(this.started, time);
     this.latest = !this.latest ? time : Math.max(this.latest, time);
     const idx = time - (time % this.windowSize);
     this.counters = this.counters.update(`${idx}`, 0, (n) => n + 1);
+    if (value !== undefined) {
+      this.sums = this.sums.update(`${idx}`, 0, (n) => n + value);
+    }
     this.gc();
     return this;
   }
 
-  compute() {
+  compute(time = now()) {
     this.gc();
     if (!this.started) {
       return List();
     }
-    const time = now();
     return Range(0, this.size)
       .map((n) => {
         let t = time - n * this.windowSize;
@@ -45,6 +49,42 @@ class Speed {
       })
       .filter((v) => v !== undefined);
   }
+
+  computeSum(time = now()) {
+    this.gc();
+    if (!this.started) {
+      return List();
+    }
+    return Range(0, this.size)
+      .map((n) => {
+        let t = time - n * this.windowSize;
+        t = t - (t % this.windowSize);
+        if (t >= this.started - (this.started % this.windowSize)) {
+          return this.sums.get(`${t}`, 0);
+        }
+      })
+      .filter((v) => v !== undefined);
+  }
+
+  toJSON() {
+    return {
+      windowSize: this.windowSize,
+      size: this.size,
+      counters: this.counters.toObject(),
+      sums: this.sums.toObject(),
+      started: this.started,
+      latest: this.latest,
+    };
+  }
+
+  static fromJSON(data) {
+    const speed = new Speed(data.windowSize, data.size);
+    speed.counters = Map(data.counters);
+    speed.sums = Map(data.sums);
+    speed.started = data.started;
+    speed.latest = data.latest;
+    return speed;
+  }
 }
 
 module.exports = {

package/src/lib/util.js

@@ -34,12 +34,28 @@ exports.createLog = (req, res) => {
   });
 };
 
-exports.aggregateSpeed = (entry, key) =>
+exports.aggregateCount = (entry, key) =>
   entry
     .getIn(['speed', key])
     .compute()
     .reduce((p, c) => p + c, 0);
 
+exports.aggregateSum = (entry, key) =>
+  entry
+    .getIn(['speed', key])
+    .computeSum()
+    .reduce((p, c) => p + c, 0);
+
+exports.formatDuration = (ms) => {
+  const totalSeconds = ms / 1000;
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = Math.round(totalSeconds % 60);
+  if (minutes > 0) {
+    return seconds > 0 ? `${minutes}m${seconds}s` : `${minutes}m`;
+  }
+  return `${totalSeconds.toFixed(1)}s`;
+};
+
 exports.formatTable = (data) => {
   if (!data || data.length === 0) {
     return '';

package/src/modules/agent.js

@@ -70,7 +70,7 @@ function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('agent', agentFormat, {
-    before: '15m',
+    before: 'count15m',
     color: 'grey',
   });
   aggregator.defaultFormatter.insertFormat('os', osFormat, {

package/src/modules/history.js

@@ -0,0 +1,86 @@
+const { api } = require('../app');
+const constants = require('../constants');
+const pipeline = require('../lib/pipeline');
+
+class CircularBuffer {
+  constructor(capacity = 1000) {
+    this.capacity = capacity;
+    this.buffer = new Array(capacity);
+    this.pointer = 0;
+    this.size = 0;
+  }
+
+  push(item) {
+    this.buffer[this.pointer] = item;
+    this.pointer = (this.pointer + 1) % this.capacity;
+    if (this.size < this.capacity) {
+      this.size++;
+    }
+  }
+
+  toArray() {
+    if (this.size < this.capacity) {
+      return this.buffer.slice(0, this.size).reverse();
+    }
+    return [
+      ...this.buffer.slice(this.pointer),
+      ...this.buffer.slice(0, this.pointer),
+    ].reverse();
+  }
+}
+
+function start() {
+  const capacity =
+    (constants.modules.history && constants.modules.history.capacity) || 1000;
+  const buffers = {};
+
+  function registerNodeHistory(name, node) {
+    const buffer = new CircularBuffer(capacity);
+    buffers[name] = buffer;
+
+    node.map((log) => {
+      buffer.push(log);
+      return log;
+    });
+
+    api.get(`/history/${name}.json`, (req, res) => {
+      const { identity, signature, address } = req.query;
+      const limit = parseInt(req.query.limit, 10) || 100;
+
+      let logs = buffer.toArray();
+
+      if (identity) {
+        logs = logs.filter((log) => log.get('identity') === identity);
+      }
+      if (signature) {
+        logs = logs.filter(
+          (log) => log.getIn(['signature', 'id']) === signature
+        );
+      }
+      if (address) {
+        logs = logs.filter(
+          (log) => log.getIn(['address', 'value']) === address
+        );
+      }
+
+      logs = logs.slice(0, limit);
+
+      res.json(logs);
+    });
+  }
+
+  for (const [name, node] of Object.entries(pipeline.nodes)) {
+    registerNodeHistory(name, node);
+  }
+
+  // Auto-register future nodes
+  const originalRegisterNode = pipeline.registerNode.bind(pipeline);
+  pipeline.registerNode = function (name, node) {
+    originalRegisterNode(name, node);
+    if (!buffers[name]) {
+      registerNodeHistory(name, node);
+    }
+  };
+}
+
+module.exports = { start };

package/src/modules/identity.js

@@ -1,6 +1,14 @@
-const IPCIDR = require('ip-cidr');
+const IPCIDR = require('ip-cidr').default;
 
 const api = require('../app/api');
+// Bot IP lists for identity verification
+// Run `node scripts/fetch-openai-ips.js` to update OpenAI lists
+const amazonSearchBotIps = require('../data/amazon-searchbot-ips.json');
+const amazonUserIps = require('../data/amazon-user-ips.json');
+const amazonBotIps = require('../data/amazonbot-ips.json');
+const chatgptUserIps = require('../data/chatgpt-user-ips.json');
+const gptbotIps = require('../data/gptbot-ips.json');
+const openaiSearchbotIps = require('../data/openai-searchbot-ips.json');
 const { Aggregator } = require('../lib/aggregator');
 const pipeline = require('../lib/pipeline');
 
@@ -69,6 +77,7 @@ function augment(log) {
       return hostname && hostname.endsWith('.mojeek.com')
         ? log.set('identity', 'Mojeek')
         : log;
+    case 'AwarioBot':
     case 'BLEXBot':
       return hostname && hostname.endsWith('.webmeup.com')
         ? log.set('identity', 'WebMeUp')
@@ -89,13 +98,11 @@ function augment(log) {
       return hostname && hostname.endsWith('.naver.com')
         ? log.set('identity', 'Naver')
         : log;
-    case 'FacebookBot':
-      return hostname && hostname.endsWith('.fbsv.net')
-        ? log.set('identity', 'Facebook')
-        : log;
+
     case 'Stripe':
+    case 'Stripebot':
       return hostname && hostname.endsWith('.stripe.com')
-        ? log.set('identity', family)
+        ? log.set('identity', 'Stripe')
         : log;
     case 'UptimeRobot':
       return hostname && hostname.endsWith('.uptimerobot.com')
@@ -127,9 +134,17 @@ function augment(log) {
         ? log.set('identity', 'Sirportly')
         : log;
     case 'Bytespider':
-      return hostname && hostname.endsWith('.crawl.bytedance.com')
+      return hostname &&
+        (hostname.endsWith('.crawl.bytedance.com') ||
+          hostname.endsWith('.ap-southeast-1.compute.amazonaws.com'))
         ? log.set('identity', family)
         : log;
+    case 'TikTokSpider':
+      return hostname &&
+        (hostname.endsWith('.crawl.bytedance.com') ||
+          hostname.endsWith('.ap-southeast-1.compute.amazonaws.com'))
+        ? log.set('identity', 'TikTok')
+        : log;
     case 'Mail.RU Bot':
     case 'Mail.RU Bot Img':
     case 'Mail.RU Bot Fast':
@@ -182,6 +197,14 @@ function augment(log) {
       return hostname && hostname.endsWith('.neevabot.com')
         ? log.set('identity', 'Neevabot')
         : log;
+    case 'RootCrawl-Crawler':
+      return hostname && hostname.endsWith('.rootcrawl.org')
+        ? log.set('identity', 'RootCrawl')
+        : log;
+    case 'bl.uk ldfc bot':
+      return hostname && hostname.endsWith('.bl.uk')
+        ? log.set('identity', 'British Library')
+        : log;
     case 'DataForSeoBot':
       return hostname && hostname.endsWith('.dataforseo.com')
         ? log.set('identity', 'DataForSeo')
@@ -196,9 +219,24 @@ function augment(log) {
         ? log.set('identity', 'InfoTiger')
         : log;
     case 'Amazonbot':
-      return hostname && hostname.endsWith('.crawl.amazonbot.amazon')
+      return (hostname && hostname.endsWith('.crawl.amazonbot.amazon')) ||
+        amazonBotIps.some((cidr) => new IPCIDR(cidr).contains(address))
         ? log.set('identity', 'Amazonbot')
         : log;
+    case 'Amzn-SearchBot':
+      return (hostname && hostname.endsWith('.crawl.amazonbot.amazon')) ||
+        amazonSearchBotIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'Amazon SearchBot')
+        : log;
+    case 'Amzn-User':
+      return (hostname && hostname.endsWith('.crawl.amazonbot.amazon')) ||
+        amazonUserIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'Amazon User')
+        : log;
+    case 'SERankingBacklinksBot':
+      return hostname && hostname.endsWith('.blex.seranking.com')
+        ? log.set('identity', 'SE Ranking')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -211,6 +249,11 @@ function augment(log) {
         (address && new IPCIDR('2a02:598::/32').contains(address))
         ? log.set('identity', 'Seznam')
         : log;
+    case 'FacebookBot':
+      return (hostname && hostname.endsWith('.fbsv.net')) ||
+        (address && new IPCIDR('2a03:2880::/29').contains(address))
+        ? log.set('identity', 'Facebook')
+        : log;
 
     // Per CIDR
     case 'github-camo':
@@ -221,6 +264,10 @@ function augment(log) {
       return address && new IPCIDR('216.244.64.0/19').contains(address)
         ? log.set('identity', 'Moz')
         : log;
+    case 'AliyunSecBot':
+      return address && new IPCIDR('8.217.0.0/16').contains(address)
+        ? log.set('identity', family)
+        : log;
     case '360Spider':
       return address && new IPCIDR('42.236.10.0/24').contains(address)
         ? log.set('identity', family)
@@ -229,6 +276,28 @@ function augment(log) {
       return address && new IPCIDR('203.133.160.0/19').contains(address)
         ? log.set('identity', family)
         : log;
+    case 'OAI-SearchBot':
+      return openaiSearchbotIps.some((cidr) =>
+        new IPCIDR(cidr).contains(address)
+      )
+        ? log.set('identity', 'OpenAI SearchBot')
+        : log;
+    case 'GPTBot':
+      return gptbotIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'OpenAI GPTBot')
+        : log;
+    case 'ChatGPT-User':
+      // https://openai.com/chatgpt-user.json
+      return chatgptUserIps.some((cidr) => new IPCIDR(cidr).contains(address))
+        ? log.set('identity', 'ChatGPT')
+        : log;
+    case 'meta-externalagent':
+    case 'meta-webindexer':
+      return address &&
+        (new IPCIDR('2a03:2880::/29').contains(address) ||
+          new IPCIDR('2a06:98c0:3600::/48').contains(address))
+        ? log.set('identity', 'Meta')
+        : log;
 
     // EC2
     case 'Raven':
@@ -270,12 +339,32 @@ function augment(log) {
       return hostname && hostname.endsWith('.us-east-2.compute.amazonaws.com')
         ? log.set('identity', 'Claude')
         : log;
+    case 'PerplexityBot':
+      return hostname && hostname.endsWith('.compute-1.amazonaws.com')
+        ? log.set('identity', 'Perplexity')
+        : log;
 
     // GCE
+    case 'Aranet-SearchBot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Aranet')
+        : log;
+    case 'Discordbot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Discord')
+        : log;
     case 'VelenPublicWebCrawler':
       return hostname && hostname.endsWith('.googleusercontent.com')
         ? log.set('identity', 'Velen')
         : log;
+    case 'SleepBot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'SleepBot')
+        : log;
+    case 'DolfeEngineCrawler':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Dolfe')
+        : log;
 
     // Hetzner
     case 'Ubermetrics':
@@ -290,6 +379,18 @@ function augment(log) {
       return hostname && hostname.endsWith('.clients.your-server.de')
         ? log.set('identity', 'MegaIndex.ru')
         : log;
+    case 'ev-crawler':
+      return hostname && hostname.endsWith('.headline.com')
+        ? log.set('identity', 'Headline')
+        : log;
+    case 'SentryUptimeBot':
+      return hostname && hostname.endsWith('.googleusercontent.com')
+        ? log.set('identity', 'Sentry')
+        : log;
+    case 'FlipboardProxy':
+      return hostname && hostname.endsWith('.flipboard.com')
+        ? log.set('identity', 'Flipboard')
+        : log;
   }
 
   // Hostname only

package/src/modules/index.js

@@ -16,6 +16,8 @@ function get(module) {
       return require('./dnsbl');
     case 'geoip':
       return require('./geoip');
+    case 'history':
+      return require('./history');
     case 'hostname':
       return require('./hostname');
     case 'identity':

package/src/modules/language.js

@@ -33,7 +33,7 @@ function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('language', language, {
-    before: '15m',
+    before: 'count15m',
     color: 'grey',
   });
 }

package/src/modules/signature.js

@@ -4,7 +4,12 @@ const api = require('../app/api');
 const { Aggregator } = require('../lib/aggregator');
 const { Formatter } = require('../lib/formatter');
 const pipeline = require('../lib/pipeline');
-const { aggregateSpeed, md5 } = require('../lib/util');
+const {
+  aggregateCount,
+  aggregateSum,
+  formatDuration,
+  md5,
+} = require('../lib/util');
 
 const { agentFormat } = require('./agent');
 
@@ -91,12 +96,18 @@ function start() {
       },
     ],
 
-    ['15m', (entry) => aggregateSpeed(entry, 'per_minute')],
-    ['24h', (entry) => aggregateSpeed(entry, 'per_hour')],
+    ['count15m', (entry) => aggregateCount(entry, 'per_minute')],
+    ['count24h', (entry) => aggregateCount(entry, 'per_hour')],
+
+    [
+      'execTime15m',
+      (entry) => formatDuration(aggregateSum(entry, 'per_minute')),
+    ],
+    ['execTime24h', (entry) => formatDuration(aggregateSum(entry, 'per_hour'))],
   ]);
 
   signatureFormatter.insertFormat('agent', agentFormat, {
-    before: '15m',
+    before: 'count15m',
     color: 'grey',
   });