@adalo/metrics 0.1.162 → 0.1.164

package/src/metrics/metricsRedisClient.js

@@ -9,6 +9,9 @@ const {
 const redisConnectionStableFields = ['name', 'flags', 'cmd']
 const redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']
 
+/** Stream entries older than this (ms) are trimmed so messages are not kept in Redis. Fixed 60s. */
+const GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000
+
 /**
  * RedisMetricsClient extends BaseMetricsClient to collect
  * Redis metrics periodically and push them to Prometheus Pushgateway.
@@ -32,7 +35,11 @@ class RedisMetricsClient extends BaseMetricsClient {
    * @param {function} [options.startupValidation] - Function to validate startup (from BaseMetricsClient)
    * @param {boolean} [options.disablePushgateway] - Disable pushing to Pushgateway (use HTTP scraping instead)
    */
-  constructor({ redisClient, gracefulShutdownRedis, ...metricsConfig } = {}) {
+  constructor({ redisClient, ...metricsConfig } = {}) {
+    if (redisClient == null) {
+      throw new Error('RedisMetricsClient requires redisClient')
+    }
+
     const intervalSec =
       metricsConfig.intervalSec ||
       parseInt(process.env.METRICS_QUEUE_INTERVAL_SEC || '', 10) ||
@@ -42,36 +49,16 @@ class RedisMetricsClient extends BaseMetricsClient {
       ...metricsConfig,
       processType: metricsConfig.processType || 'queue-metrics',
       intervalSec,
-      skipFirstPush: metricsConfig.skipFirstPush !== false,
     })
 
     /** Redis client used for metrics */
     this.redisClient = redisClient
     this.redisClientType = getRedisClientType(redisClient)
 
-    /** When true, new instance publishes once to Redis on start; old instances subscribed to same channel exit immediately. Channel = metrics:graceful-shutdown:{app}:{process_type}. Default true; set METRICS_GRACEFUL_SHUTDOWN_REDIS=false or gracefulShutdownRedis: false to disable. */
-    const disabledByParam = gracefulShutdownRedis === false
-    const disabledByEnv =
-      process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false'
-    this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv
-    this._gracefulShutdownChannel = this._gracefulShutdownRedis
-      ? `metrics:graceful-shutdown:${this.appName}:${this.processType}`
-      : null
     /** Dedicated Redis connection for subscribe (subscriber mode); closed in cleanup. */
     this._subClient = null
 
-    /** Log prefix for graceful-shutdown messages (always logged, not behind METRICS_LOG_VALUES). */
-    this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`
-
-    console.info(
-      `${this._gracefulShutdownLogPrefix} startup: graceful_shutdown_redis=${
-        this._gracefulShutdownRedis
-      } channel=${this._gracefulShutdownChannel || 'none'}`
-    )
-
-    if (this._gracefulShutdownRedis && this._gracefulShutdownChannel) {
-      this._setupGracefulShutdownSubscribe()
-    }
+    this._initGracefulShutdown(metricsConfig.gracefulShutdownRedis)
 
     /** Counter for Redis client connections */
     this.redisConnectionsGauge = this.createGauge({
@@ -113,139 +100,207 @@ class RedisMetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Set up Redis subscribe for graceful shutdown. Uses one connection: either redisClient.duplicate() (ioredis) or a client created from REDIS_URL (node-redis). Caller passes only redisClient.
+   * Initialize graceful-shutdown state and subscribe when enabled. Called from constructor.
+   * @param {boolean} [gracefulShutdownRedis] - Explicit false to disable; otherwise enabled unless METRICS_GRACEFUL_SHUTDOWN_REDIS=false.
    */
-  _setupGracefulShutdownSubscribe() {
-    const channel = this._gracefulShutdownChannel
-    if (!channel) return
-
-    let subClient = null
-    if (this.redisClient && typeof this.redisClient.duplicate === 'function') {
-      subClient = this.redisClient.duplicate()
-      console.info(
-        `${this._gracefulShutdownLogPrefix} setup_subscribe: using redisClient.duplicate() for channel=${channel}`
-      )
-    } else if (process.env.REDIS_URL) {
+  _initGracefulShutdown(gracefulShutdownRedis) {
+    const disabledByParam = gracefulShutdownRedis === false
+    const disabledByEnv =
+      process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false'
+    this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv
+    this._gracefulShutdownStream = this._gracefulShutdownRedis
+      ? `metrics:graceful-shutdown:${this.appName}:${this.processType}`
+      : null
+    this._gracefulShutdownAckChannel = this._gracefulShutdownRedis
+      ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}`
+      : null
+    this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`
+    if (this._gracefulShutdownRedis && this._gracefulShutdownStream) {
+      this._setupGracefulShutdownSubscribe()
+    }
+  }
+
+  /**
+   * Create a dedicated Redis client for subscribe (pub/sub). Uses duplicate() when available, else createClient from REDIS_URL. Branches by redisClientType (IOREDIS, REDIS_V3, REDIS_V4).
+   * @returns {any|null} Subscriber client or null if not possible.
+   */
+  _createSubscriberClient() {
+    if (this.redisClientType === IOREDIS && this.redisClient && typeof this.redisClient.duplicate === 'function') {
+      return this.redisClient.duplicate()
+    }
+    if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && this.redisClient && typeof this.redisClient.duplicate === 'function') {
+      return this.redisClient.duplicate()
+    }
+    if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && process.env.REDIS_URL) {
       try {
         const redis = require('redis')
-        subClient = redis.createClient({ url: process.env.REDIS_URL })
-        if (subClient && typeof subClient.on === 'function') {
-          subClient.on('error', err => {
-            console.error(
-              `${this._gracefulShutdownLogPrefix} subscriber client error:`,
-              err.message
-            )
-          })
+        if (typeof redis.createClient !== 'function') return null
+        try {
+          return redis.createClient({ url: process.env.REDIS_URL })
+        } catch {
+          return redis.createClient(process.env.REDIS_URL)
         }
-        console.info(
-          `${this._gracefulShutdownLogPrefix} setup_subscribe: created subscriber from REDIS_URL for channel=${channel}`
-        )
-      } catch (e) {
-        console.info(
-          `${this._gracefulShutdownLogPrefix} setup_subscribe: could not create subscriber from REDIS_URL:`,
-          e.message
-        )
+      } catch {
+        return null
       }
     }
+    return null
+  }
 
-    if (!subClient) {
-      console.info(
-        `${this._gracefulShutdownLogPrefix} No subscriber (need ioredis or REDIS_URL). Graceful shutdown via Redis disabled.`
-      )
-      return
+  /**
+   * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).
+   */
+  /**
+   * Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.
+   */
+  _setupGracefulShutdownSubscribe() {
+    const streamKey = this._gracefulShutdownStream
+    if (!streamKey) return
+
+    const subClient = this._createSubscriberClient()
+    if (subClient && typeof subClient.on === 'function') {
+      subClient.on('error', () => {})
     }
 
-    console.info(
-      `${this._gracefulShutdownLogPrefix} setup_subscribe: channel=${channel} hasSubClient=true`
-    )
+    if (!subClient) return
 
     this._subClient = subClient
+    this._streamReadLoop(streamKey)
+  }
 
-    const onMessage = () => {
-      console.info(
-        `${this._gracefulShutdownLogPrefix} OLD: received new-instance message on channel ${channel}; exiting.`
-      )
-      this.cleanup()
+  _streamReadLoop(streamKey) {
+    const self = this
+    const blockMs = 5000
+
+    const runRead = () => {
+      if (!self._subClient) return
+      self._xreadBlock(streamKey, blockMs)
+        .then((entries) => {
+          if (!entries || entries.length === 0) {
+            setImmediate(runRead)
+            return
+          }
+          self._cleanupAndPublishAck()
+        })
+        .catch(() => {
+          setImmediate(runRead)
+        })
     }
+    runRead()
+  }
+
+  _xreadBlock(streamKey, blockMs) {
+    const client = this._subClient || this.redisClient
+    if (!client) return Promise.resolve([])
 
     if (this.redisClientType === REDIS_V3) {
-      subClient.on('subscribe', () => {
-        console.info(
-          `${this._gracefulShutdownLogPrefix} Subscribed to channel=${channel} (waiting for new-instance message).`
-        )
-      })
-      subClient.on('message', (ch, _msg) => {
-        if (ch === channel) onMessage()
-      })
-      subClient.subscribe(channel)
-    } else if (this.redisClientType === REDIS_V4) {
-      subClient.on('subscribe', () => {
-        console.info(
-          `${this._gracefulShutdownLogPrefix} Subscribed to channel=${channel} (waiting for new-instance message).`
-        )
-      })
-      subClient.on('message', (ch, _msg) => {
-        if (ch === channel) onMessage()
-      })
-      subClient.subscribe(channel)
-    } else if (this.redisClientType === IOREDIS) {
-      subClient.subscribe(channel, (err, count) => {
-        if (err) {
-          console.error(
-            `${this._gracefulShutdownLogPrefix} Subscribe error:`,
-            err
-          )
-          return
-        }
-        console.info(
-          `${this._gracefulShutdownLogPrefix} Subscribed to channel=${channel} (waiting for new-instance message).`
-        )
-      })
-      subClient.on('message', (ch, _msg) => {
-        if (ch === channel) onMessage()
+      return new Promise((resolve, reject) => {
+        client.send_command('XREAD', ['BLOCK', String(blockMs), 'STREAMS', streamKey, '$'], (err, result) => {
+          if (err) return reject(err)
+          resolve(this._parseXreadReply(result, streamKey))
+        })
       })
     }
+    if (this.redisClientType === REDIS_V4) {
+      const p = client.sendCommand(['XREAD', 'BLOCK', String(blockMs), 'STREAMS', streamKey, '$'])
+      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])
+    }
+    if (this.redisClientType === IOREDIS) {
+      const p = client.call('XREAD', 'BLOCK', blockMs, 'STREAMS', streamKey, '$')
+      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])
+    }
+    return Promise.resolve([])
+  }
+
+  _parseXreadReply(reply, streamKey) {
+    if (!reply || !Array.isArray(reply)) return []
+    const streamReply = reply.find(r => r && r[0] === streamKey)
+    if (!streamReply || !Array.isArray(streamReply[1])) return []
+    return streamReply[1].map(entry => (Array.isArray(entry) ? [entry[0], entry[1] || []] : [entry, []]))
   }
 
   /**
-   * Publish one-time "new instance started" so old instances (subscribed to the same channel) exit. Call from new instance after startPush.
+   * Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.
    */
-  _publishNewInstanceStarted() {
-    if (!this._gracefulShutdownChannel || !this.redisClient) return
-    const channel = this._gracefulShutdownChannel
-    const message = this.dynoId || '1'
-
-    const done = err => {
-      if (err) {
-        console.info(
-          `${this._gracefulShutdownLogPrefix} NEW: publish failed:`,
-          err.message
-        )
-      } else {
-        console.info(
-          `${this._gracefulShutdownLogPrefix} NEW: published to channel=${channel} (new instance started; old instances will exit).`
-        )
+  async _cleanupAndPublishAck() {
+    this.stopPush()
+    if (this.enabled) {
+      await this.gatewayDelete()
+    }
+    await this._publishAck()
+    if (this._subClient) {
+      try {
+        if (this.redisClientType === REDIS_V3) {
+          await new Promise((resolve, reject) => {
+            if (typeof this._subClient.quit === 'function') {
+              this._subClient.quit(err => (err ? reject(err) : resolve()))
+            } else resolve()
+          })
+        } else if (this.redisClientType === REDIS_V4) {
+          if (this._subClient.quit) await this._subClient.quit()
+        } else if (this.redisClientType === IOREDIS) {
+          if (this._subClient.disconnect) await this._subClient.disconnect()
+        }
+      } catch {
+        // ignore
       }
+      this._subClient = null
     }
+    try {
+      if (this.redisClient) {
+        if (this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) {
+          await this.redisClient.quit()
+        } else if (this.redisClientType === IOREDIS) {
+          await this.redisClient.disconnect()
+        }
+      }
+    } catch {
+      // ignore
+    }
+    process.exit(0)
+  }
 
-    console.info(
-      `${
-        this._gracefulShutdownLogPrefix
-      } NEW: publishing to channel=${channel} dynoId=${this.dynoId || '1'}`
+  _publishAck() {
+    const ackChannel = this._gracefulShutdownAckChannel
+    if (!ackChannel || !this.redisClient) return Promise.resolve()
+    console.warn(
+      `${this._gracefulShutdownLogPrefix} OLD: clearing metrics and exiting.`
     )
+    const msg = this.dynoId || 'stopped'
+    if (this.redisClientType === REDIS_V3) {
+      return new Promise((resolve) => {
+        this.redisClient.send_command('PUBLISH', [ackChannel, msg], () => resolve())
+      })
+    }
+    if (this.redisClientType === REDIS_V4) {
+      return this.redisClient.sendCommand(['PUBLISH', ackChannel, msg]).catch(() => {})
+    }
+    if (this.redisClientType === IOREDIS) {
+      return this.redisClient.publish(ackChannel, msg).catch(() => {})
+    }
+    return Promise.resolve()
+  }
+
+  /**
+   * Publish "new instance started" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.
+   */
+  _publishNewInstanceStarted() {
+    const streamKey = this._gracefulShutdownStream
+    if (!streamKey || !this.redisClient) return
+    const value = this.dynoId || '1'
+    const noop = () => {}
+    const maxAgeMs = GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS
+    const minId = `${Date.now() - maxAgeMs}-0`
 
     if (this.redisClientType === REDIS_V3) {
-      this.redisClient.send_command('PUBLISH', [channel, message], done)
+      this.redisClient.send_command('XADD', [streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value], noop)
+      this.redisClient.send_command('XTRIM', [streamKey, 'MINID', minId], noop)
     } else if (this.redisClientType === REDIS_V4) {
-      this.redisClient
-        .sendCommand(['PUBLISH', channel, message])
-        .then(() => done())
-        .catch(done)
+      this.redisClient.sendCommand(['XADD', streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value]).catch(noop)
+      this.redisClient.sendCommand(['XTRIM', streamKey, 'MINID', minId]).catch(noop)
     } else if (this.redisClientType === IOREDIS) {
-      this.redisClient
-        .publish(channel, message)
-        .then(() => done())
-        .catch(done)
+      this.redisClient.call('XADD', streamKey, 'MAXLEN', '~', 10, '*', 'dyno_id', value).catch(noop)
+      this.redisClient.call('XTRIM', streamKey, 'MINID', minId).catch(noop)
     }
   }
 
@@ -498,7 +553,7 @@ class RedisMetricsClient extends BaseMetricsClient {
         )
       }
     } catch (error) {
-      console.info(
+      console.error(
         `[queue-metrics] Failed to collect Redis metrics:`,
         error.message
       )
@@ -514,14 +569,6 @@ class RedisMetricsClient extends BaseMetricsClient {
       await this.collectRedisMetrics()
       await this.gatewayPush()
       this.clearAllCounters()
-
-      if (this.metricsLogValues) {
-        const metricObjects = await this.registry.getMetricsAsJSON()
-        console.info(
-          `[queue-metrics] Collected metrics for Redis`,
-          JSON.stringify(metricObjects, null, 2)
-        )
-      }
     } catch (error) {
       console.error(
         `[queue-metrics] Failed to collect Redis metrics: ${error.message}`
@@ -531,8 +578,7 @@ class RedisMetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Start periodic collection.
-   * @param {number} [intervalSec=this.intervalSec] - Interval in seconds
+   * Start periodic collection. When graceful shutdown is on: new publishes "new started" so old exits and clears; new does not push until after one interval (skipFirstPush), giving old time to stop.
    */
   startPush = (intervalSec = this.intervalSec) => {
     this._startPush(intervalSec, () => {
@@ -587,7 +633,7 @@ class RedisMetricsClient extends BaseMetricsClient {
     } catch (err) {
       console.error('[queue-metrics] Error closing Redis client:', err)
     }
-    process.exit(0)
+    await super.cleanup()
   }
 
   _setCleanupHandlers = () => {