@adalo/metrics 0.1.163 → 0.1.165

package/__tests__/metricsRedisClient.test.js

@@ -0,0 +1,221 @@
+const http = require('http')
+const https = require('https')
+const { RedisMetricsClient } = require('../src/metrics/metricsRedisClient')
+const { IOREDIS, REDIS_V3, REDIS_V4 } = require('../src/redisUtils')
+
+jest.mock('../src/redisUtils', () => ({
+  ...jest.requireActual('../src/redisUtils'),
+  getRedisClientType: jest.fn(),
+  IOREDIS: 'IOREDIS',
+  REDIS_V3: 'REDIS_V3',
+  REDIS_V4: 'REDIS_V4',
+}))
+
+const { getRedisClientType } = require('../src/redisUtils')
+
+describe('RedisMetricsClient', () => {
+  const originalEnv = process.env
+  let mockRedisClient
+  let mockHttpRequest
+  let mockHttpsRequest
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    process.env = { ...originalEnv }
+    process.exit = jest.fn()
+
+    mockRedisClient = {
+      duplicate: jest.fn(() => ({
+        on: jest.fn(),
+        subscribe: jest.fn(),
+        disconnect: jest.fn(),
+        call: jest.fn().mockResolvedValue(null),
+        sendCommand: jest.fn().mockResolvedValue(null),
+        send_command: jest.fn((cmd, args, cb) => (cb ? cb(null, null) : undefined)),
+      })),
+      publish: jest.fn().mockResolvedValue(1),
+      sendCommand: jest.fn().mockResolvedValue(1),
+      send_command: jest.fn((cmd, args, cb) => (cb ? cb(null, 1) : undefined)),
+      disconnect: jest.fn(),
+      quit: jest.fn().mockResolvedValue(undefined),
+    }
+
+    const createMockRequest = (cb) => {
+      const req = {
+        setHeader: jest.fn(),
+        end: jest.fn(),
+        on: jest.fn(),
+      }
+      setImmediate(() => {
+        const res = { statusCode: 200, on: jest.fn() }
+        if (cb) cb(res)
+      })
+      return req
+    }
+    mockHttpRequest = jest.spyOn(http, 'request').mockImplementation((opts, cb) => {
+      const req = createMockRequest((res) => {
+        if (typeof cb === 'function') cb(res)
+      })
+      return req
+    })
+    mockHttpsRequest = jest.spyOn(https, 'request').mockImplementation((opts, cb) => {
+      const req = createMockRequest((res) => {
+        if (typeof cb === 'function') cb(res)
+      })
+      return req
+    })
+  })
+
+  afterEach(() => {
+    process.env = originalEnv
+    mockHttpRequest.mockRestore()
+    mockHttpsRequest.mockRestore()
+  })
+
+  describe('constructor', () => {
+    it('throws when redisClient is missing', () => {
+      expect(() => new RedisMetricsClient({})).toThrow('RedisMetricsClient requires redisClient')
+      expect(() => new RedisMetricsClient()).toThrow('RedisMetricsClient requires redisClient')
+      expect(() => new RedisMetricsClient({ redisClient: null })).toThrow('RedisMetricsClient requires redisClient')
+    })
+
+    it('accepts redisClient and passes other options via metricsConfig', () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        appName: 'my-app',
+        processType: 'web',
+        pushgatewayUrl: 'http://vm:8428/api/v1/import/prometheus',
+      })
+      expect(client.redisClient).toBe(mockRedisClient)
+      expect(client.appName).toBe('my-app')
+      expect(client.processType).toBe('web')
+      expect(client.pushgatewayUrl).toBe('http://vm:8428/api/v1/import/prometheus')
+    })
+
+    it('reads gracefulShutdownRedis from metricsConfig', () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        gracefulShutdownRedis: false,
+      })
+      expect(client._gracefulShutdownRedis).toBe(false)
+      expect(client._gracefulShutdownStream).toBeNull()
+    })
+
+    it('enables graceful shutdown by default', () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        appName: 'test-app',
+        processType: 'worker',
+      })
+      expect(client._gracefulShutdownRedis).toBe(true)
+      expect(client._gracefulShutdownStream).toBe('metrics:graceful-shutdown:test-app:worker')
+      expect(client._gracefulShutdownAckChannel).toBe('metrics:graceful-shutdown-ack:test-app:worker')
+    })
+  })
+
+  describe('_initGracefulShutdown', () => {
+    it('sets channels and log prefix when enabled', () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        appName: 'myapp',
+        processType: 'queue-metrics',
+      })
+      expect(client._gracefulShutdownLogPrefix).toContain('myapp')
+      expect(client._gracefulShutdownLogPrefix).toContain('queue-metrics')
+    })
+
+    it('respects METRICS_GRACEFUL_SHUTDOWN_REDIS=false', () => {
+      process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS = 'false'
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+      })
+      expect(client._gracefulShutdownRedis).toBe(false)
+    })
+  })
+
+  describe('_createSubscriberClient', () => {
+    it('uses duplicate() for IOREDIS client', () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const dup = {
+        on: jest.fn(),
+        subscribe: jest.fn(),
+        call: jest.fn().mockResolvedValue(null),
+      }
+      mockRedisClient.duplicate.mockReturnValue(dup)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        gracefulShutdownRedis: true,
+      })
+      expect(client._subClient).toBe(dup)
+      expect(mockRedisClient.duplicate).toHaveBeenCalled()
+    })
+
+    it('uses duplicate() for node-redis v4 when available', () => {
+      getRedisClientType.mockReturnValue(REDIS_V4)
+      const dup = {
+        on: jest.fn(),
+        subscribe: jest.fn(),
+        quit: jest.fn().mockResolvedValue(),
+        sendCommand: jest.fn().mockResolvedValue(null),
+      }
+      mockRedisClient.duplicate = jest.fn(() => dup)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        gracefulShutdownRedis: true,
+      })
+      expect(client._subClient).toBe(dup)
+      expect(mockRedisClient.duplicate).toHaveBeenCalled()
+    })
+
+    it('does not override skipFirstPush from config', () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        skipFirstPush: false,
+      })
+      expect(client.skipFirstPush).toBe(false)
+    })
+  })
+
+  describe('gateway push (mocked)', () => {
+    it('calls http.request when pushgatewayUrl is set', async () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        pushgatewayUrl: 'http://localhost:8428/api/v1/import/prometheus',
+        enabled: true,
+        cleanupExitsProcess: false,
+      })
+      await client.gatewayPush()
+      expect(mockHttpRequest).toHaveBeenCalled()
+      expect(mockHttpRequest.mock.calls[0][0].method).toBe('POST')
+      expect(mockHttpRequest.mock.calls[0][0].path).toContain('import/prometheus')
+    })
+  })
+
+  describe('gateway delete (mocked)', () => {
+    it('calls http.request for delete_series when removeOldMetrics and pushgatewayUrl set', async () => {
+      getRedisClientType.mockReturnValue(IOREDIS)
+      const client = new RedisMetricsClient({
+        redisClient: mockRedisClient,
+        pushgatewayUrl: 'http://localhost:8428/api/v1/import/prometheus',
+        removeOldMetrics: true,
+        enabled: true,
+        appName: 'test-app',
+        dynoId: 'dyno-1',
+        processType: 'web',
+        cleanupExitsProcess: false,
+      })
+      await client.gatewayDelete()
+      expect(mockHttpRequest).toHaveBeenCalled()
+      const call = mockHttpRequest.mock.calls[0][0]
+      expect(call.path).toContain('delete_series')
+      expect(call.path).toContain('match')
+    })
+  })
+})

package/lib/metrics/metricsRedisClient.d.ts

@@ -56,7 +56,7 @@ export class RedisMetricsClient extends BaseMetricsClient {
      */
     _initGracefulShutdown(gracefulShutdownRedis?: boolean | undefined): void;
     _gracefulShutdownRedis: boolean | undefined;
-    _gracefulShutdownChannel: string | null | undefined;
+    _gracefulShutdownStream: string | null | undefined;
     _gracefulShutdownAckChannel: string | null | undefined;
     _gracefulShutdownLogPrefix: string | undefined;
     /**
@@ -67,14 +67,20 @@ export class RedisMetricsClient extends BaseMetricsClient {
     /**
      * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).
      */
+    /**
+     * Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.
+     */
     _setupGracefulShutdownSubscribe(): void;
+    _streamReadLoop(streamKey: any): void;
+    _xreadBlock(streamKey: any, blockMs: any): Promise<any>;
+    _parseXreadReply(reply: any, streamKey: any): any[][];
     /**
      * Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.
      */
     _cleanupAndPublishAck(): Promise<void>;
     _publishAck(): any;
     /**
-     * Publish "new instance started" so old instances exit and clear metrics. No push waiting.
+     * Publish "new instance started" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.
      */
     _publishNewInstanceStarted(): void;
     getRedisConnections: () => Promise<any>;

package/lib/metrics/metricsRedisClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metricsRedisClient.d.ts","sourceRoot":"","sources":["../../src/metrics/metricsRedisClient.js"],"names":[],"mappings":"AAWA;;;;;GAKG;AACH;IACE;;;;;;;;;;;;;;;OAeG;IACH;QAdwB,WAAW,EAAxB,GAAG;QACe,qBAAqB;QACtB,OAAO;QACP,MAAM;QACN,WAAW;QACV,OAAO;QACP,SAAS;QACV,cAAc;QACd,iBAAiB;QACjB,WAAW;QACV,gBAAgB;QACf,iBAAiB;QAClB,kBAAkB;OAgE9C;IA9CC,oCAAoC;IACpC,iBAA8B;IAC9B,wBAAsD;IAEtD,qFAAqF;IACrF,gBAAsB;IAItB,2CAA2C;IAC3C,2DAIE;IAEF,iEAIE;IAEF,mCAAmC;IACnC,sDAIE;IAEF,sCAAsC;IACtC,qDAIE;IAQF,6BAAmC;IACnC,+BAAqC;IAKvC;;;OAGG;IACH,yEAeC;IAXC,4CAAgE;IAChE,oDAEQ;IACR,uDAEQ;IACR,+CAAgH;IAMlH;;;OAGG;IACH,2BAFa,GAAG,GAAC,IAAI,CAuBpB;IAED;;OAEG;IACH,wCA+BC;IAED;;OAEG;IACH,uCAoCC;IAED,mBAmBC;IAED;;OAEG;IACH,mCAYC;IAED,wCAgCC;IAED,6CAuBC;IAED,gDAkBC;IAED;;;OAGG;IACH,2BAFa,QAAQ,IAAI,CAAC,CA6KzB;IAED;;;OAGG;IACH,wBAFa,QAAQ,IAAI,CAAC,CAazB;IAED;;OAEG;IACH,0CASC;CAmDF"}
+{"version":3,"file":"metricsRedisClient.d.ts","sourceRoot":"","sources":["../../src/metrics/metricsRedisClient.js"],"names":[],"mappings":"AAcA;;;;;GAKG;AACH;IACE;;;;;;;;;;;;;;;OAeG;IACH;QAdwB,WAAW,EAAxB,GAAG;QACe,qBAAqB;QACtB,OAAO;QACP,MAAM;QACN,WAAW;QACV,OAAO;QACP,SAAS;QACV,cAAc;QACd,iBAAiB;QACjB,WAAW;QACV,gBAAgB;QACf,iBAAiB;QAClB,kBAAkB;OAgE9C;IA9CC,oCAAoC;IACpC,iBAA8B;IAC9B,wBAAsD;IAEtD,qFAAqF;IACrF,gBAAsB;IAItB,2CAA2C;IAC3C,2DAIE;IAEF,iEAIE;IAEF,mCAAmC;IACnC,sDAIE;IAEF,sCAAsC;IACtC,qDAIE;IAQF,6BAAmC;IACnC,+BAAqC;IAKvC;;;OAGG;IACH,yEAeC;IAXC,4CAAgE;IAChE,mDAEQ;IACR,uDAEQ;IACR,+CAAgH;IAMlH;;;OAGG;IACH,2BAFa,GAAG,GAAC,IAAI,CAuBpB;IAED;;OAEG;IACH;;OAEG;IACH,wCAaC;IAED,sCAmBC;IAED,wDAqBC;IAED,sDAKC;IAED;;OAEG;IACH,uCAoCC;IAED,mBAmBC;IAED;;OAEG;IACH,mCAkBC;IAED,wCAgCC;IAED,6CAuBC;IAED,gDAkBC;IAED;;;OAGG;IACH,2BAFa,QAAQ,IAAI,CAAC,CA6KzB;IAED;;;OAGG;IACH,wBAFa,QAAQ,IAAI,CAAC,CAazB;IAED;;OAEG;IACH,0CASC;CAmDF"}

package/lib/metrics/metricsRedisClient.js

@@ -12,6 +12,9 @@ const {
 const redisConnectionStableFields = ['name', 'flags', 'cmd'];
 const redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd'];
 
+/** Stream entries older than this (ms) are trimmed so messages are not kept in Redis. Fixed 60s. */
+const GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000;
+
 /**
  * RedisMetricsClient extends BaseMetricsClient to collect
  * Redis metrics periodically and push them to Prometheus Pushgateway.
@@ -102,10 +105,10 @@ class RedisMetricsClient extends BaseMetricsClient {
     const disabledByParam = gracefulShutdownRedis === false;
     const disabledByEnv = process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false';
     this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv;
-    this._gracefulShutdownChannel = this._gracefulShutdownRedis ? `metrics:graceful-shutdown:${this.appName}:${this.processType}` : null;
+    this._gracefulShutdownStream = this._gracefulShutdownRedis ? `metrics:graceful-shutdown:${this.appName}:${this.processType}` : null;
     this._gracefulShutdownAckChannel = this._gracefulShutdownRedis ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}` : null;
     this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`;
-    if (this._gracefulShutdownRedis && this._gracefulShutdownChannel) {
+    if (this._gracefulShutdownRedis && this._gracefulShutdownStream) {
       this._setupGracefulShutdownSubscribe();
     }
   }
@@ -142,31 +145,63 @@ class RedisMetricsClient extends BaseMetricsClient {
   /**
    * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).
    */
+  /**
+   * Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.
+   */
   _setupGracefulShutdownSubscribe() {
-    const channel = this._gracefulShutdownChannel;
-    if (!channel) return;
-    let subClient = this._createSubscriberClient();
+    const streamKey = this._gracefulShutdownStream;
+    if (!streamKey) return;
+    const subClient = this._createSubscriberClient();
     if (subClient && typeof subClient.on === 'function') {
       subClient.on('error', () => {});
     }
     if (!subClient) return;
     this._subClient = subClient;
-    const onNewStarted = () => {
-      this._cleanupAndPublishAck();
-    };
-    const onMessage = (ch, _msg) => {
-      if (ch === channel) onNewStarted();
+    this._streamReadLoop(streamKey);
+  }
+  _streamReadLoop(streamKey) {
+    const self = this;
+    const blockMs = 5000;
+    const runRead = () => {
+      if (!self._subClient) return;
+      self._xreadBlock(streamKey, blockMs).then(entries => {
+        if (!entries || entries.length === 0) {
+          setImmediate(runRead);
+          return;
+        }
+        self._cleanupAndPublishAck();
+      }).catch(() => {
+        setImmediate(runRead);
+      });
     };
+    runRead();
+  }
+  _xreadBlock(streamKey, blockMs) {
+    const client = this._subClient || this.redisClient;
+    if (!client) return Promise.resolve([]);
     if (this.redisClientType === REDIS_V3) {
-      subClient.on('message', onMessage);
-      subClient.subscribe(channel);
-    } else if (this.redisClientType === REDIS_V4) {
-      subClient.on('message', onMessage);
-      subClient.subscribe(channel);
-    } else if (this.redisClientType === IOREDIS) {
-      subClient.on('message', onMessage);
-      subClient.subscribe(channel);
+      return new Promise((resolve, reject) => {
+        client.send_command('XREAD', ['BLOCK', String(blockMs), 'STREAMS', streamKey, '$'], (err, result) => {
+          if (err) return reject(err);
+          resolve(this._parseXreadReply(result, streamKey));
+        });
+      });
+    }
+    if (this.redisClientType === REDIS_V4) {
+      const p = client.sendCommand(['XREAD', 'BLOCK', String(blockMs), 'STREAMS', streamKey, '$']);
+      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => []);
     }
+    if (this.redisClientType === IOREDIS) {
+      const p = client.call('XREAD', 'BLOCK', blockMs, 'STREAMS', streamKey, '$');
+      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => []);
+    }
+    return Promise.resolve([]);
+  }
+  _parseXreadReply(reply, streamKey) {
+    if (!reply || !Array.isArray(reply)) return [];
+    const streamReply = reply.find(r => r && r[0] === streamKey);
+    if (!streamReply || !Array.isArray(streamReply[1])) return [];
+    return streamReply[1].map(entry => Array.isArray(entry) ? [entry[0], entry[1] || []] : [entry, []]);
   }
 
   /**
@@ -229,19 +264,24 @@ class RedisMetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Publish "new instance started" so old instances exit and clear metrics. No push waiting.
+   * Publish "new instance started" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.
    */
   _publishNewInstanceStarted() {
-    if (!this._gracefulShutdownChannel || !this.redisClient) return;
-    const channel = this._gracefulShutdownChannel;
-    const message = this.dynoId || '1';
+    const streamKey = this._gracefulShutdownStream;
+    if (!streamKey || !this.redisClient) return;
+    const value = this.dynoId || '1';
     const noop = () => {};
+    const maxAgeMs = GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS;
+    const minId = `${Date.now() - maxAgeMs}-0`;
     if (this.redisClientType === REDIS_V3) {
-      this.redisClient.send_command('PUBLISH', [channel, message], noop);
+      this.redisClient.send_command('XADD', [streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value], noop);
+      this.redisClient.send_command('XTRIM', [streamKey, 'MINID', minId], noop);
     } else if (this.redisClientType === REDIS_V4) {
-      this.redisClient.sendCommand(['PUBLISH', channel, message]).catch(noop);
+      this.redisClient.sendCommand(['XADD', streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value]).catch(noop);
+      this.redisClient.sendCommand(['XTRIM', streamKey, 'MINID', minId]).catch(noop);
     } else if (this.redisClientType === IOREDIS) {
-      this.redisClient.publish(channel, message).catch(noop);
+      this.redisClient.call('XADD', streamKey, 'MAXLEN', '~', 10, '*', 'dyno_id', value).catch(noop);
+      this.redisClient.call('XTRIM', streamKey, 'MINID', minId).catch(noop);
     }
   }
   getRedisConnections = async () => {
@@ -528,7 +568,7 @@ class RedisMetricsClient extends BaseMetricsClient {
     } catch (err) {
       console.error('[queue-metrics] Error closing Redis client:', err);
     }
-    process.exit(0);
+    await super.cleanup();
   };
   _setCleanupHandlers = () => {
     process.on('SIGINT', this.cleanup);

package/lib/metrics/metricsRedisClient.js.map

@@ -1 +1 @@
-{"version":3,"file":"metricsRedisClient.js","names":["BaseMetricsClient","require","getRedisClientType","REDIS_V4","IOREDIS","REDIS_V3","redisConnectionStableFields","redisConnectionFields","RedisMetricsClient","constructor","redisClient","metricsConfig","Error","intervalSec","parseInt","process","env","METRICS_QUEUE_INTERVAL_SEC","processType","redisClientType","_subClient","_initGracefulShutdown","gracefulShutdownRedis","redisConnectionsGauge","createGauge","name","help","labelNames","withDefaultLabels","redisConnectionsMemoryGauge","redisMemoryGauge","redisStatsGauge","_redisConnSeenKeys","Set","_redisConnZeroedKeys","_setCleanupHandlers","disabledByParam","disabledByEnv","METRICS_GRACEFUL_SHUTDOWN_REDIS","_gracefulShutdownRedis","_gracefulShutdownChannel","appName","_gracefulShutdownAckChannel","_gracefulShutdownLogPrefix","dynoId","_setupGracefulShutdownSubscribe","_createSubscriberClient","duplicate","REDIS_URL","redis","createClient","url","channel","subClient","on","onNewStarted","_cleanupAndPublishAck","onMessage","ch","_msg","subscribe","stopPush","enabled","gatewayDelete","_publishAck","Promise","resolve","reject","quit","err","disconnect","exit","ackChannel","console","warn","msg","send_command","sendCommand","catch","publish","_publishNewInstanceStarted","message","noop","getRedisConnections","result","client","getRedisInfo","section","info","parseRedisConnections","clientsStr","split","filter","line","trim","map","parts","forEach","p","eqIdx","indexOf","k","slice","v","includes","collectRedisMetrics","memoryInfoStr","statsInfoStr","connectionsInfoStr","all","labels","getDefaultLabels","connections","logValues","log","length","JSON","stringify","missing","flags","cmd","c","grouped","conn","totMem","key","count","memory","mem","Number","isFinite","currentKeys","Object","keys","add","has","delete","reset","valueLabels","parse","set","groups","values","groupedTotal","reduce","sum","g","size","parseRedisInfo","infoStr","fromEntries","startsWith","stats","used_memory","memory_type","maxmemory","instantaneous_ops_per_sec","operation","error","pushRedisMetrics","gatewayPush","clearAllCounters","startPush","_startPush","cleanup","module","exports"],"sources":["../../src/metrics/metricsRedisClient.js"],"sourcesContent":["const { BaseMetricsClient } = require('./baseMetricsClient')\nconst {\n  getRedisClientType,\n  REDIS_V4,\n  IOREDIS,\n  REDIS_V3,\n} = require('../redisUtils')\n\nconst redisConnectionStableFields = ['name', 'flags', 'cmd']\nconst redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']\n\n/**\n * RedisMetricsClient extends BaseMetricsClient to collect\n * Redis metrics periodically and push them to Prometheus Pushgateway.\n *\n * @extends BaseMetricsClient\n */\nclass RedisMetricsClient extends BaseMetricsClient {\n  /**\n   * @param {Object} options\n   * @param {any} options.redisClient - Redis client instance (required). Used for metrics and publish. Subscriber is created internally (duplicate() or REDIS_URL).\n   * @param {boolean} [options.gracefulShutdownRedis] - Default true. When true, new instance publishes on start and old instances exit on message. Set false or METRICS_GRACEFUL_SHUTDOWN_REDIS=false to disable.\n   * @param {string} [options.appName] - Application name (from BaseMetricsClient)\n   * @param {string} [options.dynoId] - Dyno/instance ID (from BaseMetricsClient)\n   * @param {string} [options.processType] - Process type (from BaseMetricsClient)\n   * @param {boolean} [options.enabled] - Enable metrics collection (from BaseMetricsClient)\n   * @param {boolean} [options.logValues] - Log metrics values (from BaseMetricsClient)\n   * @param {string} [options.pushgatewayUrl] - PushGateway URL (from BaseMetricsClient)\n   * @param {string} [options.pushgatewaySecret] - PushGateway secret token (from BaseMetricsClient)\n   * @param {number} [options.intervalSec] - Interval in seconds for pushing metrics (from BaseMetricsClient)\n   * @param {boolean} [options.removeOldMetrics] - Remove old metrics by service (from BaseMetricsClient)\n   * @param {function} [options.startupValidation] - Function to validate startup (from BaseMetricsClient)\n   * @param {boolean} [options.disablePushgateway] - Disable pushing to Pushgateway (use HTTP scraping instead)\n   */\n  constructor({ redisClient, ...metricsConfig } = {}) {\n    if (redisClient == null) {\n      throw new Error('RedisMetricsClient requires redisClient')\n    }\n\n    const intervalSec =\n      metricsConfig.intervalSec ||\n      parseInt(process.env.METRICS_QUEUE_INTERVAL_SEC || '', 10) ||\n      5\n\n    super({\n      ...metricsConfig,\n      processType: metricsConfig.processType || 'queue-metrics',\n      intervalSec,\n    })\n\n    /** Redis client used for metrics */\n    this.redisClient = redisClient\n    this.redisClientType = getRedisClientType(redisClient)\n\n    /** Dedicated Redis connection for subscribe (subscriber mode); closed in cleanup. */\n    this._subClient = null\n\n    this._initGracefulShutdown(metricsConfig.gracefulShutdownRedis)\n\n    /** Counter for Redis client connections */\n    this.redisConnectionsGauge = this.createGauge({\n      name: 'app_redis_connections_count',\n      help: 'Redis client connections',\n      labelNames: this.withDefaultLabels(redisConnectionStableFields),\n    })\n\n    this.redisConnectionsMemoryGauge = this.createGauge({\n      name: 'app_redis_connections_memory_usage_count',\n      help: 'Redis client connections',\n      labelNames: this.withDefaultLabels(redisConnectionStableFields),\n    })\n\n    /** Gauge for Redis memory usage */\n    this.redisMemoryGauge = this.createGauge({\n      name: 'app_redis_memory_bytes',\n      help: 'Redis memory usage in bytes',\n      labelNames: this.withDefaultLabels(['memory_type']),\n    })\n\n    /** Gauge for Redis operation stats */\n    this.redisStatsGauge = this.createGauge({\n      name: 'app_redis_stats_total',\n      help: 'Redis operation statistics',\n      labelNames: this.withDefaultLabels(['operation']),\n    })\n\n    // Track emitted connection label combinations so we can:\n    // - emit a one-shot 0 for series that disappear (overwrite last non-zero sample)\n    // - then stop emitting them on subsequent scrapes\n    //\n    // This is mainly needed because we label connections by `cmd`,\n    // and Redis `CLIENT LIST` `cmd` is volatile.\n    this._redisConnSeenKeys = new Set()\n    this._redisConnZeroedKeys = new Set()\n\n    this._setCleanupHandlers()\n  }\n\n  /**\n   * Initialize graceful-shutdown state and subscribe when enabled. Called from constructor.\n   * @param {boolean} [gracefulShutdownRedis] - Explicit false to disable; otherwise enabled unless METRICS_GRACEFUL_SHUTDOWN_REDIS=false.\n   */\n  _initGracefulShutdown(gracefulShutdownRedis) {\n    const disabledByParam = gracefulShutdownRedis === false\n    const disabledByEnv =\n      process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false'\n    this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv\n    this._gracefulShutdownChannel = this._gracefulShutdownRedis\n      ? `metrics:graceful-shutdown:${this.appName}:${this.processType}`\n      : null\n    this._gracefulShutdownAckChannel = this._gracefulShutdownRedis\n      ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}`\n      : null\n    this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`\n    if (this._gracefulShutdownRedis && this._gracefulShutdownChannel) {\n      this._setupGracefulShutdownSubscribe()\n    }\n  }\n\n  /**\n   * Create a dedicated Redis client for subscribe (pub/sub). Uses duplicate() when available, else createClient from REDIS_URL. Branches by redisClientType (IOREDIS, REDIS_V3, REDIS_V4).\n   * @returns {any|null} Subscriber client or null if not possible.\n   */\n  _createSubscriberClient() {\n    if (this.redisClientType === IOREDIS && this.redisClient && typeof this.redisClient.duplicate === 'function') {\n      return this.redisClient.duplicate()\n    }\n    if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && this.redisClient && typeof this.redisClient.duplicate === 'function') {\n      return this.redisClient.duplicate()\n    }\n    if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && process.env.REDIS_URL) {\n      try {\n        const redis = require('redis')\n        if (typeof redis.createClient !== 'function') return null\n        try {\n          return redis.createClient({ url: process.env.REDIS_URL })\n        } catch {\n          return redis.createClient(process.env.REDIS_URL)\n        }\n      } catch {\n        return null\n      }\n    }\n    return null\n  }\n\n  /**\n   * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).\n   */\n  _setupGracefulShutdownSubscribe() {\n    const channel = this._gracefulShutdownChannel\n    if (!channel) return\n\n    let subClient = this._createSubscriberClient()\n    if (subClient && typeof subClient.on === 'function') {\n      subClient.on('error', () => {})\n    }\n\n    if (!subClient) return\n\n    this._subClient = subClient\n\n    const onNewStarted = () => {\n      this._cleanupAndPublishAck()\n    }\n\n    const onMessage = (ch, _msg) => {\n      if (ch === channel) onNewStarted()\n    }\n\n    if (this.redisClientType === REDIS_V3) {\n      subClient.on('message', onMessage)\n      subClient.subscribe(channel)\n    } else if (this.redisClientType === REDIS_V4) {\n      subClient.on('message', onMessage)\n      subClient.subscribe(channel)\n    } else if (this.redisClientType === IOREDIS) {\n      subClient.on('message', onMessage)\n      subClient.subscribe(channel)\n    }\n  }\n\n  /**\n   * Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.\n   */\n  async _cleanupAndPublishAck() {\n    this.stopPush()\n    if (this.enabled) {\n      await this.gatewayDelete()\n    }\n    await this._publishAck()\n    if (this._subClient) {\n      try {\n        if (this.redisClientType === REDIS_V3) {\n          await new Promise((resolve, reject) => {\n            if (typeof this._subClient.quit === 'function') {\n              this._subClient.quit(err => (err ? reject(err) : resolve()))\n            } else resolve()\n          })\n        } else if (this.redisClientType === REDIS_V4) {\n          if (this._subClient.quit) await this._subClient.quit()\n        } else if (this.redisClientType === IOREDIS) {\n          if (this._subClient.disconnect) await this._subClient.disconnect()\n        }\n      } catch {\n        // ignore\n      }\n      this._subClient = null\n    }\n    try {\n      if (this.redisClient) {\n        if (this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) {\n          await this.redisClient.quit()\n        } else if (this.redisClientType === IOREDIS) {\n          await this.redisClient.disconnect()\n        }\n      }\n    } catch {\n      // ignore\n    }\n    process.exit(0)\n  }\n\n  _publishAck() {\n    const ackChannel = this._gracefulShutdownAckChannel\n    if (!ackChannel || !this.redisClient) return Promise.resolve()\n    console.warn(\n      `${this._gracefulShutdownLogPrefix} OLD: clearing metrics and exiting.`\n    )\n    const msg = this.dynoId || 'stopped'\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve) => {\n        this.redisClient.send_command('PUBLISH', [ackChannel, msg], () => resolve())\n      })\n    }\n    if (this.redisClientType === REDIS_V4) {\n      return this.redisClient.sendCommand(['PUBLISH', ackChannel, msg]).catch(() => {})\n    }\n    if (this.redisClientType === IOREDIS) {\n      return this.redisClient.publish(ackChannel, msg).catch(() => {})\n    }\n    return Promise.resolve()\n  }\n\n  /**\n   * Publish \"new instance started\" so old instances exit and clear metrics. No push waiting.\n   */\n  _publishNewInstanceStarted() {\n    if (!this._gracefulShutdownChannel || !this.redisClient) return\n    const channel = this._gracefulShutdownChannel\n    const message = this.dynoId || '1'\n    const noop = () => {}\n    if (this.redisClientType === REDIS_V3) {\n      this.redisClient.send_command('PUBLISH', [channel, message], noop)\n    } else if (this.redisClientType === REDIS_V4) {\n      this.redisClient.sendCommand(['PUBLISH', channel, message]).catch(noop)\n    } else if (this.redisClientType === IOREDIS) {\n      this.redisClient.publish(channel, message).catch(noop)\n    }\n  }\n\n  getRedisConnections = async () => {\n    if (!this.redisClient) throw new Error('Redis client not provided')\n\n    // node-redis v3 (uses callback)\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve, reject) => {\n        this.redisClient.send_command('CLIENT', ['LIST'], (err, result) => {\n          if (err) {\n            reject(new Error(`Failed to get CLIENT LIST: ${err.message}`))\n          } else resolve(result)\n        })\n      })\n    }\n\n    // node-redis v4\n    if (this.redisClientType === REDIS_V4) {\n      try {\n        return this.redisClient.sendCommand(['CLIENT', 'LIST'])\n      } catch (err) {\n        throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n      }\n    }\n\n    if (this.redisClientType === IOREDIS) {\n      try {\n        return this.redisClient.client('LIST')\n      } catch (err) {\n        throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n      }\n    }\n\n    throw new Error('Unsupported Redis client type')\n  }\n\n  getRedisInfo = async section => {\n    if (!this.redisClient) throw new Error('Redis client not provided')\n\n    // node-redis v3 (uses callback)\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve, reject) => {\n        this.redisClient.info(section, (err, result) => {\n          if (err) reject(err)\n          else resolve(result)\n        })\n      })\n    }\n\n    // node-redis v4 or ioredis (info returns Promise)\n    if (this.redisClientType === REDIS_V4 || this.redisClientType === IOREDIS) {\n      try {\n        return this.redisClient.info(section)\n      } catch (err) {\n        throw new Error(`Failed to get Redis INFO: ${err.message}`)\n      }\n    }\n\n    throw new Error('Unsupported Redis client type')\n  }\n\n  parseRedisConnections = clientsStr => {\n    return clientsStr\n      .split('\\n')\n      .filter(line => line.trim() !== '')\n      .map(line => {\n        const parts = line.split(' ')\n        const client = {}\n        parts.forEach(p => {\n          const eqIdx = p.indexOf('=')\n          if (eqIdx === -1) return\n          const k = p.slice(0, eqIdx)\n          const v = p.slice(eqIdx + 1)\n          if (redisConnectionFields.includes(k)) {\n            client[k] = v\n          }\n        })\n        return client\n      })\n  }\n\n  /**\n   * Collect basic Redis INFO metrics: clients, memory, stats\n   * @returns {Promise<void>}\n   */\n  collectRedisMetrics = async () => {\n    try {\n      const [memoryInfoStr, statsInfoStr, connectionsInfoStr] =\n        await Promise.all([\n          this.getRedisInfo('memory'),\n          this.getRedisInfo('stats'),\n          this.getRedisConnections(),\n        ])\n\n      const labels = this.getDefaultLabels()\n\n      const connections = this.parseRedisConnections(connectionsInfoStr)\n\n      if (this.logValues) {\n        // \"IN\" data: raw client list from Redis\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (raw):\\n',\n          connectionsInfoStr\n        )\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (raw) length:',\n          connectionsInfoStr.length\n        )\n\n        // \"OUT\" data: parsed fields used for metrics\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (parsed) count:',\n          connections.length\n        )\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (parsed sample, first 10):',\n          JSON.stringify(connections.slice(0, 10), null, 2)\n        )\n\n        const missing = { name: 0, flags: 0, cmd: 0, 'tot-mem': 0 }\n        connections.forEach(c => {\n          if (!c.name) missing.name += 1\n          if (!c.flags) missing.flags += 1\n          if (!c.cmd) missing.cmd += 1\n          if (!c['tot-mem']) missing['tot-mem'] += 1\n        })\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (parsed) missing fields:',\n          missing\n        )\n      }\n\n      const grouped = {}\n      connections.forEach(conn => {\n        const { name, flags, 'tot-mem': totMem, cmd } = conn\n\n        // build grouping key (includes default gauge labels later)\n        const key = JSON.stringify({ name, flags, cmd })\n\n        if (!grouped[key]) {\n          grouped[key] = {\n            labels: { name, flags, cmd },\n            count: 0,\n            memory: 0,\n          }\n        }\n\n        grouped[key].count += 1\n        const mem = parseInt(totMem, 10)\n        grouped[key].memory += Number.isFinite(mem) ? mem : 0\n      })\n\n      // Ensure sets exist even if older instance is hot-reloaded.\n      if (!this._redisConnSeenKeys) this._redisConnSeenKeys = new Set()\n      if (!this._redisConnZeroedKeys) this._redisConnZeroedKeys = new Set()\n\n      const currentKeys = new Set(Object.keys(grouped))\n      for (const k of currentKeys) {\n        this._redisConnSeenKeys.add(k)\n        if (this._redisConnZeroedKeys.has(k)) {\n          this._redisConnZeroedKeys.delete(k)\n        }\n      }\n\n      // Reset gauges each scrape so we only export:\n      // - current snapshot series\n      // - one-shot zeros for recently disappeared series\n      this.redisConnectionsGauge.reset()\n      this.redisConnectionsMemoryGauge.reset()\n\n      // Emit one-shot zeros for series that disappeared since last seen.\n      for (const k of this._redisConnSeenKeys) {\n        if (currentKeys.has(k)) continue\n        if (this._redisConnZeroedKeys.has(k)) continue\n        try {\n          const valueLabels = JSON.parse(k)\n          this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, 0)\n          this.redisConnectionsMemoryGauge.set({ ...labels, ...valueLabels }, 0)\n          this._redisConnZeroedKeys.add(k)\n        } catch {\n          // ignore malformed key\n        }\n      }\n\n      if (this.logValues) {\n        const groups = Object.values(grouped)\n        const groupedTotal = groups.reduce((sum, g) => sum + (g.count || 0), 0)\n        console.log(\n          '[queue-metrics] Redis connections grouped buckets:',\n          groups.length\n        )\n        console.log(\n          '[queue-metrics] Redis connections grouped total:',\n          groupedTotal\n        )\n        console.log(\n          '[queue-metrics] Redis connections seen keys:',\n          this._redisConnSeenKeys.size\n        )\n        console.log(\n          '[queue-metrics] Redis connections zeroed keys:',\n          this._redisConnZeroedKeys.size\n        )\n        console.log(\n          '[queue-metrics] Redis connections grouped sample (first 10):',\n          JSON.stringify(groups.slice(0, 10), null, 2)\n        )\n      }\n\n      Object.values(grouped).forEach(\n        ({ labels: valueLabels, count, memory }) => {\n          this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, count)\n          this.redisConnectionsMemoryGauge.set(\n            { ...labels, ...valueLabels },\n            memory\n          )\n        }\n      )\n\n      const parseRedisInfo = infoStr =>\n        Object.fromEntries(\n          infoStr\n            .split('\\r\\n')\n            .filter(line => line && !line.startsWith('#'))\n            .map(line => line.split(':', 2))\n            .filter(parts => parts.length === 2 && parts[0] && parts[1])\n        )\n\n      const memory = parseRedisInfo(memoryInfoStr)\n      const stats = parseRedisInfo(statsInfoStr)\n\n      if (memory.used_memory) {\n        this.redisMemoryGauge.set(\n          { ...labels, memory_type: 'used' },\n          parseInt(memory.used_memory, 10) || 0\n        )\n      }\n      if (memory.maxmemory) {\n        this.redisMemoryGauge.set(\n          { ...labels, memory_type: 'max' },\n          parseInt(memory.maxmemory, 10) || 0\n        )\n      }\n\n      if (stats.instantaneous_ops_per_sec) {\n        this.redisStatsGauge.set(\n          { ...labels, operation: 'ops_per_sec' },\n          parseInt(stats.instantaneous_ops_per_sec, 10) || 0\n        )\n      }\n    } catch (error) {\n      console.error(\n        `[queue-metrics] Failed to collect Redis metrics:`,\n        error.message\n      )\n    }\n  }\n\n  /**\n   * Collect metrics for all Redis and push to Prometheus Pushgateway\n   * @returns {Promise<void>}\n   */\n  pushRedisMetrics = async () => {\n    try {\n      await this.collectRedisMetrics()\n      await this.gatewayPush()\n      this.clearAllCounters()\n    } catch (error) {\n      console.error(\n        `[queue-metrics] Failed to collect Redis metrics: ${error.message}`\n      )\n      throw error\n    }\n  }\n\n  /**\n   * Start periodic collection. When graceful shutdown is on: new publishes \"new started\" so old exits and clears; new does not push until after one interval (skipFirstPush), giving old time to stop.\n   */\n  startPush = (intervalSec = this.intervalSec) => {\n    this._startPush(intervalSec, () => {\n      this.pushRedisMetrics().catch(err => {\n        console.error(`[queue-metrics] Failed to push Redis metrics:`, err)\n      })\n    })\n    if (this._gracefulShutdownRedis) {\n      this._publishNewInstanceStarted()\n    }\n  }\n\n  /**\n   * Cleanup Redis client and exit process.\n   * Stops push, deletes this instance's metrics from VM (if removeOldMetrics), closes subscriber and main Redis, then exits.\n   * @returns {Promise<void>}\n   */\n  cleanup = async () => {\n    this.stopPush()\n    if (this.enabled) {\n      await this.gatewayDelete()\n    }\n    if (this._subClient) {\n      try {\n        if (this.redisClientType === REDIS_V3) {\n          await new Promise((resolve, reject) => {\n            if (typeof this._subClient.quit === 'function') {\n              this._subClient.quit(err => (err ? reject(err) : resolve()))\n            } else resolve()\n          })\n        } else if (this.redisClientType === REDIS_V4) {\n          if (this._subClient.quit) await this._subClient.quit()\n        } else if (this.redisClientType === IOREDIS) {\n          if (this._subClient.disconnect) await this._subClient.disconnect()\n        }\n      } catch (err) {\n        console.error('[queue-metrics] Error closing subscriber client:', err)\n      }\n      this._subClient = null\n    }\n    try {\n      if (!this.redisClient) return\n\n      if (\n        this.redisClientType === REDIS_V3 ||\n        this.redisClientType === REDIS_V4\n      ) {\n        await this.redisClient.quit()\n      } else if (this.redisClientType === IOREDIS) {\n        await this.redisClient.disconnect()\n      }\n    } catch (err) {\n      console.error('[queue-metrics] Error closing Redis client:', err)\n    }\n    process.exit(0)\n  }\n\n  _setCleanupHandlers = () => {\n    process.on('SIGINT', this.cleanup)\n    process.on('SIGTERM', this.cleanup)\n  }\n}\n\nmodule.exports = { RedisMetricsClient }\n"],"mappings":";;AAAA,MAAM;EAAEA;AAAkB,CAAC,GAAGC,OAAO,CAAC,qBAAqB,CAAC;AAC5D,MAAM;EACJC,kBAAkB;EAClBC,QAAQ;EACRC,OAAO;EACPC;AACF,CAAC,GAAGJ,OAAO,CAAC,eAAe,CAAC;AAE5B,MAAMK,2BAA2B,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC;AAC5D,MAAMC,qBAAqB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;;AAEjE;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,kBAAkB,SAASR,iBAAiB,CAAC;EACjD;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACES,WAAWA,CAAC;IAAEC,WAAW;IAAE,GAAGC;EAAc,CAAC,GAAG,CAAC,CAAC,EAAE;IAClD,IAAID,WAAW,IAAI,IAAI,EAAE;MACvB,MAAM,IAAIE,KAAK,CAAC,yCAAyC,CAAC;IAC5D;IAEA,MAAMC,WAAW,GACfF,aAAa,CAACE,WAAW,IACzBC,QAAQ,CAACC,OAAO,CAACC,GAAG,CAACC,0BAA0B,IAAI,EAAE,EAAE,EAAE,CAAC,IAC1D,CAAC;IAEH,KAAK,CAAC;MACJ,GAAGN,aAAa;MAChBO,WAAW,EAAEP,aAAa,CAACO,WAAW,IAAI,eAAe;MACzDL;IACF,CAAC,CAAC;;IAEF;IACA,IAAI,CAACH,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACS,eAAe,GAAGjB,kBAAkB,CAACQ,WAAW,CAAC;;IAEtD;IACA,IAAI,CAACU,UAAU,GAAG,IAAI;IAEtB,IAAI,CAACC,qBAAqB,CAACV,aAAa,CAACW,qBAAqB,CAAC;;IAE/D;IACA,IAAI,CAACC,qBAAqB,GAAG,IAAI,CAACC,WAAW,CAAC;MAC5CC,IAAI,EAAE,6BAA6B;MACnCC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAACtB,2BAA2B;IAChE,CAAC,CAAC;IAEF,IAAI,CAACuB,2BAA2B,GAAG,IAAI,CAACL,WAAW,CAAC;MAClDC,IAAI,EAAE,0CAA0C;MAChDC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAACtB,2BAA2B;IAChE,CAAC,CAAC;;IAEF;IACA,IAAI,CAACwB,gBAAgB,GAAG,IAAI,CAACN,WAAW,CAAC;MACvCC,IAAI,EAAE,wBAAwB;MAC9BC,IAAI,EAAE,6BAA6B;MACnCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAAC,CAAC,aAAa,CAAC;IACpD,CAAC,CAAC;;IAEF;IACA,IAAI,CAACG,eAAe,GAAG,IAAI,CAACP,WAAW,CAAC;MACtCC,IAAI,EAAE,uBAAuB;MAC7BC,IAAI,EAAE,4BAA4B;MAClCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAAC,CAAC,WAAW,CAAC;IAClD,CAAC,CAAC;;IAEF;IACA;IACA;IACA;IACA;IACA;IACA,IAAI,CAACI,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;IACnC,IAAI,CAACC,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;IAErC,IAAI,CAACE,mBAAmB,CAAC,CAAC;EAC5B;;EAEA;AACF;AACA;AACA;EACEd,qBAAqBA,CAACC,qBAAqB,EAAE;IAC3C,MAAMc,eAAe,GAAGd,qBAAqB,KAAK,KAAK;IACvD,MAAMe,aAAa,GACjBtB,OAAO,CAACC,GAAG,CAACsB,+BAA+B,KAAK,OAAO;IACzD,IAAI,CAACC,sBAAsB,GAAG,CAACH,eAAe,IAAI,CAACC,aAAa;IAChE,IAAI,CAACG,wBAAwB,GAAG,IAAI,CAACD,sBAAsB,GACvD,6BAA6B,IAAI,CAACE,OAAO,IAAI,IAAI,CAACvB,WAAW,EAAE,GAC/D,IAAI;IACR,IAAI,CAACwB,2BAA2B,GAAG,IAAI,CAACH,sBAAsB,GAC1D,iCAAiC,IAAI,CAACE,OAAO,IAAI,IAAI,CAACvB,WAAW,EAAE,GACnE,IAAI;IACR,IAAI,CAACyB,0BAA0B,GAAG,wBAAwB,IAAI,CAACzB,WAAW,MAAM,IAAI,CAACuB,OAAO,MAAM,IAAI,CAACG,MAAM,GAAG;IAChH,IAAI,IAAI,CAACL,sBAAsB,IAAI,IAAI,CAACC,wBAAwB,EAAE;MAChE,IAAI,CAACK,+BAA+B,CAAC,CAAC;IACxC;EACF;;EAEA;AACF;AACA;AACA;EACEC,uBAAuBA,CAAA,EAAG;IACxB,IAAI,IAAI,CAAC3B,eAAe,KAAKf,OAAO,IAAI,IAAI,CAACM,WAAW,IAAI,OAAO,IAAI,CAACA,WAAW,CAACqC,SAAS,KAAK,UAAU,EAAE;MAC5G,OAAO,IAAI,CAACrC,WAAW,CAACqC,SAAS,CAAC,CAAC;IACrC;IACA,IAAI,CAAC,IAAI,CAAC5B,eAAe,KAAKd,QAAQ,IAAI,IAAI,CAACc,eAAe,KAAKhB,QAAQ,KAAK,IAAI,CAACO,WAAW,IAAI,OAAO,IAAI,CAACA,WAAW,CAACqC,SAAS,KAAK,UAAU,EAAE;MACpJ,OAAO,IAAI,CAACrC,WAAW,CAACqC,SAAS,CAAC,CAAC;IACrC;IACA,IAAI,CAAC,IAAI,CAAC5B,eAAe,KAAKd,QAAQ,IAAI,IAAI,CAACc,eAAe,KAAKhB,QAAQ,KAAKY,OAAO,CAACC,GAAG,CAACgC,SAAS,EAAE;MACrG,IAAI;QACF,MAAMC,KAAK,GAAGhD,OAAO,CAAC,OAAO,CAAC;QAC9B,IAAI,OAAOgD,KAAK,CAACC,YAAY,KAAK,UAAU,EAAE,OAAO,IAAI;QACzD,IAAI;UACF,OAAOD,KAAK,CAACC,YAAY,CAAC;YAAEC,GAAG,EAAEpC,OAAO,CAACC,GAAG,CAACgC;UAAU,CAAC,CAAC;QAC3D,CAAC,CAAC,MAAM;UACN,OAAOC,KAAK,CAACC,YAAY,CAACnC,OAAO,CAACC,GAAG,CAACgC,SAAS,CAAC;QAClD;MACF,CAAC,CAAC,MAAM;QACN,OAAO,IAAI;MACb;IACF;IACA,OAAO,IAAI;EACb;;EAEA;AACF;AACA;EACEH,+BAA+BA,CAAA,EAAG;IAChC,MAAMO,OAAO,GAAG,IAAI,CAACZ,wBAAwB;IAC7C,IAAI,CAACY,OAAO,EAAE;IAEd,IAAIC,SAAS,GAAG,IAAI,CAACP,uBAAuB,CAAC,CAAC;IAC9C,IAAIO,SAAS,IAAI,OAAOA,SAAS,CAACC,EAAE,KAAK,UAAU,EAAE;MACnDD,SAAS,CAACC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACjC;IAEA,IAAI,CAACD,SAAS,EAAE;IAEhB,IAAI,CAACjC,UAAU,GAAGiC,SAAS;IAE3B,MAAME,YAAY,GAAGA,CAAA,KAAM;MACzB,IAAI,CAACC,qBAAqB,CAAC,CAAC;IAC9B,CAAC;IAED,MAAMC,SAAS,GAAGA,CAACC,EAAE,EAAEC,IAAI,KAAK;MAC9B,IAAID,EAAE,KAAKN,OAAO,EAAEG,YAAY,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,CAACpC,eAAe,KAAKd,QAAQ,EAAE;MACrCgD,SAAS,CAACC,EAAE,CAAC,SAAS,EAAEG,SAAS,CAAC;MAClCJ,SAAS,CAACO,SAAS,CAACR,OAAO,CAAC;IAC9B,CAAC,MAAM,IAAI,IAAI,CAACjC,eAAe,KAAKhB,QAAQ,EAAE;MAC5CkD,SAAS,CAACC,EAAE,CAAC,SAAS,EAAEG,SAAS,CAAC;MAClCJ,SAAS,CAACO,SAAS,CAACR,OAAO,CAAC;IAC9B,CAAC,MAAM,IAAI,IAAI,CAACjC,eAAe,KAAKf,OAAO,EAAE;MAC3CiD,SAAS,CAACC,EAAE,CAAC,SAAS,EAAEG,SAAS,CAAC;MAClCJ,SAAS,CAACO,SAAS,CAACR,OAAO,CAAC;IAC9B;EACF;;EAEA;AACF;AACA;EACE,MAAMI,qBAAqBA,CAAA,EAAG;IAC5B,IAAI,CAACK,QAAQ,CAAC,CAAC;IACf,IAAI,IAAI,CAACC,OAAO,EAAE;MAChB,MAAM,IAAI,CAACC,aAAa,CAAC,CAAC;IAC5B;IACA,MAAM,IAAI,CAACC,WAAW,CAAC,CAAC;IACxB,IAAI,IAAI,CAAC5C,UAAU,EAAE;MACnB,IAAI;QACF,IAAI,IAAI,CAACD,eAAe,KAAKd,QAAQ,EAAE;UACrC,MAAM,IAAI4D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;YACrC,IAAI,OAAO,IAAI,CAAC/C,UAAU,CAACgD,IAAI,KAAK,UAAU,EAAE;cAC9C,IAAI,CAAChD,UAAU,CAACgD,IAAI,CAACC,GAAG,IAAKA,GAAG,GAAGF,MAAM,CAACE,GAAG,CAAC,GAAGH,OAAO,CAAC,CAAE,CAAC;YAC9D,CAAC,MAAMA,OAAO,CAAC,CAAC;UAClB,CAAC,CAAC;QACJ,CAAC,MAAM,IAAI,IAAI,CAAC/C,eAAe,KAAKhB,QAAQ,EAAE;UAC5C,IAAI,IAAI,CAACiB,UAAU,CAACgD,IAAI,EAAE,MAAM,IAAI,CAAChD,UAAU,CAACgD,IAAI,CAAC,CAAC;QACxD,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKf,OAAO,EAAE;UAC3C,IAAI,IAAI,CAACgB,UAAU,CAACkD,UAAU,EAAE,MAAM,IAAI,CAAClD,UAAU,CAACkD,UAAU,CAAC,CAAC;QACpE;MACF,CAAC,CAAC,MAAM;QACN;MAAA;MAEF,IAAI,CAAClD,UAAU,GAAG,IAAI;IACxB;IACA,IAAI;MACF,IAAI,IAAI,CAACV,WAAW,EAAE;QACpB,IAAI,IAAI,CAACS,eAAe,KAAKd,QAAQ,IAAI,IAAI,CAACc,eAAe,KAAKhB,QAAQ,EAAE;UAC1E,MAAM,IAAI,CAACO,WAAW,CAAC0D,IAAI,CAAC,CAAC;QAC/B,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKf,OAAO,EAAE;UAC3C,MAAM,IAAI,CAACM,WAAW,CAAC4D,UAAU,CAAC,CAAC;QACrC;MACF;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEFvD,OAAO,CAACwD,IAAI,CAAC,CAAC,CAAC;EACjB;EAEAP,WAAWA,CAAA,EAAG;IACZ,MAAMQ,UAAU,GAAG,IAAI,CAAC9B,2BAA2B;IACnD,IAAI,CAAC8B,UAAU,IAAI,CAAC,IAAI,CAAC9D,WAAW,EAAE,OAAOuD,OAAO,CAACC,OAAO,CAAC,CAAC;IAC9DO,OAAO,CAACC,IAAI,CACV,GAAG,IAAI,CAAC/B,0BAA0B,qCACpC,CAAC;IACD,MAAMgC,GAAG,GAAG,IAAI,CAAC/B,MAAM,IAAI,SAAS;IACpC,IAAI,IAAI,CAACzB,eAAe,KAAKd,QAAQ,EAAE;MACrC,OAAO,IAAI4D,OAAO,CAAEC,OAAO,IAAK;QAC9B,IAAI,CAACxD,WAAW,CAACkE,YAAY,CAAC,SAAS,EAAE,CAACJ,UAAU,EAAEG,GAAG,CAAC,EAAE,MAAMT,OAAO,CAAC,CAAC,CAAC;MAC9E,CAAC,CAAC;IACJ;IACA,IAAI,IAAI,CAAC/C,eAAe,KAAKhB,QAAQ,EAAE;MACrC,OAAO,IAAI,CAACO,WAAW,CAACmE,WAAW,CAAC,CAAC,SAAS,EAAEL,UAAU,EAAEG,GAAG,CAAC,CAAC,CAACG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACnF;IACA,IAAI,IAAI,CAAC3D,eAAe,KAAKf,OAAO,EAAE;MACpC,OAAO,IAAI,CAACM,WAAW,CAACqE,OAAO,CAACP,UAAU,EAAEG,GAAG,CAAC,CAACG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE;IACA,OAAOb,OAAO,CAACC,OAAO,CAAC,CAAC;EAC1B;;EAEA;AACF;AACA;EACEc,0BAA0BA,CAAA,EAAG;IAC3B,IAAI,CAAC,IAAI,CAACxC,wBAAwB,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;IACzD,MAAM0C,OAAO,GAAG,IAAI,CAACZ,wBAAwB;IAC7C,MAAMyC,OAAO,GAAG,IAAI,CAACrC,MAAM,IAAI,GAAG;IAClC,MAAMsC,IAAI,GAAGA,CAAA,KAAM,CAAC,CAAC;IACrB,IAAI,IAAI,CAAC/D,eAAe,KAAKd,QAAQ,EAAE;MACrC,IAAI,CAACK,WAAW,CAACkE,YAAY,CAAC,SAAS,EAAE,CAACxB,OAAO,EAAE6B,OAAO,CAAC,EAAEC,IAAI,CAAC;IACpE,CAAC,MAAM,IAAI,IAAI,CAAC/D,eAAe,KAAKhB,QAAQ,EAAE;MAC5C,IAAI,CAACO,WAAW,CAACmE,WAAW,CAAC,CAAC,SAAS,EAAEzB,OAAO,EAAE6B,OAAO,CAAC,CAAC,CAACH,KAAK,CAACI,IAAI,CAAC;IACzE,CAAC,MAAM,IAAI,IAAI,CAAC/D,eAAe,KAAKf,OAAO,EAAE;MAC3C,IAAI,CAACM,WAAW,CAACqE,OAAO,CAAC3B,OAAO,EAAE6B,OAAO,CAAC,CAACH,KAAK,CAACI,IAAI,CAAC;IACxD;EACF;EAEAC,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI,CAAC,IAAI,CAACzE,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKd,QAAQ,EAAE;MACrC,OAAO,IAAI4D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAACzD,WAAW,CAACkE,YAAY,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,CAACP,GAAG,EAAEe,MAAM,KAAK;UACjE,IAAIf,GAAG,EAAE;YACPF,MAAM,CAAC,IAAIvD,KAAK,CAAC,8BAA8ByD,GAAG,CAACY,OAAO,EAAE,CAAC,CAAC;UAChE,CAAC,MAAMf,OAAO,CAACkB,MAAM,CAAC;QACxB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACjE,eAAe,KAAKhB,QAAQ,EAAE;MACrC,IAAI;QACF,OAAO,IAAI,CAACO,WAAW,CAACmE,WAAW,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;MACzD,CAAC,CAAC,OAAOR,GAAG,EAAE;QACZ,MAAM,IAAIzD,KAAK,CAAC,8BAA8ByD,GAAG,CAACY,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,IAAI,IAAI,CAAC9D,eAAe,KAAKf,OAAO,EAAE;MACpC,IAAI;QACF,OAAO,IAAI,CAACM,WAAW,CAAC2E,MAAM,CAAC,MAAM,CAAC;MACxC,CAAC,CAAC,OAAOhB,GAAG,EAAE;QACZ,MAAM,IAAIzD,KAAK,CAAC,8BAA8ByD,GAAG,CAACY,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,MAAM,IAAIrE,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAED0E,YAAY,GAAG,MAAMC,OAAO,IAAI;IAC9B,IAAI,CAAC,IAAI,CAAC7E,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKd,QAAQ,EAAE;MACrC,OAAO,IAAI4D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAACzD,WAAW,CAAC8E,IAAI,CAACD,OAAO,EAAE,CAAClB,GAAG,EAAEe,MAAM,KAAK;UAC9C,IAAIf,GAAG,EAAEF,MAAM,CAACE,GAAG,CAAC,MACfH,OAAO,CAACkB,MAAM,CAAC;QACtB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACjE,eAAe,KAAKhB,QAAQ,IAAI,IAAI,CAACgB,eAAe,KAAKf,OAAO,EAAE;MACzE,IAAI;QACF,OAAO,IAAI,CAACM,WAAW,CAAC8E,IAAI,CAACD,OAAO,CAAC;MACvC,CAAC,CAAC,OAAOlB,GAAG,EAAE;QACZ,MAAM,IAAIzD,KAAK,CAAC,6BAA6ByD,GAAG,CAACY,OAAO,EAAE,CAAC;MAC7D;IACF;IAEA,MAAM,IAAIrE,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAED6E,qBAAqB,GAAGC,UAAU,IAAI;IACpC,OAAOA,UAAU,CACdC,KAAK,CAAC,IAAI,CAAC,CACXC,MAAM,CAACC,IAAI,IAAIA,IAAI,CAACC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAClCC,GAAG,CAACF,IAAI,IAAI;MACX,MAAMG,KAAK,GAAGH,IAAI,CAACF,KAAK,CAAC,GAAG,CAAC;MAC7B,MAAMN,MAAM,GAAG,CAAC,CAAC;MACjBW,KAAK,CAACC,OAAO,CAACC,CAAC,IAAI;QACjB,MAAMC,KAAK,GAAGD,CAAC,CAACE,OAAO,CAAC,GAAG,CAAC;QAC5B,IAAID,KAAK,KAAK,CAAC,CAAC,EAAE;QAClB,MAAME,CAAC,GAAGH,CAAC,CAACI,KAAK,CAAC,CAAC,EAAEH,KAAK,CAAC;QAC3B,MAAMI,CAAC,GAAGL,CAAC,CAACI,KAAK,CAACH,KAAK,GAAG,CAAC,CAAC;QAC5B,IAAI5F,qBAAqB,CAACiG,QAAQ,CAACH,CAAC,CAAC,EAAE;UACrChB,MAAM,CAACgB,CAAC,CAAC,GAAGE,CAAC;QACf;MACF,CAAC,CAAC;MACF,OAAOlB,MAAM;IACf,CAAC,CAAC;EACN,CAAC;;EAED;AACF;AACA;AACA;EACEoB,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI;MACF,MAAM,CAACC,aAAa,EAAEC,YAAY,EAAEC,kBAAkB,CAAC,GACrD,MAAM3C,OAAO,CAAC4C,GAAG,CAAC,CAChB,IAAI,CAACvB,YAAY,CAAC,QAAQ,CAAC,EAC3B,IAAI,CAACA,YAAY,CAAC,OAAO,CAAC,EAC1B,IAAI,CAACH,mBAAmB,CAAC,CAAC,CAC3B,CAAC;MAEJ,MAAM2B,MAAM,GAAG,IAAI,CAACC,gBAAgB,CAAC,CAAC;MAEtC,MAAMC,WAAW,GAAG,IAAI,CAACvB,qBAAqB,CAACmB,kBAAkB,CAAC;MAElE,IAAI,IAAI,CAACK,SAAS,EAAE;QAClB;QACAxC,OAAO,CAACyC,GAAG,CACT,4CAA4C,EAC5CN,kBACF,CAAC;QACDnC,OAAO,CAACyC,GAAG,CACT,iDAAiD,EACjDN,kBAAkB,CAACO,MACrB,CAAC;;QAED;QACA1C,OAAO,CAACyC,GAAG,CACT,mDAAmD,EACnDF,WAAW,CAACG,MACd,CAAC;QACD1C,OAAO,CAACyC,GAAG,CACT,8DAA8D,EAC9DE,IAAI,CAACC,SAAS,CAACL,WAAW,CAACV,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAClD,CAAC;QAED,MAAMgB,OAAO,GAAG;UAAE7F,IAAI,EAAE,CAAC;UAAE8F,KAAK,EAAE,CAAC;UAAEC,GAAG,EAAE,CAAC;UAAE,SAAS,EAAE;QAAE,CAAC;QAC3DR,WAAW,CAACf,OAAO,CAACwB,CAAC,IAAI;UACvB,IAAI,CAACA,CAAC,CAAChG,IAAI,EAAE6F,OAAO,CAAC7F,IAAI,IAAI,CAAC;UAC9B,IAAI,CAACgG,CAAC,CAACF,KAAK,EAAED,OAAO,CAACC,KAAK,IAAI,CAAC;UAChC,IAAI,CAACE,CAAC,CAACD,GAAG,EAAEF,OAAO,CAACE,GAAG,IAAI,CAAC;UAC5B,IAAI,CAACC,CAAC,CAAC,SAAS,CAAC,EAAEH,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;QACF7C,OAAO,CAACyC,GAAG,CACT,4DAA4D,EAC5DI,OACF,CAAC;MACH;MAEA,MAAMI,OAAO,GAAG,CAAC,CAAC;MAClBV,WAAW,CAACf,OAAO,CAAC0B,IAAI,IAAI;QAC1B,MAAM;UAAElG,IAAI;UAAE8F,KAAK;UAAE,SAAS,EAAEK,MAAM;UAAEJ;QAAI,CAAC,GAAGG,IAAI;;QAEpD;QACA,MAAME,GAAG,GAAGT,IAAI,CAACC,SAAS,CAAC;UAAE5F,IAAI;UAAE8F,KAAK;UAAEC;QAAI,CAAC,CAAC;QAEhD,IAAI,CAACE,OAAO,CAACG,GAAG,CAAC,EAAE;UACjBH,OAAO,CAACG,GAAG,CAAC,GAAG;YACbf,MAAM,EAAE;cAAErF,IAAI;cAAE8F,KAAK;cAAEC;YAAI,CAAC;YAC5BM,KAAK,EAAE,CAAC;YACRC,MAAM,EAAE;UACV,CAAC;QACH;QAEAL,OAAO,CAACG,GAAG,CAAC,CAACC,KAAK,IAAI,CAAC;QACvB,MAAME,GAAG,GAAGlH,QAAQ,CAAC8G,MAAM,EAAE,EAAE,CAAC;QAChCF,OAAO,CAACG,GAAG,CAAC,CAACE,MAAM,IAAIE,MAAM,CAACC,QAAQ,CAACF,GAAG,CAAC,GAAGA,GAAG,GAAG,CAAC;MACvD,CAAC,CAAC;;MAEF;MACA,IAAI,CAAC,IAAI,CAAChG,kBAAkB,EAAE,IAAI,CAACA,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;MACjE,IAAI,CAAC,IAAI,CAACC,oBAAoB,EAAE,IAAI,CAACA,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;MAErE,MAAMkG,WAAW,GAAG,IAAIlG,GAAG,CAACmG,MAAM,CAACC,IAAI,CAACX,OAAO,CAAC,CAAC;MACjD,KAAK,MAAMrB,CAAC,IAAI8B,WAAW,EAAE;QAC3B,IAAI,CAACnG,kBAAkB,CAACsG,GAAG,CAACjC,CAAC,CAAC;QAC9B,IAAI,IAAI,CAACnE,oBAAoB,CAACqG,GAAG,CAAClC,CAAC,CAAC,EAAE;UACpC,IAAI,CAACnE,oBAAoB,CAACsG,MAAM,CAACnC,CAAC,CAAC;QACrC;MACF;;MAEA;MACA;MACA;MACA,IAAI,CAAC9E,qBAAqB,CAACkH,KAAK,CAAC,CAAC;MAClC,IAAI,CAAC5G,2BAA2B,CAAC4G,KAAK,CAAC,CAAC;;MAExC;MACA,KAAK,MAAMpC,CAAC,IAAI,IAAI,CAACrE,kBAAkB,EAAE;QACvC,IAAImG,WAAW,CAACI,GAAG,CAAClC,CAAC,CAAC,EAAE;QACxB,IAAI,IAAI,CAACnE,oBAAoB,CAACqG,GAAG,CAAClC,CAAC,CAAC,EAAE;QACtC,IAAI;UACF,MAAMqC,WAAW,GAAGtB,IAAI,CAACuB,KAAK,CAACtC,CAAC,CAAC;UACjC,IAAI,CAAC9E,qBAAqB,CAACqH,GAAG,CAAC;YAAE,GAAG9B,MAAM;YAAE,GAAG4B;UAAY,CAAC,EAAE,CAAC,CAAC;UAChE,IAAI,CAAC7G,2BAA2B,CAAC+G,GAAG,CAAC;YAAE,GAAG9B,MAAM;YAAE,GAAG4B;UAAY,CAAC,EAAE,CAAC,CAAC;UACtE,IAAI,CAACxG,oBAAoB,CAACoG,GAAG,CAACjC,CAAC,CAAC;QAClC,CAAC,CAAC,MAAM;UACN;QAAA;MAEJ;MAEA,IAAI,IAAI,CAACY,SAAS,EAAE;QAClB,MAAM4B,MAAM,GAAGT,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC;QACrC,MAAMqB,YAAY,GAAGF,MAAM,CAACG,MAAM,CAAC,CAACC,GAAG,EAAEC,CAAC,KAAKD,GAAG,IAAIC,CAAC,CAACpB,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACvErD,OAAO,CAACyC,GAAG,CACT,oDAAoD,EACpD2B,MAAM,CAAC1B,MACT,CAAC;QACD1C,OAAO,CAACyC,GAAG,CACT,kDAAkD,EAClD6B,YACF,CAAC;QACDtE,OAAO,CAACyC,GAAG,CACT,8CAA8C,EAC9C,IAAI,CAAClF,kBAAkB,CAACmH,IAC1B,CAAC;QACD1E,OAAO,CAACyC,GAAG,CACT,gDAAgD,EAChD,IAAI,CAAChF,oBAAoB,CAACiH,IAC5B,CAAC;QACD1E,OAAO,CAACyC,GAAG,CACT,8DAA8D,EAC9DE,IAAI,CAACC,SAAS,CAACwB,MAAM,CAACvC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAC7C,CAAC;MACH;MAEA8B,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC,CAACzB,OAAO,CAC5B,CAAC;QAAEa,MAAM,EAAE4B,WAAW;QAAEZ,KAAK;QAAEC;MAAO,CAAC,KAAK;QAC1C,IAAI,CAACxG,qBAAqB,CAACqH,GAAG,CAAC;UAAE,GAAG9B,MAAM;UAAE,GAAG4B;QAAY,CAAC,EAAEZ,KAAK,CAAC;QACpE,IAAI,CAACjG,2BAA2B,CAAC+G,GAAG,CAClC;UAAE,GAAG9B,MAAM;UAAE,GAAG4B;QAAY,CAAC,EAC7BX,MACF,CAAC;MACH,CACF,CAAC;MAED,MAAMqB,cAAc,GAAGC,OAAO,IAC5BjB,MAAM,CAACkB,WAAW,CAChBD,OAAO,CACJ1D,KAAK,CAAC,MAAM,CAAC,CACbC,MAAM,CAACC,IAAI,IAAIA,IAAI,IAAI,CAACA,IAAI,CAAC0D,UAAU,CAAC,GAAG,CAAC,CAAC,CAC7CxD,GAAG,CAACF,IAAI,IAAIA,IAAI,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAC/BC,MAAM,CAACI,KAAK,IAAIA,KAAK,CAACmB,MAAM,KAAK,CAAC,IAAInB,KAAK,CAAC,CAAC,CAAC,IAAIA,KAAK,CAAC,CAAC,CAAC,CAC/D,CAAC;MAEH,MAAM+B,MAAM,GAAGqB,cAAc,CAAC1C,aAAa,CAAC;MAC5C,MAAM8C,KAAK,GAAGJ,cAAc,CAACzC,YAAY,CAAC;MAE1C,IAAIoB,MAAM,CAAC0B,WAAW,EAAE;QACtB,IAAI,CAAC3H,gBAAgB,CAAC8G,GAAG,CACvB;UAAE,GAAG9B,MAAM;UAAE4C,WAAW,EAAE;QAAO,CAAC,EAClC5I,QAAQ,CAACiH,MAAM,CAAC0B,WAAW,EAAE,EAAE,CAAC,IAAI,CACtC,CAAC;MACH;MACA,IAAI1B,MAAM,CAAC4B,SAAS,EAAE;QACpB,IAAI,CAAC7H,gBAAgB,CAAC8G,GAAG,CACvB;UAAE,GAAG9B,MAAM;UAAE4C,WAAW,EAAE;QAAM,CAAC,EACjC5I,QAAQ,CAACiH,MAAM,CAAC4B,SAAS,EAAE,EAAE,CAAC,IAAI,CACpC,CAAC;MACH;MAEA,IAAIH,KAAK,CAACI,yBAAyB,EAAE;QACnC,IAAI,CAAC7H,eAAe,CAAC6G,GAAG,CACtB;UAAE,GAAG9B,MAAM;UAAE+C,SAAS,EAAE;QAAc,CAAC,EACvC/I,QAAQ,CAAC0I,KAAK,CAACI,yBAAyB,EAAE,EAAE,CAAC,IAAI,CACnD,CAAC;MACH;IACF,CAAC,CAAC,OAAOE,KAAK,EAAE;MACdrF,OAAO,CAACqF,KAAK,CACX,kDAAkD,EAClDA,KAAK,CAAC7E,OACR,CAAC;IACH;EACF,CAAC;;EAED;AACF;AACA;AACA;EACE8E,gBAAgB,GAAG,MAAAA,CAAA,KAAY;IAC7B,IAAI;MACF,MAAM,IAAI,CAACtD,mBAAmB,CAAC,CAAC;MAChC,MAAM,IAAI,CAACuD,WAAW,CAAC,CAAC;MACxB,IAAI,CAACC,gBAAgB,CAAC,CAAC;IACzB,CAAC,CAAC,OAAOH,KAAK,EAAE;MACdrF,OAAO,CAACqF,KAAK,CACX,oDAAoDA,KAAK,CAAC7E,OAAO,EACnE,CAAC;MACD,MAAM6E,KAAK;IACb;EACF,CAAC;;EAED;AACF;AACA;EACEI,SAAS,GAAGA,CAACrJ,WAAW,GAAG,IAAI,CAACA,WAAW,KAAK;IAC9C,IAAI,CAACsJ,UAAU,CAACtJ,WAAW,EAAE,MAAM;MACjC,IAAI,CAACkJ,gBAAgB,CAAC,CAAC,CAACjF,KAAK,CAACT,GAAG,IAAI;QACnCI,OAAO,CAACqF,KAAK,CAAC,+CAA+C,EAAEzF,GAAG,CAAC;MACrE,CAAC,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,IAAI,CAAC9B,sBAAsB,EAAE;MAC/B,IAAI,CAACyC,0BAA0B,CAAC,CAAC;IACnC;EACF,CAAC;;EAED;AACF;AACA;AACA;AACA;EACEoF,OAAO,GAAG,MAAAA,CAAA,KAAY;IACpB,IAAI,CAACvG,QAAQ,CAAC,CAAC;IACf,IAAI,IAAI,CAACC,OAAO,EAAE;MAChB,MAAM,IAAI,CAACC,aAAa,CAAC,CAAC;IAC5B;IACA,IAAI,IAAI,CAAC3C,UAAU,EAAE;MACnB,IAAI;QACF,IAAI,IAAI,CAACD,eAAe,KAAKd,QAAQ,EAAE;UACrC,MAAM,IAAI4D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;YACrC,IAAI,OAAO,IAAI,CAAC/C,UAAU,CAACgD,IAAI,KAAK,UAAU,EAAE;cAC9C,IAAI,CAAChD,UAAU,CAACgD,IAAI,CAACC,GAAG,IAAKA,GAAG,GAAGF,MAAM,CAACE,GAAG,CAAC,GAAGH,OAAO,CAAC,CAAE,CAAC;YAC9D,CAAC,MAAMA,OAAO,CAAC,CAAC;UAClB,CAAC,CAAC;QACJ,CAAC,MAAM,IAAI,IAAI,CAAC/C,eAAe,KAAKhB,QAAQ,EAAE;UAC5C,IAAI,IAAI,CAACiB,UAAU,CAACgD,IAAI,EAAE,MAAM,IAAI,CAAChD,UAAU,CAACgD,IAAI,CAAC,CAAC;QACxD,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKf,OAAO,EAAE;UAC3C,IAAI,IAAI,CAACgB,UAAU,CAACkD,UAAU,EAAE,MAAM,IAAI,CAAClD,UAAU,CAACkD,UAAU,CAAC,CAAC;QACpE;MACF,CAAC,CAAC,OAAOD,GAAG,EAAE;QACZI,OAAO,CAACqF,KAAK,CAAC,kDAAkD,EAAEzF,GAAG,CAAC;MACxE;MACA,IAAI,CAACjD,UAAU,GAAG,IAAI;IACxB;IACA,IAAI;MACF,IAAI,CAAC,IAAI,CAACV,WAAW,EAAE;MAEvB,IACE,IAAI,CAACS,eAAe,KAAKd,QAAQ,IACjC,IAAI,CAACc,eAAe,KAAKhB,QAAQ,EACjC;QACA,MAAM,IAAI,CAACO,WAAW,CAAC0D,IAAI,CAAC,CAAC;MAC/B,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKf,OAAO,EAAE;QAC3C,MAAM,IAAI,CAACM,WAAW,CAAC4D,UAAU,CAAC,CAAC;MACrC;IACF,CAAC,CAAC,OAAOD,GAAG,EAAE;MACZI,OAAO,CAACqF,KAAK,CAAC,6CAA6C,EAAEzF,GAAG,CAAC;IACnE;IACAtD,OAAO,CAACwD,IAAI,CAAC,CAAC,CAAC;EACjB,CAAC;EAEDpC,mBAAmB,GAAGA,CAAA,KAAM;IAC1BpB,OAAO,CAACuC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC8G,OAAO,CAAC;IAClCrJ,OAAO,CAACuC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC8G,OAAO,CAAC;EACrC,CAAC;AACH;AAEAC,MAAM,CAACC,OAAO,GAAG;EAAE9J;AAAmB,CAAC","ignoreList":[]}
+{"version":3,"file":"metricsRedisClient.js","names":["BaseMetricsClient","require","getRedisClientType","REDIS_V4","IOREDIS","REDIS_V3","redisConnectionStableFields","redisConnectionFields","GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS","RedisMetricsClient","constructor","redisClient","metricsConfig","Error","intervalSec","parseInt","process","env","METRICS_QUEUE_INTERVAL_SEC","processType","redisClientType","_subClient","_initGracefulShutdown","gracefulShutdownRedis","redisConnectionsGauge","createGauge","name","help","labelNames","withDefaultLabels","redisConnectionsMemoryGauge","redisMemoryGauge","redisStatsGauge","_redisConnSeenKeys","Set","_redisConnZeroedKeys","_setCleanupHandlers","disabledByParam","disabledByEnv","METRICS_GRACEFUL_SHUTDOWN_REDIS","_gracefulShutdownRedis","_gracefulShutdownStream","appName","_gracefulShutdownAckChannel","_gracefulShutdownLogPrefix","dynoId","_setupGracefulShutdownSubscribe","_createSubscriberClient","duplicate","REDIS_URL","redis","createClient","url","streamKey","subClient","on","_streamReadLoop","self","blockMs","runRead","_xreadBlock","then","entries","length","setImmediate","_cleanupAndPublishAck","catch","client","Promise","resolve","reject","send_command","String","err","result","_parseXreadReply","p","sendCommand","call","reply","Array","isArray","streamReply","find","r","map","entry","stopPush","enabled","gatewayDelete","_publishAck","quit","disconnect","exit","ackChannel","console","warn","msg","publish","_publishNewInstanceStarted","value","noop","maxAgeMs","minId","Date","now","getRedisConnections","message","getRedisInfo","section","info","parseRedisConnections","clientsStr","split","filter","line","trim","parts","forEach","eqIdx","indexOf","k","slice","v","includes","collectRedisMetrics","memoryInfoStr","statsInfoStr","connectionsInfoStr","all","labels","getDefaultLabels","connections","logValues","log","JSON","stringify","missing","flags","cmd","c","grouped","conn","totMem","key","count","memory","mem","Number","isFinite","currentKeys","Object","keys","add","has","delete","reset","valueLabels","parse","set","groups","values","groupedTotal","reduce","sum","g","size","parseRedisInfo","infoStr","fromEntries","startsWith","stats","used_memory","memory_type","maxmemory","instantaneous_ops_per_sec","operation","error","pushRedisMetrics","gatewayPush","clearAllCounters","startPush","_startPush","cleanup","module","exports"],"sources":["../../src/metrics/metricsRedisClient.js"],"sourcesContent":["const { BaseMetricsClient } = require('./baseMetricsClient')\nconst {\n  getRedisClientType,\n  REDIS_V4,\n  IOREDIS,\n  REDIS_V3,\n} = require('../redisUtils')\n\nconst redisConnectionStableFields = ['name', 'flags', 'cmd']\nconst redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']\n\n/** Stream entries older than this (ms) are trimmed so messages are not kept in Redis. Fixed 60s. */\nconst GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000\n\n/**\n * RedisMetricsClient extends BaseMetricsClient to collect\n * Redis metrics periodically and push them to Prometheus Pushgateway.\n *\n * @extends BaseMetricsClient\n */\nclass RedisMetricsClient extends BaseMetricsClient {\n  /**\n   * @param {Object} options\n   * @param {any} options.redisClient - Redis client instance (required). Used for metrics and publish. Subscriber is created internally (duplicate() or REDIS_URL).\n   * @param {boolean} [options.gracefulShutdownRedis] - Default true. When true, new instance publishes on start and old instances exit on message. Set false or METRICS_GRACEFUL_SHUTDOWN_REDIS=false to disable.\n   * @param {string} [options.appName] - Application name (from BaseMetricsClient)\n   * @param {string} [options.dynoId] - Dyno/instance ID (from BaseMetricsClient)\n   * @param {string} [options.processType] - Process type (from BaseMetricsClient)\n   * @param {boolean} [options.enabled] - Enable metrics collection (from BaseMetricsClient)\n   * @param {boolean} [options.logValues] - Log metrics values (from BaseMetricsClient)\n   * @param {string} [options.pushgatewayUrl] - PushGateway URL (from BaseMetricsClient)\n   * @param {string} [options.pushgatewaySecret] - PushGateway secret token (from BaseMetricsClient)\n   * @param {number} [options.intervalSec] - Interval in seconds for pushing metrics (from BaseMetricsClient)\n   * @param {boolean} [options.removeOldMetrics] - Remove old metrics by service (from BaseMetricsClient)\n   * @param {function} [options.startupValidation] - Function to validate startup (from BaseMetricsClient)\n   * @param {boolean} [options.disablePushgateway] - Disable pushing to Pushgateway (use HTTP scraping instead)\n   */\n  constructor({ redisClient, ...metricsConfig } = {}) {\n    if (redisClient == null) {\n      throw new Error('RedisMetricsClient requires redisClient')\n    }\n\n    const intervalSec =\n      metricsConfig.intervalSec ||\n      parseInt(process.env.METRICS_QUEUE_INTERVAL_SEC || '', 10) ||\n      5\n\n    super({\n      ...metricsConfig,\n      processType: metricsConfig.processType || 'queue-metrics',\n      intervalSec,\n    })\n\n    /** Redis client used for metrics */\n    this.redisClient = redisClient\n    this.redisClientType = getRedisClientType(redisClient)\n\n    /** Dedicated Redis connection for subscribe (subscriber mode); closed in cleanup. */\n    this._subClient = null\n\n    this._initGracefulShutdown(metricsConfig.gracefulShutdownRedis)\n\n    /** Counter for Redis client connections */\n    this.redisConnectionsGauge = this.createGauge({\n      name: 'app_redis_connections_count',\n      help: 'Redis client connections',\n      labelNames: this.withDefaultLabels(redisConnectionStableFields),\n    })\n\n    this.redisConnectionsMemoryGauge = this.createGauge({\n      name: 'app_redis_connections_memory_usage_count',\n      help: 'Redis client connections',\n      labelNames: this.withDefaultLabels(redisConnectionStableFields),\n    })\n\n    /** Gauge for Redis memory usage */\n    this.redisMemoryGauge = this.createGauge({\n      name: 'app_redis_memory_bytes',\n      help: 'Redis memory usage in bytes',\n      labelNames: this.withDefaultLabels(['memory_type']),\n    })\n\n    /** Gauge for Redis operation stats */\n    this.redisStatsGauge = this.createGauge({\n      name: 'app_redis_stats_total',\n      help: 'Redis operation statistics',\n      labelNames: this.withDefaultLabels(['operation']),\n    })\n\n    // Track emitted connection label combinations so we can:\n    // - emit a one-shot 0 for series that disappear (overwrite last non-zero sample)\n    // - then stop emitting them on subsequent scrapes\n    //\n    // This is mainly needed because we label connections by `cmd`,\n    // and Redis `CLIENT LIST` `cmd` is volatile.\n    this._redisConnSeenKeys = new Set()\n    this._redisConnZeroedKeys = new Set()\n\n    this._setCleanupHandlers()\n  }\n\n  /**\n   * Initialize graceful-shutdown state and subscribe when enabled. Called from constructor.\n   * @param {boolean} [gracefulShutdownRedis] - Explicit false to disable; otherwise enabled unless METRICS_GRACEFUL_SHUTDOWN_REDIS=false.\n   */\n  _initGracefulShutdown(gracefulShutdownRedis) {\n    const disabledByParam = gracefulShutdownRedis === false\n    const disabledByEnv =\n      process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false'\n    this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv\n    this._gracefulShutdownStream = this._gracefulShutdownRedis\n      ? `metrics:graceful-shutdown:${this.appName}:${this.processType}`\n      : null\n    this._gracefulShutdownAckChannel = this._gracefulShutdownRedis\n      ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}`\n      : null\n    this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`\n    if (this._gracefulShutdownRedis && this._gracefulShutdownStream) {\n      this._setupGracefulShutdownSubscribe()\n    }\n  }\n\n  /**\n   * Create a dedicated Redis client for subscribe (pub/sub). Uses duplicate() when available, else createClient from REDIS_URL. Branches by redisClientType (IOREDIS, REDIS_V3, REDIS_V4).\n   * @returns {any|null} Subscriber client or null if not possible.\n   */\n  _createSubscriberClient() {\n    if (this.redisClientType === IOREDIS && this.redisClient && typeof this.redisClient.duplicate === 'function') {\n      return this.redisClient.duplicate()\n    }\n    if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && this.redisClient && typeof this.redisClient.duplicate === 'function') {\n      return this.redisClient.duplicate()\n    }\n    if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && process.env.REDIS_URL) {\n      try {\n        const redis = require('redis')\n        if (typeof redis.createClient !== 'function') return null\n        try {\n          return redis.createClient({ url: process.env.REDIS_URL })\n        } catch {\n          return redis.createClient(process.env.REDIS_URL)\n        }\n      } catch {\n        return null\n      }\n    }\n    return null\n  }\n\n  /**\n   * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).\n   */\n  /**\n   * Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.\n   */\n  _setupGracefulShutdownSubscribe() {\n    const streamKey = this._gracefulShutdownStream\n    if (!streamKey) return\n\n    const subClient = this._createSubscriberClient()\n    if (subClient && typeof subClient.on === 'function') {\n      subClient.on('error', () => {})\n    }\n\n    if (!subClient) return\n\n    this._subClient = subClient\n    this._streamReadLoop(streamKey)\n  }\n\n  _streamReadLoop(streamKey) {\n    const self = this\n    const blockMs = 5000\n\n    const runRead = () => {\n      if (!self._subClient) return\n      self._xreadBlock(streamKey, blockMs)\n        .then((entries) => {\n          if (!entries || entries.length === 0) {\n            setImmediate(runRead)\n            return\n          }\n          self._cleanupAndPublishAck()\n        })\n        .catch(() => {\n          setImmediate(runRead)\n        })\n    }\n    runRead()\n  }\n\n  _xreadBlock(streamKey, blockMs) {\n    const client = this._subClient || this.redisClient\n    if (!client) return Promise.resolve([])\n\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve, reject) => {\n        client.send_command('XREAD', ['BLOCK', String(blockMs), 'STREAMS', streamKey, '$'], (err, result) => {\n          if (err) return reject(err)\n          resolve(this._parseXreadReply(result, streamKey))\n        })\n      })\n    }\n    if (this.redisClientType === REDIS_V4) {\n      const p = client.sendCommand(['XREAD', 'BLOCK', String(blockMs), 'STREAMS', streamKey, '$'])\n      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])\n    }\n    if (this.redisClientType === IOREDIS) {\n      const p = client.call('XREAD', 'BLOCK', blockMs, 'STREAMS', streamKey, '$')\n      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])\n    }\n    return Promise.resolve([])\n  }\n\n  _parseXreadReply(reply, streamKey) {\n    if (!reply || !Array.isArray(reply)) return []\n    const streamReply = reply.find(r => r && r[0] === streamKey)\n    if (!streamReply || !Array.isArray(streamReply[1])) return []\n    return streamReply[1].map(entry => (Array.isArray(entry) ? [entry[0], entry[1] || []] : [entry, []]))\n  }\n\n  /**\n   * Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.\n   */\n  async _cleanupAndPublishAck() {\n    this.stopPush()\n    if (this.enabled) {\n      await this.gatewayDelete()\n    }\n    await this._publishAck()\n    if (this._subClient) {\n      try {\n        if (this.redisClientType === REDIS_V3) {\n          await new Promise((resolve, reject) => {\n            if (typeof this._subClient.quit === 'function') {\n              this._subClient.quit(err => (err ? reject(err) : resolve()))\n            } else resolve()\n          })\n        } else if (this.redisClientType === REDIS_V4) {\n          if (this._subClient.quit) await this._subClient.quit()\n        } else if (this.redisClientType === IOREDIS) {\n          if (this._subClient.disconnect) await this._subClient.disconnect()\n        }\n      } catch {\n        // ignore\n      }\n      this._subClient = null\n    }\n    try {\n      if (this.redisClient) {\n        if (this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) {\n          await this.redisClient.quit()\n        } else if (this.redisClientType === IOREDIS) {\n          await this.redisClient.disconnect()\n        }\n      }\n    } catch {\n      // ignore\n    }\n    process.exit(0)\n  }\n\n  _publishAck() {\n    const ackChannel = this._gracefulShutdownAckChannel\n    if (!ackChannel || !this.redisClient) return Promise.resolve()\n    console.warn(\n      `${this._gracefulShutdownLogPrefix} OLD: clearing metrics and exiting.`\n    )\n    const msg = this.dynoId || 'stopped'\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve) => {\n        this.redisClient.send_command('PUBLISH', [ackChannel, msg], () => resolve())\n      })\n    }\n    if (this.redisClientType === REDIS_V4) {\n      return this.redisClient.sendCommand(['PUBLISH', ackChannel, msg]).catch(() => {})\n    }\n    if (this.redisClientType === IOREDIS) {\n      return this.redisClient.publish(ackChannel, msg).catch(() => {})\n    }\n    return Promise.resolve()\n  }\n\n  /**\n   * Publish \"new instance started\" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.\n   */\n  _publishNewInstanceStarted() {\n    const streamKey = this._gracefulShutdownStream\n    if (!streamKey || !this.redisClient) return\n    const value = this.dynoId || '1'\n    const noop = () => {}\n    const maxAgeMs = GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS\n    const minId = `${Date.now() - maxAgeMs}-0`\n\n    if (this.redisClientType === REDIS_V3) {\n      this.redisClient.send_command('XADD', [streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value], noop)\n      this.redisClient.send_command('XTRIM', [streamKey, 'MINID', minId], noop)\n    } else if (this.redisClientType === REDIS_V4) {\n      this.redisClient.sendCommand(['XADD', streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value]).catch(noop)\n      this.redisClient.sendCommand(['XTRIM', streamKey, 'MINID', minId]).catch(noop)\n    } else if (this.redisClientType === IOREDIS) {\n      this.redisClient.call('XADD', streamKey, 'MAXLEN', '~', 10, '*', 'dyno_id', value).catch(noop)\n      this.redisClient.call('XTRIM', streamKey, 'MINID', minId).catch(noop)\n    }\n  }\n\n  getRedisConnections = async () => {\n    if (!this.redisClient) throw new Error('Redis client not provided')\n\n    // node-redis v3 (uses callback)\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve, reject) => {\n        this.redisClient.send_command('CLIENT', ['LIST'], (err, result) => {\n          if (err) {\n            reject(new Error(`Failed to get CLIENT LIST: ${err.message}`))\n          } else resolve(result)\n        })\n      })\n    }\n\n    // node-redis v4\n    if (this.redisClientType === REDIS_V4) {\n      try {\n        return this.redisClient.sendCommand(['CLIENT', 'LIST'])\n      } catch (err) {\n        throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n      }\n    }\n\n    if (this.redisClientType === IOREDIS) {\n      try {\n        return this.redisClient.client('LIST')\n      } catch (err) {\n        throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n      }\n    }\n\n    throw new Error('Unsupported Redis client type')\n  }\n\n  getRedisInfo = async section => {\n    if (!this.redisClient) throw new Error('Redis client not provided')\n\n    // node-redis v3 (uses callback)\n    if (this.redisClientType === REDIS_V3) {\n      return new Promise((resolve, reject) => {\n        this.redisClient.info(section, (err, result) => {\n          if (err) reject(err)\n          else resolve(result)\n        })\n      })\n    }\n\n    // node-redis v4 or ioredis (info returns Promise)\n    if (this.redisClientType === REDIS_V4 || this.redisClientType === IOREDIS) {\n      try {\n        return this.redisClient.info(section)\n      } catch (err) {\n        throw new Error(`Failed to get Redis INFO: ${err.message}`)\n      }\n    }\n\n    throw new Error('Unsupported Redis client type')\n  }\n\n  parseRedisConnections = clientsStr => {\n    return clientsStr\n      .split('\\n')\n      .filter(line => line.trim() !== '')\n      .map(line => {\n        const parts = line.split(' ')\n        const client = {}\n        parts.forEach(p => {\n          const eqIdx = p.indexOf('=')\n          if (eqIdx === -1) return\n          const k = p.slice(0, eqIdx)\n          const v = p.slice(eqIdx + 1)\n          if (redisConnectionFields.includes(k)) {\n            client[k] = v\n          }\n        })\n        return client\n      })\n  }\n\n  /**\n   * Collect basic Redis INFO metrics: clients, memory, stats\n   * @returns {Promise<void>}\n   */\n  collectRedisMetrics = async () => {\n    try {\n      const [memoryInfoStr, statsInfoStr, connectionsInfoStr] =\n        await Promise.all([\n          this.getRedisInfo('memory'),\n          this.getRedisInfo('stats'),\n          this.getRedisConnections(),\n        ])\n\n      const labels = this.getDefaultLabels()\n\n      const connections = this.parseRedisConnections(connectionsInfoStr)\n\n      if (this.logValues) {\n        // \"IN\" data: raw client list from Redis\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (raw):\\n',\n          connectionsInfoStr\n        )\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (raw) length:',\n          connectionsInfoStr.length\n        )\n\n        // \"OUT\" data: parsed fields used for metrics\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (parsed) count:',\n          connections.length\n        )\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (parsed sample, first 10):',\n          JSON.stringify(connections.slice(0, 10), null, 2)\n        )\n\n        const missing = { name: 0, flags: 0, cmd: 0, 'tot-mem': 0 }\n        connections.forEach(c => {\n          if (!c.name) missing.name += 1\n          if (!c.flags) missing.flags += 1\n          if (!c.cmd) missing.cmd += 1\n          if (!c['tot-mem']) missing['tot-mem'] += 1\n        })\n        console.log(\n          '[queue-metrics] Redis CLIENT LIST (parsed) missing fields:',\n          missing\n        )\n      }\n\n      const grouped = {}\n      connections.forEach(conn => {\n        const { name, flags, 'tot-mem': totMem, cmd } = conn\n\n        // build grouping key (includes default gauge labels later)\n        const key = JSON.stringify({ name, flags, cmd })\n\n        if (!grouped[key]) {\n          grouped[key] = {\n            labels: { name, flags, cmd },\n            count: 0,\n            memory: 0,\n          }\n        }\n\n        grouped[key].count += 1\n        const mem = parseInt(totMem, 10)\n        grouped[key].memory += Number.isFinite(mem) ? mem : 0\n      })\n\n      // Ensure sets exist even if older instance is hot-reloaded.\n      if (!this._redisConnSeenKeys) this._redisConnSeenKeys = new Set()\n      if (!this._redisConnZeroedKeys) this._redisConnZeroedKeys = new Set()\n\n      const currentKeys = new Set(Object.keys(grouped))\n      for (const k of currentKeys) {\n        this._redisConnSeenKeys.add(k)\n        if (this._redisConnZeroedKeys.has(k)) {\n          this._redisConnZeroedKeys.delete(k)\n        }\n      }\n\n      // Reset gauges each scrape so we only export:\n      // - current snapshot series\n      // - one-shot zeros for recently disappeared series\n      this.redisConnectionsGauge.reset()\n      this.redisConnectionsMemoryGauge.reset()\n\n      // Emit one-shot zeros for series that disappeared since last seen.\n      for (const k of this._redisConnSeenKeys) {\n        if (currentKeys.has(k)) continue\n        if (this._redisConnZeroedKeys.has(k)) continue\n        try {\n          const valueLabels = JSON.parse(k)\n          this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, 0)\n          this.redisConnectionsMemoryGauge.set({ ...labels, ...valueLabels }, 0)\n          this._redisConnZeroedKeys.add(k)\n        } catch {\n          // ignore malformed key\n        }\n      }\n\n      if (this.logValues) {\n        const groups = Object.values(grouped)\n        const groupedTotal = groups.reduce((sum, g) => sum + (g.count || 0), 0)\n        console.log(\n          '[queue-metrics] Redis connections grouped buckets:',\n          groups.length\n        )\n        console.log(\n          '[queue-metrics] Redis connections grouped total:',\n          groupedTotal\n        )\n        console.log(\n          '[queue-metrics] Redis connections seen keys:',\n          this._redisConnSeenKeys.size\n        )\n        console.log(\n          '[queue-metrics] Redis connections zeroed keys:',\n          this._redisConnZeroedKeys.size\n        )\n        console.log(\n          '[queue-metrics] Redis connections grouped sample (first 10):',\n          JSON.stringify(groups.slice(0, 10), null, 2)\n        )\n      }\n\n      Object.values(grouped).forEach(\n        ({ labels: valueLabels, count, memory }) => {\n          this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, count)\n          this.redisConnectionsMemoryGauge.set(\n            { ...labels, ...valueLabels },\n            memory\n          )\n        }\n      )\n\n      const parseRedisInfo = infoStr =>\n        Object.fromEntries(\n          infoStr\n            .split('\\r\\n')\n            .filter(line => line && !line.startsWith('#'))\n            .map(line => line.split(':', 2))\n            .filter(parts => parts.length === 2 && parts[0] && parts[1])\n        )\n\n      const memory = parseRedisInfo(memoryInfoStr)\n      const stats = parseRedisInfo(statsInfoStr)\n\n      if (memory.used_memory) {\n        this.redisMemoryGauge.set(\n          { ...labels, memory_type: 'used' },\n          parseInt(memory.used_memory, 10) || 0\n        )\n      }\n      if (memory.maxmemory) {\n        this.redisMemoryGauge.set(\n          { ...labels, memory_type: 'max' },\n          parseInt(memory.maxmemory, 10) || 0\n        )\n      }\n\n      if (stats.instantaneous_ops_per_sec) {\n        this.redisStatsGauge.set(\n          { ...labels, operation: 'ops_per_sec' },\n          parseInt(stats.instantaneous_ops_per_sec, 10) || 0\n        )\n      }\n    } catch (error) {\n      console.error(\n        `[queue-metrics] Failed to collect Redis metrics:`,\n        error.message\n      )\n    }\n  }\n\n  /**\n   * Collect metrics for all Redis and push to Prometheus Pushgateway\n   * @returns {Promise<void>}\n   */\n  pushRedisMetrics = async () => {\n    try {\n      await this.collectRedisMetrics()\n      await this.gatewayPush()\n      this.clearAllCounters()\n    } catch (error) {\n      console.error(\n        `[queue-metrics] Failed to collect Redis metrics: ${error.message}`\n      )\n      throw error\n    }\n  }\n\n  /**\n   * Start periodic collection. When graceful shutdown is on: new publishes \"new started\" so old exits and clears; new does not push until after one interval (skipFirstPush), giving old time to stop.\n   */\n  startPush = (intervalSec = this.intervalSec) => {\n    this._startPush(intervalSec, () => {\n      this.pushRedisMetrics().catch(err => {\n        console.error(`[queue-metrics] Failed to push Redis metrics:`, err)\n      })\n    })\n    if (this._gracefulShutdownRedis) {\n      this._publishNewInstanceStarted()\n    }\n  }\n\n  /**\n   * Cleanup Redis client and exit process.\n   * Stops push, deletes this instance's metrics from VM (if removeOldMetrics), closes subscriber and main Redis, then exits.\n   * @returns {Promise<void>}\n   */\n  cleanup = async () => {\n    this.stopPush()\n    if (this.enabled) {\n      await this.gatewayDelete()\n    }\n    if (this._subClient) {\n      try {\n        if (this.redisClientType === REDIS_V3) {\n          await new Promise((resolve, reject) => {\n            if (typeof this._subClient.quit === 'function') {\n              this._subClient.quit(err => (err ? reject(err) : resolve()))\n            } else resolve()\n          })\n        } else if (this.redisClientType === REDIS_V4) {\n          if (this._subClient.quit) await this._subClient.quit()\n        } else if (this.redisClientType === IOREDIS) {\n          if (this._subClient.disconnect) await this._subClient.disconnect()\n        }\n      } catch (err) {\n        console.error('[queue-metrics] Error closing subscriber client:', err)\n      }\n      this._subClient = null\n    }\n    try {\n      if (!this.redisClient) return\n\n      if (\n        this.redisClientType === REDIS_V3 ||\n        this.redisClientType === REDIS_V4\n      ) {\n        await this.redisClient.quit()\n      } else if (this.redisClientType === IOREDIS) {\n        await this.redisClient.disconnect()\n      }\n    } catch (err) {\n      console.error('[queue-metrics] Error closing Redis client:', err)\n    }\n    await super.cleanup()\n  }\n\n  _setCleanupHandlers = () => {\n    process.on('SIGINT', this.cleanup)\n    process.on('SIGTERM', this.cleanup)\n  }\n}\n\nmodule.exports = { RedisMetricsClient }\n"],"mappings":";;AAAA,MAAM;EAAEA;AAAkB,CAAC,GAAGC,OAAO,CAAC,qBAAqB,CAAC;AAC5D,MAAM;EACJC,kBAAkB;EAClBC,QAAQ;EACRC,OAAO;EACPC;AACF,CAAC,GAAGJ,OAAO,CAAC,eAAe,CAAC;AAE5B,MAAMK,2BAA2B,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC;AAC5D,MAAMC,qBAAqB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;;AAEjE;AACA,MAAMC,kCAAkC,GAAG,KAAK;;AAEhD;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,kBAAkB,SAAST,iBAAiB,CAAC;EACjD;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEU,WAAWA,CAAC;IAAEC,WAAW;IAAE,GAAGC;EAAc,CAAC,GAAG,CAAC,CAAC,EAAE;IAClD,IAAID,WAAW,IAAI,IAAI,EAAE;MACvB,MAAM,IAAIE,KAAK,CAAC,yCAAyC,CAAC;IAC5D;IAEA,MAAMC,WAAW,GACfF,aAAa,CAACE,WAAW,IACzBC,QAAQ,CAACC,OAAO,CAACC,GAAG,CAACC,0BAA0B,IAAI,EAAE,EAAE,EAAE,CAAC,IAC1D,CAAC;IAEH,KAAK,CAAC;MACJ,GAAGN,aAAa;MAChBO,WAAW,EAAEP,aAAa,CAACO,WAAW,IAAI,eAAe;MACzDL;IACF,CAAC,CAAC;;IAEF;IACA,IAAI,CAACH,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACS,eAAe,GAAGlB,kBAAkB,CAACS,WAAW,CAAC;;IAEtD;IACA,IAAI,CAACU,UAAU,GAAG,IAAI;IAEtB,IAAI,CAACC,qBAAqB,CAACV,aAAa,CAACW,qBAAqB,CAAC;;IAE/D;IACA,IAAI,CAACC,qBAAqB,GAAG,IAAI,CAACC,WAAW,CAAC;MAC5CC,IAAI,EAAE,6BAA6B;MACnCC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAACvB,2BAA2B;IAChE,CAAC,CAAC;IAEF,IAAI,CAACwB,2BAA2B,GAAG,IAAI,CAACL,WAAW,CAAC;MAClDC,IAAI,EAAE,0CAA0C;MAChDC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAACvB,2BAA2B;IAChE,CAAC,CAAC;;IAEF;IACA,IAAI,CAACyB,gBAAgB,GAAG,IAAI,CAACN,WAAW,CAAC;MACvCC,IAAI,EAAE,wBAAwB;MAC9BC,IAAI,EAAE,6BAA6B;MACnCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAAC,CAAC,aAAa,CAAC;IACpD,CAAC,CAAC;;IAEF;IACA,IAAI,CAACG,eAAe,GAAG,IAAI,CAACP,WAAW,CAAC;MACtCC,IAAI,EAAE,uBAAuB;MAC7BC,IAAI,EAAE,4BAA4B;MAClCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAAC,CAAC,WAAW,CAAC;IAClD,CAAC,CAAC;;IAEF;IACA;IACA;IACA;IACA;IACA;IACA,IAAI,CAACI,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;IACnC,IAAI,CAACC,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;IAErC,IAAI,CAACE,mBAAmB,CAAC,CAAC;EAC5B;;EAEA;AACF;AACA;AACA;EACEd,qBAAqBA,CAACC,qBAAqB,EAAE;IAC3C,MAAMc,eAAe,GAAGd,qBAAqB,KAAK,KAAK;IACvD,MAAMe,aAAa,GACjBtB,OAAO,CAACC,GAAG,CAACsB,+BAA+B,KAAK,OAAO;IACzD,IAAI,CAACC,sBAAsB,GAAG,CAACH,eAAe,IAAI,CAACC,aAAa;IAChE,IAAI,CAACG,uBAAuB,GAAG,IAAI,CAACD,sBAAsB,GACtD,6BAA6B,IAAI,CAACE,OAAO,IAAI,IAAI,CAACvB,WAAW,EAAE,GAC/D,IAAI;IACR,IAAI,CAACwB,2BAA2B,GAAG,IAAI,CAACH,sBAAsB,GAC1D,iCAAiC,IAAI,CAACE,OAAO,IAAI,IAAI,CAACvB,WAAW,EAAE,GACnE,IAAI;IACR,IAAI,CAACyB,0BAA0B,GAAG,wBAAwB,IAAI,CAACzB,WAAW,MAAM,IAAI,CAACuB,OAAO,MAAM,IAAI,CAACG,MAAM,GAAG;IAChH,IAAI,IAAI,CAACL,sBAAsB,IAAI,IAAI,CAACC,uBAAuB,EAAE;MAC/D,IAAI,CAACK,+BAA+B,CAAC,CAAC;IACxC;EACF;;EAEA;AACF;AACA;AACA;EACEC,uBAAuBA,CAAA,EAAG;IACxB,IAAI,IAAI,CAAC3B,eAAe,KAAKhB,OAAO,IAAI,IAAI,CAACO,WAAW,IAAI,OAAO,IAAI,CAACA,WAAW,CAACqC,SAAS,KAAK,UAAU,EAAE;MAC5G,OAAO,IAAI,CAACrC,WAAW,CAACqC,SAAS,CAAC,CAAC;IACrC;IACA,IAAI,CAAC,IAAI,CAAC5B,eAAe,KAAKf,QAAQ,IAAI,IAAI,CAACe,eAAe,KAAKjB,QAAQ,KAAK,IAAI,CAACQ,WAAW,IAAI,OAAO,IAAI,CAACA,WAAW,CAACqC,SAAS,KAAK,UAAU,EAAE;MACpJ,OAAO,IAAI,CAACrC,WAAW,CAACqC,SAAS,CAAC,CAAC;IACrC;IACA,IAAI,CAAC,IAAI,CAAC5B,eAAe,KAAKf,QAAQ,IAAI,IAAI,CAACe,eAAe,KAAKjB,QAAQ,KAAKa,OAAO,CAACC,GAAG,CAACgC,SAAS,EAAE;MACrG,IAAI;QACF,MAAMC,KAAK,GAAGjD,OAAO,CAAC,OAAO,CAAC;QAC9B,IAAI,OAAOiD,KAAK,CAACC,YAAY,KAAK,UAAU,EAAE,OAAO,IAAI;QACzD,IAAI;UACF,OAAOD,KAAK,CAACC,YAAY,CAAC;YAAEC,GAAG,EAAEpC,OAAO,CAACC,GAAG,CAACgC;UAAU,CAAC,CAAC;QAC3D,CAAC,CAAC,MAAM;UACN,OAAOC,KAAK,CAACC,YAAY,CAACnC,OAAO,CAACC,GAAG,CAACgC,SAAS,CAAC;QAClD;MACF,CAAC,CAAC,MAAM;QACN,OAAO,IAAI;MACb;IACF;IACA,OAAO,IAAI;EACb;;EAEA;AACF;AACA;EACE;AACF;AACA;EACEH,+BAA+BA,CAAA,EAAG;IAChC,MAAMO,SAAS,GAAG,IAAI,CAACZ,uBAAuB;IAC9C,IAAI,CAACY,SAAS,EAAE;IAEhB,MAAMC,SAAS,GAAG,IAAI,CAACP,uBAAuB,CAAC,CAAC;IAChD,IAAIO,SAAS,IAAI,OAAOA,SAAS,CAACC,EAAE,KAAK,UAAU,EAAE;MACnDD,SAAS,CAACC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACjC;IAEA,IAAI,CAACD,SAAS,EAAE;IAEhB,IAAI,CAACjC,UAAU,GAAGiC,SAAS;IAC3B,IAAI,CAACE,eAAe,CAACH,SAAS,CAAC;EACjC;EAEAG,eAAeA,CAACH,SAAS,EAAE;IACzB,MAAMI,IAAI,GAAG,IAAI;IACjB,MAAMC,OAAO,GAAG,IAAI;IAEpB,MAAMC,OAAO,GAAGA,CAAA,KAAM;MACpB,IAAI,CAACF,IAAI,CAACpC,UAAU,EAAE;MACtBoC,IAAI,CAACG,WAAW,CAACP,SAAS,EAAEK,OAAO,CAAC,CACjCG,IAAI,CAAEC,OAAO,IAAK;QACjB,IAAI,CAACA,OAAO,IAAIA,OAAO,CAACC,MAAM,KAAK,CAAC,EAAE;UACpCC,YAAY,CAACL,OAAO,CAAC;UACrB;QACF;QACAF,IAAI,CAACQ,qBAAqB,CAAC,CAAC;MAC9B,CAAC,CAAC,CACDC,KAAK,CAAC,MAAM;QACXF,YAAY,CAACL,OAAO,CAAC;MACvB,CAAC,CAAC;IACN,CAAC;IACDA,OAAO,CAAC,CAAC;EACX;EAEAC,WAAWA,CAACP,SAAS,EAAEK,OAAO,EAAE;IAC9B,MAAMS,MAAM,GAAG,IAAI,CAAC9C,UAAU,IAAI,IAAI,CAACV,WAAW;IAClD,IAAI,CAACwD,MAAM,EAAE,OAAOC,OAAO,CAACC,OAAO,CAAC,EAAE,CAAC;IAEvC,IAAI,IAAI,CAACjD,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtCH,MAAM,CAACI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAEC,MAAM,CAACd,OAAO,CAAC,EAAE,SAAS,EAAEL,SAAS,EAAE,GAAG,CAAC,EAAE,CAACoB,GAAG,EAAEC,MAAM,KAAK;UACnG,IAAID,GAAG,EAAE,OAAOH,MAAM,CAACG,GAAG,CAAC;UAC3BJ,OAAO,CAAC,IAAI,CAACM,gBAAgB,CAACD,MAAM,EAAErB,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;IACA,IAAI,IAAI,CAACjC,eAAe,KAAKjB,QAAQ,EAAE;MACrC,MAAMyE,CAAC,GAAGT,MAAM,CAACU,WAAW,CAAC,CAAC,OAAO,EAAE,OAAO,EAAEL,MAAM,CAACd,OAAO,CAAC,EAAE,SAAS,EAAEL,SAAS,EAAE,GAAG,CAAC,CAAC;MAC5F,OAAOe,OAAO,CAACC,OAAO,CAACO,CAAC,CAAC,CAACf,IAAI,CAACa,MAAM,IAAI,IAAI,CAACC,gBAAgB,CAACD,MAAM,EAAErB,SAAS,CAAC,CAAC,CAACa,KAAK,CAAC,MAAM,EAAE,CAAC;IACpG;IACA,IAAI,IAAI,CAAC9C,eAAe,KAAKhB,OAAO,EAAE;MACpC,MAAMwE,CAAC,GAAGT,MAAM,CAACW,IAAI,CAAC,OAAO,EAAE,OAAO,EAAEpB,OAAO,EAAE,SAAS,EAAEL,SAAS,EAAE,GAAG,CAAC;MAC3E,OAAOe,OAAO,CAACC,OAAO,CAACO,CAAC,CAAC,CAACf,IAAI,CAACa,MAAM,IAAI,IAAI,CAACC,gBAAgB,CAACD,MAAM,EAAErB,SAAS,CAAC,CAAC,CAACa,KAAK,CAAC,MAAM,EAAE,CAAC;IACpG;IACA,OAAOE,OAAO,CAACC,OAAO,CAAC,EAAE,CAAC;EAC5B;EAEAM,gBAAgBA,CAACI,KAAK,EAAE1B,SAAS,EAAE;IACjC,IAAI,CAAC0B,KAAK,IAAI,CAACC,KAAK,CAACC,OAAO,CAACF,KAAK,CAAC,EAAE,OAAO,EAAE;IAC9C,MAAMG,WAAW,GAAGH,KAAK,CAACI,IAAI,CAACC,CAAC,IAAIA,CAAC,IAAIA,CAAC,CAAC,CAAC,CAAC,KAAK/B,SAAS,CAAC;IAC5D,IAAI,CAAC6B,WAAW,IAAI,CAACF,KAAK,CAACC,OAAO,CAACC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE;IAC7D,OAAOA,WAAW,CAAC,CAAC,CAAC,CAACG,GAAG,CAACC,KAAK,IAAKN,KAAK,CAACC,OAAO,CAACK,KAAK,CAAC,GAAG,CAACA,KAAK,CAAC,CAAC,CAAC,EAAEA,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAACA,KAAK,EAAE,EAAE,CAAE,CAAC;EACvG;;EAEA;AACF;AACA;EACE,MAAMrB,qBAAqBA,CAAA,EAAG;IAC5B,IAAI,CAACsB,QAAQ,CAAC,CAAC;IACf,IAAI,IAAI,CAACC,OAAO,EAAE;MAChB,MAAM,IAAI,CAACC,aAAa,CAAC,CAAC;IAC5B;IACA,MAAM,IAAI,CAACC,WAAW,CAAC,CAAC;IACxB,IAAI,IAAI,CAACrE,UAAU,EAAE;MACnB,IAAI;QACF,IAAI,IAAI,CAACD,eAAe,KAAKf,QAAQ,EAAE;UACrC,MAAM,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;YACrC,IAAI,OAAO,IAAI,CAACjD,UAAU,CAACsE,IAAI,KAAK,UAAU,EAAE;cAC9C,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAClB,GAAG,IAAKA,GAAG,GAAGH,MAAM,CAACG,GAAG,CAAC,GAAGJ,OAAO,CAAC,CAAE,CAAC;YAC9D,CAAC,MAAMA,OAAO,CAAC,CAAC;UAClB,CAAC,CAAC;QACJ,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKjB,QAAQ,EAAE;UAC5C,IAAI,IAAI,CAACkB,UAAU,CAACsE,IAAI,EAAE,MAAM,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAC,CAAC;QACxD,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;UAC3C,IAAI,IAAI,CAACiB,UAAU,CAACuE,UAAU,EAAE,MAAM,IAAI,CAACvE,UAAU,CAACuE,UAAU,CAAC,CAAC;QACpE;MACF,CAAC,CAAC,MAAM;QACN;MAAA;MAEF,IAAI,CAACvE,UAAU,GAAG,IAAI;IACxB;IACA,IAAI;MACF,IAAI,IAAI,CAACV,WAAW,EAAE;QACpB,IAAI,IAAI,CAACS,eAAe,KAAKf,QAAQ,IAAI,IAAI,CAACe,eAAe,KAAKjB,QAAQ,EAAE;UAC1E,MAAM,IAAI,CAACQ,WAAW,CAACgF,IAAI,CAAC,CAAC;QAC/B,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;UAC3C,MAAM,IAAI,CAACO,WAAW,CAACiF,UAAU,CAAC,CAAC;QACrC;MACF;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEF5E,OAAO,CAAC6E,IAAI,CAAC,CAAC,CAAC;EACjB;EAEAH,WAAWA,CAAA,EAAG;IACZ,MAAMI,UAAU,GAAG,IAAI,CAACnD,2BAA2B;IACnD,IAAI,CAACmD,UAAU,IAAI,CAAC,IAAI,CAACnF,WAAW,EAAE,OAAOyD,OAAO,CAACC,OAAO,CAAC,CAAC;IAC9D0B,OAAO,CAACC,IAAI,CACV,GAAG,IAAI,CAACpD,0BAA0B,qCACpC,CAAC;IACD,MAAMqD,GAAG,GAAG,IAAI,CAACpD,MAAM,IAAI,SAAS;IACpC,IAAI,IAAI,CAACzB,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAEC,OAAO,IAAK;QAC9B,IAAI,CAAC1D,WAAW,CAAC4D,YAAY,CAAC,SAAS,EAAE,CAACuB,UAAU,EAAEG,GAAG,CAAC,EAAE,MAAM5B,OAAO,CAAC,CAAC,CAAC;MAC9E,CAAC,CAAC;IACJ;IACA,IAAI,IAAI,CAACjD,eAAe,KAAKjB,QAAQ,EAAE;MACrC,OAAO,IAAI,CAACQ,WAAW,CAACkE,WAAW,CAAC,CAAC,SAAS,EAAEiB,UAAU,EAAEG,GAAG,CAAC,CAAC,CAAC/B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACnF;IACA,IAAI,IAAI,CAAC9C,eAAe,KAAKhB,OAAO,EAAE;MACpC,OAAO,IAAI,CAACO,WAAW,CAACuF,OAAO,CAACJ,UAAU,EAAEG,GAAG,CAAC,CAAC/B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE;IACA,OAAOE,OAAO,CAACC,OAAO,CAAC,CAAC;EAC1B;;EAEA;AACF;AACA;EACE8B,0BAA0BA,CAAA,EAAG;IAC3B,MAAM9C,SAAS,GAAG,IAAI,CAACZ,uBAAuB;IAC9C,IAAI,CAACY,SAAS,IAAI,CAAC,IAAI,CAAC1C,WAAW,EAAE;IACrC,MAAMyF,KAAK,GAAG,IAAI,CAACvD,MAAM,IAAI,GAAG;IAChC,MAAMwD,IAAI,GAAGA,CAAA,KAAM,CAAC,CAAC;IACrB,MAAMC,QAAQ,GAAG9F,kCAAkC;IACnD,MAAM+F,KAAK,GAAG,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGH,QAAQ,IAAI;IAE1C,IAAI,IAAI,CAAClF,eAAe,KAAKf,QAAQ,EAAE;MACrC,IAAI,CAACM,WAAW,CAAC4D,YAAY,CAAC,MAAM,EAAE,CAAClB,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE+C,KAAK,CAAC,EAAEC,IAAI,CAAC;MACpG,IAAI,CAAC1F,WAAW,CAAC4D,YAAY,CAAC,OAAO,EAAE,CAAClB,SAAS,EAAE,OAAO,EAAEkD,KAAK,CAAC,EAAEF,IAAI,CAAC;IAC3E,CAAC,MAAM,IAAI,IAAI,CAACjF,eAAe,KAAKjB,QAAQ,EAAE;MAC5C,IAAI,CAACQ,WAAW,CAACkE,WAAW,CAAC,CAAC,MAAM,EAAExB,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE+C,KAAK,CAAC,CAAC,CAAClC,KAAK,CAACmC,IAAI,CAAC;MACzG,IAAI,CAAC1F,WAAW,CAACkE,WAAW,CAAC,CAAC,OAAO,EAAExB,SAAS,EAAE,OAAO,EAAEkD,KAAK,CAAC,CAAC,CAACrC,KAAK,CAACmC,IAAI,CAAC;IAChF,CAAC,MAAM,IAAI,IAAI,CAACjF,eAAe,KAAKhB,OAAO,EAAE;MAC3C,IAAI,CAACO,WAAW,CAACmE,IAAI,CAAC,MAAM,EAAEzB,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE+C,KAAK,CAAC,CAAClC,KAAK,CAACmC,IAAI,CAAC;MAC9F,IAAI,CAAC1F,WAAW,CAACmE,IAAI,CAAC,OAAO,EAAEzB,SAAS,EAAE,OAAO,EAAEkD,KAAK,CAAC,CAACrC,KAAK,CAACmC,IAAI,CAAC;IACvE;EACF;EAEAK,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI,CAAC,IAAI,CAAC/F,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAAC3D,WAAW,CAAC4D,YAAY,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,CAACE,GAAG,EAAEC,MAAM,KAAK;UACjE,IAAID,GAAG,EAAE;YACPH,MAAM,CAAC,IAAIzD,KAAK,CAAC,8BAA8B4D,GAAG,CAACkC,OAAO,EAAE,CAAC,CAAC;UAChE,CAAC,MAAMtC,OAAO,CAACK,MAAM,CAAC;QACxB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACtD,eAAe,KAAKjB,QAAQ,EAAE;MACrC,IAAI;QACF,OAAO,IAAI,CAACQ,WAAW,CAACkE,WAAW,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;MACzD,CAAC,CAAC,OAAOJ,GAAG,EAAE;QACZ,MAAM,IAAI5D,KAAK,CAAC,8BAA8B4D,GAAG,CAACkC,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,IAAI,IAAI,CAACvF,eAAe,KAAKhB,OAAO,EAAE;MACpC,IAAI;QACF,OAAO,IAAI,CAACO,WAAW,CAACwD,MAAM,CAAC,MAAM,CAAC;MACxC,CAAC,CAAC,OAAOM,GAAG,EAAE;QACZ,MAAM,IAAI5D,KAAK,CAAC,8BAA8B4D,GAAG,CAACkC,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,MAAM,IAAI9F,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAED+F,YAAY,GAAG,MAAMC,OAAO,IAAI;IAC9B,IAAI,CAAC,IAAI,CAAClG,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAAC3D,WAAW,CAACmG,IAAI,CAACD,OAAO,EAAE,CAACpC,GAAG,EAAEC,MAAM,KAAK;UAC9C,IAAID,GAAG,EAAEH,MAAM,CAACG,GAAG,CAAC,MACfJ,OAAO,CAACK,MAAM,CAAC;QACtB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACtD,eAAe,KAAKjB,QAAQ,IAAI,IAAI,CAACiB,eAAe,KAAKhB,OAAO,EAAE;MACzE,IAAI;QACF,OAAO,IAAI,CAACO,WAAW,CAACmG,IAAI,CAACD,OAAO,CAAC;MACvC,CAAC,CAAC,OAAOpC,GAAG,EAAE;QACZ,MAAM,IAAI5D,KAAK,CAAC,6BAA6B4D,GAAG,CAACkC,OAAO,EAAE,CAAC;MAC7D;IACF;IAEA,MAAM,IAAI9F,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAEDkG,qBAAqB,GAAGC,UAAU,IAAI;IACpC,OAAOA,UAAU,CACdC,KAAK,CAAC,IAAI,CAAC,CACXC,MAAM,CAACC,IAAI,IAAIA,IAAI,CAACC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAClC/B,GAAG,CAAC8B,IAAI,IAAI;MACX,MAAME,KAAK,GAAGF,IAAI,CAACF,KAAK,CAAC,GAAG,CAAC;MAC7B,MAAM9C,MAAM,GAAG,CAAC,CAAC;MACjBkD,KAAK,CAACC,OAAO,CAAC1C,CAAC,IAAI;QACjB,MAAM2C,KAAK,GAAG3C,CAAC,CAAC4C,OAAO,CAAC,GAAG,CAAC;QAC5B,IAAID,KAAK,KAAK,CAAC,CAAC,EAAE;QAClB,MAAME,CAAC,GAAG7C,CAAC,CAAC8C,KAAK,CAAC,CAAC,EAAEH,KAAK,CAAC;QAC3B,MAAMI,CAAC,GAAG/C,CAAC,CAAC8C,KAAK,CAACH,KAAK,GAAG,CAAC,CAAC;QAC5B,IAAIhH,qBAAqB,CAACqH,QAAQ,CAACH,CAAC,CAAC,EAAE;UACrCtD,MAAM,CAACsD,CAAC,CAAC,GAAGE,CAAC;QACf;MACF,CAAC,CAAC;MACF,OAAOxD,MAAM;IACf,CAAC,CAAC;EACN,CAAC;;EAED;AACF;AACA;AACA;EACE0D,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI;MACF,MAAM,CAACC,aAAa,EAAEC,YAAY,EAAEC,kBAAkB,CAAC,GACrD,MAAM5D,OAAO,CAAC6D,GAAG,CAAC,CAChB,IAAI,CAACrB,YAAY,CAAC,QAAQ,CAAC,EAC3B,IAAI,CAACA,YAAY,CAAC,OAAO,CAAC,EAC1B,IAAI,CAACF,mBAAmB,CAAC,CAAC,CAC3B,CAAC;MAEJ,MAAMwB,MAAM,GAAG,IAAI,CAACC,gBAAgB,CAAC,CAAC;MAEtC,MAAMC,WAAW,GAAG,IAAI,CAACrB,qBAAqB,CAACiB,kBAAkB,CAAC;MAElE,IAAI,IAAI,CAACK,SAAS,EAAE;QAClB;QACAtC,OAAO,CAACuC,GAAG,CACT,4CAA4C,EAC5CN,kBACF,CAAC;QACDjC,OAAO,CAACuC,GAAG,CACT,iDAAiD,EACjDN,kBAAkB,CAACjE,MACrB,CAAC;;QAED;QACAgC,OAAO,CAACuC,GAAG,CACT,mDAAmD,EACnDF,WAAW,CAACrE,MACd,CAAC;QACDgC,OAAO,CAACuC,GAAG,CACT,8DAA8D,EAC9DC,IAAI,CAACC,SAAS,CAACJ,WAAW,CAACV,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAClD,CAAC;QAED,MAAMe,OAAO,GAAG;UAAE/G,IAAI,EAAE,CAAC;UAAEgH,KAAK,EAAE,CAAC;UAAEC,GAAG,EAAE,CAAC;UAAE,SAAS,EAAE;QAAE,CAAC;QAC3DP,WAAW,CAACd,OAAO,CAACsB,CAAC,IAAI;UACvB,IAAI,CAACA,CAAC,CAAClH,IAAI,EAAE+G,OAAO,CAAC/G,IAAI,IAAI,CAAC;UAC9B,IAAI,CAACkH,CAAC,CAACF,KAAK,EAAED,OAAO,CAACC,KAAK,IAAI,CAAC;UAChC,IAAI,CAACE,CAAC,CAACD,GAAG,EAAEF,OAAO,CAACE,GAAG,IAAI,CAAC;UAC5B,IAAI,CAACC,CAAC,CAAC,SAAS,CAAC,EAAEH,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;QACF1C,OAAO,CAACuC,GAAG,CACT,4DAA4D,EAC5DG,OACF,CAAC;MACH;MAEA,MAAMI,OAAO,GAAG,CAAC,CAAC;MAClBT,WAAW,CAACd,OAAO,CAACwB,IAAI,IAAI;QAC1B,MAAM;UAAEpH,IAAI;UAAEgH,KAAK;UAAE,SAAS,EAAEK,MAAM;UAAEJ;QAAI,CAAC,GAAGG,IAAI;;QAEpD;QACA,MAAME,GAAG,GAAGT,IAAI,CAACC,SAAS,CAAC;UAAE9G,IAAI;UAAEgH,KAAK;UAAEC;QAAI,CAAC,CAAC;QAEhD,IAAI,CAACE,OAAO,CAACG,GAAG,CAAC,EAAE;UACjBH,OAAO,CAACG,GAAG,CAAC,GAAG;YACbd,MAAM,EAAE;cAAExG,IAAI;cAAEgH,KAAK;cAAEC;YAAI,CAAC;YAC5BM,KAAK,EAAE,CAAC;YACRC,MAAM,EAAE;UACV,CAAC;QACH;QAEAL,OAAO,CAACG,GAAG,CAAC,CAACC,KAAK,IAAI,CAAC;QACvB,MAAME,GAAG,GAAGpI,QAAQ,CAACgI,MAAM,EAAE,EAAE,CAAC;QAChCF,OAAO,CAACG,GAAG,CAAC,CAACE,MAAM,IAAIE,MAAM,CAACC,QAAQ,CAACF,GAAG,CAAC,GAAGA,GAAG,GAAG,CAAC;MACvD,CAAC,CAAC;;MAEF;MACA,IAAI,CAAC,IAAI,CAAClH,kBAAkB,EAAE,IAAI,CAACA,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;MACjE,IAAI,CAAC,IAAI,CAACC,oBAAoB,EAAE,IAAI,CAACA,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;MAErE,MAAMoH,WAAW,GAAG,IAAIpH,GAAG,CAACqH,MAAM,CAACC,IAAI,CAACX,OAAO,CAAC,CAAC;MACjD,KAAK,MAAMpB,CAAC,IAAI6B,WAAW,EAAE;QAC3B,IAAI,CAACrH,kBAAkB,CAACwH,GAAG,CAAChC,CAAC,CAAC;QAC9B,IAAI,IAAI,CAACtF,oBAAoB,CAACuH,GAAG,CAACjC,CAAC,CAAC,EAAE;UACpC,IAAI,CAACtF,oBAAoB,CAACwH,MAAM,CAAClC,CAAC,CAAC;QACrC;MACF;;MAEA;MACA;MACA;MACA,IAAI,CAACjG,qBAAqB,CAACoI,KAAK,CAAC,CAAC;MAClC,IAAI,CAAC9H,2BAA2B,CAAC8H,KAAK,CAAC,CAAC;;MAExC;MACA,KAAK,MAAMnC,CAAC,IAAI,IAAI,CAACxF,kBAAkB,EAAE;QACvC,IAAIqH,WAAW,CAACI,GAAG,CAACjC,CAAC,CAAC,EAAE;QACxB,IAAI,IAAI,CAACtF,oBAAoB,CAACuH,GAAG,CAACjC,CAAC,CAAC,EAAE;QACtC,IAAI;UACF,MAAMoC,WAAW,GAAGtB,IAAI,CAACuB,KAAK,CAACrC,CAAC,CAAC;UACjC,IAAI,CAACjG,qBAAqB,CAACuI,GAAG,CAAC;YAAE,GAAG7B,MAAM;YAAE,GAAG2B;UAAY,CAAC,EAAE,CAAC,CAAC;UAChE,IAAI,CAAC/H,2BAA2B,CAACiI,GAAG,CAAC;YAAE,GAAG7B,MAAM;YAAE,GAAG2B;UAAY,CAAC,EAAE,CAAC,CAAC;UACtE,IAAI,CAAC1H,oBAAoB,CAACsH,GAAG,CAAChC,CAAC,CAAC;QAClC,CAAC,CAAC,MAAM;UACN;QAAA;MAEJ;MAEA,IAAI,IAAI,CAACY,SAAS,EAAE;QAClB,MAAM2B,MAAM,GAAGT,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC;QACrC,MAAMqB,YAAY,GAAGF,MAAM,CAACG,MAAM,CAAC,CAACC,GAAG,EAAEC,CAAC,KAAKD,GAAG,IAAIC,CAAC,CAACpB,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACvElD,OAAO,CAACuC,GAAG,CACT,oDAAoD,EACpD0B,MAAM,CAACjG,MACT,CAAC;QACDgC,OAAO,CAACuC,GAAG,CACT,kDAAkD,EAClD4B,YACF,CAAC;QACDnE,OAAO,CAACuC,GAAG,CACT,8CAA8C,EAC9C,IAAI,CAACrG,kBAAkB,CAACqI,IAC1B,CAAC;QACDvE,OAAO,CAACuC,GAAG,CACT,gDAAgD,EAChD,IAAI,CAACnG,oBAAoB,CAACmI,IAC5B,CAAC;QACDvE,OAAO,CAACuC,GAAG,CACT,8DAA8D,EAC9DC,IAAI,CAACC,SAAS,CAACwB,MAAM,CAACtC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAC7C,CAAC;MACH;MAEA6B,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC,CAACvB,OAAO,CAC5B,CAAC;QAAEY,MAAM,EAAE2B,WAAW;QAAEZ,KAAK;QAAEC;MAAO,CAAC,KAAK;QAC1C,IAAI,CAAC1H,qBAAqB,CAACuI,GAAG,CAAC;UAAE,GAAG7B,MAAM;UAAE,GAAG2B;QAAY,CAAC,EAAEZ,KAAK,CAAC;QACpE,IAAI,CAACnH,2BAA2B,CAACiI,GAAG,CAClC;UAAE,GAAG7B,MAAM;UAAE,GAAG2B;QAAY,CAAC,EAC7BX,MACF,CAAC;MACH,CACF,CAAC;MAED,MAAMqB,cAAc,GAAGC,OAAO,IAC5BjB,MAAM,CAACkB,WAAW,CAChBD,OAAO,CACJvD,KAAK,CAAC,MAAM,CAAC,CACbC,MAAM,CAACC,IAAI,IAAIA,IAAI,IAAI,CAACA,IAAI,CAACuD,UAAU,CAAC,GAAG,CAAC,CAAC,CAC7CrF,GAAG,CAAC8B,IAAI,IAAIA,IAAI,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAC/BC,MAAM,CAACG,KAAK,IAAIA,KAAK,CAACtD,MAAM,KAAK,CAAC,IAAIsD,KAAK,CAAC,CAAC,CAAC,IAAIA,KAAK,CAAC,CAAC,CAAC,CAC/D,CAAC;MAEH,MAAM6B,MAAM,GAAGqB,cAAc,CAACzC,aAAa,CAAC;MAC5C,MAAM6C,KAAK,GAAGJ,cAAc,CAACxC,YAAY,CAAC;MAE1C,IAAImB,MAAM,CAAC0B,WAAW,EAAE;QACtB,IAAI,CAAC7I,gBAAgB,CAACgI,GAAG,CACvB;UAAE,GAAG7B,MAAM;UAAE2C,WAAW,EAAE;QAAO,CAAC,EAClC9J,QAAQ,CAACmI,MAAM,CAAC0B,WAAW,EAAE,EAAE,CAAC,IAAI,CACtC,CAAC;MACH;MACA,IAAI1B,MAAM,CAAC4B,SAAS,EAAE;QACpB,IAAI,CAAC/I,gBAAgB,CAACgI,GAAG,CACvB;UAAE,GAAG7B,MAAM;UAAE2C,WAAW,EAAE;QAAM,CAAC,EACjC9J,QAAQ,CAACmI,MAAM,CAAC4B,SAAS,EAAE,EAAE,CAAC,IAAI,CACpC,CAAC;MACH;MAEA,IAAIH,KAAK,CAACI,yBAAyB,EAAE;QACnC,IAAI,CAAC/I,eAAe,CAAC+H,GAAG,CACtB;UAAE,GAAG7B,MAAM;UAAE8C,SAAS,EAAE;QAAc,CAAC,EACvCjK,QAAQ,CAAC4J,KAAK,CAACI,yBAAyB,EAAE,EAAE,CAAC,IAAI,CACnD,CAAC;MACH;IACF,CAAC,CAAC,OAAOE,KAAK,EAAE;MACdlF,OAAO,CAACkF,KAAK,CACX,kDAAkD,EAClDA,KAAK,CAACtE,OACR,CAAC;IACH;EACF,CAAC;;EAED;AACF;AACA;AACA;EACEuE,gBAAgB,GAAG,MAAAA,CAAA,KAAY;IAC7B,IAAI;MACF,MAAM,IAAI,CAACrD,mBAAmB,CAAC,CAAC;MAChC,MAAM,IAAI,CAACsD,WAAW,CAAC,CAAC;MACxB,IAAI,CAACC,gBAAgB,CAAC,CAAC;IACzB,CAAC,CAAC,OAAOH,KAAK,EAAE;MACdlF,OAAO,CAACkF,KAAK,CACX,oDAAoDA,KAAK,CAACtE,OAAO,EACnE,CAAC;MACD,MAAMsE,KAAK;IACb;EACF,CAAC;;EAED;AACF;AACA;EACEI,SAAS,GAAGA,CAACvK,WAAW,GAAG,IAAI,CAACA,WAAW,KAAK;IAC9C,IAAI,CAACwK,UAAU,CAACxK,WAAW,EAAE,MAAM;MACjC,IAAI,CAACoK,gBAAgB,CAAC,CAAC,CAAChH,KAAK,CAACO,GAAG,IAAI;QACnCsB,OAAO,CAACkF,KAAK,CAAC,+CAA+C,EAAExG,GAAG,CAAC;MACrE,CAAC,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,IAAI,CAACjC,sBAAsB,EAAE;MAC/B,IAAI,CAAC2D,0BAA0B,CAAC,CAAC;IACnC;EACF,CAAC;;EAED;AACF;AACA;AACA;AACA;EACEoF,OAAO,GAAG,MAAAA,CAAA,KAAY;IACpB,IAAI,CAAChG,QAAQ,CAAC,CAAC;IACf,IAAI,IAAI,CAACC,OAAO,EAAE;MAChB,MAAM,IAAI,CAACC,aAAa,CAAC,CAAC;IAC5B;IACA,IAAI,IAAI,CAACpE,UAAU,EAAE;MACnB,IAAI;QACF,IAAI,IAAI,CAACD,eAAe,KAAKf,QAAQ,EAAE;UACrC,MAAM,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;YACrC,IAAI,OAAO,IAAI,CAACjD,UAAU,CAACsE,IAAI,KAAK,UAAU,EAAE;cAC9C,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAClB,GAAG,IAAKA,GAAG,GAAGH,MAAM,CAACG,GAAG,CAAC,GAAGJ,OAAO,CAAC,CAAE,CAAC;YAC9D,CAAC,MAAMA,OAAO,CAAC,CAAC;UAClB,CAAC,CAAC;QACJ,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKjB,QAAQ,EAAE;UAC5C,IAAI,IAAI,CAACkB,UAAU,CAACsE,IAAI,EAAE,MAAM,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAC,CAAC;QACxD,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;UAC3C,IAAI,IAAI,CAACiB,UAAU,CAACuE,UAAU,EAAE,MAAM,IAAI,CAACvE,UAAU,CAACuE,UAAU,CAAC,CAAC;QACpE;MACF,CAAC,CAAC,OAAOnB,GAAG,EAAE;QACZsB,OAAO,CAACkF,KAAK,CAAC,kDAAkD,EAAExG,GAAG,CAAC;MACxE;MACA,IAAI,CAACpD,UAAU,GAAG,IAAI;IACxB;IACA,IAAI;MACF,IAAI,CAAC,IAAI,CAACV,WAAW,EAAE;MAEvB,IACE,IAAI,CAACS,eAAe,KAAKf,QAAQ,IACjC,IAAI,CAACe,eAAe,KAAKjB,QAAQ,EACjC;QACA,MAAM,IAAI,CAACQ,WAAW,CAACgF,IAAI,CAAC,CAAC;MAC/B,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;QAC3C,MAAM,IAAI,CAACO,WAAW,CAACiF,UAAU,CAAC,CAAC;MACrC;IACF,CAAC,CAAC,OAAOnB,GAAG,EAAE;MACZsB,OAAO,CAACkF,KAAK,CAAC,6CAA6C,EAAExG,GAAG,CAAC;IACnE;IACA,MAAM,KAAK,CAAC8G,OAAO,CAAC,CAAC;EACvB,CAAC;EAEDnJ,mBAAmB,GAAGA,CAAA,KAAM;IAC1BpB,OAAO,CAACuC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAACgI,OAAO,CAAC;IAClCvK,OAAO,CAACuC,EAAE,CAAC,SAAS,EAAE,IAAI,CAACgI,OAAO,CAAC;EACrC,CAAC;AACH;AAEAC,MAAM,CAACC,OAAO,GAAG;EAAEhL;AAAmB,CAAC","ignoreList":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adalo/metrics",
-  "version": "0.1.163",
+  "version": "0.1.165",
   "description": "Reusable metrics utilities for Node.js and Laravel apps",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -31,10 +31,10 @@
     "@types/pg": "^8.15.6"
   },
   "devDependencies": {
-    "@babel/cli": "7.18.6",
-    "@babel/core": "7.18.6",
-    "@babel/preset-env": "7.18.6",
-    "@babel/preset-typescript": "7.18.6",
+    "@babel/cli": "^7.24.0",
+    "@babel/core": "^7.24.0",
+    "@babel/preset-env": "^7.24.0",
+    "@babel/preset-typescript": "^7.24.0",
     "@types/ioredis": "^5.0.0",
     "@types/jest": "28.1.6",
     "@types/node": "18.0.4",

package/src/metrics/metricsRedisClient.js

@@ -9,6 +9,9 @@ const {
 const redisConnectionStableFields = ['name', 'flags', 'cmd']
 const redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']
 
+/** Stream entries older than this (ms) are trimmed so messages are not kept in Redis. Fixed 60s. */
+const GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000
+
 /**
  * RedisMetricsClient extends BaseMetricsClient to collect
  * Redis metrics periodically and push them to Prometheus Pushgateway.
@@ -105,14 +108,14 @@ class RedisMetricsClient extends BaseMetricsClient {
     const disabledByEnv =
       process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false'
     this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv
-    this._gracefulShutdownChannel = this._gracefulShutdownRedis
+    this._gracefulShutdownStream = this._gracefulShutdownRedis
       ? `metrics:graceful-shutdown:${this.appName}:${this.processType}`
       : null
     this._gracefulShutdownAckChannel = this._gracefulShutdownRedis
       ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}`
       : null
     this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`
-    if (this._gracefulShutdownRedis && this._gracefulShutdownChannel) {
+    if (this._gracefulShutdownRedis && this._gracefulShutdownStream) {
       this._setupGracefulShutdownSubscribe()
     }
   }
@@ -147,11 +150,14 @@ class RedisMetricsClient extends BaseMetricsClient {
   /**
    * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).
    */
+  /**
+   * Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.
+   */
   _setupGracefulShutdownSubscribe() {
-    const channel = this._gracefulShutdownChannel
-    if (!channel) return
+    const streamKey = this._gracefulShutdownStream
+    if (!streamKey) return
 
-    let subClient = this._createSubscriberClient()
+    const subClient = this._createSubscriberClient()
     if (subClient && typeof subClient.on === 'function') {
       subClient.on('error', () => {})
     }
@@ -159,25 +165,58 @@ class RedisMetricsClient extends BaseMetricsClient {
     if (!subClient) return
 
     this._subClient = subClient
+    this._streamReadLoop(streamKey)
+  }
 
-    const onNewStarted = () => {
-      this._cleanupAndPublishAck()
+  _streamReadLoop(streamKey) {
+    const self = this
+    const blockMs = 5000
+
+    const runRead = () => {
+      if (!self._subClient) return
+      self._xreadBlock(streamKey, blockMs)
+        .then((entries) => {
+          if (!entries || entries.length === 0) {
+            setImmediate(runRead)
+            return
+          }
+          self._cleanupAndPublishAck()
+        })
+        .catch(() => {
+          setImmediate(runRead)
+        })
     }
+    runRead()
+  }
 
-    const onMessage = (ch, _msg) => {
-      if (ch === channel) onNewStarted()
-    }
+  _xreadBlock(streamKey, blockMs) {
+    const client = this._subClient || this.redisClient
+    if (!client) return Promise.resolve([])
 
     if (this.redisClientType === REDIS_V3) {
-      subClient.on('message', onMessage)
-      subClient.subscribe(channel)
-    } else if (this.redisClientType === REDIS_V4) {
-      subClient.on('message', onMessage)
-      subClient.subscribe(channel)
-    } else if (this.redisClientType === IOREDIS) {
-      subClient.on('message', onMessage)
-      subClient.subscribe(channel)
+      return new Promise((resolve, reject) => {
+        client.send_command('XREAD', ['BLOCK', String(blockMs), 'STREAMS', streamKey, '$'], (err, result) => {
+          if (err) return reject(err)
+          resolve(this._parseXreadReply(result, streamKey))
+        })
+      })
     }
+    if (this.redisClientType === REDIS_V4) {
+      const p = client.sendCommand(['XREAD', 'BLOCK', String(blockMs), 'STREAMS', streamKey, '$'])
+      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])
+    }
+    if (this.redisClientType === IOREDIS) {
+      const p = client.call('XREAD', 'BLOCK', blockMs, 'STREAMS', streamKey, '$')
+      return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])
+    }
+    return Promise.resolve([])
+  }
+
+  _parseXreadReply(reply, streamKey) {
+    if (!reply || !Array.isArray(reply)) return []
+    const streamReply = reply.find(r => r && r[0] === streamKey)
+    if (!streamReply || !Array.isArray(streamReply[1])) return []
+    return streamReply[1].map(entry => (Array.isArray(entry) ? [entry[0], entry[1] || []] : [entry, []]))
   }
 
   /**
@@ -243,19 +282,25 @@ class RedisMetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Publish "new instance started" so old instances exit and clear metrics. No push waiting.
+   * Publish "new instance started" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.
    */
   _publishNewInstanceStarted() {
-    if (!this._gracefulShutdownChannel || !this.redisClient) return
-    const channel = this._gracefulShutdownChannel
-    const message = this.dynoId || '1'
+    const streamKey = this._gracefulShutdownStream
+    if (!streamKey || !this.redisClient) return
+    const value = this.dynoId || '1'
     const noop = () => {}
+    const maxAgeMs = GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS
+    const minId = `${Date.now() - maxAgeMs}-0`
+
     if (this.redisClientType === REDIS_V3) {
-      this.redisClient.send_command('PUBLISH', [channel, message], noop)
+      this.redisClient.send_command('XADD', [streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value], noop)
+      this.redisClient.send_command('XTRIM', [streamKey, 'MINID', minId], noop)
     } else if (this.redisClientType === REDIS_V4) {
-      this.redisClient.sendCommand(['PUBLISH', channel, message]).catch(noop)
+      this.redisClient.sendCommand(['XADD', streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value]).catch(noop)
+      this.redisClient.sendCommand(['XTRIM', streamKey, 'MINID', minId]).catch(noop)
     } else if (this.redisClientType === IOREDIS) {
-      this.redisClient.publish(channel, message).catch(noop)
+      this.redisClient.call('XADD', streamKey, 'MAXLEN', '~', 10, '*', 'dyno_id', value).catch(noop)
+      this.redisClient.call('XTRIM', streamKey, 'MINID', minId).catch(noop)
     }
   }
 
@@ -588,7 +633,7 @@ class RedisMetricsClient extends BaseMetricsClient {
     } catch (err) {
       console.error('[queue-metrics] Error closing Redis client:', err)
     }
-    process.exit(0)
+    await super.cleanup()
   }
 
   _setCleanupHandlers = () => {