@adalo/metrics 0.1.165 → 0.1.166
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +0 -1
- package/__tests__/metricsRedisClient.test.js +7 -90
- package/lib/metrics/baseMetricsClient.d.ts +1 -15
- package/lib/metrics/baseMetricsClient.d.ts.map +1 -1
- package/lib/metrics/baseMetricsClient.js +10 -40
- package/lib/metrics/baseMetricsClient.js.map +1 -1
- package/lib/metrics/metricsQueueRedisClient.d.ts.map +1 -1
- package/lib/metrics/metricsQueueRedisClient.js +1 -3
- package/lib/metrics/metricsQueueRedisClient.js.map +1 -1
- package/lib/metrics/metricsRedisClient.d.ts +7 -41
- package/lib/metrics/metricsRedisClient.d.ts.map +1 -1
- package/lib/metrics/metricsRedisClient.js +20 -231
- package/lib/metrics/metricsRedisClient.js.map +1 -1
- package/package.json +1 -1
- package/src/metrics/baseMetricsClient.js +10 -51
- package/src/metrics/metricsQueueRedisClient.js +1 -3
- package/src/metrics/metricsRedisClient.js +21 -249
|
@@ -7,8 +7,7 @@
|
|
|
7
7
|
export class RedisMetricsClient extends BaseMetricsClient {
|
|
8
8
|
/**
|
|
9
9
|
* @param {Object} options
|
|
10
|
-
* @param {any} options.redisClient - Redis client instance (required)
|
|
11
|
-
* @param {boolean} [options.gracefulShutdownRedis] - Default true. When true, new instance publishes on start and old instances exit on message. Set false or METRICS_GRACEFUL_SHUTDOWN_REDIS=false to disable.
|
|
10
|
+
* @param {any} options.redisClient - Redis client instance (required)
|
|
12
11
|
* @param {string} [options.appName] - Application name (from BaseMetricsClient)
|
|
13
12
|
* @param {string} [options.dynoId] - Dyno/instance ID (from BaseMetricsClient)
|
|
14
13
|
* @param {string} [options.processType] - Process type (from BaseMetricsClient)
|
|
@@ -23,7 +22,6 @@ export class RedisMetricsClient extends BaseMetricsClient {
|
|
|
23
22
|
*/
|
|
24
23
|
constructor({ redisClient, ...metricsConfig }?: {
|
|
25
24
|
redisClient: any;
|
|
26
|
-
gracefulShutdownRedis?: boolean | undefined;
|
|
27
25
|
appName?: string | undefined;
|
|
28
26
|
dynoId?: string | undefined;
|
|
29
27
|
processType?: string | undefined;
|
|
@@ -39,9 +37,9 @@ export class RedisMetricsClient extends BaseMetricsClient {
|
|
|
39
37
|
/** Redis client used for metrics */
|
|
40
38
|
redisClient: any;
|
|
41
39
|
redisClientType: string;
|
|
42
|
-
/**
|
|
43
|
-
|
|
44
|
-
/** Counter for Redis
|
|
40
|
+
/** Label names for Redis metrics: app + process_type only (no dyno_id). */
|
|
41
|
+
_redisLabelNames: string[];
|
|
42
|
+
/** Counter for Redis connection metrics */
|
|
45
43
|
redisConnectionsGauge: import("prom-client").Gauge<string>;
|
|
46
44
|
redisConnectionsMemoryGauge: import("prom-client").Gauge<string>;
|
|
47
45
|
/** Gauge for Redis memory usage */
|
|
@@ -50,39 +48,6 @@ export class RedisMetricsClient extends BaseMetricsClient {
|
|
|
50
48
|
redisStatsGauge: import("prom-client").Gauge<string>;
|
|
51
49
|
_redisConnSeenKeys: Set<any>;
|
|
52
50
|
_redisConnZeroedKeys: Set<any>;
|
|
53
|
-
/**
|
|
54
|
-
* Initialize graceful-shutdown state and subscribe when enabled. Called from constructor.
|
|
55
|
-
* @param {boolean} [gracefulShutdownRedis] - Explicit false to disable; otherwise enabled unless METRICS_GRACEFUL_SHUTDOWN_REDIS=false.
|
|
56
|
-
*/
|
|
57
|
-
_initGracefulShutdown(gracefulShutdownRedis?: boolean | undefined): void;
|
|
58
|
-
_gracefulShutdownRedis: boolean | undefined;
|
|
59
|
-
_gracefulShutdownStream: string | null | undefined;
|
|
60
|
-
_gracefulShutdownAckChannel: string | null | undefined;
|
|
61
|
-
_gracefulShutdownLogPrefix: string | undefined;
|
|
62
|
-
/**
|
|
63
|
-
* Create a dedicated Redis client for subscribe (pub/sub). Uses duplicate() when available, else createClient from REDIS_URL. Branches by redisClientType (IOREDIS, REDIS_V3, REDIS_V4).
|
|
64
|
-
* @returns {any|null} Subscriber client or null if not possible.
|
|
65
|
-
*/
|
|
66
|
-
_createSubscriberClient(): any | null;
|
|
67
|
-
/**
|
|
68
|
-
* Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).
|
|
69
|
-
*/
|
|
70
|
-
/**
|
|
71
|
-
* Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.
|
|
72
|
-
*/
|
|
73
|
-
_setupGracefulShutdownSubscribe(): void;
|
|
74
|
-
_streamReadLoop(streamKey: any): void;
|
|
75
|
-
_xreadBlock(streamKey: any, blockMs: any): Promise<any>;
|
|
76
|
-
_parseXreadReply(reply: any, streamKey: any): any[][];
|
|
77
|
-
/**
|
|
78
|
-
* Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.
|
|
79
|
-
*/
|
|
80
|
-
_cleanupAndPublishAck(): Promise<void>;
|
|
81
|
-
_publishAck(): any;
|
|
82
|
-
/**
|
|
83
|
-
* Publish "new instance started" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.
|
|
84
|
-
*/
|
|
85
|
-
_publishNewInstanceStarted(): void;
|
|
86
51
|
getRedisConnections: () => Promise<any>;
|
|
87
52
|
getRedisInfo: (section: any) => Promise<any>;
|
|
88
53
|
parseRedisConnections: (clientsStr: any) => any;
|
|
@@ -97,9 +62,10 @@ export class RedisMetricsClient extends BaseMetricsClient {
|
|
|
97
62
|
*/
|
|
98
63
|
pushRedisMetrics: () => Promise<void>;
|
|
99
64
|
/**
|
|
100
|
-
* Start periodic collection.
|
|
65
|
+
* Start periodic collection.
|
|
66
|
+
* @param {number} [intervalSec=this.intervalSec] - Interval in seconds
|
|
101
67
|
*/
|
|
102
|
-
startPush: (intervalSec?: number) => void;
|
|
68
|
+
startPush: (intervalSec?: number | undefined) => void;
|
|
103
69
|
}
|
|
104
70
|
import { BaseMetricsClient } from "./baseMetricsClient";
|
|
105
71
|
//# sourceMappingURL=metricsRedisClient.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metricsRedisClient.d.ts","sourceRoot":"","sources":["../../src/metrics/metricsRedisClient.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"metricsRedisClient.d.ts","sourceRoot":"","sources":["../../src/metrics/metricsRedisClient.js"],"names":[],"mappings":"AAWA;;;;;GAKG;AACH;IACE;;;;;;;;;;;;;;OAcG;IACH;QAbwB,WAAW,EAAxB,GAAG;QACc,OAAO;QACP,MAAM;QACN,WAAW;QACV,OAAO;QACP,SAAS;QACV,cAAc;QACd,iBAAiB;QACjB,WAAW;QACV,gBAAgB;QACf,iBAAiB;QAClB,kBAAkB;OA8D9C;IA5CC,oCAAoC;IACpC,iBAA8B;IAC9B,wBAAsD;IAEtD,2EAA2E;IAC3E,2BAA+C;IAE/C,2CAA2C;IAC3C,2DAIE;IAEF,iEAIE;IAEF,mCAAmC;IACnC,sDAIE;IAEF,sCAAsC;IACtC,qDAIE;IAQF,6BAAmC;IACnC,+BAAqC;IAKvC,wCAgCC;IAED,6CAuBC;IAED,gDAkBC;IAED;;;OAGG;IACH,2BAFa,QAAQ,IAAI,CAAC,CA6KzB;IAED;;;OAGG;IACH,wBAFa,QAAQ,IAAI,CAAC,CAqBzB;IAED;;;OAGG;IACH,sDAMC;CA4BF"}
|
|
@@ -12,9 +12,6 @@ const {
|
|
|
12
12
|
const redisConnectionStableFields = ['name', 'flags', 'cmd'];
|
|
13
13
|
const redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd'];
|
|
14
14
|
|
|
15
|
-
/** Stream entries older than this (ms) are trimmed so messages are not kept in Redis. Fixed 60s. */
|
|
16
|
-
const GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000;
|
|
17
|
-
|
|
18
15
|
/**
|
|
19
16
|
* RedisMetricsClient extends BaseMetricsClient to collect
|
|
20
17
|
* Redis metrics periodically and push them to Prometheus Pushgateway.
|
|
@@ -24,8 +21,7 @@ const GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000;
|
|
|
24
21
|
class RedisMetricsClient extends BaseMetricsClient {
|
|
25
22
|
/**
|
|
26
23
|
* @param {Object} options
|
|
27
|
-
* @param {any} options.redisClient - Redis client instance (required)
|
|
28
|
-
* @param {boolean} [options.gracefulShutdownRedis] - Default true. When true, new instance publishes on start and old instances exit on message. Set false or METRICS_GRACEFUL_SHUTDOWN_REDIS=false to disable.
|
|
24
|
+
* @param {any} options.redisClient - Redis client instance (required)
|
|
29
25
|
* @param {string} [options.appName] - Application name (from BaseMetricsClient)
|
|
30
26
|
* @param {string} [options.dynoId] - Dyno/instance ID (from BaseMetricsClient)
|
|
31
27
|
* @param {string} [options.processType] - Process type (from BaseMetricsClient)
|
|
@@ -56,34 +52,33 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
56
52
|
this.redisClient = redisClient;
|
|
57
53
|
this.redisClientType = getRedisClientType(redisClient);
|
|
58
54
|
|
|
59
|
-
/**
|
|
60
|
-
this.
|
|
61
|
-
this._initGracefulShutdown(metricsConfig.gracefulShutdownRedis);
|
|
55
|
+
/** Label names for Redis metrics: app + process_type only (no dyno_id). */
|
|
56
|
+
this._redisLabelNames = ['app', 'process_type'];
|
|
62
57
|
|
|
63
|
-
/** Counter for Redis
|
|
58
|
+
/** Counter for Redis connection metrics */
|
|
64
59
|
this.redisConnectionsGauge = this.createGauge({
|
|
65
60
|
name: 'app_redis_connections_count',
|
|
66
61
|
help: 'Redis client connections',
|
|
67
|
-
labelNames: this.
|
|
62
|
+
labelNames: [...this._redisLabelNames, ...redisConnectionStableFields]
|
|
68
63
|
});
|
|
69
64
|
this.redisConnectionsMemoryGauge = this.createGauge({
|
|
70
65
|
name: 'app_redis_connections_memory_usage_count',
|
|
71
66
|
help: 'Redis client connections',
|
|
72
|
-
labelNames: this.
|
|
67
|
+
labelNames: [...this._redisLabelNames, ...redisConnectionStableFields]
|
|
73
68
|
});
|
|
74
69
|
|
|
75
70
|
/** Gauge for Redis memory usage */
|
|
76
71
|
this.redisMemoryGauge = this.createGauge({
|
|
77
72
|
name: 'app_redis_memory_bytes',
|
|
78
73
|
help: 'Redis memory usage in bytes',
|
|
79
|
-
labelNames: this.
|
|
74
|
+
labelNames: [...this._redisLabelNames, 'memory_type']
|
|
80
75
|
});
|
|
81
76
|
|
|
82
77
|
/** Gauge for Redis operation stats */
|
|
83
78
|
this.redisStatsGauge = this.createGauge({
|
|
84
79
|
name: 'app_redis_stats_total',
|
|
85
80
|
help: 'Redis operation statistics',
|
|
86
|
-
labelNames: this.
|
|
81
|
+
labelNames: [...this._redisLabelNames, 'operation']
|
|
87
82
|
});
|
|
88
83
|
|
|
89
84
|
// Track emitted connection label combinations so we can:
|
|
@@ -96,194 +91,6 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
96
91
|
this._redisConnZeroedKeys = new Set();
|
|
97
92
|
this._setCleanupHandlers();
|
|
98
93
|
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Initialize graceful-shutdown state and subscribe when enabled. Called from constructor.
|
|
102
|
-
* @param {boolean} [gracefulShutdownRedis] - Explicit false to disable; otherwise enabled unless METRICS_GRACEFUL_SHUTDOWN_REDIS=false.
|
|
103
|
-
*/
|
|
104
|
-
_initGracefulShutdown(gracefulShutdownRedis) {
|
|
105
|
-
const disabledByParam = gracefulShutdownRedis === false;
|
|
106
|
-
const disabledByEnv = process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false';
|
|
107
|
-
this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv;
|
|
108
|
-
this._gracefulShutdownStream = this._gracefulShutdownRedis ? `metrics:graceful-shutdown:${this.appName}:${this.processType}` : null;
|
|
109
|
-
this._gracefulShutdownAckChannel = this._gracefulShutdownRedis ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}` : null;
|
|
110
|
-
this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`;
|
|
111
|
-
if (this._gracefulShutdownRedis && this._gracefulShutdownStream) {
|
|
112
|
-
this._setupGracefulShutdownSubscribe();
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* Create a dedicated Redis client for subscribe (pub/sub). Uses duplicate() when available, else createClient from REDIS_URL. Branches by redisClientType (IOREDIS, REDIS_V3, REDIS_V4).
|
|
118
|
-
* @returns {any|null} Subscriber client or null if not possible.
|
|
119
|
-
*/
|
|
120
|
-
_createSubscriberClient() {
|
|
121
|
-
if (this.redisClientType === IOREDIS && this.redisClient && typeof this.redisClient.duplicate === 'function') {
|
|
122
|
-
return this.redisClient.duplicate();
|
|
123
|
-
}
|
|
124
|
-
if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && this.redisClient && typeof this.redisClient.duplicate === 'function') {
|
|
125
|
-
return this.redisClient.duplicate();
|
|
126
|
-
}
|
|
127
|
-
if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && process.env.REDIS_URL) {
|
|
128
|
-
try {
|
|
129
|
-
const redis = require('redis');
|
|
130
|
-
if (typeof redis.createClient !== 'function') return null;
|
|
131
|
-
try {
|
|
132
|
-
return redis.createClient({
|
|
133
|
-
url: process.env.REDIS_URL
|
|
134
|
-
});
|
|
135
|
-
} catch {
|
|
136
|
-
return redis.createClient(process.env.REDIS_URL);
|
|
137
|
-
}
|
|
138
|
-
} catch {
|
|
139
|
-
return null;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
return null;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).
|
|
147
|
-
*/
|
|
148
|
-
/**
|
|
149
|
-
* Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.
|
|
150
|
-
*/
|
|
151
|
-
_setupGracefulShutdownSubscribe() {
|
|
152
|
-
const streamKey = this._gracefulShutdownStream;
|
|
153
|
-
if (!streamKey) return;
|
|
154
|
-
const subClient = this._createSubscriberClient();
|
|
155
|
-
if (subClient && typeof subClient.on === 'function') {
|
|
156
|
-
subClient.on('error', () => {});
|
|
157
|
-
}
|
|
158
|
-
if (!subClient) return;
|
|
159
|
-
this._subClient = subClient;
|
|
160
|
-
this._streamReadLoop(streamKey);
|
|
161
|
-
}
|
|
162
|
-
_streamReadLoop(streamKey) {
|
|
163
|
-
const self = this;
|
|
164
|
-
const blockMs = 5000;
|
|
165
|
-
const runRead = () => {
|
|
166
|
-
if (!self._subClient) return;
|
|
167
|
-
self._xreadBlock(streamKey, blockMs).then(entries => {
|
|
168
|
-
if (!entries || entries.length === 0) {
|
|
169
|
-
setImmediate(runRead);
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
self._cleanupAndPublishAck();
|
|
173
|
-
}).catch(() => {
|
|
174
|
-
setImmediate(runRead);
|
|
175
|
-
});
|
|
176
|
-
};
|
|
177
|
-
runRead();
|
|
178
|
-
}
|
|
179
|
-
_xreadBlock(streamKey, blockMs) {
|
|
180
|
-
const client = this._subClient || this.redisClient;
|
|
181
|
-
if (!client) return Promise.resolve([]);
|
|
182
|
-
if (this.redisClientType === REDIS_V3) {
|
|
183
|
-
return new Promise((resolve, reject) => {
|
|
184
|
-
client.send_command('XREAD', ['BLOCK', String(blockMs), 'STREAMS', streamKey, '$'], (err, result) => {
|
|
185
|
-
if (err) return reject(err);
|
|
186
|
-
resolve(this._parseXreadReply(result, streamKey));
|
|
187
|
-
});
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
if (this.redisClientType === REDIS_V4) {
|
|
191
|
-
const p = client.sendCommand(['XREAD', 'BLOCK', String(blockMs), 'STREAMS', streamKey, '$']);
|
|
192
|
-
return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => []);
|
|
193
|
-
}
|
|
194
|
-
if (this.redisClientType === IOREDIS) {
|
|
195
|
-
const p = client.call('XREAD', 'BLOCK', blockMs, 'STREAMS', streamKey, '$');
|
|
196
|
-
return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => []);
|
|
197
|
-
}
|
|
198
|
-
return Promise.resolve([]);
|
|
199
|
-
}
|
|
200
|
-
_parseXreadReply(reply, streamKey) {
|
|
201
|
-
if (!reply || !Array.isArray(reply)) return [];
|
|
202
|
-
const streamReply = reply.find(r => r && r[0] === streamKey);
|
|
203
|
-
if (!streamReply || !Array.isArray(streamReply[1])) return [];
|
|
204
|
-
return streamReply[1].map(entry => Array.isArray(entry) ? [entry[0], entry[1] || []] : [entry, []]);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/**
|
|
208
|
-
* Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.
|
|
209
|
-
*/
|
|
210
|
-
async _cleanupAndPublishAck() {
|
|
211
|
-
this.stopPush();
|
|
212
|
-
if (this.enabled) {
|
|
213
|
-
await this.gatewayDelete();
|
|
214
|
-
}
|
|
215
|
-
await this._publishAck();
|
|
216
|
-
if (this._subClient) {
|
|
217
|
-
try {
|
|
218
|
-
if (this.redisClientType === REDIS_V3) {
|
|
219
|
-
await new Promise((resolve, reject) => {
|
|
220
|
-
if (typeof this._subClient.quit === 'function') {
|
|
221
|
-
this._subClient.quit(err => err ? reject(err) : resolve());
|
|
222
|
-
} else resolve();
|
|
223
|
-
});
|
|
224
|
-
} else if (this.redisClientType === REDIS_V4) {
|
|
225
|
-
if (this._subClient.quit) await this._subClient.quit();
|
|
226
|
-
} else if (this.redisClientType === IOREDIS) {
|
|
227
|
-
if (this._subClient.disconnect) await this._subClient.disconnect();
|
|
228
|
-
}
|
|
229
|
-
} catch {
|
|
230
|
-
// ignore
|
|
231
|
-
}
|
|
232
|
-
this._subClient = null;
|
|
233
|
-
}
|
|
234
|
-
try {
|
|
235
|
-
if (this.redisClient) {
|
|
236
|
-
if (this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) {
|
|
237
|
-
await this.redisClient.quit();
|
|
238
|
-
} else if (this.redisClientType === IOREDIS) {
|
|
239
|
-
await this.redisClient.disconnect();
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
} catch {
|
|
243
|
-
// ignore
|
|
244
|
-
}
|
|
245
|
-
process.exit(0);
|
|
246
|
-
}
|
|
247
|
-
_publishAck() {
|
|
248
|
-
const ackChannel = this._gracefulShutdownAckChannel;
|
|
249
|
-
if (!ackChannel || !this.redisClient) return Promise.resolve();
|
|
250
|
-
console.warn(`${this._gracefulShutdownLogPrefix} OLD: clearing metrics and exiting.`);
|
|
251
|
-
const msg = this.dynoId || 'stopped';
|
|
252
|
-
if (this.redisClientType === REDIS_V3) {
|
|
253
|
-
return new Promise(resolve => {
|
|
254
|
-
this.redisClient.send_command('PUBLISH', [ackChannel, msg], () => resolve());
|
|
255
|
-
});
|
|
256
|
-
}
|
|
257
|
-
if (this.redisClientType === REDIS_V4) {
|
|
258
|
-
return this.redisClient.sendCommand(['PUBLISH', ackChannel, msg]).catch(() => {});
|
|
259
|
-
}
|
|
260
|
-
if (this.redisClientType === IOREDIS) {
|
|
261
|
-
return this.redisClient.publish(ackChannel, msg).catch(() => {});
|
|
262
|
-
}
|
|
263
|
-
return Promise.resolve();
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
/**
|
|
267
|
-
* Publish "new instance started" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.
|
|
268
|
-
*/
|
|
269
|
-
_publishNewInstanceStarted() {
|
|
270
|
-
const streamKey = this._gracefulShutdownStream;
|
|
271
|
-
if (!streamKey || !this.redisClient) return;
|
|
272
|
-
const value = this.dynoId || '1';
|
|
273
|
-
const noop = () => {};
|
|
274
|
-
const maxAgeMs = GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS;
|
|
275
|
-
const minId = `${Date.now() - maxAgeMs}-0`;
|
|
276
|
-
if (this.redisClientType === REDIS_V3) {
|
|
277
|
-
this.redisClient.send_command('XADD', [streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value], noop);
|
|
278
|
-
this.redisClient.send_command('XTRIM', [streamKey, 'MINID', minId], noop);
|
|
279
|
-
} else if (this.redisClientType === REDIS_V4) {
|
|
280
|
-
this.redisClient.sendCommand(['XADD', streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value]).catch(noop);
|
|
281
|
-
this.redisClient.sendCommand(['XTRIM', streamKey, 'MINID', minId]).catch(noop);
|
|
282
|
-
} else if (this.redisClientType === IOREDIS) {
|
|
283
|
-
this.redisClient.call('XADD', streamKey, 'MAXLEN', '~', 10, '*', 'dyno_id', value).catch(noop);
|
|
284
|
-
this.redisClient.call('XTRIM', streamKey, 'MINID', minId).catch(noop);
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
94
|
getRedisConnections = async () => {
|
|
288
95
|
if (!this.redisClient) throw new Error('Redis client not provided');
|
|
289
96
|
|
|
@@ -361,7 +168,10 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
361
168
|
collectRedisMetrics = async () => {
|
|
362
169
|
try {
|
|
363
170
|
const [memoryInfoStr, statsInfoStr, connectionsInfoStr] = await Promise.all([this.getRedisInfo('memory'), this.getRedisInfo('stats'), this.getRedisConnections()]);
|
|
364
|
-
const labels =
|
|
171
|
+
const labels = {
|
|
172
|
+
app: this.appName,
|
|
173
|
+
process_type: this.processType
|
|
174
|
+
};
|
|
365
175
|
const connections = this.parseRedisConnections(connectionsInfoStr);
|
|
366
176
|
if (this.logValues) {
|
|
367
177
|
// "IN" data: raw client list from Redis
|
|
@@ -497,7 +307,7 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
497
307
|
}, parseInt(stats.instantaneous_ops_per_sec, 10) || 0);
|
|
498
308
|
}
|
|
499
309
|
} catch (error) {
|
|
500
|
-
console.
|
|
310
|
+
console.warn(`[queue-metrics] Failed to collect Redis metrics:`, error.message);
|
|
501
311
|
}
|
|
502
312
|
};
|
|
503
313
|
|
|
@@ -510,6 +320,10 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
510
320
|
await this.collectRedisMetrics();
|
|
511
321
|
await this.gatewayPush();
|
|
512
322
|
this.clearAllCounters();
|
|
323
|
+
if (this.metricsLogValues) {
|
|
324
|
+
const metricObjects = await this.registry.getMetricsAsJSON();
|
|
325
|
+
console.info(`[queue-metrics] Collected metrics for Redis`, JSON.stringify(metricObjects, null, 2));
|
|
326
|
+
}
|
|
513
327
|
} catch (error) {
|
|
514
328
|
console.error(`[queue-metrics] Failed to collect Redis metrics: ${error.message}`);
|
|
515
329
|
throw error;
|
|
@@ -517,7 +331,8 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
517
331
|
};
|
|
518
332
|
|
|
519
333
|
/**
|
|
520
|
-
* Start periodic collection.
|
|
334
|
+
* Start periodic collection.
|
|
335
|
+
* @param {number} [intervalSec=this.intervalSec] - Interval in seconds
|
|
521
336
|
*/
|
|
522
337
|
startPush = (intervalSec = this.intervalSec) => {
|
|
523
338
|
this._startPush(intervalSec, () => {
|
|
@@ -525,39 +340,13 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
525
340
|
console.error(`[queue-metrics] Failed to push Redis metrics:`, err);
|
|
526
341
|
});
|
|
527
342
|
});
|
|
528
|
-
if (this._gracefulShutdownRedis) {
|
|
529
|
-
this._publishNewInstanceStarted();
|
|
530
|
-
}
|
|
531
343
|
};
|
|
532
344
|
|
|
533
345
|
/**
|
|
534
346
|
* Cleanup Redis client and exit process.
|
|
535
|
-
* Stops push, deletes this instance's metrics from VM (if removeOldMetrics), closes subscriber and main Redis, then exits.
|
|
536
347
|
* @returns {Promise<void>}
|
|
537
348
|
*/
|
|
538
349
|
cleanup = async () => {
|
|
539
|
-
this.stopPush();
|
|
540
|
-
if (this.enabled) {
|
|
541
|
-
await this.gatewayDelete();
|
|
542
|
-
}
|
|
543
|
-
if (this._subClient) {
|
|
544
|
-
try {
|
|
545
|
-
if (this.redisClientType === REDIS_V3) {
|
|
546
|
-
await new Promise((resolve, reject) => {
|
|
547
|
-
if (typeof this._subClient.quit === 'function') {
|
|
548
|
-
this._subClient.quit(err => err ? reject(err) : resolve());
|
|
549
|
-
} else resolve();
|
|
550
|
-
});
|
|
551
|
-
} else if (this.redisClientType === REDIS_V4) {
|
|
552
|
-
if (this._subClient.quit) await this._subClient.quit();
|
|
553
|
-
} else if (this.redisClientType === IOREDIS) {
|
|
554
|
-
if (this._subClient.disconnect) await this._subClient.disconnect();
|
|
555
|
-
}
|
|
556
|
-
} catch (err) {
|
|
557
|
-
console.error('[queue-metrics] Error closing subscriber client:', err);
|
|
558
|
-
}
|
|
559
|
-
this._subClient = null;
|
|
560
|
-
}
|
|
561
350
|
try {
|
|
562
351
|
if (!this.redisClient) return;
|
|
563
352
|
if (this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) {
|
|
@@ -568,7 +357,7 @@ class RedisMetricsClient extends BaseMetricsClient {
|
|
|
568
357
|
} catch (err) {
|
|
569
358
|
console.error('[queue-metrics] Error closing Redis client:', err);
|
|
570
359
|
}
|
|
571
|
-
|
|
360
|
+
process.exit(0);
|
|
572
361
|
};
|
|
573
362
|
_setCleanupHandlers = () => {
|
|
574
363
|
process.on('SIGINT', this.cleanup);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metricsRedisClient.js","names":["BaseMetricsClient","require","getRedisClientType","REDIS_V4","IOREDIS","REDIS_V3","redisConnectionStableFields","redisConnectionFields","GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS","RedisMetricsClient","constructor","redisClient","metricsConfig","Error","intervalSec","parseInt","process","env","METRICS_QUEUE_INTERVAL_SEC","processType","redisClientType","_subClient","_initGracefulShutdown","gracefulShutdownRedis","redisConnectionsGauge","createGauge","name","help","labelNames","withDefaultLabels","redisConnectionsMemoryGauge","redisMemoryGauge","redisStatsGauge","_redisConnSeenKeys","Set","_redisConnZeroedKeys","_setCleanupHandlers","disabledByParam","disabledByEnv","METRICS_GRACEFUL_SHUTDOWN_REDIS","_gracefulShutdownRedis","_gracefulShutdownStream","appName","_gracefulShutdownAckChannel","_gracefulShutdownLogPrefix","dynoId","_setupGracefulShutdownSubscribe","_createSubscriberClient","duplicate","REDIS_URL","redis","createClient","url","streamKey","subClient","on","_streamReadLoop","self","blockMs","runRead","_xreadBlock","then","entries","length","setImmediate","_cleanupAndPublishAck","catch","client","Promise","resolve","reject","send_command","String","err","result","_parseXreadReply","p","sendCommand","call","reply","Array","isArray","streamReply","find","r","map","entry","stopPush","enabled","gatewayDelete","_publishAck","quit","disconnect","exit","ackChannel","console","warn","msg","publish","_publishNewInstanceStarted","value","noop","maxAgeMs","minId","Date","now","getRedisConnections","message","getRedisInfo","section","info","parseRedisConnections","clientsStr","split","filter","line","trim","parts","forEach","eqIdx","indexOf","k","slice","v","includes","collectRedisMetrics","memoryInfoStr","statsInfoStr","connectionsInfoStr","all","labels","getDefaultLabels","connections","logValues","log","JSON","stringify","missing","flags","cmd","c","grouped","conn","totMem","key","count","memory","mem","Number","isFinite","currentKeys","Object","keys","add","has","delete","reset","valueLabels","parse","set","groups","values","groupedTotal","reduce","sum","g","size","parseRedisInfo","infoStr","fromEntries","startsWith","stats","used_memory","memory_type","maxmemory","instantaneous_ops_per_sec","operation","error","pushRedisMetrics","gatewayPush","clearAllCounters","startPush","_startPush","cleanup","module","exports"],"sources":["../../src/metrics/metricsRedisClient.js"],"sourcesContent":["const { BaseMetricsClient } = require('./baseMetricsClient')\nconst {\n getRedisClientType,\n REDIS_V4,\n IOREDIS,\n REDIS_V3,\n} = require('../redisUtils')\n\nconst redisConnectionStableFields = ['name', 'flags', 'cmd']\nconst redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']\n\n/** Stream entries older than this (ms) are trimmed so messages are not kept in Redis. Fixed 60s. */\nconst GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS = 60000\n\n/**\n * RedisMetricsClient extends BaseMetricsClient to collect\n * Redis metrics periodically and push them to Prometheus Pushgateway.\n *\n * @extends BaseMetricsClient\n */\nclass RedisMetricsClient extends BaseMetricsClient {\n /**\n * @param {Object} options\n * @param {any} options.redisClient - Redis client instance (required). Used for metrics and publish. Subscriber is created internally (duplicate() or REDIS_URL).\n * @param {boolean} [options.gracefulShutdownRedis] - Default true. When true, new instance publishes on start and old instances exit on message. Set false or METRICS_GRACEFUL_SHUTDOWN_REDIS=false to disable.\n * @param {string} [options.appName] - Application name (from BaseMetricsClient)\n * @param {string} [options.dynoId] - Dyno/instance ID (from BaseMetricsClient)\n * @param {string} [options.processType] - Process type (from BaseMetricsClient)\n * @param {boolean} [options.enabled] - Enable metrics collection (from BaseMetricsClient)\n * @param {boolean} [options.logValues] - Log metrics values (from BaseMetricsClient)\n * @param {string} [options.pushgatewayUrl] - PushGateway URL (from BaseMetricsClient)\n * @param {string} [options.pushgatewaySecret] - PushGateway secret token (from BaseMetricsClient)\n * @param {number} [options.intervalSec] - Interval in seconds for pushing metrics (from BaseMetricsClient)\n * @param {boolean} [options.removeOldMetrics] - Remove old metrics by service (from BaseMetricsClient)\n * @param {function} [options.startupValidation] - Function to validate startup (from BaseMetricsClient)\n * @param {boolean} [options.disablePushgateway] - Disable pushing to Pushgateway (use HTTP scraping instead)\n */\n constructor({ redisClient, ...metricsConfig } = {}) {\n if (redisClient == null) {\n throw new Error('RedisMetricsClient requires redisClient')\n }\n\n const intervalSec =\n metricsConfig.intervalSec ||\n parseInt(process.env.METRICS_QUEUE_INTERVAL_SEC || '', 10) ||\n 5\n\n super({\n ...metricsConfig,\n processType: metricsConfig.processType || 'queue-metrics',\n intervalSec,\n })\n\n /** Redis client used for metrics */\n this.redisClient = redisClient\n this.redisClientType = getRedisClientType(redisClient)\n\n /** Dedicated Redis connection for subscribe (subscriber mode); closed in cleanup. */\n this._subClient = null\n\n this._initGracefulShutdown(metricsConfig.gracefulShutdownRedis)\n\n /** Counter for Redis client connections */\n this.redisConnectionsGauge = this.createGauge({\n name: 'app_redis_connections_count',\n help: 'Redis client connections',\n labelNames: this.withDefaultLabels(redisConnectionStableFields),\n })\n\n this.redisConnectionsMemoryGauge = this.createGauge({\n name: 'app_redis_connections_memory_usage_count',\n help: 'Redis client connections',\n labelNames: this.withDefaultLabels(redisConnectionStableFields),\n })\n\n /** Gauge for Redis memory usage */\n this.redisMemoryGauge = this.createGauge({\n name: 'app_redis_memory_bytes',\n help: 'Redis memory usage in bytes',\n labelNames: this.withDefaultLabels(['memory_type']),\n })\n\n /** Gauge for Redis operation stats */\n this.redisStatsGauge = this.createGauge({\n name: 'app_redis_stats_total',\n help: 'Redis operation statistics',\n labelNames: this.withDefaultLabels(['operation']),\n })\n\n // Track emitted connection label combinations so we can:\n // - emit a one-shot 0 for series that disappear (overwrite last non-zero sample)\n // - then stop emitting them on subsequent scrapes\n //\n // This is mainly needed because we label connections by `cmd`,\n // and Redis `CLIENT LIST` `cmd` is volatile.\n this._redisConnSeenKeys = new Set()\n this._redisConnZeroedKeys = new Set()\n\n this._setCleanupHandlers()\n }\n\n /**\n * Initialize graceful-shutdown state and subscribe when enabled. Called from constructor.\n * @param {boolean} [gracefulShutdownRedis] - Explicit false to disable; otherwise enabled unless METRICS_GRACEFUL_SHUTDOWN_REDIS=false.\n */\n _initGracefulShutdown(gracefulShutdownRedis) {\n const disabledByParam = gracefulShutdownRedis === false\n const disabledByEnv =\n process.env.METRICS_GRACEFUL_SHUTDOWN_REDIS === 'false'\n this._gracefulShutdownRedis = !disabledByParam && !disabledByEnv\n this._gracefulShutdownStream = this._gracefulShutdownRedis\n ? `metrics:graceful-shutdown:${this.appName}:${this.processType}`\n : null\n this._gracefulShutdownAckChannel = this._gracefulShutdownRedis\n ? `metrics:graceful-shutdown-ack:${this.appName}:${this.processType}`\n : null\n this._gracefulShutdownLogPrefix = `[graceful-shutdown] [${this.processType}] [${this.appName}] [${this.dynoId}]`\n if (this._gracefulShutdownRedis && this._gracefulShutdownStream) {\n this._setupGracefulShutdownSubscribe()\n }\n }\n\n /**\n * Create a dedicated Redis client for subscribe (pub/sub). Uses duplicate() when available, else createClient from REDIS_URL. Branches by redisClientType (IOREDIS, REDIS_V3, REDIS_V4).\n * @returns {any|null} Subscriber client or null if not possible.\n */\n _createSubscriberClient() {\n if (this.redisClientType === IOREDIS && this.redisClient && typeof this.redisClient.duplicate === 'function') {\n return this.redisClient.duplicate()\n }\n if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && this.redisClient && typeof this.redisClient.duplicate === 'function') {\n return this.redisClient.duplicate()\n }\n if ((this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) && process.env.REDIS_URL) {\n try {\n const redis = require('redis')\n if (typeof redis.createClient !== 'function') return null\n try {\n return redis.createClient({ url: process.env.REDIS_URL })\n } catch {\n return redis.createClient(process.env.REDIS_URL)\n }\n } catch {\n return null\n }\n }\n return null\n }\n\n /**\n * Set up Redis subscribe for graceful shutdown. Uses _createSubscriberClient() so subscriber matches client type (ioredis vs node-redis v3/v4).\n */\n /**\n * Set up Redis stream read for graceful shutdown. Uses _createSubscriberClient() and XREAD BLOCK; stream is trimmed (MAXLEN and MINID) so messages are not kept in Redis forever.\n */\n _setupGracefulShutdownSubscribe() {\n const streamKey = this._gracefulShutdownStream\n if (!streamKey) return\n\n const subClient = this._createSubscriberClient()\n if (subClient && typeof subClient.on === 'function') {\n subClient.on('error', () => {})\n }\n\n if (!subClient) return\n\n this._subClient = subClient\n this._streamReadLoop(streamKey)\n }\n\n _streamReadLoop(streamKey) {\n const self = this\n const blockMs = 5000\n\n const runRead = () => {\n if (!self._subClient) return\n self._xreadBlock(streamKey, blockMs)\n .then((entries) => {\n if (!entries || entries.length === 0) {\n setImmediate(runRead)\n return\n }\n self._cleanupAndPublishAck()\n })\n .catch(() => {\n setImmediate(runRead)\n })\n }\n runRead()\n }\n\n _xreadBlock(streamKey, blockMs) {\n const client = this._subClient || this.redisClient\n if (!client) return Promise.resolve([])\n\n if (this.redisClientType === REDIS_V3) {\n return new Promise((resolve, reject) => {\n client.send_command('XREAD', ['BLOCK', String(blockMs), 'STREAMS', streamKey, '$'], (err, result) => {\n if (err) return reject(err)\n resolve(this._parseXreadReply(result, streamKey))\n })\n })\n }\n if (this.redisClientType === REDIS_V4) {\n const p = client.sendCommand(['XREAD', 'BLOCK', String(blockMs), 'STREAMS', streamKey, '$'])\n return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])\n }\n if (this.redisClientType === IOREDIS) {\n const p = client.call('XREAD', 'BLOCK', blockMs, 'STREAMS', streamKey, '$')\n return Promise.resolve(p).then(result => this._parseXreadReply(result, streamKey)).catch(() => [])\n }\n return Promise.resolve([])\n }\n\n _parseXreadReply(reply, streamKey) {\n if (!reply || !Array.isArray(reply)) return []\n const streamReply = reply.find(r => r && r[0] === streamKey)\n if (!streamReply || !Array.isArray(streamReply[1])) return []\n return streamReply[1].map(entry => (Array.isArray(entry) ? [entry[0], entry[1] || []] : [entry, []]))\n }\n\n /**\n * Old instance: stop push, clear metrics from VM, then publish ack so new can start; then exit.\n */\n async _cleanupAndPublishAck() {\n this.stopPush()\n if (this.enabled) {\n await this.gatewayDelete()\n }\n await this._publishAck()\n if (this._subClient) {\n try {\n if (this.redisClientType === REDIS_V3) {\n await new Promise((resolve, reject) => {\n if (typeof this._subClient.quit === 'function') {\n this._subClient.quit(err => (err ? reject(err) : resolve()))\n } else resolve()\n })\n } else if (this.redisClientType === REDIS_V4) {\n if (this._subClient.quit) await this._subClient.quit()\n } else if (this.redisClientType === IOREDIS) {\n if (this._subClient.disconnect) await this._subClient.disconnect()\n }\n } catch {\n // ignore\n }\n this._subClient = null\n }\n try {\n if (this.redisClient) {\n if (this.redisClientType === REDIS_V3 || this.redisClientType === REDIS_V4) {\n await this.redisClient.quit()\n } else if (this.redisClientType === IOREDIS) {\n await this.redisClient.disconnect()\n }\n }\n } catch {\n // ignore\n }\n process.exit(0)\n }\n\n _publishAck() {\n const ackChannel = this._gracefulShutdownAckChannel\n if (!ackChannel || !this.redisClient) return Promise.resolve()\n console.warn(\n `${this._gracefulShutdownLogPrefix} OLD: clearing metrics and exiting.`\n )\n const msg = this.dynoId || 'stopped'\n if (this.redisClientType === REDIS_V3) {\n return new Promise((resolve) => {\n this.redisClient.send_command('PUBLISH', [ackChannel, msg], () => resolve())\n })\n }\n if (this.redisClientType === REDIS_V4) {\n return this.redisClient.sendCommand(['PUBLISH', ackChannel, msg]).catch(() => {})\n }\n if (this.redisClientType === IOREDIS) {\n return this.redisClient.publish(ackChannel, msg).catch(() => {})\n }\n return Promise.resolve()\n }\n\n /**\n * Publish \"new instance started\" to stream so old instances exit. Stream is trimmed (MAXLEN ~ 10 and MINID older than 1 min) so messages are not kept in Redis forever.\n */\n _publishNewInstanceStarted() {\n const streamKey = this._gracefulShutdownStream\n if (!streamKey || !this.redisClient) return\n const value = this.dynoId || '1'\n const noop = () => {}\n const maxAgeMs = GRACEFUL_SHUTDOWN_STREAM_MAXAGE_MS\n const minId = `${Date.now() - maxAgeMs}-0`\n\n if (this.redisClientType === REDIS_V3) {\n this.redisClient.send_command('XADD', [streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value], noop)\n this.redisClient.send_command('XTRIM', [streamKey, 'MINID', minId], noop)\n } else if (this.redisClientType === REDIS_V4) {\n this.redisClient.sendCommand(['XADD', streamKey, 'MAXLEN', '~', '10', '*', 'dyno_id', value]).catch(noop)\n this.redisClient.sendCommand(['XTRIM', streamKey, 'MINID', minId]).catch(noop)\n } else if (this.redisClientType === IOREDIS) {\n this.redisClient.call('XADD', streamKey, 'MAXLEN', '~', 10, '*', 'dyno_id', value).catch(noop)\n this.redisClient.call('XTRIM', streamKey, 'MINID', minId).catch(noop)\n }\n }\n\n getRedisConnections = async () => {\n if (!this.redisClient) throw new Error('Redis client not provided')\n\n // node-redis v3 (uses callback)\n if (this.redisClientType === REDIS_V3) {\n return new Promise((resolve, reject) => {\n this.redisClient.send_command('CLIENT', ['LIST'], (err, result) => {\n if (err) {\n reject(new Error(`Failed to get CLIENT LIST: ${err.message}`))\n } else resolve(result)\n })\n })\n }\n\n // node-redis v4\n if (this.redisClientType === REDIS_V4) {\n try {\n return this.redisClient.sendCommand(['CLIENT', 'LIST'])\n } catch (err) {\n throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n }\n }\n\n if (this.redisClientType === IOREDIS) {\n try {\n return this.redisClient.client('LIST')\n } catch (err) {\n throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n }\n }\n\n throw new Error('Unsupported Redis client type')\n }\n\n getRedisInfo = async section => {\n if (!this.redisClient) throw new Error('Redis client not provided')\n\n // node-redis v3 (uses callback)\n if (this.redisClientType === REDIS_V3) {\n return new Promise((resolve, reject) => {\n this.redisClient.info(section, (err, result) => {\n if (err) reject(err)\n else resolve(result)\n })\n })\n }\n\n // node-redis v4 or ioredis (info returns Promise)\n if (this.redisClientType === REDIS_V4 || this.redisClientType === IOREDIS) {\n try {\n return this.redisClient.info(section)\n } catch (err) {\n throw new Error(`Failed to get Redis INFO: ${err.message}`)\n }\n }\n\n throw new Error('Unsupported Redis client type')\n }\n\n parseRedisConnections = clientsStr => {\n return clientsStr\n .split('\\n')\n .filter(line => line.trim() !== '')\n .map(line => {\n const parts = line.split(' ')\n const client = {}\n parts.forEach(p => {\n const eqIdx = p.indexOf('=')\n if (eqIdx === -1) return\n const k = p.slice(0, eqIdx)\n const v = p.slice(eqIdx + 1)\n if (redisConnectionFields.includes(k)) {\n client[k] = v\n }\n })\n return client\n })\n }\n\n /**\n * Collect basic Redis INFO metrics: clients, memory, stats\n * @returns {Promise<void>}\n */\n collectRedisMetrics = async () => {\n try {\n const [memoryInfoStr, statsInfoStr, connectionsInfoStr] =\n await Promise.all([\n this.getRedisInfo('memory'),\n this.getRedisInfo('stats'),\n this.getRedisConnections(),\n ])\n\n const labels = this.getDefaultLabels()\n\n const connections = this.parseRedisConnections(connectionsInfoStr)\n\n if (this.logValues) {\n // \"IN\" data: raw client list from Redis\n console.log(\n '[queue-metrics] Redis CLIENT LIST (raw):\\n',\n connectionsInfoStr\n )\n console.log(\n '[queue-metrics] Redis CLIENT LIST (raw) length:',\n connectionsInfoStr.length\n )\n\n // \"OUT\" data: parsed fields used for metrics\n console.log(\n '[queue-metrics] Redis CLIENT LIST (parsed) count:',\n connections.length\n )\n console.log(\n '[queue-metrics] Redis CLIENT LIST (parsed sample, first 10):',\n JSON.stringify(connections.slice(0, 10), null, 2)\n )\n\n const missing = { name: 0, flags: 0, cmd: 0, 'tot-mem': 0 }\n connections.forEach(c => {\n if (!c.name) missing.name += 1\n if (!c.flags) missing.flags += 1\n if (!c.cmd) missing.cmd += 1\n if (!c['tot-mem']) missing['tot-mem'] += 1\n })\n console.log(\n '[queue-metrics] Redis CLIENT LIST (parsed) missing fields:',\n missing\n )\n }\n\n const grouped = {}\n connections.forEach(conn => {\n const { name, flags, 'tot-mem': totMem, cmd } = conn\n\n // build grouping key (includes default gauge labels later)\n const key = JSON.stringify({ name, flags, cmd })\n\n if (!grouped[key]) {\n grouped[key] = {\n labels: { name, flags, cmd },\n count: 0,\n memory: 0,\n }\n }\n\n grouped[key].count += 1\n const mem = parseInt(totMem, 10)\n grouped[key].memory += Number.isFinite(mem) ? mem : 0\n })\n\n // Ensure sets exist even if older instance is hot-reloaded.\n if (!this._redisConnSeenKeys) this._redisConnSeenKeys = new Set()\n if (!this._redisConnZeroedKeys) this._redisConnZeroedKeys = new Set()\n\n const currentKeys = new Set(Object.keys(grouped))\n for (const k of currentKeys) {\n this._redisConnSeenKeys.add(k)\n if (this._redisConnZeroedKeys.has(k)) {\n this._redisConnZeroedKeys.delete(k)\n }\n }\n\n // Reset gauges each scrape so we only export:\n // - current snapshot series\n // - one-shot zeros for recently disappeared series\n this.redisConnectionsGauge.reset()\n this.redisConnectionsMemoryGauge.reset()\n\n // Emit one-shot zeros for series that disappeared since last seen.\n for (const k of this._redisConnSeenKeys) {\n if (currentKeys.has(k)) continue\n if (this._redisConnZeroedKeys.has(k)) continue\n try {\n const valueLabels = JSON.parse(k)\n this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, 0)\n this.redisConnectionsMemoryGauge.set({ ...labels, ...valueLabels }, 0)\n this._redisConnZeroedKeys.add(k)\n } catch {\n // ignore malformed key\n }\n }\n\n if (this.logValues) {\n const groups = Object.values(grouped)\n const groupedTotal = groups.reduce((sum, g) => sum + (g.count || 0), 0)\n console.log(\n '[queue-metrics] Redis connections grouped buckets:',\n groups.length\n )\n console.log(\n '[queue-metrics] Redis connections grouped total:',\n groupedTotal\n )\n console.log(\n '[queue-metrics] Redis connections seen keys:',\n this._redisConnSeenKeys.size\n )\n console.log(\n '[queue-metrics] Redis connections zeroed keys:',\n this._redisConnZeroedKeys.size\n )\n console.log(\n '[queue-metrics] Redis connections grouped sample (first 10):',\n JSON.stringify(groups.slice(0, 10), null, 2)\n )\n }\n\n Object.values(grouped).forEach(\n ({ labels: valueLabels, count, memory }) => {\n this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, count)\n this.redisConnectionsMemoryGauge.set(\n { ...labels, ...valueLabels },\n memory\n )\n }\n )\n\n const parseRedisInfo = infoStr =>\n Object.fromEntries(\n infoStr\n .split('\\r\\n')\n .filter(line => line && !line.startsWith('#'))\n .map(line => line.split(':', 2))\n .filter(parts => parts.length === 2 && parts[0] && parts[1])\n )\n\n const memory = parseRedisInfo(memoryInfoStr)\n const stats = parseRedisInfo(statsInfoStr)\n\n if (memory.used_memory) {\n this.redisMemoryGauge.set(\n { ...labels, memory_type: 'used' },\n parseInt(memory.used_memory, 10) || 0\n )\n }\n if (memory.maxmemory) {\n this.redisMemoryGauge.set(\n { ...labels, memory_type: 'max' },\n parseInt(memory.maxmemory, 10) || 0\n )\n }\n\n if (stats.instantaneous_ops_per_sec) {\n this.redisStatsGauge.set(\n { ...labels, operation: 'ops_per_sec' },\n parseInt(stats.instantaneous_ops_per_sec, 10) || 0\n )\n }\n } catch (error) {\n console.error(\n `[queue-metrics] Failed to collect Redis metrics:`,\n error.message\n )\n }\n }\n\n /**\n * Collect metrics for all Redis and push to Prometheus Pushgateway\n * @returns {Promise<void>}\n */\n pushRedisMetrics = async () => {\n try {\n await this.collectRedisMetrics()\n await this.gatewayPush()\n this.clearAllCounters()\n } catch (error) {\n console.error(\n `[queue-metrics] Failed to collect Redis metrics: ${error.message}`\n )\n throw error\n }\n }\n\n /**\n * Start periodic collection. When graceful shutdown is on: new publishes \"new started\" so old exits and clears; new does not push until after one interval (skipFirstPush), giving old time to stop.\n */\n startPush = (intervalSec = this.intervalSec) => {\n this._startPush(intervalSec, () => {\n this.pushRedisMetrics().catch(err => {\n console.error(`[queue-metrics] Failed to push Redis metrics:`, err)\n })\n })\n if (this._gracefulShutdownRedis) {\n this._publishNewInstanceStarted()\n }\n }\n\n /**\n * Cleanup Redis client and exit process.\n * Stops push, deletes this instance's metrics from VM (if removeOldMetrics), closes subscriber and main Redis, then exits.\n * @returns {Promise<void>}\n */\n cleanup = async () => {\n this.stopPush()\n if (this.enabled) {\n await this.gatewayDelete()\n }\n if (this._subClient) {\n try {\n if (this.redisClientType === REDIS_V3) {\n await new Promise((resolve, reject) => {\n if (typeof this._subClient.quit === 'function') {\n this._subClient.quit(err => (err ? reject(err) : resolve()))\n } else resolve()\n })\n } else if (this.redisClientType === REDIS_V4) {\n if (this._subClient.quit) await this._subClient.quit()\n } else if (this.redisClientType === IOREDIS) {\n if (this._subClient.disconnect) await this._subClient.disconnect()\n }\n } catch (err) {\n console.error('[queue-metrics] Error closing subscriber client:', err)\n }\n this._subClient = null\n }\n try {\n if (!this.redisClient) return\n\n if (\n this.redisClientType === REDIS_V3 ||\n this.redisClientType === REDIS_V4\n ) {\n await this.redisClient.quit()\n } else if (this.redisClientType === IOREDIS) {\n await this.redisClient.disconnect()\n }\n } catch (err) {\n console.error('[queue-metrics] Error closing Redis client:', err)\n }\n await super.cleanup()\n }\n\n _setCleanupHandlers = () => {\n process.on('SIGINT', this.cleanup)\n process.on('SIGTERM', this.cleanup)\n }\n}\n\nmodule.exports = { RedisMetricsClient }\n"],"mappings":";;AAAA,MAAM;EAAEA;AAAkB,CAAC,GAAGC,OAAO,CAAC,qBAAqB,CAAC;AAC5D,MAAM;EACJC,kBAAkB;EAClBC,QAAQ;EACRC,OAAO;EACPC;AACF,CAAC,GAAGJ,OAAO,CAAC,eAAe,CAAC;AAE5B,MAAMK,2BAA2B,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC;AAC5D,MAAMC,qBAAqB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;;AAEjE;AACA,MAAMC,kCAAkC,GAAG,KAAK;;AAEhD;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,kBAAkB,SAAST,iBAAiB,CAAC;EACjD;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEU,WAAWA,CAAC;IAAEC,WAAW;IAAE,GAAGC;EAAc,CAAC,GAAG,CAAC,CAAC,EAAE;IAClD,IAAID,WAAW,IAAI,IAAI,EAAE;MACvB,MAAM,IAAIE,KAAK,CAAC,yCAAyC,CAAC;IAC5D;IAEA,MAAMC,WAAW,GACfF,aAAa,CAACE,WAAW,IACzBC,QAAQ,CAACC,OAAO,CAACC,GAAG,CAACC,0BAA0B,IAAI,EAAE,EAAE,EAAE,CAAC,IAC1D,CAAC;IAEH,KAAK,CAAC;MACJ,GAAGN,aAAa;MAChBO,WAAW,EAAEP,aAAa,CAACO,WAAW,IAAI,eAAe;MACzDL;IACF,CAAC,CAAC;;IAEF;IACA,IAAI,CAACH,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACS,eAAe,GAAGlB,kBAAkB,CAACS,WAAW,CAAC;;IAEtD;IACA,IAAI,CAACU,UAAU,GAAG,IAAI;IAEtB,IAAI,CAACC,qBAAqB,CAACV,aAAa,CAACW,qBAAqB,CAAC;;IAE/D;IACA,IAAI,CAACC,qBAAqB,GAAG,IAAI,CAACC,WAAW,CAAC;MAC5CC,IAAI,EAAE,6BAA6B;MACnCC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAACvB,2BAA2B;IAChE,CAAC,CAAC;IAEF,IAAI,CAACwB,2BAA2B,GAAG,IAAI,CAACL,WAAW,CAAC;MAClDC,IAAI,EAAE,0CAA0C;MAChDC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAACvB,2BAA2B;IAChE,CAAC,CAAC;;IAEF;IACA,IAAI,CAACyB,gBAAgB,GAAG,IAAI,CAACN,WAAW,CAAC;MACvCC,IAAI,EAAE,wBAAwB;MAC9BC,IAAI,EAAE,6BAA6B;MACnCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAAC,CAAC,aAAa,CAAC;IACpD,CAAC,CAAC;;IAEF;IACA,IAAI,CAACG,eAAe,GAAG,IAAI,CAACP,WAAW,CAAC;MACtCC,IAAI,EAAE,uBAAuB;MAC7BC,IAAI,EAAE,4BAA4B;MAClCC,UAAU,EAAE,IAAI,CAACC,iBAAiB,CAAC,CAAC,WAAW,CAAC;IAClD,CAAC,CAAC;;IAEF;IACA;IACA;IACA;IACA;IACA;IACA,IAAI,CAACI,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;IACnC,IAAI,CAACC,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;IAErC,IAAI,CAACE,mBAAmB,CAAC,CAAC;EAC5B;;EAEA;AACF;AACA;AACA;EACEd,qBAAqBA,CAACC,qBAAqB,EAAE;IAC3C,MAAMc,eAAe,GAAGd,qBAAqB,KAAK,KAAK;IACvD,MAAMe,aAAa,GACjBtB,OAAO,CAACC,GAAG,CAACsB,+BAA+B,KAAK,OAAO;IACzD,IAAI,CAACC,sBAAsB,GAAG,CAACH,eAAe,IAAI,CAACC,aAAa;IAChE,IAAI,CAACG,uBAAuB,GAAG,IAAI,CAACD,sBAAsB,GACtD,6BAA6B,IAAI,CAACE,OAAO,IAAI,IAAI,CAACvB,WAAW,EAAE,GAC/D,IAAI;IACR,IAAI,CAACwB,2BAA2B,GAAG,IAAI,CAACH,sBAAsB,GAC1D,iCAAiC,IAAI,CAACE,OAAO,IAAI,IAAI,CAACvB,WAAW,EAAE,GACnE,IAAI;IACR,IAAI,CAACyB,0BAA0B,GAAG,wBAAwB,IAAI,CAACzB,WAAW,MAAM,IAAI,CAACuB,OAAO,MAAM,IAAI,CAACG,MAAM,GAAG;IAChH,IAAI,IAAI,CAACL,sBAAsB,IAAI,IAAI,CAACC,uBAAuB,EAAE;MAC/D,IAAI,CAACK,+BAA+B,CAAC,CAAC;IACxC;EACF;;EAEA;AACF;AACA;AACA;EACEC,uBAAuBA,CAAA,EAAG;IACxB,IAAI,IAAI,CAAC3B,eAAe,KAAKhB,OAAO,IAAI,IAAI,CAACO,WAAW,IAAI,OAAO,IAAI,CAACA,WAAW,CAACqC,SAAS,KAAK,UAAU,EAAE;MAC5G,OAAO,IAAI,CAACrC,WAAW,CAACqC,SAAS,CAAC,CAAC;IACrC;IACA,IAAI,CAAC,IAAI,CAAC5B,eAAe,KAAKf,QAAQ,IAAI,IAAI,CAACe,eAAe,KAAKjB,QAAQ,KAAK,IAAI,CAACQ,WAAW,IAAI,OAAO,IAAI,CAACA,WAAW,CAACqC,SAAS,KAAK,UAAU,EAAE;MACpJ,OAAO,IAAI,CAACrC,WAAW,CAACqC,SAAS,CAAC,CAAC;IACrC;IACA,IAAI,CAAC,IAAI,CAAC5B,eAAe,KAAKf,QAAQ,IAAI,IAAI,CAACe,eAAe,KAAKjB,QAAQ,KAAKa,OAAO,CAACC,GAAG,CAACgC,SAAS,EAAE;MACrG,IAAI;QACF,MAAMC,KAAK,GAAGjD,OAAO,CAAC,OAAO,CAAC;QAC9B,IAAI,OAAOiD,KAAK,CAACC,YAAY,KAAK,UAAU,EAAE,OAAO,IAAI;QACzD,IAAI;UACF,OAAOD,KAAK,CAACC,YAAY,CAAC;YAAEC,GAAG,EAAEpC,OAAO,CAACC,GAAG,CAACgC;UAAU,CAAC,CAAC;QAC3D,CAAC,CAAC,MAAM;UACN,OAAOC,KAAK,CAACC,YAAY,CAACnC,OAAO,CAACC,GAAG,CAACgC,SAAS,CAAC;QAClD;MACF,CAAC,CAAC,MAAM;QACN,OAAO,IAAI;MACb;IACF;IACA,OAAO,IAAI;EACb;;EAEA;AACF;AACA;EACE;AACF;AACA;EACEH,+BAA+BA,CAAA,EAAG;IAChC,MAAMO,SAAS,GAAG,IAAI,CAACZ,uBAAuB;IAC9C,IAAI,CAACY,SAAS,EAAE;IAEhB,MAAMC,SAAS,GAAG,IAAI,CAACP,uBAAuB,CAAC,CAAC;IAChD,IAAIO,SAAS,IAAI,OAAOA,SAAS,CAACC,EAAE,KAAK,UAAU,EAAE;MACnDD,SAAS,CAACC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACjC;IAEA,IAAI,CAACD,SAAS,EAAE;IAEhB,IAAI,CAACjC,UAAU,GAAGiC,SAAS;IAC3B,IAAI,CAACE,eAAe,CAACH,SAAS,CAAC;EACjC;EAEAG,eAAeA,CAACH,SAAS,EAAE;IACzB,MAAMI,IAAI,GAAG,IAAI;IACjB,MAAMC,OAAO,GAAG,IAAI;IAEpB,MAAMC,OAAO,GAAGA,CAAA,KAAM;MACpB,IAAI,CAACF,IAAI,CAACpC,UAAU,EAAE;MACtBoC,IAAI,CAACG,WAAW,CAACP,SAAS,EAAEK,OAAO,CAAC,CACjCG,IAAI,CAAEC,OAAO,IAAK;QACjB,IAAI,CAACA,OAAO,IAAIA,OAAO,CAACC,MAAM,KAAK,CAAC,EAAE;UACpCC,YAAY,CAACL,OAAO,CAAC;UACrB;QACF;QACAF,IAAI,CAACQ,qBAAqB,CAAC,CAAC;MAC9B,CAAC,CAAC,CACDC,KAAK,CAAC,MAAM;QACXF,YAAY,CAACL,OAAO,CAAC;MACvB,CAAC,CAAC;IACN,CAAC;IACDA,OAAO,CAAC,CAAC;EACX;EAEAC,WAAWA,CAACP,SAAS,EAAEK,OAAO,EAAE;IAC9B,MAAMS,MAAM,GAAG,IAAI,CAAC9C,UAAU,IAAI,IAAI,CAACV,WAAW;IAClD,IAAI,CAACwD,MAAM,EAAE,OAAOC,OAAO,CAACC,OAAO,CAAC,EAAE,CAAC;IAEvC,IAAI,IAAI,CAACjD,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtCH,MAAM,CAACI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAEC,MAAM,CAACd,OAAO,CAAC,EAAE,SAAS,EAAEL,SAAS,EAAE,GAAG,CAAC,EAAE,CAACoB,GAAG,EAAEC,MAAM,KAAK;UACnG,IAAID,GAAG,EAAE,OAAOH,MAAM,CAACG,GAAG,CAAC;UAC3BJ,OAAO,CAAC,IAAI,CAACM,gBAAgB,CAACD,MAAM,EAAErB,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;IACA,IAAI,IAAI,CAACjC,eAAe,KAAKjB,QAAQ,EAAE;MACrC,MAAMyE,CAAC,GAAGT,MAAM,CAACU,WAAW,CAAC,CAAC,OAAO,EAAE,OAAO,EAAEL,MAAM,CAACd,OAAO,CAAC,EAAE,SAAS,EAAEL,SAAS,EAAE,GAAG,CAAC,CAAC;MAC5F,OAAOe,OAAO,CAACC,OAAO,CAACO,CAAC,CAAC,CAACf,IAAI,CAACa,MAAM,IAAI,IAAI,CAACC,gBAAgB,CAACD,MAAM,EAAErB,SAAS,CAAC,CAAC,CAACa,KAAK,CAAC,MAAM,EAAE,CAAC;IACpG;IACA,IAAI,IAAI,CAAC9C,eAAe,KAAKhB,OAAO,EAAE;MACpC,MAAMwE,CAAC,GAAGT,MAAM,CAACW,IAAI,CAAC,OAAO,EAAE,OAAO,EAAEpB,OAAO,EAAE,SAAS,EAAEL,SAAS,EAAE,GAAG,CAAC;MAC3E,OAAOe,OAAO,CAACC,OAAO,CAACO,CAAC,CAAC,CAACf,IAAI,CAACa,MAAM,IAAI,IAAI,CAACC,gBAAgB,CAACD,MAAM,EAAErB,SAAS,CAAC,CAAC,CAACa,KAAK,CAAC,MAAM,EAAE,CAAC;IACpG;IACA,OAAOE,OAAO,CAACC,OAAO,CAAC,EAAE,CAAC;EAC5B;EAEAM,gBAAgBA,CAACI,KAAK,EAAE1B,SAAS,EAAE;IACjC,IAAI,CAAC0B,KAAK,IAAI,CAACC,KAAK,CAACC,OAAO,CAACF,KAAK,CAAC,EAAE,OAAO,EAAE;IAC9C,MAAMG,WAAW,GAAGH,KAAK,CAACI,IAAI,CAACC,CAAC,IAAIA,CAAC,IAAIA,CAAC,CAAC,CAAC,CAAC,KAAK/B,SAAS,CAAC;IAC5D,IAAI,CAAC6B,WAAW,IAAI,CAACF,KAAK,CAACC,OAAO,CAACC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE;IAC7D,OAAOA,WAAW,CAAC,CAAC,CAAC,CAACG,GAAG,CAACC,KAAK,IAAKN,KAAK,CAACC,OAAO,CAACK,KAAK,CAAC,GAAG,CAACA,KAAK,CAAC,CAAC,CAAC,EAAEA,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAACA,KAAK,EAAE,EAAE,CAAE,CAAC;EACvG;;EAEA;AACF;AACA;EACE,MAAMrB,qBAAqBA,CAAA,EAAG;IAC5B,IAAI,CAACsB,QAAQ,CAAC,CAAC;IACf,IAAI,IAAI,CAACC,OAAO,EAAE;MAChB,MAAM,IAAI,CAACC,aAAa,CAAC,CAAC;IAC5B;IACA,MAAM,IAAI,CAACC,WAAW,CAAC,CAAC;IACxB,IAAI,IAAI,CAACrE,UAAU,EAAE;MACnB,IAAI;QACF,IAAI,IAAI,CAACD,eAAe,KAAKf,QAAQ,EAAE;UACrC,MAAM,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;YACrC,IAAI,OAAO,IAAI,CAACjD,UAAU,CAACsE,IAAI,KAAK,UAAU,EAAE;cAC9C,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAClB,GAAG,IAAKA,GAAG,GAAGH,MAAM,CAACG,GAAG,CAAC,GAAGJ,OAAO,CAAC,CAAE,CAAC;YAC9D,CAAC,MAAMA,OAAO,CAAC,CAAC;UAClB,CAAC,CAAC;QACJ,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKjB,QAAQ,EAAE;UAC5C,IAAI,IAAI,CAACkB,UAAU,CAACsE,IAAI,EAAE,MAAM,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAC,CAAC;QACxD,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;UAC3C,IAAI,IAAI,CAACiB,UAAU,CAACuE,UAAU,EAAE,MAAM,IAAI,CAACvE,UAAU,CAACuE,UAAU,CAAC,CAAC;QACpE;MACF,CAAC,CAAC,MAAM;QACN;MAAA;MAEF,IAAI,CAACvE,UAAU,GAAG,IAAI;IACxB;IACA,IAAI;MACF,IAAI,IAAI,CAACV,WAAW,EAAE;QACpB,IAAI,IAAI,CAACS,eAAe,KAAKf,QAAQ,IAAI,IAAI,CAACe,eAAe,KAAKjB,QAAQ,EAAE;UAC1E,MAAM,IAAI,CAACQ,WAAW,CAACgF,IAAI,CAAC,CAAC;QAC/B,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;UAC3C,MAAM,IAAI,CAACO,WAAW,CAACiF,UAAU,CAAC,CAAC;QACrC;MACF;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEF5E,OAAO,CAAC6E,IAAI,CAAC,CAAC,CAAC;EACjB;EAEAH,WAAWA,CAAA,EAAG;IACZ,MAAMI,UAAU,GAAG,IAAI,CAACnD,2BAA2B;IACnD,IAAI,CAACmD,UAAU,IAAI,CAAC,IAAI,CAACnF,WAAW,EAAE,OAAOyD,OAAO,CAACC,OAAO,CAAC,CAAC;IAC9D0B,OAAO,CAACC,IAAI,CACV,GAAG,IAAI,CAACpD,0BAA0B,qCACpC,CAAC;IACD,MAAMqD,GAAG,GAAG,IAAI,CAACpD,MAAM,IAAI,SAAS;IACpC,IAAI,IAAI,CAACzB,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAEC,OAAO,IAAK;QAC9B,IAAI,CAAC1D,WAAW,CAAC4D,YAAY,CAAC,SAAS,EAAE,CAACuB,UAAU,EAAEG,GAAG,CAAC,EAAE,MAAM5B,OAAO,CAAC,CAAC,CAAC;MAC9E,CAAC,CAAC;IACJ;IACA,IAAI,IAAI,CAACjD,eAAe,KAAKjB,QAAQ,EAAE;MACrC,OAAO,IAAI,CAACQ,WAAW,CAACkE,WAAW,CAAC,CAAC,SAAS,EAAEiB,UAAU,EAAEG,GAAG,CAAC,CAAC,CAAC/B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACnF;IACA,IAAI,IAAI,CAAC9C,eAAe,KAAKhB,OAAO,EAAE;MACpC,OAAO,IAAI,CAACO,WAAW,CAACuF,OAAO,CAACJ,UAAU,EAAEG,GAAG,CAAC,CAAC/B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE;IACA,OAAOE,OAAO,CAACC,OAAO,CAAC,CAAC;EAC1B;;EAEA;AACF;AACA;EACE8B,0BAA0BA,CAAA,EAAG;IAC3B,MAAM9C,SAAS,GAAG,IAAI,CAACZ,uBAAuB;IAC9C,IAAI,CAACY,SAAS,IAAI,CAAC,IAAI,CAAC1C,WAAW,EAAE;IACrC,MAAMyF,KAAK,GAAG,IAAI,CAACvD,MAAM,IAAI,GAAG;IAChC,MAAMwD,IAAI,GAAGA,CAAA,KAAM,CAAC,CAAC;IACrB,MAAMC,QAAQ,GAAG9F,kCAAkC;IACnD,MAAM+F,KAAK,GAAG,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGH,QAAQ,IAAI;IAE1C,IAAI,IAAI,CAAClF,eAAe,KAAKf,QAAQ,EAAE;MACrC,IAAI,CAACM,WAAW,CAAC4D,YAAY,CAAC,MAAM,EAAE,CAAClB,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE+C,KAAK,CAAC,EAAEC,IAAI,CAAC;MACpG,IAAI,CAAC1F,WAAW,CAAC4D,YAAY,CAAC,OAAO,EAAE,CAAClB,SAAS,EAAE,OAAO,EAAEkD,KAAK,CAAC,EAAEF,IAAI,CAAC;IAC3E,CAAC,MAAM,IAAI,IAAI,CAACjF,eAAe,KAAKjB,QAAQ,EAAE;MAC5C,IAAI,CAACQ,WAAW,CAACkE,WAAW,CAAC,CAAC,MAAM,EAAExB,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE+C,KAAK,CAAC,CAAC,CAAClC,KAAK,CAACmC,IAAI,CAAC;MACzG,IAAI,CAAC1F,WAAW,CAACkE,WAAW,CAAC,CAAC,OAAO,EAAExB,SAAS,EAAE,OAAO,EAAEkD,KAAK,CAAC,CAAC,CAACrC,KAAK,CAACmC,IAAI,CAAC;IAChF,CAAC,MAAM,IAAI,IAAI,CAACjF,eAAe,KAAKhB,OAAO,EAAE;MAC3C,IAAI,CAACO,WAAW,CAACmE,IAAI,CAAC,MAAM,EAAEzB,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE+C,KAAK,CAAC,CAAClC,KAAK,CAACmC,IAAI,CAAC;MAC9F,IAAI,CAAC1F,WAAW,CAACmE,IAAI,CAAC,OAAO,EAAEzB,SAAS,EAAE,OAAO,EAAEkD,KAAK,CAAC,CAACrC,KAAK,CAACmC,IAAI,CAAC;IACvE;EACF;EAEAK,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI,CAAC,IAAI,CAAC/F,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAAC3D,WAAW,CAAC4D,YAAY,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,CAACE,GAAG,EAAEC,MAAM,KAAK;UACjE,IAAID,GAAG,EAAE;YACPH,MAAM,CAAC,IAAIzD,KAAK,CAAC,8BAA8B4D,GAAG,CAACkC,OAAO,EAAE,CAAC,CAAC;UAChE,CAAC,MAAMtC,OAAO,CAACK,MAAM,CAAC;QACxB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACtD,eAAe,KAAKjB,QAAQ,EAAE;MACrC,IAAI;QACF,OAAO,IAAI,CAACQ,WAAW,CAACkE,WAAW,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;MACzD,CAAC,CAAC,OAAOJ,GAAG,EAAE;QACZ,MAAM,IAAI5D,KAAK,CAAC,8BAA8B4D,GAAG,CAACkC,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,IAAI,IAAI,CAACvF,eAAe,KAAKhB,OAAO,EAAE;MACpC,IAAI;QACF,OAAO,IAAI,CAACO,WAAW,CAACwD,MAAM,CAAC,MAAM,CAAC;MACxC,CAAC,CAAC,OAAOM,GAAG,EAAE;QACZ,MAAM,IAAI5D,KAAK,CAAC,8BAA8B4D,GAAG,CAACkC,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,MAAM,IAAI9F,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAED+F,YAAY,GAAG,MAAMC,OAAO,IAAI;IAC9B,IAAI,CAAC,IAAI,CAAClG,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKf,QAAQ,EAAE;MACrC,OAAO,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAAC3D,WAAW,CAACmG,IAAI,CAACD,OAAO,EAAE,CAACpC,GAAG,EAAEC,MAAM,KAAK;UAC9C,IAAID,GAAG,EAAEH,MAAM,CAACG,GAAG,CAAC,MACfJ,OAAO,CAACK,MAAM,CAAC;QACtB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACtD,eAAe,KAAKjB,QAAQ,IAAI,IAAI,CAACiB,eAAe,KAAKhB,OAAO,EAAE;MACzE,IAAI;QACF,OAAO,IAAI,CAACO,WAAW,CAACmG,IAAI,CAACD,OAAO,CAAC;MACvC,CAAC,CAAC,OAAOpC,GAAG,EAAE;QACZ,MAAM,IAAI5D,KAAK,CAAC,6BAA6B4D,GAAG,CAACkC,OAAO,EAAE,CAAC;MAC7D;IACF;IAEA,MAAM,IAAI9F,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAEDkG,qBAAqB,GAAGC,UAAU,IAAI;IACpC,OAAOA,UAAU,CACdC,KAAK,CAAC,IAAI,CAAC,CACXC,MAAM,CAACC,IAAI,IAAIA,IAAI,CAACC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAClC/B,GAAG,CAAC8B,IAAI,IAAI;MACX,MAAME,KAAK,GAAGF,IAAI,CAACF,KAAK,CAAC,GAAG,CAAC;MAC7B,MAAM9C,MAAM,GAAG,CAAC,CAAC;MACjBkD,KAAK,CAACC,OAAO,CAAC1C,CAAC,IAAI;QACjB,MAAM2C,KAAK,GAAG3C,CAAC,CAAC4C,OAAO,CAAC,GAAG,CAAC;QAC5B,IAAID,KAAK,KAAK,CAAC,CAAC,EAAE;QAClB,MAAME,CAAC,GAAG7C,CAAC,CAAC8C,KAAK,CAAC,CAAC,EAAEH,KAAK,CAAC;QAC3B,MAAMI,CAAC,GAAG/C,CAAC,CAAC8C,KAAK,CAACH,KAAK,GAAG,CAAC,CAAC;QAC5B,IAAIhH,qBAAqB,CAACqH,QAAQ,CAACH,CAAC,CAAC,EAAE;UACrCtD,MAAM,CAACsD,CAAC,CAAC,GAAGE,CAAC;QACf;MACF,CAAC,CAAC;MACF,OAAOxD,MAAM;IACf,CAAC,CAAC;EACN,CAAC;;EAED;AACF;AACA;AACA;EACE0D,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI;MACF,MAAM,CAACC,aAAa,EAAEC,YAAY,EAAEC,kBAAkB,CAAC,GACrD,MAAM5D,OAAO,CAAC6D,GAAG,CAAC,CAChB,IAAI,CAACrB,YAAY,CAAC,QAAQ,CAAC,EAC3B,IAAI,CAACA,YAAY,CAAC,OAAO,CAAC,EAC1B,IAAI,CAACF,mBAAmB,CAAC,CAAC,CAC3B,CAAC;MAEJ,MAAMwB,MAAM,GAAG,IAAI,CAACC,gBAAgB,CAAC,CAAC;MAEtC,MAAMC,WAAW,GAAG,IAAI,CAACrB,qBAAqB,CAACiB,kBAAkB,CAAC;MAElE,IAAI,IAAI,CAACK,SAAS,EAAE;QAClB;QACAtC,OAAO,CAACuC,GAAG,CACT,4CAA4C,EAC5CN,kBACF,CAAC;QACDjC,OAAO,CAACuC,GAAG,CACT,iDAAiD,EACjDN,kBAAkB,CAACjE,MACrB,CAAC;;QAED;QACAgC,OAAO,CAACuC,GAAG,CACT,mDAAmD,EACnDF,WAAW,CAACrE,MACd,CAAC;QACDgC,OAAO,CAACuC,GAAG,CACT,8DAA8D,EAC9DC,IAAI,CAACC,SAAS,CAACJ,WAAW,CAACV,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAClD,CAAC;QAED,MAAMe,OAAO,GAAG;UAAE/G,IAAI,EAAE,CAAC;UAAEgH,KAAK,EAAE,CAAC;UAAEC,GAAG,EAAE,CAAC;UAAE,SAAS,EAAE;QAAE,CAAC;QAC3DP,WAAW,CAACd,OAAO,CAACsB,CAAC,IAAI;UACvB,IAAI,CAACA,CAAC,CAAClH,IAAI,EAAE+G,OAAO,CAAC/G,IAAI,IAAI,CAAC;UAC9B,IAAI,CAACkH,CAAC,CAACF,KAAK,EAAED,OAAO,CAACC,KAAK,IAAI,CAAC;UAChC,IAAI,CAACE,CAAC,CAACD,GAAG,EAAEF,OAAO,CAACE,GAAG,IAAI,CAAC;UAC5B,IAAI,CAACC,CAAC,CAAC,SAAS,CAAC,EAAEH,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;QACF1C,OAAO,CAACuC,GAAG,CACT,4DAA4D,EAC5DG,OACF,CAAC;MACH;MAEA,MAAMI,OAAO,GAAG,CAAC,CAAC;MAClBT,WAAW,CAACd,OAAO,CAACwB,IAAI,IAAI;QAC1B,MAAM;UAAEpH,IAAI;UAAEgH,KAAK;UAAE,SAAS,EAAEK,MAAM;UAAEJ;QAAI,CAAC,GAAGG,IAAI;;QAEpD;QACA,MAAME,GAAG,GAAGT,IAAI,CAACC,SAAS,CAAC;UAAE9G,IAAI;UAAEgH,KAAK;UAAEC;QAAI,CAAC,CAAC;QAEhD,IAAI,CAACE,OAAO,CAACG,GAAG,CAAC,EAAE;UACjBH,OAAO,CAACG,GAAG,CAAC,GAAG;YACbd,MAAM,EAAE;cAAExG,IAAI;cAAEgH,KAAK;cAAEC;YAAI,CAAC;YAC5BM,KAAK,EAAE,CAAC;YACRC,MAAM,EAAE;UACV,CAAC;QACH;QAEAL,OAAO,CAACG,GAAG,CAAC,CAACC,KAAK,IAAI,CAAC;QACvB,MAAME,GAAG,GAAGpI,QAAQ,CAACgI,MAAM,EAAE,EAAE,CAAC;QAChCF,OAAO,CAACG,GAAG,CAAC,CAACE,MAAM,IAAIE,MAAM,CAACC,QAAQ,CAACF,GAAG,CAAC,GAAGA,GAAG,GAAG,CAAC;MACvD,CAAC,CAAC;;MAEF;MACA,IAAI,CAAC,IAAI,CAAClH,kBAAkB,EAAE,IAAI,CAACA,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;MACjE,IAAI,CAAC,IAAI,CAACC,oBAAoB,EAAE,IAAI,CAACA,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;MAErE,MAAMoH,WAAW,GAAG,IAAIpH,GAAG,CAACqH,MAAM,CAACC,IAAI,CAACX,OAAO,CAAC,CAAC;MACjD,KAAK,MAAMpB,CAAC,IAAI6B,WAAW,EAAE;QAC3B,IAAI,CAACrH,kBAAkB,CAACwH,GAAG,CAAChC,CAAC,CAAC;QAC9B,IAAI,IAAI,CAACtF,oBAAoB,CAACuH,GAAG,CAACjC,CAAC,CAAC,EAAE;UACpC,IAAI,CAACtF,oBAAoB,CAACwH,MAAM,CAAClC,CAAC,CAAC;QACrC;MACF;;MAEA;MACA;MACA;MACA,IAAI,CAACjG,qBAAqB,CAACoI,KAAK,CAAC,CAAC;MAClC,IAAI,CAAC9H,2BAA2B,CAAC8H,KAAK,CAAC,CAAC;;MAExC;MACA,KAAK,MAAMnC,CAAC,IAAI,IAAI,CAACxF,kBAAkB,EAAE;QACvC,IAAIqH,WAAW,CAACI,GAAG,CAACjC,CAAC,CAAC,EAAE;QACxB,IAAI,IAAI,CAACtF,oBAAoB,CAACuH,GAAG,CAACjC,CAAC,CAAC,EAAE;QACtC,IAAI;UACF,MAAMoC,WAAW,GAAGtB,IAAI,CAACuB,KAAK,CAACrC,CAAC,CAAC;UACjC,IAAI,CAACjG,qBAAqB,CAACuI,GAAG,CAAC;YAAE,GAAG7B,MAAM;YAAE,GAAG2B;UAAY,CAAC,EAAE,CAAC,CAAC;UAChE,IAAI,CAAC/H,2BAA2B,CAACiI,GAAG,CAAC;YAAE,GAAG7B,MAAM;YAAE,GAAG2B;UAAY,CAAC,EAAE,CAAC,CAAC;UACtE,IAAI,CAAC1H,oBAAoB,CAACsH,GAAG,CAAChC,CAAC,CAAC;QAClC,CAAC,CAAC,MAAM;UACN;QAAA;MAEJ;MAEA,IAAI,IAAI,CAACY,SAAS,EAAE;QAClB,MAAM2B,MAAM,GAAGT,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC;QACrC,MAAMqB,YAAY,GAAGF,MAAM,CAACG,MAAM,CAAC,CAACC,GAAG,EAAEC,CAAC,KAAKD,GAAG,IAAIC,CAAC,CAACpB,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACvElD,OAAO,CAACuC,GAAG,CACT,oDAAoD,EACpD0B,MAAM,CAACjG,MACT,CAAC;QACDgC,OAAO,CAACuC,GAAG,CACT,kDAAkD,EAClD4B,YACF,CAAC;QACDnE,OAAO,CAACuC,GAAG,CACT,8CAA8C,EAC9C,IAAI,CAACrG,kBAAkB,CAACqI,IAC1B,CAAC;QACDvE,OAAO,CAACuC,GAAG,CACT,gDAAgD,EAChD,IAAI,CAACnG,oBAAoB,CAACmI,IAC5B,CAAC;QACDvE,OAAO,CAACuC,GAAG,CACT,8DAA8D,EAC9DC,IAAI,CAACC,SAAS,CAACwB,MAAM,CAACtC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAC7C,CAAC;MACH;MAEA6B,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC,CAACvB,OAAO,CAC5B,CAAC;QAAEY,MAAM,EAAE2B,WAAW;QAAEZ,KAAK;QAAEC;MAAO,CAAC,KAAK;QAC1C,IAAI,CAAC1H,qBAAqB,CAACuI,GAAG,CAAC;UAAE,GAAG7B,MAAM;UAAE,GAAG2B;QAAY,CAAC,EAAEZ,KAAK,CAAC;QACpE,IAAI,CAACnH,2BAA2B,CAACiI,GAAG,CAClC;UAAE,GAAG7B,MAAM;UAAE,GAAG2B;QAAY,CAAC,EAC7BX,MACF,CAAC;MACH,CACF,CAAC;MAED,MAAMqB,cAAc,GAAGC,OAAO,IAC5BjB,MAAM,CAACkB,WAAW,CAChBD,OAAO,CACJvD,KAAK,CAAC,MAAM,CAAC,CACbC,MAAM,CAACC,IAAI,IAAIA,IAAI,IAAI,CAACA,IAAI,CAACuD,UAAU,CAAC,GAAG,CAAC,CAAC,CAC7CrF,GAAG,CAAC8B,IAAI,IAAIA,IAAI,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAC/BC,MAAM,CAACG,KAAK,IAAIA,KAAK,CAACtD,MAAM,KAAK,CAAC,IAAIsD,KAAK,CAAC,CAAC,CAAC,IAAIA,KAAK,CAAC,CAAC,CAAC,CAC/D,CAAC;MAEH,MAAM6B,MAAM,GAAGqB,cAAc,CAACzC,aAAa,CAAC;MAC5C,MAAM6C,KAAK,GAAGJ,cAAc,CAACxC,YAAY,CAAC;MAE1C,IAAImB,MAAM,CAAC0B,WAAW,EAAE;QACtB,IAAI,CAAC7I,gBAAgB,CAACgI,GAAG,CACvB;UAAE,GAAG7B,MAAM;UAAE2C,WAAW,EAAE;QAAO,CAAC,EAClC9J,QAAQ,CAACmI,MAAM,CAAC0B,WAAW,EAAE,EAAE,CAAC,IAAI,CACtC,CAAC;MACH;MACA,IAAI1B,MAAM,CAAC4B,SAAS,EAAE;QACpB,IAAI,CAAC/I,gBAAgB,CAACgI,GAAG,CACvB;UAAE,GAAG7B,MAAM;UAAE2C,WAAW,EAAE;QAAM,CAAC,EACjC9J,QAAQ,CAACmI,MAAM,CAAC4B,SAAS,EAAE,EAAE,CAAC,IAAI,CACpC,CAAC;MACH;MAEA,IAAIH,KAAK,CAACI,yBAAyB,EAAE;QACnC,IAAI,CAAC/I,eAAe,CAAC+H,GAAG,CACtB;UAAE,GAAG7B,MAAM;UAAE8C,SAAS,EAAE;QAAc,CAAC,EACvCjK,QAAQ,CAAC4J,KAAK,CAACI,yBAAyB,EAAE,EAAE,CAAC,IAAI,CACnD,CAAC;MACH;IACF,CAAC,CAAC,OAAOE,KAAK,EAAE;MACdlF,OAAO,CAACkF,KAAK,CACX,kDAAkD,EAClDA,KAAK,CAACtE,OACR,CAAC;IACH;EACF,CAAC;;EAED;AACF;AACA;AACA;EACEuE,gBAAgB,GAAG,MAAAA,CAAA,KAAY;IAC7B,IAAI;MACF,MAAM,IAAI,CAACrD,mBAAmB,CAAC,CAAC;MAChC,MAAM,IAAI,CAACsD,WAAW,CAAC,CAAC;MACxB,IAAI,CAACC,gBAAgB,CAAC,CAAC;IACzB,CAAC,CAAC,OAAOH,KAAK,EAAE;MACdlF,OAAO,CAACkF,KAAK,CACX,oDAAoDA,KAAK,CAACtE,OAAO,EACnE,CAAC;MACD,MAAMsE,KAAK;IACb;EACF,CAAC;;EAED;AACF;AACA;EACEI,SAAS,GAAGA,CAACvK,WAAW,GAAG,IAAI,CAACA,WAAW,KAAK;IAC9C,IAAI,CAACwK,UAAU,CAACxK,WAAW,EAAE,MAAM;MACjC,IAAI,CAACoK,gBAAgB,CAAC,CAAC,CAAChH,KAAK,CAACO,GAAG,IAAI;QACnCsB,OAAO,CAACkF,KAAK,CAAC,+CAA+C,EAAExG,GAAG,CAAC;MACrE,CAAC,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,IAAI,CAACjC,sBAAsB,EAAE;MAC/B,IAAI,CAAC2D,0BAA0B,CAAC,CAAC;IACnC;EACF,CAAC;;EAED;AACF;AACA;AACA;AACA;EACEoF,OAAO,GAAG,MAAAA,CAAA,KAAY;IACpB,IAAI,CAAChG,QAAQ,CAAC,CAAC;IACf,IAAI,IAAI,CAACC,OAAO,EAAE;MAChB,MAAM,IAAI,CAACC,aAAa,CAAC,CAAC;IAC5B;IACA,IAAI,IAAI,CAACpE,UAAU,EAAE;MACnB,IAAI;QACF,IAAI,IAAI,CAACD,eAAe,KAAKf,QAAQ,EAAE;UACrC,MAAM,IAAI+D,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;YACrC,IAAI,OAAO,IAAI,CAACjD,UAAU,CAACsE,IAAI,KAAK,UAAU,EAAE;cAC9C,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAClB,GAAG,IAAKA,GAAG,GAAGH,MAAM,CAACG,GAAG,CAAC,GAAGJ,OAAO,CAAC,CAAE,CAAC;YAC9D,CAAC,MAAMA,OAAO,CAAC,CAAC;UAClB,CAAC,CAAC;QACJ,CAAC,MAAM,IAAI,IAAI,CAACjD,eAAe,KAAKjB,QAAQ,EAAE;UAC5C,IAAI,IAAI,CAACkB,UAAU,CAACsE,IAAI,EAAE,MAAM,IAAI,CAACtE,UAAU,CAACsE,IAAI,CAAC,CAAC;QACxD,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;UAC3C,IAAI,IAAI,CAACiB,UAAU,CAACuE,UAAU,EAAE,MAAM,IAAI,CAACvE,UAAU,CAACuE,UAAU,CAAC,CAAC;QACpE;MACF,CAAC,CAAC,OAAOnB,GAAG,EAAE;QACZsB,OAAO,CAACkF,KAAK,CAAC,kDAAkD,EAAExG,GAAG,CAAC;MACxE;MACA,IAAI,CAACpD,UAAU,GAAG,IAAI;IACxB;IACA,IAAI;MACF,IAAI,CAAC,IAAI,CAACV,WAAW,EAAE;MAEvB,IACE,IAAI,CAACS,eAAe,KAAKf,QAAQ,IACjC,IAAI,CAACe,eAAe,KAAKjB,QAAQ,EACjC;QACA,MAAM,IAAI,CAACQ,WAAW,CAACgF,IAAI,CAAC,CAAC;MAC/B,CAAC,MAAM,IAAI,IAAI,CAACvE,eAAe,KAAKhB,OAAO,EAAE;QAC3C,MAAM,IAAI,CAACO,WAAW,CAACiF,UAAU,CAAC,CAAC;MACrC;IACF,CAAC,CAAC,OAAOnB,GAAG,EAAE;MACZsB,OAAO,CAACkF,KAAK,CAAC,6CAA6C,EAAExG,GAAG,CAAC;IACnE;IACA,MAAM,KAAK,CAAC8G,OAAO,CAAC,CAAC;EACvB,CAAC;EAEDnJ,mBAAmB,GAAGA,CAAA,KAAM;IAC1BpB,OAAO,CAACuC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAACgI,OAAO,CAAC;IAClCvK,OAAO,CAACuC,EAAE,CAAC,SAAS,EAAE,IAAI,CAACgI,OAAO,CAAC;EACrC,CAAC;AACH;AAEAC,MAAM,CAACC,OAAO,GAAG;EAAEhL;AAAmB,CAAC","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"metricsRedisClient.js","names":["BaseMetricsClient","require","getRedisClientType","REDIS_V4","IOREDIS","REDIS_V3","redisConnectionStableFields","redisConnectionFields","RedisMetricsClient","constructor","redisClient","metricsConfig","Error","intervalSec","parseInt","process","env","METRICS_QUEUE_INTERVAL_SEC","processType","redisClientType","_redisLabelNames","redisConnectionsGauge","createGauge","name","help","labelNames","redisConnectionsMemoryGauge","redisMemoryGauge","redisStatsGauge","_redisConnSeenKeys","Set","_redisConnZeroedKeys","_setCleanupHandlers","getRedisConnections","Promise","resolve","reject","send_command","err","result","message","sendCommand","client","getRedisInfo","section","info","parseRedisConnections","clientsStr","split","filter","line","trim","map","parts","forEach","p","eqIdx","indexOf","k","slice","v","includes","collectRedisMetrics","memoryInfoStr","statsInfoStr","connectionsInfoStr","all","labels","app","appName","process_type","connections","logValues","console","log","length","JSON","stringify","missing","flags","cmd","c","grouped","conn","totMem","key","count","memory","mem","Number","isFinite","currentKeys","Object","keys","add","has","delete","reset","valueLabels","parse","set","groups","values","groupedTotal","reduce","sum","g","size","parseRedisInfo","infoStr","fromEntries","startsWith","stats","used_memory","memory_type","maxmemory","instantaneous_ops_per_sec","operation","error","warn","pushRedisMetrics","gatewayPush","clearAllCounters","metricsLogValues","metricObjects","registry","getMetricsAsJSON","startPush","_startPush","catch","cleanup","quit","disconnect","exit","on","module","exports"],"sources":["../../src/metrics/metricsRedisClient.js"],"sourcesContent":["const { BaseMetricsClient } = require('./baseMetricsClient')\nconst {\n getRedisClientType,\n REDIS_V4,\n IOREDIS,\n REDIS_V3,\n} = require('../redisUtils')\n\nconst redisConnectionStableFields = ['name', 'flags', 'cmd']\nconst redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']\n\n/**\n * RedisMetricsClient extends BaseMetricsClient to collect\n * Redis metrics periodically and push them to Prometheus Pushgateway.\n *\n * @extends BaseMetricsClient\n */\nclass RedisMetricsClient extends BaseMetricsClient {\n /**\n * @param {Object} options\n * @param {any} options.redisClient - Redis client instance (required)\n * @param {string} [options.appName] - Application name (from BaseMetricsClient)\n * @param {string} [options.dynoId] - Dyno/instance ID (from BaseMetricsClient)\n * @param {string} [options.processType] - Process type (from BaseMetricsClient)\n * @param {boolean} [options.enabled] - Enable metrics collection (from BaseMetricsClient)\n * @param {boolean} [options.logValues] - Log metrics values (from BaseMetricsClient)\n * @param {string} [options.pushgatewayUrl] - PushGateway URL (from BaseMetricsClient)\n * @param {string} [options.pushgatewaySecret] - PushGateway secret token (from BaseMetricsClient)\n * @param {number} [options.intervalSec] - Interval in seconds for pushing metrics (from BaseMetricsClient)\n * @param {boolean} [options.removeOldMetrics] - Remove old metrics by service (from BaseMetricsClient)\n * @param {function} [options.startupValidation] - Function to validate startup (from BaseMetricsClient)\n * @param {boolean} [options.disablePushgateway] - Disable pushing to Pushgateway (use HTTP scraping instead)\n */\n constructor({ redisClient, ...metricsConfig } = {}) {\n if (redisClient == null) {\n throw new Error('RedisMetricsClient requires redisClient')\n }\n\n const intervalSec =\n metricsConfig.intervalSec ||\n parseInt(process.env.METRICS_QUEUE_INTERVAL_SEC || '', 10) ||\n 5\n\n super({\n ...metricsConfig,\n processType: metricsConfig.processType || 'queue-metrics',\n intervalSec,\n })\n\n /** Redis client used for metrics */\n this.redisClient = redisClient\n this.redisClientType = getRedisClientType(redisClient)\n\n /** Label names for Redis metrics: app + process_type only (no dyno_id). */\n this._redisLabelNames = ['app', 'process_type']\n\n /** Counter for Redis connection metrics */\n this.redisConnectionsGauge = this.createGauge({\n name: 'app_redis_connections_count',\n help: 'Redis client connections',\n labelNames: [...this._redisLabelNames, ...redisConnectionStableFields],\n })\n\n this.redisConnectionsMemoryGauge = this.createGauge({\n name: 'app_redis_connections_memory_usage_count',\n help: 'Redis client connections',\n labelNames: [...this._redisLabelNames, ...redisConnectionStableFields],\n })\n\n /** Gauge for Redis memory usage */\n this.redisMemoryGauge = this.createGauge({\n name: 'app_redis_memory_bytes',\n help: 'Redis memory usage in bytes',\n labelNames: [...this._redisLabelNames, 'memory_type'],\n })\n\n /** Gauge for Redis operation stats */\n this.redisStatsGauge = this.createGauge({\n name: 'app_redis_stats_total',\n help: 'Redis operation statistics',\n labelNames: [...this._redisLabelNames, 'operation'],\n })\n\n // Track emitted connection label combinations so we can:\n // - emit a one-shot 0 for series that disappear (overwrite last non-zero sample)\n // - then stop emitting them on subsequent scrapes\n //\n // This is mainly needed because we label connections by `cmd`,\n // and Redis `CLIENT LIST` `cmd` is volatile.\n this._redisConnSeenKeys = new Set()\n this._redisConnZeroedKeys = new Set()\n\n this._setCleanupHandlers()\n }\n\n getRedisConnections = async () => {\n if (!this.redisClient) throw new Error('Redis client not provided')\n\n // node-redis v3 (uses callback)\n if (this.redisClientType === REDIS_V3) {\n return new Promise((resolve, reject) => {\n this.redisClient.send_command('CLIENT', ['LIST'], (err, result) => {\n if (err) {\n reject(new Error(`Failed to get CLIENT LIST: ${err.message}`))\n } else resolve(result)\n })\n })\n }\n\n // node-redis v4\n if (this.redisClientType === REDIS_V4) {\n try {\n return this.redisClient.sendCommand(['CLIENT', 'LIST'])\n } catch (err) {\n throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n }\n }\n\n if (this.redisClientType === IOREDIS) {\n try {\n return this.redisClient.client('LIST')\n } catch (err) {\n throw new Error(`Failed to get CLIENT LIST: ${err.message}`)\n }\n }\n\n throw new Error('Unsupported Redis client type')\n }\n\n getRedisInfo = async section => {\n if (!this.redisClient) throw new Error('Redis client not provided')\n\n // node-redis v3 (uses callback)\n if (this.redisClientType === REDIS_V3) {\n return new Promise((resolve, reject) => {\n this.redisClient.info(section, (err, result) => {\n if (err) reject(err)\n else resolve(result)\n })\n })\n }\n\n // node-redis v4 or ioredis (info returns Promise)\n if (this.redisClientType === REDIS_V4 || this.redisClientType === IOREDIS) {\n try {\n return this.redisClient.info(section)\n } catch (err) {\n throw new Error(`Failed to get Redis INFO: ${err.message}`)\n }\n }\n\n throw new Error('Unsupported Redis client type')\n }\n\n parseRedisConnections = clientsStr => {\n return clientsStr\n .split('\\n')\n .filter(line => line.trim() !== '')\n .map(line => {\n const parts = line.split(' ')\n const client = {}\n parts.forEach(p => {\n const eqIdx = p.indexOf('=')\n if (eqIdx === -1) return\n const k = p.slice(0, eqIdx)\n const v = p.slice(eqIdx + 1)\n if (redisConnectionFields.includes(k)) {\n client[k] = v\n }\n })\n return client\n })\n }\n\n /**\n * Collect basic Redis INFO metrics: clients, memory, stats\n * @returns {Promise<void>}\n */\n collectRedisMetrics = async () => {\n try {\n const [memoryInfoStr, statsInfoStr, connectionsInfoStr] =\n await Promise.all([\n this.getRedisInfo('memory'),\n this.getRedisInfo('stats'),\n this.getRedisConnections(),\n ])\n\n const labels = { app: this.appName, process_type: this.processType }\n\n const connections = this.parseRedisConnections(connectionsInfoStr)\n\n if (this.logValues) {\n // \"IN\" data: raw client list from Redis\n console.log(\n '[queue-metrics] Redis CLIENT LIST (raw):\\n',\n connectionsInfoStr\n )\n console.log(\n '[queue-metrics] Redis CLIENT LIST (raw) length:',\n connectionsInfoStr.length\n )\n\n // \"OUT\" data: parsed fields used for metrics\n console.log(\n '[queue-metrics] Redis CLIENT LIST (parsed) count:',\n connections.length\n )\n console.log(\n '[queue-metrics] Redis CLIENT LIST (parsed sample, first 10):',\n JSON.stringify(connections.slice(0, 10), null, 2)\n )\n\n const missing = { name: 0, flags: 0, cmd: 0, 'tot-mem': 0 }\n connections.forEach(c => {\n if (!c.name) missing.name += 1\n if (!c.flags) missing.flags += 1\n if (!c.cmd) missing.cmd += 1\n if (!c['tot-mem']) missing['tot-mem'] += 1\n })\n console.log(\n '[queue-metrics] Redis CLIENT LIST (parsed) missing fields:',\n missing\n )\n }\n\n const grouped = {}\n connections.forEach(conn => {\n const { name, flags, 'tot-mem': totMem, cmd } = conn\n\n // build grouping key (includes default gauge labels later)\n const key = JSON.stringify({ name, flags, cmd })\n\n if (!grouped[key]) {\n grouped[key] = {\n labels: { name, flags, cmd },\n count: 0,\n memory: 0,\n }\n }\n\n grouped[key].count += 1\n const mem = parseInt(totMem, 10)\n grouped[key].memory += Number.isFinite(mem) ? mem : 0\n })\n\n // Ensure sets exist even if older instance is hot-reloaded.\n if (!this._redisConnSeenKeys) this._redisConnSeenKeys = new Set()\n if (!this._redisConnZeroedKeys) this._redisConnZeroedKeys = new Set()\n\n const currentKeys = new Set(Object.keys(grouped))\n for (const k of currentKeys) {\n this._redisConnSeenKeys.add(k)\n if (this._redisConnZeroedKeys.has(k)) {\n this._redisConnZeroedKeys.delete(k)\n }\n }\n\n // Reset gauges each scrape so we only export:\n // - current snapshot series\n // - one-shot zeros for recently disappeared series\n this.redisConnectionsGauge.reset()\n this.redisConnectionsMemoryGauge.reset()\n\n // Emit one-shot zeros for series that disappeared since last seen.\n for (const k of this._redisConnSeenKeys) {\n if (currentKeys.has(k)) continue\n if (this._redisConnZeroedKeys.has(k)) continue\n try {\n const valueLabels = JSON.parse(k)\n this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, 0)\n this.redisConnectionsMemoryGauge.set({ ...labels, ...valueLabels }, 0)\n this._redisConnZeroedKeys.add(k)\n } catch {\n // ignore malformed key\n }\n }\n\n if (this.logValues) {\n const groups = Object.values(grouped)\n const groupedTotal = groups.reduce((sum, g) => sum + (g.count || 0), 0)\n console.log(\n '[queue-metrics] Redis connections grouped buckets:',\n groups.length\n )\n console.log(\n '[queue-metrics] Redis connections grouped total:',\n groupedTotal\n )\n console.log(\n '[queue-metrics] Redis connections seen keys:',\n this._redisConnSeenKeys.size\n )\n console.log(\n '[queue-metrics] Redis connections zeroed keys:',\n this._redisConnZeroedKeys.size\n )\n console.log(\n '[queue-metrics] Redis connections grouped sample (first 10):',\n JSON.stringify(groups.slice(0, 10), null, 2)\n )\n }\n\n Object.values(grouped).forEach(\n ({ labels: valueLabels, count, memory }) => {\n this.redisConnectionsGauge.set({ ...labels, ...valueLabels }, count)\n this.redisConnectionsMemoryGauge.set(\n { ...labels, ...valueLabels },\n memory\n )\n }\n )\n\n const parseRedisInfo = infoStr =>\n Object.fromEntries(\n infoStr\n .split('\\r\\n')\n .filter(line => line && !line.startsWith('#'))\n .map(line => line.split(':', 2))\n .filter(parts => parts.length === 2 && parts[0] && parts[1])\n )\n\n const memory = parseRedisInfo(memoryInfoStr)\n const stats = parseRedisInfo(statsInfoStr)\n\n if (memory.used_memory) {\n this.redisMemoryGauge.set(\n { ...labels, memory_type: 'used' },\n parseInt(memory.used_memory, 10) || 0\n )\n }\n if (memory.maxmemory) {\n this.redisMemoryGauge.set(\n { ...labels, memory_type: 'max' },\n parseInt(memory.maxmemory, 10) || 0\n )\n }\n\n if (stats.instantaneous_ops_per_sec) {\n this.redisStatsGauge.set(\n { ...labels, operation: 'ops_per_sec' },\n parseInt(stats.instantaneous_ops_per_sec, 10) || 0\n )\n }\n } catch (error) {\n console.warn(\n `[queue-metrics] Failed to collect Redis metrics:`,\n error.message\n )\n }\n }\n\n /**\n * Collect metrics for all Redis and push to Prometheus Pushgateway\n * @returns {Promise<void>}\n */\n pushRedisMetrics = async () => {\n try {\n await this.collectRedisMetrics()\n await this.gatewayPush()\n this.clearAllCounters()\n\n if (this.metricsLogValues) {\n const metricObjects = await this.registry.getMetricsAsJSON()\n console.info(\n `[queue-metrics] Collected metrics for Redis`,\n JSON.stringify(metricObjects, null, 2)\n )\n }\n } catch (error) {\n console.error(\n `[queue-metrics] Failed to collect Redis metrics: ${error.message}`\n )\n throw error\n }\n }\n\n /**\n * Start periodic collection.\n * @param {number} [intervalSec=this.intervalSec] - Interval in seconds\n */\n startPush = (intervalSec = this.intervalSec) => {\n this._startPush(intervalSec, () => {\n this.pushRedisMetrics().catch(err => {\n console.error(`[queue-metrics] Failed to push Redis metrics:`, err)\n })\n })\n }\n\n /**\n * Cleanup Redis client and exit process.\n * @returns {Promise<void>}\n */\n cleanup = async () => {\n try {\n if (!this.redisClient) return\n\n if (\n this.redisClientType === REDIS_V3 ||\n this.redisClientType === REDIS_V4\n ) {\n await this.redisClient.quit()\n } else if (this.redisClientType === IOREDIS) {\n await this.redisClient.disconnect()\n }\n } catch (err) {\n console.error('[queue-metrics] Error closing Redis client:', err)\n }\n process.exit(0)\n }\n\n _setCleanupHandlers = () => {\n process.on('SIGINT', this.cleanup)\n process.on('SIGTERM', this.cleanup)\n }\n}\n\nmodule.exports = { RedisMetricsClient }\n"],"mappings":";;AAAA,MAAM;EAAEA;AAAkB,CAAC,GAAGC,OAAO,CAAC,qBAAqB,CAAC;AAC5D,MAAM;EACJC,kBAAkB;EAClBC,QAAQ;EACRC,OAAO;EACPC;AACF,CAAC,GAAGJ,OAAO,CAAC,eAAe,CAAC;AAE5B,MAAMK,2BAA2B,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC;AAC5D,MAAMC,qBAAqB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;;AAEjE;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,kBAAkB,SAASR,iBAAiB,CAAC;EACjD;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACES,WAAWA,CAAC;IAAEC,WAAW;IAAE,GAAGC;EAAc,CAAC,GAAG,CAAC,CAAC,EAAE;IAClD,IAAID,WAAW,IAAI,IAAI,EAAE;MACvB,MAAM,IAAIE,KAAK,CAAC,yCAAyC,CAAC;IAC5D;IAEA,MAAMC,WAAW,GACfF,aAAa,CAACE,WAAW,IACzBC,QAAQ,CAACC,OAAO,CAACC,GAAG,CAACC,0BAA0B,IAAI,EAAE,EAAE,EAAE,CAAC,IAC1D,CAAC;IAEH,KAAK,CAAC;MACJ,GAAGN,aAAa;MAChBO,WAAW,EAAEP,aAAa,CAACO,WAAW,IAAI,eAAe;MACzDL;IACF,CAAC,CAAC;;IAEF;IACA,IAAI,CAACH,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACS,eAAe,GAAGjB,kBAAkB,CAACQ,WAAW,CAAC;;IAEtD;IACA,IAAI,CAACU,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,CAAC;;IAE/C;IACA,IAAI,CAACC,qBAAqB,GAAG,IAAI,CAACC,WAAW,CAAC;MAC5CC,IAAI,EAAE,6BAA6B;MACnCC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,CAAC,GAAG,IAAI,CAACL,gBAAgB,EAAE,GAAGd,2BAA2B;IACvE,CAAC,CAAC;IAEF,IAAI,CAACoB,2BAA2B,GAAG,IAAI,CAACJ,WAAW,CAAC;MAClDC,IAAI,EAAE,0CAA0C;MAChDC,IAAI,EAAE,0BAA0B;MAChCC,UAAU,EAAE,CAAC,GAAG,IAAI,CAACL,gBAAgB,EAAE,GAAGd,2BAA2B;IACvE,CAAC,CAAC;;IAEF;IACA,IAAI,CAACqB,gBAAgB,GAAG,IAAI,CAACL,WAAW,CAAC;MACvCC,IAAI,EAAE,wBAAwB;MAC9BC,IAAI,EAAE,6BAA6B;MACnCC,UAAU,EAAE,CAAC,GAAG,IAAI,CAACL,gBAAgB,EAAE,aAAa;IACtD,CAAC,CAAC;;IAEF;IACA,IAAI,CAACQ,eAAe,GAAG,IAAI,CAACN,WAAW,CAAC;MACtCC,IAAI,EAAE,uBAAuB;MAC7BC,IAAI,EAAE,4BAA4B;MAClCC,UAAU,EAAE,CAAC,GAAG,IAAI,CAACL,gBAAgB,EAAE,WAAW;IACpD,CAAC,CAAC;;IAEF;IACA;IACA;IACA;IACA;IACA;IACA,IAAI,CAACS,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;IACnC,IAAI,CAACC,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;IAErC,IAAI,CAACE,mBAAmB,CAAC,CAAC;EAC5B;EAEAC,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI,CAAC,IAAI,CAACvB,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKd,QAAQ,EAAE;MACrC,OAAO,IAAI6B,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAAC1B,WAAW,CAAC2B,YAAY,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,CAACC,GAAG,EAAEC,MAAM,KAAK;UACjE,IAAID,GAAG,EAAE;YACPF,MAAM,CAAC,IAAIxB,KAAK,CAAC,8BAA8B0B,GAAG,CAACE,OAAO,EAAE,CAAC,CAAC;UAChE,CAAC,MAAML,OAAO,CAACI,MAAM,CAAC;QACxB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACpB,eAAe,KAAKhB,QAAQ,EAAE;MACrC,IAAI;QACF,OAAO,IAAI,CAACO,WAAW,CAAC+B,WAAW,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;MACzD,CAAC,CAAC,OAAOH,GAAG,EAAE;QACZ,MAAM,IAAI1B,KAAK,CAAC,8BAA8B0B,GAAG,CAACE,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,IAAI,IAAI,CAACrB,eAAe,KAAKf,OAAO,EAAE;MACpC,IAAI;QACF,OAAO,IAAI,CAACM,WAAW,CAACgC,MAAM,CAAC,MAAM,CAAC;MACxC,CAAC,CAAC,OAAOJ,GAAG,EAAE;QACZ,MAAM,IAAI1B,KAAK,CAAC,8BAA8B0B,GAAG,CAACE,OAAO,EAAE,CAAC;MAC9D;IACF;IAEA,MAAM,IAAI5B,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAED+B,YAAY,GAAG,MAAMC,OAAO,IAAI;IAC9B,IAAI,CAAC,IAAI,CAAClC,WAAW,EAAE,MAAM,IAAIE,KAAK,CAAC,2BAA2B,CAAC;;IAEnE;IACA,IAAI,IAAI,CAACO,eAAe,KAAKd,QAAQ,EAAE;MACrC,OAAO,IAAI6B,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;QACtC,IAAI,CAAC1B,WAAW,CAACmC,IAAI,CAACD,OAAO,EAAE,CAACN,GAAG,EAAEC,MAAM,KAAK;UAC9C,IAAID,GAAG,EAAEF,MAAM,CAACE,GAAG,CAAC,MACfH,OAAO,CAACI,MAAM,CAAC;QACtB,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,IAAI,CAACpB,eAAe,KAAKhB,QAAQ,IAAI,IAAI,CAACgB,eAAe,KAAKf,OAAO,EAAE;MACzE,IAAI;QACF,OAAO,IAAI,CAACM,WAAW,CAACmC,IAAI,CAACD,OAAO,CAAC;MACvC,CAAC,CAAC,OAAON,GAAG,EAAE;QACZ,MAAM,IAAI1B,KAAK,CAAC,6BAA6B0B,GAAG,CAACE,OAAO,EAAE,CAAC;MAC7D;IACF;IAEA,MAAM,IAAI5B,KAAK,CAAC,+BAA+B,CAAC;EAClD,CAAC;EAEDkC,qBAAqB,GAAGC,UAAU,IAAI;IACpC,OAAOA,UAAU,CACdC,KAAK,CAAC,IAAI,CAAC,CACXC,MAAM,CAACC,IAAI,IAAIA,IAAI,CAACC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAClCC,GAAG,CAACF,IAAI,IAAI;MACX,MAAMG,KAAK,GAAGH,IAAI,CAACF,KAAK,CAAC,GAAG,CAAC;MAC7B,MAAMN,MAAM,GAAG,CAAC,CAAC;MACjBW,KAAK,CAACC,OAAO,CAACC,CAAC,IAAI;QACjB,MAAMC,KAAK,GAAGD,CAAC,CAACE,OAAO,CAAC,GAAG,CAAC;QAC5B,IAAID,KAAK,KAAK,CAAC,CAAC,EAAE;QAClB,MAAME,CAAC,GAAGH,CAAC,CAACI,KAAK,CAAC,CAAC,EAAEH,KAAK,CAAC;QAC3B,MAAMI,CAAC,GAAGL,CAAC,CAACI,KAAK,CAACH,KAAK,GAAG,CAAC,CAAC;QAC5B,IAAIjD,qBAAqB,CAACsD,QAAQ,CAACH,CAAC,CAAC,EAAE;UACrChB,MAAM,CAACgB,CAAC,CAAC,GAAGE,CAAC;QACf;MACF,CAAC,CAAC;MACF,OAAOlB,MAAM;IACf,CAAC,CAAC;EACN,CAAC;;EAED;AACF;AACA;AACA;EACEoB,mBAAmB,GAAG,MAAAA,CAAA,KAAY;IAChC,IAAI;MACF,MAAM,CAACC,aAAa,EAAEC,YAAY,EAAEC,kBAAkB,CAAC,GACrD,MAAM/B,OAAO,CAACgC,GAAG,CAAC,CAChB,IAAI,CAACvB,YAAY,CAAC,QAAQ,CAAC,EAC3B,IAAI,CAACA,YAAY,CAAC,OAAO,CAAC,EAC1B,IAAI,CAACV,mBAAmB,CAAC,CAAC,CAC3B,CAAC;MAEJ,MAAMkC,MAAM,GAAG;QAAEC,GAAG,EAAE,IAAI,CAACC,OAAO;QAAEC,YAAY,EAAE,IAAI,CAACpD;MAAY,CAAC;MAEpE,MAAMqD,WAAW,GAAG,IAAI,CAACzB,qBAAqB,CAACmB,kBAAkB,CAAC;MAElE,IAAI,IAAI,CAACO,SAAS,EAAE;QAClB;QACAC,OAAO,CAACC,GAAG,CACT,4CAA4C,EAC5CT,kBACF,CAAC;QACDQ,OAAO,CAACC,GAAG,CACT,iDAAiD,EACjDT,kBAAkB,CAACU,MACrB,CAAC;;QAED;QACAF,OAAO,CAACC,GAAG,CACT,mDAAmD,EACnDH,WAAW,CAACI,MACd,CAAC;QACDF,OAAO,CAACC,GAAG,CACT,8DAA8D,EAC9DE,IAAI,CAACC,SAAS,CAACN,WAAW,CAACZ,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAClD,CAAC;QAED,MAAMmB,OAAO,GAAG;UAAEvD,IAAI,EAAE,CAAC;UAAEwD,KAAK,EAAE,CAAC;UAAEC,GAAG,EAAE,CAAC;UAAE,SAAS,EAAE;QAAE,CAAC;QAC3DT,WAAW,CAACjB,OAAO,CAAC2B,CAAC,IAAI;UACvB,IAAI,CAACA,CAAC,CAAC1D,IAAI,EAAEuD,OAAO,CAACvD,IAAI,IAAI,CAAC;UAC9B,IAAI,CAAC0D,CAAC,CAACF,KAAK,EAAED,OAAO,CAACC,KAAK,IAAI,CAAC;UAChC,IAAI,CAACE,CAAC,CAACD,GAAG,EAAEF,OAAO,CAACE,GAAG,IAAI,CAAC;UAC5B,IAAI,CAACC,CAAC,CAAC,SAAS,CAAC,EAAEH,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC;QACFL,OAAO,CAACC,GAAG,CACT,4DAA4D,EAC5DI,OACF,CAAC;MACH;MAEA,MAAMI,OAAO,GAAG,CAAC,CAAC;MAClBX,WAAW,CAACjB,OAAO,CAAC6B,IAAI,IAAI;QAC1B,MAAM;UAAE5D,IAAI;UAAEwD,KAAK;UAAE,SAAS,EAAEK,MAAM;UAAEJ;QAAI,CAAC,GAAGG,IAAI;;QAEpD;QACA,MAAME,GAAG,GAAGT,IAAI,CAACC,SAAS,CAAC;UAAEtD,IAAI;UAAEwD,KAAK;UAAEC;QAAI,CAAC,CAAC;QAEhD,IAAI,CAACE,OAAO,CAACG,GAAG,CAAC,EAAE;UACjBH,OAAO,CAACG,GAAG,CAAC,GAAG;YACblB,MAAM,EAAE;cAAE5C,IAAI;cAAEwD,KAAK;cAAEC;YAAI,CAAC;YAC5BM,KAAK,EAAE,CAAC;YACRC,MAAM,EAAE;UACV,CAAC;QACH;QAEAL,OAAO,CAACG,GAAG,CAAC,CAACC,KAAK,IAAI,CAAC;QACvB,MAAME,GAAG,GAAG1E,QAAQ,CAACsE,MAAM,EAAE,EAAE,CAAC;QAChCF,OAAO,CAACG,GAAG,CAAC,CAACE,MAAM,IAAIE,MAAM,CAACC,QAAQ,CAACF,GAAG,CAAC,GAAGA,GAAG,GAAG,CAAC;MACvD,CAAC,CAAC;;MAEF;MACA,IAAI,CAAC,IAAI,CAAC3D,kBAAkB,EAAE,IAAI,CAACA,kBAAkB,GAAG,IAAIC,GAAG,CAAC,CAAC;MACjE,IAAI,CAAC,IAAI,CAACC,oBAAoB,EAAE,IAAI,CAACA,oBAAoB,GAAG,IAAID,GAAG,CAAC,CAAC;MAErE,MAAM6D,WAAW,GAAG,IAAI7D,GAAG,CAAC8D,MAAM,CAACC,IAAI,CAACX,OAAO,CAAC,CAAC;MACjD,KAAK,MAAMxB,CAAC,IAAIiC,WAAW,EAAE;QAC3B,IAAI,CAAC9D,kBAAkB,CAACiE,GAAG,CAACpC,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC3B,oBAAoB,CAACgE,GAAG,CAACrC,CAAC,CAAC,EAAE;UACpC,IAAI,CAAC3B,oBAAoB,CAACiE,MAAM,CAACtC,CAAC,CAAC;QACrC;MACF;;MAEA;MACA;MACA;MACA,IAAI,CAACrC,qBAAqB,CAAC4E,KAAK,CAAC,CAAC;MAClC,IAAI,CAACvE,2BAA2B,CAACuE,KAAK,CAAC,CAAC;;MAExC;MACA,KAAK,MAAMvC,CAAC,IAAI,IAAI,CAAC7B,kBAAkB,EAAE;QACvC,IAAI8D,WAAW,CAACI,GAAG,CAACrC,CAAC,CAAC,EAAE;QACxB,IAAI,IAAI,CAAC3B,oBAAoB,CAACgE,GAAG,CAACrC,CAAC,CAAC,EAAE;QACtC,IAAI;UACF,MAAMwC,WAAW,GAAGtB,IAAI,CAACuB,KAAK,CAACzC,CAAC,CAAC;UACjC,IAAI,CAACrC,qBAAqB,CAAC+E,GAAG,CAAC;YAAE,GAAGjC,MAAM;YAAE,GAAG+B;UAAY,CAAC,EAAE,CAAC,CAAC;UAChE,IAAI,CAACxE,2BAA2B,CAAC0E,GAAG,CAAC;YAAE,GAAGjC,MAAM;YAAE,GAAG+B;UAAY,CAAC,EAAE,CAAC,CAAC;UACtE,IAAI,CAACnE,oBAAoB,CAAC+D,GAAG,CAACpC,CAAC,CAAC;QAClC,CAAC,CAAC,MAAM;UACN;QAAA;MAEJ;MAEA,IAAI,IAAI,CAACc,SAAS,EAAE;QAClB,MAAM6B,MAAM,GAAGT,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC;QACrC,MAAMqB,YAAY,GAAGF,MAAM,CAACG,MAAM,CAAC,CAACC,GAAG,EAAEC,CAAC,KAAKD,GAAG,IAAIC,CAAC,CAACpB,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACvEb,OAAO,CAACC,GAAG,CACT,oDAAoD,EACpD2B,MAAM,CAAC1B,MACT,CAAC;QACDF,OAAO,CAACC,GAAG,CACT,kDAAkD,EAClD6B,YACF,CAAC;QACD9B,OAAO,CAACC,GAAG,CACT,8CAA8C,EAC9C,IAAI,CAAC7C,kBAAkB,CAAC8E,IAC1B,CAAC;QACDlC,OAAO,CAACC,GAAG,CACT,gDAAgD,EAChD,IAAI,CAAC3C,oBAAoB,CAAC4E,IAC5B,CAAC;QACDlC,OAAO,CAACC,GAAG,CACT,8DAA8D,EAC9DE,IAAI,CAACC,SAAS,CAACwB,MAAM,CAAC1C,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAC7C,CAAC;MACH;MAEAiC,MAAM,CAACU,MAAM,CAACpB,OAAO,CAAC,CAAC5B,OAAO,CAC5B,CAAC;QAAEa,MAAM,EAAE+B,WAAW;QAAEZ,KAAK;QAAEC;MAAO,CAAC,KAAK;QAC1C,IAAI,CAAClE,qBAAqB,CAAC+E,GAAG,CAAC;UAAE,GAAGjC,MAAM;UAAE,GAAG+B;QAAY,CAAC,EAAEZ,KAAK,CAAC;QACpE,IAAI,CAAC5D,2BAA2B,CAAC0E,GAAG,CAClC;UAAE,GAAGjC,MAAM;UAAE,GAAG+B;QAAY,CAAC,EAC7BX,MACF,CAAC;MACH,CACF,CAAC;MAED,MAAMqB,cAAc,GAAGC,OAAO,IAC5BjB,MAAM,CAACkB,WAAW,CAChBD,OAAO,CACJ7D,KAAK,CAAC,MAAM,CAAC,CACbC,MAAM,CAACC,IAAI,IAAIA,IAAI,IAAI,CAACA,IAAI,CAAC6D,UAAU,CAAC,GAAG,CAAC,CAAC,CAC7C3D,GAAG,CAACF,IAAI,IAAIA,IAAI,CAACF,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAC/BC,MAAM,CAACI,KAAK,IAAIA,KAAK,CAACsB,MAAM,KAAK,CAAC,IAAItB,KAAK,CAAC,CAAC,CAAC,IAAIA,KAAK,CAAC,CAAC,CAAC,CAC/D,CAAC;MAEH,MAAMkC,MAAM,GAAGqB,cAAc,CAAC7C,aAAa,CAAC;MAC5C,MAAMiD,KAAK,GAAGJ,cAAc,CAAC5C,YAAY,CAAC;MAE1C,IAAIuB,MAAM,CAAC0B,WAAW,EAAE;QACtB,IAAI,CAACtF,gBAAgB,CAACyE,GAAG,CACvB;UAAE,GAAGjC,MAAM;UAAE+C,WAAW,EAAE;QAAO,CAAC,EAClCpG,QAAQ,CAACyE,MAAM,CAAC0B,WAAW,EAAE,EAAE,CAAC,IAAI,CACtC,CAAC;MACH;MACA,IAAI1B,MAAM,CAAC4B,SAAS,EAAE;QACpB,IAAI,CAACxF,gBAAgB,CAACyE,GAAG,CACvB;UAAE,GAAGjC,MAAM;UAAE+C,WAAW,EAAE;QAAM,CAAC,EACjCpG,QAAQ,CAACyE,MAAM,CAAC4B,SAAS,EAAE,EAAE,CAAC,IAAI,CACpC,CAAC;MACH;MAEA,IAAIH,KAAK,CAACI,yBAAyB,EAAE;QACnC,IAAI,CAACxF,eAAe,CAACwE,GAAG,CACtB;UAAE,GAAGjC,MAAM;UAAEkD,SAAS,EAAE;QAAc,CAAC,EACvCvG,QAAQ,CAACkG,KAAK,CAACI,yBAAyB,EAAE,EAAE,CAAC,IAAI,CACnD,CAAC;MACH;IACF,CAAC,CAAC,OAAOE,KAAK,EAAE;MACd7C,OAAO,CAAC8C,IAAI,CACV,kDAAkD,EAClDD,KAAK,CAAC9E,OACR,CAAC;IACH;EACF,CAAC;;EAED;AACF;AACA;AACA;EACEgF,gBAAgB,GAAG,MAAAA,CAAA,KAAY;IAC7B,IAAI;MACF,MAAM,IAAI,CAAC1D,mBAAmB,CAAC,CAAC;MAChC,MAAM,IAAI,CAAC2D,WAAW,CAAC,CAAC;MACxB,IAAI,CAACC,gBAAgB,CAAC,CAAC;MAEvB,IAAI,IAAI,CAACC,gBAAgB,EAAE;QACzB,MAAMC,aAAa,GAAG,MAAM,IAAI,CAACC,QAAQ,CAACC,gBAAgB,CAAC,CAAC;QAC5DrD,OAAO,CAAC5B,IAAI,CACV,6CAA6C,EAC7C+B,IAAI,CAACC,SAAS,CAAC+C,aAAa,EAAE,IAAI,EAAE,CAAC,CACvC,CAAC;MACH;IACF,CAAC,CAAC,OAAON,KAAK,EAAE;MACd7C,OAAO,CAAC6C,KAAK,CACX,oDAAoDA,KAAK,CAAC9E,OAAO,EACnE,CAAC;MACD,MAAM8E,KAAK;IACb;EACF,CAAC;;EAED;AACF;AACA;AACA;EACES,SAAS,GAAGA,CAAClH,WAAW,GAAG,IAAI,CAACA,WAAW,KAAK;IAC9C,IAAI,CAACmH,UAAU,CAACnH,WAAW,EAAE,MAAM;MACjC,IAAI,CAAC2G,gBAAgB,CAAC,CAAC,CAACS,KAAK,CAAC3F,GAAG,IAAI;QACnCmC,OAAO,CAAC6C,KAAK,CAAC,+CAA+C,EAAEhF,GAAG,CAAC;MACrE,CAAC,CAAC;IACJ,CAAC,CAAC;EACJ,CAAC;;EAED;AACF;AACA;AACA;EACE4F,OAAO,GAAG,MAAAA,CAAA,KAAY;IACpB,IAAI;MACF,IAAI,CAAC,IAAI,CAACxH,WAAW,EAAE;MAEvB,IACE,IAAI,CAACS,eAAe,KAAKd,QAAQ,IACjC,IAAI,CAACc,eAAe,KAAKhB,QAAQ,EACjC;QACA,MAAM,IAAI,CAACO,WAAW,CAACyH,IAAI,CAAC,CAAC;MAC/B,CAAC,MAAM,IAAI,IAAI,CAAChH,eAAe,KAAKf,OAAO,EAAE;QAC3C,MAAM,IAAI,CAACM,WAAW,CAAC0H,UAAU,CAAC,CAAC;MACrC;IACF,CAAC,CAAC,OAAO9F,GAAG,EAAE;MACZmC,OAAO,CAAC6C,KAAK,CAAC,6CAA6C,EAAEhF,GAAG,CAAC;IACnE;IACAvB,OAAO,CAACsH,IAAI,CAAC,CAAC,CAAC;EACjB,CAAC;EAEDrG,mBAAmB,GAAGA,CAAA,KAAM;IAC1BjB,OAAO,CAACuH,EAAE,CAAC,QAAQ,EAAE,IAAI,CAACJ,OAAO,CAAC;IAClCnH,OAAO,CAACuH,EAAE,CAAC,SAAS,EAAE,IAAI,CAACJ,OAAO,CAAC;EACrC,CAAC;AACH;AAEAK,MAAM,CAACC,OAAO,GAAG;EAAEhI;AAAmB,CAAC","ignoreList":[]}
|