@worker-manager/metrics 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -6
- package/dist/HistoryAdmin.d.ts +23 -87
- package/dist/HistoryAdmin.js +21 -330
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +9 -9
- package/dist/HistoryStore.js +22 -0
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +27 -32
- package/dist/LatencySampler.js +55 -156
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +8 -3
- package/dist/LatencyStore.js +18 -0
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +34 -10
- package/dist/MetricsRecorder.js +30 -20
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisHistoryAdmin.d.ts +95 -0
- package/dist/RedisHistoryAdmin.js +328 -0
- package/dist/RedisHistoryAdmin.js.map +1 -0
- package/dist/RedisMetricsHistoryProvider.d.ts +7 -17
- package/dist/RedisMetricsHistoryProvider.js +9 -136
- package/dist/RedisMetricsHistoryProvider.js.map +1 -1
- package/dist/RedisMetricsStore.d.ts +27 -0
- package/dist/RedisMetricsStore.js +42 -0
- package/dist/RedisMetricsStore.js.map +1 -0
- package/dist/StoreHistoryProvider.d.ts +27 -0
- package/dist/StoreHistoryProvider.js +127 -0
- package/dist/StoreHistoryProvider.js.map +1 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/jobSources.d.ts +97 -0
- package/dist/jobSources.js +248 -0
- package/dist/jobSources.js.map +1 -0
- package/dist/keys.d.ts +7 -0
- package/dist/keys.js +23 -1
- package/dist/keys.js.map +1 -1
- package/dist/postgres/PostgresMetricsHistoryProvider.d.ts +37 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js +30 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js.map +1 -0
- package/dist/postgres/PostgresMetricsStore.d.ts +65 -0
- package/dist/postgres/PostgresMetricsStore.js +88 -0
- package/dist/postgres/PostgresMetricsStore.js.map +1 -0
- package/dist/postgres/admin.d.ts +31 -0
- package/dist/postgres/admin.js +178 -0
- package/dist/postgres/admin.js.map +1 -0
- package/dist/postgres/connection.d.ts +53 -0
- package/dist/postgres/connection.js +54 -0
- package/dist/postgres/connection.js.map +1 -0
- package/dist/postgres/context.d.ts +29 -0
- package/dist/postgres/context.js +76 -0
- package/dist/postgres/context.js.map +1 -0
- package/dist/postgres/schema.d.ts +26 -0
- package/dist/postgres/schema.js +148 -0
- package/dist/postgres/schema.js.map +1 -0
- package/dist/postgres/stores.d.ts +51 -0
- package/dist/postgres/stores.js +252 -0
- package/dist/postgres/stores.js.map +1 -0
- package/dist/store.d.ts +80 -0
- package/dist/store.js +3 -0
- package/dist/store.js.map +1 -0
- package/package.json +13 -3
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisHistoryAdmin = void 0;
|
|
4
|
+
exports.parseHistoryKey = parseHistoryKey;
|
|
5
|
+
const connection_1 = require("./connection");
|
|
6
|
+
const keys_1 = require("./keys");
|
|
7
|
+
const SCAN_COUNT = 500;
|
|
8
|
+
const BATCH = 256;
|
|
9
|
+
/** Keys per pipelined `stats()` batch. Each key costs two commands. */
|
|
10
|
+
const MEASURE_BATCH = 250;
|
|
11
|
+
const METRICS = ['completed', 'failed', 'runtime', 'waittime', 'queueage'];
|
|
12
|
+
/**
|
|
13
|
+
* Metrics whose global rollup is a plain summable counter, so one queue's share can be taken
|
|
14
|
+
* back out of it with HINCRBY. The latency metrics are deliberately absent: runtime and
|
|
15
|
+
* waittime pack a whole bucket vector into a single field and queueage holds a max gauge,
|
|
16
|
+
* and neither can be corrected by a scalar decrement. See `purge`.
|
|
17
|
+
*/
|
|
18
|
+
const SUMMABLE_METRICS = ['completed', 'failed'];
|
|
19
|
+
/**
|
|
20
|
+
* Parses the three key shapes from the right, because queue names may themselves contain
|
|
21
|
+
* colons:
|
|
22
|
+
*
|
|
23
|
+
* <ns>:<queue>:<metric>:<day> minute buckets
|
|
24
|
+
* <ns>:<queue>:<metric>:hour:<day> hourly rollup
|
|
25
|
+
* <ns>:<queue>:<metric>:totals daily totals
|
|
26
|
+
*
|
|
27
|
+
* The metric segment is checked against the known set, so a queue named `hour` or one
|
|
28
|
+
* ending in `:completed` still resolves correctly. Anything that doesn't fit returns null
|
|
29
|
+
* and is then reported but never deleted, so a stray key can't be destroyed by accident.
|
|
30
|
+
*/
|
|
31
|
+
function parseHistoryKey(key, namespace) {
|
|
32
|
+
const prefix = `${namespace}:`;
|
|
33
|
+
if (!key.startsWith(prefix)) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
const parts = key.slice(prefix.length).split(':');
|
|
37
|
+
if (parts.length < 3) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
const last = parts[parts.length - 1];
|
|
41
|
+
if (last === 'totals' && METRICS.includes(parts[parts.length - 2])) {
|
|
42
|
+
const queue = parts.slice(0, -2).join(':');
|
|
43
|
+
return queue ? { queue, metric: parts[parts.length - 2], tier: 'day', day: null } : null;
|
|
44
|
+
}
|
|
45
|
+
if (!keys_1.DAY_PATTERN.test(last)) {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
if (parts.length >= 4 && parts[parts.length - 2] === keys_1.HOUR_TIER) {
|
|
49
|
+
const metric = parts[parts.length - 3];
|
|
50
|
+
const queue = parts.slice(0, -3).join(':');
|
|
51
|
+
return queue && METRICS.includes(metric) ? { queue, metric, tier: 'hour', day: last } : null;
|
|
52
|
+
}
|
|
53
|
+
const metric = parts[parts.length - 2];
|
|
54
|
+
const queue = parts.slice(0, -2).join(':');
|
|
55
|
+
return queue && METRICS.includes(metric) ? { queue, metric, tier: 'minute', day: last } : null;
|
|
56
|
+
}
|
|
57
|
+
function emptyTiers() {
|
|
58
|
+
return {
|
|
59
|
+
minute: { keys: 0, bytes: 0 },
|
|
60
|
+
hour: { keys: 0, bytes: 0 },
|
|
61
|
+
day: { keys: 0, bytes: 0 },
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** The global rollup key mirroring a per-queue key, same tier and same day. */
|
|
65
|
+
function globalKeyFor(keys, parsed) {
|
|
66
|
+
if (parsed.day === null) {
|
|
67
|
+
return keys.totals(keys_1.GLOBAL_QUEUE, parsed.metric);
|
|
68
|
+
}
|
|
69
|
+
return parsed.tier === 'hour'
|
|
70
|
+
? keys.hour(keys_1.GLOBAL_QUEUE, parsed.metric, parsed.day)
|
|
71
|
+
: keys.day(keys_1.GLOBAL_QUEUE, parsed.metric, parsed.day);
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Inspection and cleanup for the Redis keys written by `MetricsRecorder`.
|
|
75
|
+
*
|
|
76
|
+
* Every operation is confined to the recorder's namespace and driven by SCAN, so it never
|
|
77
|
+
* blocks Redis and never touches BullMQ's own keys. Deletes use UNLINK.
|
|
78
|
+
*/
|
|
79
|
+
class RedisHistoryAdmin {
|
|
80
|
+
constructor(opts) {
|
|
81
|
+
this.redis = opts.redis;
|
|
82
|
+
this.keys = opts.keys;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Per-queue footprint of the stored history.
|
|
86
|
+
*
|
|
87
|
+
* Every key has to be measured individually, since only `MEMORY USAGE` knows what a hash
|
|
88
|
+
* really costs. Issuing those one at a time would mean a round trip per key, which at a
|
|
89
|
+
* 90-day retention across a dozen queues runs into the thousands, so the measurements go
|
|
90
|
+
* out in pipelined batches instead. Still an ops-scale call rather than a hot path: it
|
|
91
|
+
* reads the whole namespace, so it belongs behind a debug endpoint, not a poll.
|
|
92
|
+
*/
|
|
93
|
+
async stats() {
|
|
94
|
+
var _a;
|
|
95
|
+
const byQueue = new Map();
|
|
96
|
+
const tiers = emptyTiers();
|
|
97
|
+
let keys = 0;
|
|
98
|
+
let bytes = 0;
|
|
99
|
+
let minutes = 0;
|
|
100
|
+
let oldestDay = null;
|
|
101
|
+
let newestDay = null;
|
|
102
|
+
const found = [];
|
|
103
|
+
for await (const key of this.scan()) {
|
|
104
|
+
const parsed = parseHistoryKey(key, this.keys.namespace);
|
|
105
|
+
if (parsed) {
|
|
106
|
+
found.push({ key, parsed });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
const measurements = await this.measure(found.map((item) => item.key));
|
|
110
|
+
for (const [index, { parsed }] of found.entries()) {
|
|
111
|
+
const { size, len } = measurements[index];
|
|
112
|
+
const entry = (_a = byQueue.get(parsed.queue)) !== null && _a !== void 0 ? _a : {
|
|
113
|
+
queue: parsed.queue,
|
|
114
|
+
keys: 0,
|
|
115
|
+
bytes: 0,
|
|
116
|
+
minutes: 0,
|
|
117
|
+
days: [],
|
|
118
|
+
tiers: emptyTiers(),
|
|
119
|
+
};
|
|
120
|
+
entry.keys += 1;
|
|
121
|
+
entry.bytes += size;
|
|
122
|
+
entry.tiers[parsed.tier].keys += 1;
|
|
123
|
+
entry.tiers[parsed.tier].bytes += size;
|
|
124
|
+
keys += 1;
|
|
125
|
+
bytes += size;
|
|
126
|
+
tiers[parsed.tier].keys += 1;
|
|
127
|
+
tiers[parsed.tier].bytes += size;
|
|
128
|
+
if (parsed.tier === 'minute') {
|
|
129
|
+
entry.minutes += len;
|
|
130
|
+
minutes += len;
|
|
131
|
+
}
|
|
132
|
+
if (parsed.day) {
|
|
133
|
+
entry.days.push(parsed.day);
|
|
134
|
+
if (oldestDay === null || parsed.day < oldestDay) {
|
|
135
|
+
oldestDay = parsed.day;
|
|
136
|
+
}
|
|
137
|
+
if (newestDay === null || parsed.day > newestDay) {
|
|
138
|
+
newestDay = parsed.day;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
byQueue.set(parsed.queue, entry);
|
|
142
|
+
}
|
|
143
|
+
const queues = [...byQueue.values()].sort((a, b) => b.bytes - a.bytes);
|
|
144
|
+
for (const queue of queues) {
|
|
145
|
+
queue.days = [...new Set(queue.days)].sort();
|
|
146
|
+
}
|
|
147
|
+
return { keys, bytes, minutes, oldestDay, newestDay, tiers, queues };
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Size and entry count for each key, in pipelined batches so the cost is a handful of
|
|
151
|
+
* round trips rather than one per key. A key that expires between the scan and the
|
|
152
|
+
* measurement simply reads as zero rather than failing the whole call.
|
|
153
|
+
*/
|
|
154
|
+
async measure(keys) {
|
|
155
|
+
var _a, _b, _c, _d;
|
|
156
|
+
const out = [];
|
|
157
|
+
for (let i = 0; i < keys.length; i += MEASURE_BATCH) {
|
|
158
|
+
const chunk = keys.slice(i, i + MEASURE_BATCH);
|
|
159
|
+
const pipeline = this.redis.pipeline();
|
|
160
|
+
for (const key of chunk) {
|
|
161
|
+
pipeline.memory('USAGE', key);
|
|
162
|
+
pipeline.hlen(key);
|
|
163
|
+
}
|
|
164
|
+
const res = await pipeline.exec();
|
|
165
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
166
|
+
out.push({
|
|
167
|
+
size: Number((_b = (_a = res === null || res === void 0 ? void 0 : res[j * 2]) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : 0) || 0,
|
|
168
|
+
len: Number((_d = (_c = res === null || res === void 0 ? void 0 : res[j * 2 + 1]) === null || _c === void 0 ? void 0 : _c[1]) !== null && _d !== void 0 ? _d : 0) || 0,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return out;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Deletes recorded history. Purging a single queue also subtracts that queue's minutes
|
|
176
|
+
* from the global rollup, so the cross-queue chart stays correct instead of keeping the
|
|
177
|
+
* removed queue's throughput folded into it forever.
|
|
178
|
+
*
|
|
179
|
+
* That correction covers the counter metrics only. The global runtime, waittime and
|
|
180
|
+
* queueage rollups keep the purged queue's contribution until their own retention drops
|
|
181
|
+
* it: a packed bucket vector cannot be decremented field by field, and a max gauge has no
|
|
182
|
+
* record of which queue produced the maximum, so there is nothing to subtract. The
|
|
183
|
+
* per-queue keys are still deleted either way. See SUMMABLE_METRICS.
|
|
184
|
+
*/
|
|
185
|
+
async purge(opts = {}) {
|
|
186
|
+
const before = opts.before === undefined ? null : (0, keys_1.toDay)(opts.before);
|
|
187
|
+
const result = { keysDeleted: 0, fieldsDeleted: 0 };
|
|
188
|
+
const dayKeys = [];
|
|
189
|
+
const totalsKeys = [];
|
|
190
|
+
for await (const key of this.scan()) {
|
|
191
|
+
const parsed = parseHistoryKey(key, this.keys.namespace);
|
|
192
|
+
if (!parsed) {
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (opts.queue !== undefined && parsed.queue !== opts.queue) {
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
if (parsed.day === null) {
|
|
199
|
+
totalsKeys.push({ key, parsed });
|
|
200
|
+
}
|
|
201
|
+
else if (before === null || parsed.day < before) {
|
|
202
|
+
dayKeys.push({ key, parsed });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
// Rewriting the global rollup only makes sense when a single queue is being removed:
|
|
206
|
+
// a full purge drops the global keys outright, along with everything else.
|
|
207
|
+
const adjustGlobal = opts.queue !== undefined && opts.queue !== keys_1.GLOBAL_QUEUE;
|
|
208
|
+
for (const { key, parsed } of dayKeys) {
|
|
209
|
+
if (adjustGlobal) {
|
|
210
|
+
result.keysDeleted += await this.subtractDayFromGlobal(key, parsed);
|
|
211
|
+
}
|
|
212
|
+
result.keysDeleted += await this.redis.unlink(key);
|
|
213
|
+
}
|
|
214
|
+
for (const { key, parsed } of totalsKeys) {
|
|
215
|
+
const stale = (await this.redis.hkeys(key)).filter((day) => before === null || day < before);
|
|
216
|
+
if (adjustGlobal && stale.length > 0) {
|
|
217
|
+
const totals = await this.redis.hmget(key, ...stale);
|
|
218
|
+
result.fieldsDeleted += await this.subtractTotalsFromGlobal(parsed.metric, stale, totals);
|
|
219
|
+
}
|
|
220
|
+
if (before === null) {
|
|
221
|
+
result.keysDeleted += await this.redis.unlink(key);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
for (let i = 0; i < stale.length; i += BATCH) {
|
|
225
|
+
result.fieldsDeleted += await this.redis.hdel(key, ...stale.slice(i, i + BATCH));
|
|
226
|
+
}
|
|
227
|
+
if ((await this.redis.hlen(key)) === 0) {
|
|
228
|
+
result.keysDeleted += await this.redis.unlink(key);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return result;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Removes one queue's buckets from the matching global key, so the cross-queue series
|
|
235
|
+
* reflects the queues that are left rather than keeping the removed queue folded in.
|
|
236
|
+
* Each tier is corrected from its own source key, because the tiers have independent
|
|
237
|
+
* retention and the minute hash may already be gone while the hourly one survives.
|
|
238
|
+
* Fields that drain to zero are dropped: the recorder never writes a zero bucket, so a
|
|
239
|
+
* leftover zero would read as recorded-but-idle instead of not recorded.
|
|
240
|
+
* Returns the number of global keys it deleted.
|
|
241
|
+
*
|
|
242
|
+
* Only the summable metrics are touched; the latency ones are skipped outright rather than
|
|
243
|
+
* silently producing a no-op subtraction of their packed values. See SUMMABLE_METRICS.
|
|
244
|
+
*/
|
|
245
|
+
async subtractDayFromGlobal(key, parsed) {
|
|
246
|
+
if (parsed.day === null || !SUMMABLE_METRICS.includes(parsed.metric)) {
|
|
247
|
+
return 0;
|
|
248
|
+
}
|
|
249
|
+
const minutes = await this.redis.hgetall(key);
|
|
250
|
+
const globalDay = globalKeyFor(this.keys, parsed);
|
|
251
|
+
const pipeline = this.redis.multi();
|
|
252
|
+
const touched = [];
|
|
253
|
+
for (const field of Object.keys(minutes)) {
|
|
254
|
+
const value = Number(minutes[field]) || 0;
|
|
255
|
+
if (value === 0) {
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
touched.push(field);
|
|
259
|
+
pipeline.hincrby(globalDay, field, -value);
|
|
260
|
+
}
|
|
261
|
+
if (touched.length === 0) {
|
|
262
|
+
return 0;
|
|
263
|
+
}
|
|
264
|
+
const res = await pipeline.exec();
|
|
265
|
+
const drained = touched.filter((_, i) => { var _a, _b; return Number((_b = (_a = res === null || res === void 0 ? void 0 : res[i]) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : 0) <= 0; });
|
|
266
|
+
for (let i = 0; i < drained.length; i += BATCH) {
|
|
267
|
+
await this.redis.hdel(globalDay, ...drained.slice(i, i + BATCH));
|
|
268
|
+
}
|
|
269
|
+
return (await this.redis.hlen(globalDay)) === 0 ? await this.redis.unlink(globalDay) : 0;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Same idea for the daily rollup: the global totals hash is the sum of the per-queue
|
|
273
|
+
* totals hashes, so it is corrected from those rather than re-derived from day hashes,
|
|
274
|
+
* which may already have expired. Returns the number of global fields it removed.
|
|
275
|
+
*
|
|
276
|
+
* Skips the latency metrics for the same reason as subtractDayFromGlobal.
|
|
277
|
+
*/
|
|
278
|
+
async subtractTotalsFromGlobal(metric, days, values) {
|
|
279
|
+
if (!SUMMABLE_METRICS.includes(metric)) {
|
|
280
|
+
return 0;
|
|
281
|
+
}
|
|
282
|
+
const globalTotals = this.keys.totals(keys_1.GLOBAL_QUEUE, metric);
|
|
283
|
+
const pipeline = this.redis.multi();
|
|
284
|
+
const touched = [];
|
|
285
|
+
days.forEach((day, i) => {
|
|
286
|
+
const value = Number(values[i]) || 0;
|
|
287
|
+
if (value === 0) {
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
touched.push(day);
|
|
291
|
+
pipeline.hincrby(globalTotals, day, -value);
|
|
292
|
+
});
|
|
293
|
+
if (touched.length === 0) {
|
|
294
|
+
return 0;
|
|
295
|
+
}
|
|
296
|
+
const res = await pipeline.exec();
|
|
297
|
+
const drained = touched.filter((_, i) => { var _a, _b; return Number((_b = (_a = res === null || res === void 0 ? void 0 : res[i]) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : 0) <= 0; });
|
|
298
|
+
let removed = 0;
|
|
299
|
+
for (let i = 0; i < drained.length; i += BATCH) {
|
|
300
|
+
removed += await this.redis.hdel(globalTotals, ...drained.slice(i, i + BATCH));
|
|
301
|
+
}
|
|
302
|
+
return removed;
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* SCAN over the namespace, once per master: SCAN carries no key, so a cluster client has
|
|
306
|
+
* no slot to route by and would answer from one arbitrary node. SCAN may also hand back the
|
|
307
|
+
* same key on more than one cursor iteration, which would double-count in `stats()`, so
|
|
308
|
+
* emissions are de-duped here. The set is bounded by queues x metrics x retention days.
|
|
309
|
+
*/
|
|
310
|
+
async *scan() {
|
|
311
|
+
const seen = new Set();
|
|
312
|
+
for (const target of (0, connection_1.scanTargets)(this.redis)) {
|
|
313
|
+
let cursor = '0';
|
|
314
|
+
do {
|
|
315
|
+
const [next, batch] = await target.scan(cursor, 'MATCH', this.keys.scanPattern, 'COUNT', SCAN_COUNT);
|
|
316
|
+
cursor = next;
|
|
317
|
+
for (const key of batch) {
|
|
318
|
+
if (!seen.has(key)) {
|
|
319
|
+
seen.add(key);
|
|
320
|
+
yield key;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
} while (cursor !== '0');
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
exports.RedisHistoryAdmin = RedisHistoryAdmin;
|
|
328
|
+
//# sourceMappingURL=RedisHistoryAdmin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RedisHistoryAdmin.js","sourceRoot":"","sources":["../src/RedisHistoryAdmin.ts"],"names":[],"mappings":";;;AA6CA,0CA0BC;AAvED,6CAA+D;AAS/D,iCAAuF;AAGvF,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,KAAK,GAAG,GAAG,CAAC;AAClB,uEAAuE;AACvE,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,OAAO,GAAG,CAAC,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAC3E;;;;;GAKG;AACH,MAAM,gBAAgB,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AAUjD;;;;;;;;;;;GAWG;AACH,SAAgB,eAAe,CAAC,GAAW,EAAE,SAAiB;IAC5D,MAAM,MAAM,GAAG,GAAG,SAAS,GAAG,CAAC;IAC/B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAErC,IAAI,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3C,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3F,CAAC;IACD,IAAI,CAAC,kBAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,gBAAS,EAAE,CAAC;QAC/D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3C,OAAO,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACjG,CAAC;AAED,SAAS,UAAU;IACjB,OAAO;QACL,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC7B,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC3B,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KAC3B,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,SAAS,YAAY,CAAC,IAAiB,EAAE,MAAiB;IACxD,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,KAAK,MAAM;QAC3B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,mBAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;;;GAKG;AACH,MAAa,iBAAiB;IAI5B,YAAY,IAAiD;QAC3D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACxB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,KAAK;;QACT,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;QACrD,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,SAAS,GAAkB,IAAI,CAAC;QAEpC,MAAM,KAAK,GAAyC,EAAE,CAAC;QACvD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,CAAC;gBACX,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAEvE,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAE1C,MAAM,KAAK,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,EAAE;gBACR,KAAK,EAAE,UAAU,EAAE;aACpB,CAAC;YACF,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YAChB,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC;YACpB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YACnC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YACvC,IAAI,IAAI,CAAC,CAAC;YACV,KAAK,IAAI,IAAI,CAAC;YACd,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YAC7B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YAEjC,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC;gBACrB,OAAO,IAAI,GAAG,CAAC;YACjB,CAAC;YACD,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,SAAS,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;oBACjD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;gBACzB,CAAC;gBACD,IAAI,SAAS,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;oBACjD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;gBACzB,CAAC;YACH,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,OAAO,CAAC,IAAc;;QAClC,MAAM,GAAG,GAAoC,EAAE,CAAC;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,aAAa,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC9B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,GAAG,CAAC,IAAI,CAAC;oBACP,IAAI,EAAE,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,GAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC;oBACzC,GAAG,EAAE,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC;iBAC7C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,OAAqB,EAAE;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,YAAK,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAgB,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QACjE,MAAM,OAAO,GAAyC,EAAE,CAAC;QACzD,MAAM,UAAU,GAAyC,EAAE,CAAC;QAE5D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC5D,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;gBACxB,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACnC,CAAC;iBAAM,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC;gBAClD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,qFAAqF;QACrF,2EAA2E;QAC3E,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,mBAAY,CAAC;QAE7E,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC;YAC7F,IAAI,YAAY,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACrD,MAAM,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;YAC5F,CAAC;YACD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACnD,SAAS;YACX,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;gBAC7C,MAAM,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;;;;OAWG;IACK,KAAK,CAAC,qBAAqB,CAAC,GAAW,EAAE,MAAiB;QAChE,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACrE,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,eAAC,OAAA,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,wBAAwB,CACpC,MAAc,EACd,IAAc,EACd,MAAyB;QAEzB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,MAAM,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO;YACT,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,eAAC,OAAA,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QAC1E,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/C,OAAO,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,CAAC,IAAI;QACjB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,MAAM,IAAI,IAAA,wBAAW,EAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,IAAI,MAAM,GAAG,GAAG,CAAC;YACjB,GAAG,CAAC;gBACF,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,MAAM,MAAM,CAAC,IAAI,CACrC,MAAM,EACN,OAAO,EACP,IAAI,CAAC,IAAI,CAAC,WAAW,EACrB,OAAO,EACP,UAAU,CACX,CAAC;gBACF,MAAM,GAAG,IAAI,CAAC;gBACd,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACd,MAAM,GAAG,CAAC;oBACZ,CAAC;gBACH,CAAC;YACH,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;QAC3B,CAAC;IACH,CAAC;CACF;AAjRD,8CAiRC"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { type Retention } from './HistoryStore';
|
|
1
|
+
import type { MetricsConnection } from './connection';
|
|
2
|
+
import type { Retention } from './store';
|
|
3
|
+
import { StoreHistoryProvider } from './StoreHistoryProvider';
|
|
5
4
|
export interface RedisMetricsHistoryProviderOptions {
|
|
6
5
|
connection: MetricsConnection;
|
|
7
6
|
/** Must match the recorder's. See `MetricsRecorderOptions.prefix`. */
|
|
@@ -10,19 +9,10 @@ export interface RedisMetricsHistoryProviderOptions {
|
|
|
10
9
|
retention?: Partial<Retention>;
|
|
11
10
|
retentionDays?: number;
|
|
12
11
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
private readonly
|
|
16
|
-
private readonly admin;
|
|
17
|
-
private readonly redis;
|
|
18
|
-
private readonly ownsRedis;
|
|
19
|
-
private readonly retentionDays;
|
|
12
|
+
/** Serves the board's history charts and storage panel from the Redis store. */
|
|
13
|
+
export declare class RedisMetricsHistoryProvider extends StoreHistoryProvider {
|
|
14
|
+
private readonly ownedStore;
|
|
20
15
|
constructor(opts: RedisMetricsHistoryProviderOptions);
|
|
16
|
+
/** Closes the Redis connection this provider opened itself. A client handed in is left open. */
|
|
21
17
|
disconnect(): void;
|
|
22
|
-
/** Backs the board's storage panel. See MetricsHistoryAdmin.stats. */
|
|
23
|
-
getUsage(): Promise<HistoryStats>;
|
|
24
|
-
/** Backs the board's "clear history" action. See MetricsHistoryAdmin.purge. */
|
|
25
|
-
purge(options?: PurgeOptions): Promise<PurgeResult>;
|
|
26
|
-
getHistory(query: MetricsHistoryQuery): Promise<MetricsHistoryPoint[]>;
|
|
27
|
-
getLatency(query: MetricsLatencyQuery): Promise<MetricsLatencyPoint[]>;
|
|
28
18
|
}
|
|
@@ -1,147 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RedisMetricsHistoryProvider = void 0;
|
|
4
|
-
const connection_1 = require("./connection");
|
|
5
|
-
const histogram_1 = require("./histogram");
|
|
6
|
-
const HistoryAdmin_1 = require("./HistoryAdmin");
|
|
7
|
-
const HistoryStore_1 = require("./HistoryStore");
|
|
8
|
-
const keys_1 = require("./keys");
|
|
9
|
-
const LatencyStore_1 = require("./LatencyStore");
|
|
10
4
|
const MetricsRecorder_1 = require("./MetricsRecorder");
|
|
11
|
-
const
|
|
12
|
-
|
|
5
|
+
const RedisMetricsStore_1 = require("./RedisMetricsStore");
|
|
6
|
+
const StoreHistoryProvider_1 = require("./StoreHistoryProvider");
|
|
7
|
+
/** Serves the board's history charts and storage panel from the Redis store. */
|
|
8
|
+
class RedisMetricsHistoryProvider extends StoreHistoryProvider_1.StoreHistoryProvider {
|
|
13
9
|
constructor(opts) {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
this.
|
|
17
|
-
const keys = (0, keys_1.metricsKeys)((0, keys_1.resolveNamespace)(opts.prefix, (0, connection_1.isCluster)(client)));
|
|
18
|
-
const retention = (0, MetricsRecorder_1.resolveRetention)(opts);
|
|
19
|
-
this.retentionDays = retention.days;
|
|
20
|
-
this.store = new HistoryStore_1.HistoryStore({ redis: this.redis, keys, retention });
|
|
21
|
-
this.latencyStore = new LatencyStore_1.LatencyStore({ redis: this.redis, keys, retention });
|
|
22
|
-
this.admin = new HistoryAdmin_1.MetricsHistoryAdmin({ connection: this.redis, prefix: opts.prefix });
|
|
10
|
+
const store = new RedisMetricsStore_1.RedisMetricsStore({ connection: opts.connection, prefix: opts.prefix });
|
|
11
|
+
super(store, (0, MetricsRecorder_1.resolveRetention)(opts));
|
|
12
|
+
this.ownedStore = store;
|
|
23
13
|
}
|
|
14
|
+
/** Closes the Redis connection this provider opened itself. A client handed in is left open. */
|
|
24
15
|
disconnect() {
|
|
25
|
-
|
|
26
|
-
this.redis.disconnect();
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
/** Backs the board's storage panel. See MetricsHistoryAdmin.stats. */
|
|
30
|
-
async getUsage() {
|
|
31
|
-
return this.admin.stats();
|
|
32
|
-
}
|
|
33
|
-
/** Backs the board's "clear history" action. See MetricsHistoryAdmin.purge. */
|
|
34
|
-
async purge(options = {}) {
|
|
35
|
-
return this.admin.purge(options);
|
|
36
|
-
}
|
|
37
|
-
async getHistory(query) {
|
|
38
|
-
var _a, _b;
|
|
39
|
-
const queue = (_a = query.queue) !== null && _a !== void 0 ? _a : keys_1.GLOBAL_QUEUE;
|
|
40
|
-
// Clamp the span to the retention window so an unbounded `from` (e.g. 0) can't make
|
|
41
|
-
// dayRange produce an unbounded number of day buckets -- older data doesn't exist anyway.
|
|
42
|
-
const maxSpanMs = (this.retentionDays + 1) * 86400000;
|
|
43
|
-
const from = Math.max(query.from, query.to - maxSpanMs);
|
|
44
|
-
const days = (0, keys_1.dayRange)(from, query.to);
|
|
45
|
-
if (query.metric === 'queueage') {
|
|
46
|
-
const ages = await this.latencyStore.readQueueAge(queue, query.granularity, days);
|
|
47
|
-
// Day points are stamped at the day's start, so an intraday `from` would drop the day
|
|
48
|
-
// it falls in. Floored, exactly as the counter path below does it.
|
|
49
|
-
const lowerBound = query.granularity === 'day' ? dayFloor(query.from) : query.from;
|
|
50
|
-
return Object.keys(ages)
|
|
51
|
-
.map((key) => ({
|
|
52
|
-
ts: query.granularity === 'day' ? (0, keys_1.dayToStartMs)(key) : Number(key) * MS_PER_HOUR,
|
|
53
|
-
value: ages[key],
|
|
54
|
-
}))
|
|
55
|
-
.filter((p) => p.ts >= lowerBound && p.ts <= query.to)
|
|
56
|
-
.sort((a, b) => a.ts - b.ts);
|
|
57
|
-
}
|
|
58
|
-
if (query.granularity === 'day') {
|
|
59
|
-
const rawTotals = await this.store.readDailyTotalsRaw(queue, query.metric, days);
|
|
60
|
-
// Empty history only when no day in range was ever recorded (all fields missing).
|
|
61
|
-
// A day with a stored '0' still counts as recorded -- otherwise the UI's empty
|
|
62
|
-
// state would be unreachable once any data exists.
|
|
63
|
-
if (rawTotals.every((value) => value == null)) {
|
|
64
|
-
return [];
|
|
65
|
-
}
|
|
66
|
-
const totals = {};
|
|
67
|
-
days.forEach((day, i) => {
|
|
68
|
-
totals[day] = Number(rawTotals[i]) || 0;
|
|
69
|
-
});
|
|
70
|
-
return days
|
|
71
|
-
.map((day) => { var _a; return ({ ts: (0, keys_1.dayToStartMs)(day), value: (_a = totals[day]) !== null && _a !== void 0 ? _a : 0 }); })
|
|
72
|
-
.filter((p) => p.ts >= dayFloor(query.from) && p.ts <= query.to);
|
|
73
|
-
}
|
|
74
|
-
const hourBuckets = new Map();
|
|
75
|
-
const dayHours = await Promise.all(days.map((day) => this.store.readDayHours(queue, query.metric, day)));
|
|
76
|
-
for (const hours of dayHours) {
|
|
77
|
-
for (const field of Object.keys(hours)) {
|
|
78
|
-
const ts = Number(field) * MS_PER_HOUR;
|
|
79
|
-
if (ts < query.from || ts > query.to) {
|
|
80
|
-
continue;
|
|
81
|
-
}
|
|
82
|
-
hourBuckets.set(ts, ((_b = hourBuckets.get(ts)) !== null && _b !== void 0 ? _b : 0) + hours[field]);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
return [...hourBuckets.entries()]
|
|
86
|
-
.map(([ts, value]) => ({ ts, value }))
|
|
87
|
-
.sort((a, b) => a.ts - b.ts);
|
|
88
|
-
}
|
|
89
|
-
async getLatency(query) {
|
|
90
|
-
var _a;
|
|
91
|
-
const queue = (_a = query.queue) !== null && _a !== void 0 ? _a : keys_1.GLOBAL_QUEUE;
|
|
92
|
-
const maxSpanMs = (this.retentionDays + 1) * 86400000;
|
|
93
|
-
const from = Math.max(query.from, query.to - maxSpanMs);
|
|
94
|
-
const days = (0, keys_1.dayRange)(from, query.to);
|
|
95
|
-
if (query.granularity === 'range') {
|
|
96
|
-
// Percentiles don't merge: averaging per-day p95s isn't the same number as the p95 of
|
|
97
|
-
// the whole range. Read the day tier's bucket vectors and merge them, then compute each
|
|
98
|
-
// requested percentile once from the summed vector.
|
|
99
|
-
const raw = await this.latencyStore.readRange(queue, query.metric, 'day', days);
|
|
100
|
-
let merged = (0, histogram_1.emptyVector)();
|
|
101
|
-
for (const day of days) {
|
|
102
|
-
const vector = raw[day];
|
|
103
|
-
if (vector) {
|
|
104
|
-
merged = (0, histogram_1.mergeVectors)(merged, vector);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
const count = (0, histogram_1.vectorTotal)(merged);
|
|
108
|
-
if (count === 0) {
|
|
109
|
-
return [];
|
|
110
|
-
}
|
|
111
|
-
const values = {};
|
|
112
|
-
for (const p of query.percentiles) {
|
|
113
|
-
values[String(p)] = (0, histogram_1.quantile)(merged, p);
|
|
114
|
-
}
|
|
115
|
-
return [{ ts: query.from, count: Math.round(count), values }];
|
|
116
|
-
}
|
|
117
|
-
const raw = await this.latencyStore.readRange(queue, query.metric, query.granularity, days);
|
|
118
|
-
// Same day-start alignment as getHistory: comparing a day bucket against a raw `from`
|
|
119
|
-
// would drop the oldest day and leave this chart one bucket shorter than the throughput
|
|
120
|
-
// chart drawn for the same range.
|
|
121
|
-
const lowerBound = query.granularity === 'day' ? dayFloor(query.from) : query.from;
|
|
122
|
-
const points = [];
|
|
123
|
-
for (const key of Object.keys(raw)) {
|
|
124
|
-
const ts = query.granularity === 'day' ? (0, keys_1.dayToStartMs)(key) : Number(key) * MS_PER_HOUR;
|
|
125
|
-
if (ts < lowerBound || ts > query.to) {
|
|
126
|
-
continue;
|
|
127
|
-
}
|
|
128
|
-
const vector = raw[key];
|
|
129
|
-
const count = (0, histogram_1.vectorTotal)(vector);
|
|
130
|
-
if (count === 0) {
|
|
131
|
-
continue;
|
|
132
|
-
}
|
|
133
|
-
const values = {};
|
|
134
|
-
for (const p of query.percentiles) {
|
|
135
|
-
values[String(p)] = (0, histogram_1.quantile)(vector, p);
|
|
136
|
-
}
|
|
137
|
-
points.push({ ts, count: Math.round(count), values });
|
|
138
|
-
}
|
|
139
|
-
return points.sort((a, b) => a.ts - b.ts);
|
|
16
|
+
void this.ownedStore.close();
|
|
140
17
|
}
|
|
141
18
|
}
|
|
142
19
|
exports.RedisMetricsHistoryProvider = RedisMetricsHistoryProvider;
|
|
143
|
-
function dayFloor(ms) {
|
|
144
|
-
const d = new Date(ms);
|
|
145
|
-
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate());
|
|
146
|
-
}
|
|
147
20
|
//# sourceMappingURL=RedisMetricsHistoryProvider.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RedisMetricsHistoryProvider.js","sourceRoot":"","sources":["../src/RedisMetricsHistoryProvider.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"RedisMetricsHistoryProvider.js","sourceRoot":"","sources":["../src/RedisMetricsHistoryProvider.ts"],"names":[],"mappings":";;;AACA,uDAAqD;AACrD,2DAAwD;AAExD,iEAA8D;AAW9D,gFAAgF;AAChF,MAAa,2BAA4B,SAAQ,2CAAoB;IAGnE,YAAY,IAAwC;QAClD,MAAM,KAAK,GAAG,IAAI,qCAAiB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,KAAK,CAAC,KAAK,EAAE,IAAA,kCAAgB,EAAC,IAAI,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,gGAAgG;IAChG,UAAU;QACR,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF;AAbD,kEAaC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type MetricsClient, type MetricsConnection } from './connection';
|
|
2
|
+
import { type MetricsKeys } from './keys';
|
|
3
|
+
import type { CounterStore, HistoryAdministration, LatencyStorage, MetricsStore, Retention } from './store';
|
|
4
|
+
export interface RedisMetricsStoreOptions {
|
|
5
|
+
/** An ioredis client (or Cluster), or options to open one. */
|
|
6
|
+
connection: MetricsConnection;
|
|
7
|
+
/** Key namespace. See `MetricsRecorderOptions.prefix`. */
|
|
8
|
+
prefix?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* History in Redis: hashes under one key namespace, expired by TTL. The default store,
|
|
12
|
+
* which `connection` on the recorder, provider and admin is shorthand for.
|
|
13
|
+
*/
|
|
14
|
+
export declare class RedisMetricsStore implements MetricsStore {
|
|
15
|
+
readonly redis: MetricsClient;
|
|
16
|
+
readonly keys: MetricsKeys;
|
|
17
|
+
/** Whether `redis` was opened here from options, and so is closed by `close()`. */
|
|
18
|
+
readonly ownsClient: boolean;
|
|
19
|
+
private closed;
|
|
20
|
+
constructor(opts: RedisMetricsStoreOptions);
|
|
21
|
+
get jobClient(): MetricsClient;
|
|
22
|
+
counterStore(retention: Retention): CounterStore;
|
|
23
|
+
latencyStore(retention: Retention): LatencyStorage;
|
|
24
|
+
administration(): HistoryAdministration;
|
|
25
|
+
/** Disconnects a client this store opened; synchronous in effect, like `redis.disconnect()`. */
|
|
26
|
+
close(): Promise<void>;
|
|
27
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisMetricsStore = void 0;
|
|
4
|
+
const connection_1 = require("./connection");
|
|
5
|
+
const HistoryStore_1 = require("./HistoryStore");
|
|
6
|
+
const keys_1 = require("./keys");
|
|
7
|
+
const LatencyStore_1 = require("./LatencyStore");
|
|
8
|
+
const RedisHistoryAdmin_1 = require("./RedisHistoryAdmin");
|
|
9
|
+
/**
|
|
10
|
+
* History in Redis: hashes under one key namespace, expired by TTL. The default store,
|
|
11
|
+
* which `connection` on the recorder, provider and admin is shorthand for.
|
|
12
|
+
*/
|
|
13
|
+
class RedisMetricsStore {
|
|
14
|
+
constructor(opts) {
|
|
15
|
+
this.closed = false;
|
|
16
|
+
const { client, owned } = (0, connection_1.resolveClient)(opts.connection);
|
|
17
|
+
this.redis = client;
|
|
18
|
+
this.ownsClient = owned;
|
|
19
|
+
this.keys = (0, keys_1.metricsKeys)((0, keys_1.resolveNamespace)(opts.prefix, (0, connection_1.isCluster)(client)));
|
|
20
|
+
}
|
|
21
|
+
get jobClient() {
|
|
22
|
+
return this.redis;
|
|
23
|
+
}
|
|
24
|
+
counterStore(retention) {
|
|
25
|
+
return new HistoryStore_1.HistoryStore({ redis: this.redis, keys: this.keys, retention });
|
|
26
|
+
}
|
|
27
|
+
latencyStore(retention) {
|
|
28
|
+
return new LatencyStore_1.LatencyStore({ redis: this.redis, keys: this.keys, retention });
|
|
29
|
+
}
|
|
30
|
+
administration() {
|
|
31
|
+
return new RedisHistoryAdmin_1.RedisHistoryAdmin({ redis: this.redis, keys: this.keys });
|
|
32
|
+
}
|
|
33
|
+
/** Disconnects a client this store opened; synchronous in effect, like `redis.disconnect()`. */
|
|
34
|
+
async close() {
|
|
35
|
+
if (this.ownsClient && !this.closed) {
|
|
36
|
+
this.redis.disconnect();
|
|
37
|
+
}
|
|
38
|
+
this.closed = true;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
exports.RedisMetricsStore = RedisMetricsStore;
|
|
42
|
+
//# sourceMappingURL=RedisMetricsStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RedisMetricsStore.js","sourceRoot":"","sources":["../src/RedisMetricsStore.ts"],"names":[],"mappings":";;;AAAA,6CAAoG;AACpG,iDAA8C;AAC9C,iCAAyE;AACzE,iDAA8C;AAC9C,2DAAwD;AAgBxD;;;GAGG;AACH,MAAa,iBAAiB;IAO5B,YAAY,IAA8B;QAFlC,WAAM,GAAG,KAAK,CAAC;QAGrB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,IAAA,0BAAa,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAA,kBAAW,EAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAA,sBAAS,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,OAAO,IAAI,2BAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,OAAO,IAAI,2BAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,qCAAiB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,gGAAgG;IAChG,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;CACF;AArCD,8CAqCC"}
|