@worker-manager/metrics 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -0
- package/dist/HistoryAdmin.d.ts +134 -0
- package/dist/HistoryAdmin.js +345 -0
- package/dist/HistoryAdmin.js.map +1 -0
- package/dist/HistoryStore.d.ts +37 -0
- package/dist/HistoryStore.js +140 -0
- package/dist/HistoryStore.js.map +1 -0
- package/dist/LatencySampler.d.ts +88 -0
- package/dist/LatencySampler.js +285 -0
- package/dist/LatencySampler.js.map +1 -0
- package/dist/LatencyStore.d.ts +21 -0
- package/dist/LatencyStore.js +218 -0
- package/dist/LatencyStore.js.map +1 -0
- package/dist/MetricsRecorder.d.ts +92 -0
- package/dist/MetricsRecorder.js +148 -0
- package/dist/MetricsRecorder.js.map +1 -0
- package/dist/RedisMetricsHistoryProvider.d.ts +28 -0
- package/dist/RedisMetricsHistoryProvider.js +147 -0
- package/dist/RedisMetricsHistoryProvider.js.map +1 -0
- package/dist/connection.d.ts +10 -0
- package/dist/connection.js +26 -0
- package/dist/connection.js.map +1 -0
- package/dist/dataMapping.d.ts +13 -0
- package/dist/dataMapping.js +31 -0
- package/dist/dataMapping.js.map +1 -0
- package/dist/histogram.d.ts +27 -0
- package/dist/histogram.js +101 -0
- package/dist/histogram.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/keys.d.ts +21 -0
- package/dist/keys.js +76 -0
- package/dist/keys.js.map +1 -0
- package/package.json +53 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LatencyStore = exports.QUEUE_AGE_METRIC = void 0;
|
|
4
|
+
const histogram_1 = require("./histogram");
|
|
5
|
+
const keys_1 = require("./keys");
|
|
6
|
+
exports.QUEUE_AGE_METRIC = 'queueage';
|
|
7
|
+
const SECONDS_PER_DAY = 86400;
|
|
8
|
+
const MS_PER_HOUR = 3600000;
|
|
9
|
+
function ttl(days) {
|
|
10
|
+
return String(Math.max(1, Math.floor(days * SECONDS_PER_DAY)));
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Merges one tick's bucket vector into the queue's hour hash, its day totals, and the
|
|
14
|
+
* global rollup of both, in a single round trip.
|
|
15
|
+
*
|
|
16
|
+
* Packing all 18 counts into one hash field rather than one field per bucket is what keeps
|
|
17
|
+
* this affordable: Redis hash entry overhead dwarfs a two-byte count, so the packed form
|
|
18
|
+
* costs roughly an eighth of the field-per-bucket form. The trade is that HINCRBY no longer
|
|
19
|
+
* applies, hence the read-modify-write here.
|
|
20
|
+
*
|
|
21
|
+
* Unlike the counter path there is no delta trick, because the caller supplies increments
|
|
22
|
+
* rather than absolute values. Two recorders writing the same tick would double-count, which
|
|
23
|
+
* is what LatencySampler's lease prevents.
|
|
24
|
+
*
|
|
25
|
+
* KEYS[1] queue hour hash ARGV[1] hour field ARGV[4] hour-tier ttl
|
|
26
|
+
* KEYS[2] queue totals hash ARGV[2] day field ARGV[5] day-tier ttl
|
|
27
|
+
* KEYS[3] global hour hash ARGV[3] packed vector ARGV[6] oldest day to keep
|
|
28
|
+
* KEYS[4] global totals hash ARGV[7] bucket count
|
|
29
|
+
*/
|
|
30
|
+
const MERGE_VECTOR = `
|
|
31
|
+
local function merge(key, field, incoming, width)
|
|
32
|
+
local current = redis.call('HGET', key, field)
|
|
33
|
+
local out = {}
|
|
34
|
+
local i = 1
|
|
35
|
+
if current then
|
|
36
|
+
for value in string.gmatch(current, '([^,]+)') do
|
|
37
|
+
out[i] = tonumber(value) or 0
|
|
38
|
+
i = i + 1
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
while i <= width do
|
|
42
|
+
out[i] = 0
|
|
43
|
+
i = i + 1
|
|
44
|
+
end
|
|
45
|
+
local j = 1
|
|
46
|
+
for value in string.gmatch(incoming, '([^,]+)') do
|
|
47
|
+
out[j] = (out[j] or 0) + (tonumber(value) or 0)
|
|
48
|
+
j = j + 1
|
|
49
|
+
end
|
|
50
|
+
redis.call('HSET', key, field, table.concat(out, ','))
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
local function trim(key, cutoff)
|
|
54
|
+
local fields = redis.call('HKEYS', key)
|
|
55
|
+
local stale = {}
|
|
56
|
+
for i = 1, #fields do
|
|
57
|
+
if fields[i] < cutoff then
|
|
58
|
+
stale[#stale + 1] = fields[i]
|
|
59
|
+
if #stale == 256 then
|
|
60
|
+
redis.call('HDEL', key, unpack(stale))
|
|
61
|
+
stale = {}
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
if #stale > 0 then
|
|
66
|
+
redis.call('HDEL', key, unpack(stale))
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
local width = tonumber(ARGV[7])
|
|
71
|
+
local newDay = redis.call('HEXISTS', KEYS[2], ARGV[2]) == 0
|
|
72
|
+
|
|
73
|
+
merge(KEYS[1], ARGV[1], ARGV[3], width)
|
|
74
|
+
merge(KEYS[2], ARGV[2], ARGV[3], width)
|
|
75
|
+
merge(KEYS[3], ARGV[1], ARGV[3], width)
|
|
76
|
+
merge(KEYS[4], ARGV[2], ARGV[3], width)
|
|
77
|
+
|
|
78
|
+
redis.call('EXPIRE', KEYS[1], ARGV[4])
|
|
79
|
+
redis.call('EXPIRE', KEYS[2], ARGV[5])
|
|
80
|
+
redis.call('EXPIRE', KEYS[3], ARGV[4])
|
|
81
|
+
redis.call('EXPIRE', KEYS[4], ARGV[5])
|
|
82
|
+
|
|
83
|
+
if newDay then
|
|
84
|
+
trim(KEYS[2], ARGV[6])
|
|
85
|
+
trim(KEYS[4], ARGV[6])
|
|
86
|
+
end
|
|
87
|
+
return 1
|
|
88
|
+
`;
|
|
89
|
+
/**
|
|
90
|
+
* Queue age is a gauge, so an hour holds the worst backlog seen in it and a day holds the
|
|
91
|
+
* worst of its hours. Summing would be meaningless -- which is also why the global rollup
|
|
92
|
+
* here is a MAX across queues rather than the four-key SUM MERGE_VECTOR does: "the oldest
|
|
93
|
+
* job waiting anywhere on this board" is the meaningful cross-queue number, adding queues'
|
|
94
|
+
* ages together would not be.
|
|
95
|
+
*
|
|
96
|
+
* Because it's "set if greater", a bucket's value never goes down within its own hour or
|
|
97
|
+
* day once a high value lands -- that's the correct meaning of "the worst backlog seen in
|
|
98
|
+
* this hour/day", both for a single queue and for the global rollup. A later, healthier
|
|
99
|
+
* queue doesn't erase an earlier spike from the same bucket; it only wins the buckets it
|
|
100
|
+
* writes to itself.
|
|
101
|
+
*
|
|
102
|
+
* The totals hashes need the same day cutoff MERGE_VECTOR applies: their TTL is refreshed on
|
|
103
|
+
* every tick, so without a trim they would accumulate one field per day for as long as the
|
|
104
|
+
* queue exists rather than for the retention window.
|
|
105
|
+
*
|
|
106
|
+
* KEYS[1] queue hour hash ARGV[1] hour field ARGV[4] hour-tier ttl
|
|
107
|
+
* KEYS[2] queue totals hash ARGV[2] day field ARGV[5] day-tier ttl
|
|
108
|
+
* KEYS[3] global hour hash ARGV[3] value ARGV[6] oldest day to keep
|
|
109
|
+
* KEYS[4] global totals hash
|
|
110
|
+
*/
|
|
111
|
+
const MAX_GAUGE = `
|
|
112
|
+
local function setMax(key, field, value)
|
|
113
|
+
local current = tonumber(redis.call('HGET', key, field) or '-1')
|
|
114
|
+
if value > current then
|
|
115
|
+
redis.call('HSET', key, field, value)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
local function trim(key, cutoff)
|
|
120
|
+
local fields = redis.call('HKEYS', key)
|
|
121
|
+
local stale = {}
|
|
122
|
+
for i = 1, #fields do
|
|
123
|
+
if fields[i] < cutoff then
|
|
124
|
+
stale[#stale + 1] = fields[i]
|
|
125
|
+
if #stale == 256 then
|
|
126
|
+
redis.call('HDEL', key, unpack(stale))
|
|
127
|
+
stale = {}
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
if #stale > 0 then
|
|
132
|
+
redis.call('HDEL', key, unpack(stale))
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
local value = tonumber(ARGV[3])
|
|
137
|
+
local newDay = redis.call('HEXISTS', KEYS[2], ARGV[2]) == 0
|
|
138
|
+
|
|
139
|
+
setMax(KEYS[1], ARGV[1], value)
|
|
140
|
+
setMax(KEYS[2], ARGV[2], value)
|
|
141
|
+
setMax(KEYS[3], ARGV[1], value)
|
|
142
|
+
setMax(KEYS[4], ARGV[2], value)
|
|
143
|
+
|
|
144
|
+
redis.call('EXPIRE', KEYS[1], ARGV[4])
|
|
145
|
+
redis.call('EXPIRE', KEYS[2], ARGV[5])
|
|
146
|
+
redis.call('EXPIRE', KEYS[3], ARGV[4])
|
|
147
|
+
redis.call('EXPIRE', KEYS[4], ARGV[5])
|
|
148
|
+
|
|
149
|
+
if newDay then
|
|
150
|
+
trim(KEYS[2], ARGV[6])
|
|
151
|
+
trim(KEYS[4], ARGV[6])
|
|
152
|
+
end
|
|
153
|
+
return 1
|
|
154
|
+
`;
|
|
155
|
+
class LatencyStore {
|
|
156
|
+
constructor(opts) {
|
|
157
|
+
this.redis = opts.redis;
|
|
158
|
+
this.keys = opts.keys;
|
|
159
|
+
this.retention = opts.retention;
|
|
160
|
+
}
|
|
161
|
+
async addSamples(queue, metric, hour, vector) {
|
|
162
|
+
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
163
|
+
await this.redis.eval(MERGE_VECTOR, 4, this.keys.hour(queue, metric, day), this.keys.totals(queue, metric), this.keys.hour(keys_1.GLOBAL_QUEUE, metric, day), this.keys.totals(keys_1.GLOBAL_QUEUE, metric), String(hour), day, (0, histogram_1.packVector)(vector), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days), String(histogram_1.BUCKET_COUNT));
|
|
164
|
+
}
|
|
165
|
+
async recordQueueAge(queue, hour, ms) {
|
|
166
|
+
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
167
|
+
await this.redis.eval(MAX_GAUGE, 4, this.keys.hour(queue, exports.QUEUE_AGE_METRIC, day), this.keys.totals(queue, exports.QUEUE_AGE_METRIC), this.keys.hour(keys_1.GLOBAL_QUEUE, exports.QUEUE_AGE_METRIC, day), this.keys.totals(keys_1.GLOBAL_QUEUE, exports.QUEUE_AGE_METRIC), String(hour), day, String(Math.max(0, Math.round(ms))), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days));
|
|
168
|
+
}
|
|
169
|
+
async readRange(queue, metric, granularity, days) {
|
|
170
|
+
const out = {};
|
|
171
|
+
if (granularity === 'day') {
|
|
172
|
+
const raw = await this.redis.hgetall(this.keys.totals(queue, metric));
|
|
173
|
+
for (const day of days) {
|
|
174
|
+
if (raw[day] !== undefined) {
|
|
175
|
+
out[day] = (0, histogram_1.unpackVector)(raw[day]);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return out;
|
|
179
|
+
}
|
|
180
|
+
// Up to one key per retention day, so these go out together rather than as a serial
|
|
181
|
+
// chain of round trips, matching how the counter path reads its day hashes.
|
|
182
|
+
const perDay = await Promise.all(days.map((day) => this.redis.hgetall(this.keys.hour(queue, metric, day))));
|
|
183
|
+
for (const raw of perDay) {
|
|
184
|
+
for (const field of Object.keys(raw)) {
|
|
185
|
+
out[field] = out[field]
|
|
186
|
+
? (0, histogram_1.mergeVectors)(out[field], (0, histogram_1.unpackVector)(raw[field]))
|
|
187
|
+
: (0, histogram_1.unpackVector)(raw[field]);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return out;
|
|
191
|
+
}
|
|
192
|
+
async readQueueAge(queue, granularity, days) {
|
|
193
|
+
var _a;
|
|
194
|
+
const out = {};
|
|
195
|
+
if (granularity === 'day') {
|
|
196
|
+
const raw = await this.redis.hgetall(this.keys.totals(queue, exports.QUEUE_AGE_METRIC));
|
|
197
|
+
for (const day of days) {
|
|
198
|
+
if (raw[day] !== undefined) {
|
|
199
|
+
out[day] = Number(raw[day]) || 0;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return out;
|
|
203
|
+
}
|
|
204
|
+
const perDay = await Promise.all(days.map((day) => this.redis.hgetall(this.keys.hour(queue, exports.QUEUE_AGE_METRIC, day))));
|
|
205
|
+
for (const raw of perDay) {
|
|
206
|
+
for (const field of Object.keys(raw)) {
|
|
207
|
+
out[field] = Math.max((_a = out[field]) !== null && _a !== void 0 ? _a : 0, Number(raw[field]) || 0);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return out;
|
|
211
|
+
}
|
|
212
|
+
/** Epoch ms for an absolute hour index, for building response timestamps. */
|
|
213
|
+
static hourToMs(hour) {
|
|
214
|
+
return hour * MS_PER_HOUR;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
exports.LatencyStore = LatencyStore;
|
|
218
|
+
//# sourceMappingURL=LatencyStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LatencyStore.js","sourceRoot":"","sources":["../src/LatencyStore.ts"],"names":[],"mappings":";;;AACA,2CAAmF;AAEnF,iCAA+E;AAIlE,QAAA,gBAAgB,GAAG,UAAU,CAAC;AAE3C,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0DpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CjB,CAAC;AAEF,MAAa,YAAY;IAKvB,YAAY,IAAuE;QACjF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAa,EACb,MAAqB,EACrB,IAAY,EACZ,MAAgB;QAEhB,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,YAAY,EACZ,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAClC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,MAAM,CAAC,EACtC,MAAM,CAAC,IAAI,CAAC,EACZ,GAAG,EACH,IAAA,sBAAU,EAAC,MAAM,CAAC,EAClB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACnC,MAAM,CAAC,wBAAY,CAAC,CACrB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,IAAY,EAAE,EAAU;QAC1D,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,SAAS,EACT,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,wBAAgB,EAAE,GAAG,CAAC,EAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,wBAAgB,CAAC,EACzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAY,EAAE,wBAAgB,EAAE,GAAG,CAAC,EACnD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,wBAAgB,CAAC,EAChD,MAAM,CAAC,IAAI,CAAC,EACZ,GAAG,EACH,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EACnC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAa,EACb,MAAqB,EACrB,WAA2B,EAC3B,IAAc;QAEd,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;YACtE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,IAAA,wBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAC1E,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBACrB,CAAC,CAAC,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC,CAAC,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,WAA2B,EAC3B,IAAc;;QAEd,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,wBAAgB,CAAC,CAAC,CAAC;YAChF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,wBAAgB,EAAE,GAAG,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAA,GAAG,CAAC,KAAK,CAAC,mCAAI,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,QAAQ,CAAC,IAAY;QAC1B,OAAO,IAAI,GAAG,WAAW,CAAC;IAC5B,CAAC;CACF;AAlHD,oCAkHC"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { BaseAdapter } from '@worker-manager/api/baseAdapter';
|
|
2
|
+
import { type MetricsConnection } from './connection';
|
|
3
|
+
import { type Retention } from './HistoryStore';
|
|
4
|
+
/**
|
|
5
|
+
* Minute detail is the expensive tier by two orders of magnitude, so it defaults to a week
|
|
6
|
+
* rather than the full window: long enough to match the recommended `MetricsTime.ONE_WEEK`
|
|
7
|
+
* worker buffer, so the recorder can be down for a week and still catch up completely.
|
|
8
|
+
* The hourly and daily rollups are cheap enough to keep for the whole window.
|
|
9
|
+
*/
|
|
10
|
+
export declare const DEFAULT_RETENTION: Retention;
|
|
11
|
+
export interface MetricsRecorderOptions {
|
|
12
|
+
/**
|
|
13
|
+
* A function is resolved on every tick, for a board whose queue set changes while it
|
|
14
|
+
* runs. An array is read once, at construction.
|
|
15
|
+
*/
|
|
16
|
+
queues: BaseAdapter[] | (() => BaseAdapter[]);
|
|
17
|
+
connection: MetricsConnection;
|
|
18
|
+
/**
|
|
19
|
+
* Redis key namespace, defaulting to `bull-board:metrics`. Set it to separate two boards
|
|
20
|
+
* sharing one Redis, and give the reading `RedisMetricsHistoryProvider` the same value.
|
|
21
|
+
*
|
|
22
|
+
* On a Redis Cluster the namespace has to sit in one hash slot, since the rollup scripts
|
|
23
|
+
* write a queue's keys and the cross-queue keys in one EVAL. A prefix with no `{...}` hash
|
|
24
|
+
* tag is wrapped in one, so `staging:metrics` becomes `{staging:metrics}`; supply your own
|
|
25
|
+
* tag to choose the slot yourself.
|
|
26
|
+
*/
|
|
27
|
+
prefix?: string;
|
|
28
|
+
/** Per-resolution retention in days. Unspecified tiers fall back to the defaults. */
|
|
29
|
+
retention?: Partial<Retention>;
|
|
30
|
+
/**
|
|
31
|
+
* Shorthand that sets the daily and hourly windows. Minute retention stays at its
|
|
32
|
+
* default unless raised explicitly, since that is the tier that drives storage size.
|
|
33
|
+
*/
|
|
34
|
+
retentionDays?: number;
|
|
35
|
+
snapshotIntervalMs?: number;
|
|
36
|
+
/**
|
|
37
|
+
* Latency histograms and the queue-age gauge. On by default: the package exists to give
|
|
38
|
+
* boards without a metrics stack something useful, and an opt-in feature is one nobody
|
|
39
|
+
* finds. At default retention this costs roughly 250 to 300KB per queue under typical
|
|
40
|
+
* traffic, up to about 575KB in a pathological worst case, plus a one-off shared cost of
|
|
41
|
+
* roughly 224KB for the cross-queue rollup regardless of queue count.
|
|
42
|
+
*/
|
|
43
|
+
latency?: boolean;
|
|
44
|
+
/** Above this many finished jobs in one tick, the sampler subsamples. */
|
|
45
|
+
maxLatencySamplesPerTick?: number;
|
|
46
|
+
/**
|
|
47
|
+
* Test-oriented escape hatch: overrides the sampler's default 5s safety margin (see
|
|
48
|
+
* `LatencySampler`'s `SAFETY_MARGIN_MS`), which otherwise excludes jobs that finished
|
|
49
|
+
* just before a scan. Lets a test read back a sample immediately instead of sleeping
|
|
50
|
+
* past the margin.
|
|
51
|
+
*/
|
|
52
|
+
latencySafetyMarginMs?: number;
|
|
53
|
+
/**
|
|
54
|
+
* Notified whenever a latency tick fails. Latency errors are swallowed on purpose so a
|
|
55
|
+
* failing scan cannot take the counter snapshot with it, which also means a collector
|
|
56
|
+
* broken since startup looks the same as a board with no traffic. Default stays silent;
|
|
57
|
+
* wire this to your logger to tell an empty chart from a broken one.
|
|
58
|
+
*/
|
|
59
|
+
onLatencyError?: (error: unknown, queueName: string) => void;
|
|
60
|
+
}
|
|
61
|
+
export declare function resolveRetention(opts: {
|
|
62
|
+
retention?: Partial<Retention>;
|
|
63
|
+
retentionDays?: number;
|
|
64
|
+
}): Retention;
|
|
65
|
+
export declare class MetricsRecorder {
|
|
66
|
+
private readonly resolveQueues;
|
|
67
|
+
private readonly store;
|
|
68
|
+
private readonly redis;
|
|
69
|
+
private readonly ownsRedis;
|
|
70
|
+
private readonly intervalMs;
|
|
71
|
+
private readonly lastMinute;
|
|
72
|
+
private timer;
|
|
73
|
+
private running;
|
|
74
|
+
private stopped;
|
|
75
|
+
readonly latencyEnabled: boolean;
|
|
76
|
+
private readonly latencySampler;
|
|
77
|
+
constructor(opts: MetricsRecorderOptions);
|
|
78
|
+
get retention(): Retention;
|
|
79
|
+
start(): void;
|
|
80
|
+
stop(): void;
|
|
81
|
+
snapshot(): Promise<void>;
|
|
82
|
+
/**
|
|
83
|
+
* Incrementally copies BullMQ's per-minute ring buffer into long-retention storage.
|
|
84
|
+
* `seenUpTo` is a per-(queue, metric) watermark of the newest minute already written.
|
|
85
|
+
* getMetrics() returns points newest-first, so we walk from the newest and stop at the
|
|
86
|
+
* first minute we've already stored: everything past it is older and stored too. Fresh
|
|
87
|
+
* minutes are upserted (safe against overlapping windows across ticks), then the
|
|
88
|
+
* watermark advances. So the first tick backfills the buffer and every later tick only
|
|
89
|
+
* writes the minutes that appeared since.
|
|
90
|
+
*/
|
|
91
|
+
private snapshotOne;
|
|
92
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MetricsRecorder = exports.DEFAULT_RETENTION = void 0;
|
|
4
|
+
exports.resolveRetention = resolveRetention;
|
|
5
|
+
const connection_1 = require("./connection");
|
|
6
|
+
const dataMapping_1 = require("./dataMapping");
|
|
7
|
+
const HistoryStore_1 = require("./HistoryStore");
|
|
8
|
+
const keys_1 = require("./keys");
|
|
9
|
+
const LatencySampler_1 = require("./LatencySampler");
|
|
10
|
+
const LatencyStore_1 = require("./LatencyStore");
|
|
11
|
+
const METRICS = ['completed', 'failed'];
|
|
12
|
+
const MS_PER_MINUTE = 60000;
|
|
13
|
+
const MINUTES_PER_DAY = 1440;
|
|
14
|
+
/**
|
|
15
|
+
* Minute detail is the expensive tier by two orders of magnitude, so it defaults to a week
|
|
16
|
+
* rather than the full window: long enough to match the recommended `MetricsTime.ONE_WEEK`
|
|
17
|
+
* worker buffer, so the recorder can be down for a week and still catch up completely.
|
|
18
|
+
* The hourly and daily rollups are cheap enough to keep for the whole window.
|
|
19
|
+
*/
|
|
20
|
+
exports.DEFAULT_RETENTION = { minutes: 7, hours: 90, days: 90 };
|
|
21
|
+
function resolveRetention(opts) {
|
|
22
|
+
const base = opts.retentionDays === undefined
|
|
23
|
+
? exports.DEFAULT_RETENTION
|
|
24
|
+
: {
|
|
25
|
+
minutes: Math.min(exports.DEFAULT_RETENTION.minutes, opts.retentionDays),
|
|
26
|
+
hours: opts.retentionDays,
|
|
27
|
+
days: opts.retentionDays,
|
|
28
|
+
};
|
|
29
|
+
return { ...base, ...opts.retention };
|
|
30
|
+
}
|
|
31
|
+
class MetricsRecorder {
|
|
32
|
+
constructor(opts) {
|
|
33
|
+
var _a;
|
|
34
|
+
this.lastMinute = new Map();
|
|
35
|
+
this.timer = null;
|
|
36
|
+
this.running = false;
|
|
37
|
+
this.stopped = false;
|
|
38
|
+
const { queues } = opts;
|
|
39
|
+
this.resolveQueues = typeof queues === 'function' ? queues : () => queues;
|
|
40
|
+
this.intervalMs = (_a = opts.snapshotIntervalMs) !== null && _a !== void 0 ? _a : 60000;
|
|
41
|
+
const { client, owned } = (0, connection_1.resolveClient)(opts.connection);
|
|
42
|
+
this.redis = client;
|
|
43
|
+
this.ownsRedis = owned;
|
|
44
|
+
const keys = (0, keys_1.metricsKeys)((0, keys_1.resolveNamespace)(opts.prefix, (0, connection_1.isCluster)(client)));
|
|
45
|
+
this.store = new HistoryStore_1.HistoryStore({ redis: this.redis, keys, retention: resolveRetention(opts) });
|
|
46
|
+
this.latencyEnabled = opts.latency !== false;
|
|
47
|
+
this.latencySampler = this.latencyEnabled
|
|
48
|
+
? new LatencySampler_1.LatencySampler({
|
|
49
|
+
redis: this.redis,
|
|
50
|
+
keys,
|
|
51
|
+
store: new LatencyStore_1.LatencyStore({ redis: this.redis, keys, retention: resolveRetention(opts) }),
|
|
52
|
+
tickMs: this.intervalMs,
|
|
53
|
+
maxSamplesPerTick: opts.maxLatencySamplesPerTick,
|
|
54
|
+
safetyMarginMs: opts.latencySafetyMarginMs,
|
|
55
|
+
onError: opts.onLatencyError,
|
|
56
|
+
})
|
|
57
|
+
: null;
|
|
58
|
+
}
|
|
59
|
+
get retention() {
|
|
60
|
+
return this.store.retention;
|
|
61
|
+
}
|
|
62
|
+
start() {
|
|
63
|
+
if (this.timer) {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
this.timer = setInterval(() => {
|
|
67
|
+
void this.snapshot();
|
|
68
|
+
}, this.intervalMs);
|
|
69
|
+
// Do not keep the event loop alive solely for the recorder.
|
|
70
|
+
if (typeof this.timer.unref === 'function') {
|
|
71
|
+
this.timer.unref();
|
|
72
|
+
}
|
|
73
|
+
void this.snapshot();
|
|
74
|
+
}
|
|
75
|
+
stop() {
|
|
76
|
+
if (this.timer) {
|
|
77
|
+
clearInterval(this.timer);
|
|
78
|
+
this.timer = null;
|
|
79
|
+
}
|
|
80
|
+
if (this.ownsRedis && !this.stopped) {
|
|
81
|
+
this.redis.disconnect();
|
|
82
|
+
}
|
|
83
|
+
this.stopped = true;
|
|
84
|
+
}
|
|
85
|
+
async snapshot() {
|
|
86
|
+
if (this.running) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
this.running = true;
|
|
90
|
+
try {
|
|
91
|
+
for (const adapter of this.resolveQueues()) {
|
|
92
|
+
const name = adapter.getName();
|
|
93
|
+
for (const metric of METRICS) {
|
|
94
|
+
await this.snapshotOne(adapter, name, metric);
|
|
95
|
+
}
|
|
96
|
+
if (this.latencySampler) {
|
|
97
|
+
await this.latencySampler.sample(adapter);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
this.running = false;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Incrementally copies BullMQ's per-minute ring buffer into long-retention storage.
|
|
107
|
+
* `seenUpTo` is a per-(queue, metric) watermark of the newest minute already written.
|
|
108
|
+
* getMetrics() returns points newest-first, so we walk from the newest and stop at the
|
|
109
|
+
* first minute we've already stored: everything past it is older and stored too. Fresh
|
|
110
|
+
* minutes are upserted (safe against overlapping windows across ticks), then the
|
|
111
|
+
* watermark advances. So the first tick backfills the buffer and every later tick only
|
|
112
|
+
* writes the minutes that appeared since.
|
|
113
|
+
*/
|
|
114
|
+
async snapshotOne(adapter, name, metric) {
|
|
115
|
+
var _a;
|
|
116
|
+
const cursorKey = `${name}:${metric}`;
|
|
117
|
+
const seenUpTo = (_a = this.lastMinute.get(cursorKey)) !== null && _a !== void 0 ? _a : -1;
|
|
118
|
+
const metrics = await adapter.getMetrics(metric).catch(() => null);
|
|
119
|
+
const points = (0, dataMapping_1.metricsToMinutePoints)(metrics);
|
|
120
|
+
if (points.length === 0) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
// Correctness guard, not an optimization. Idempotency comes from the minute hash
|
|
124
|
+
// holding the previously written value, so a minute whose hash has already expired
|
|
125
|
+
// would look brand new and be added to the hourly and daily rollups a second time.
|
|
126
|
+
// That can only happen when the worker's metrics buffer reaches further back than the
|
|
127
|
+
// minute window (say a two-week buffer against a one-week window) and the recorder
|
|
128
|
+
// restarts, losing its in-memory watermark. Refusing to write past the window closes
|
|
129
|
+
// it. Nothing is lost that could have been retained anyway.
|
|
130
|
+
const oldestWritable = Math.floor(Date.now() / MS_PER_MINUTE) - this.store.retention.minutes * MINUTES_PER_DAY;
|
|
131
|
+
let newest = seenUpTo;
|
|
132
|
+
for (const point of points) {
|
|
133
|
+
if (point.minute <= seenUpTo) {
|
|
134
|
+
break; // points are newest-first; everything older is already stored
|
|
135
|
+
}
|
|
136
|
+
if (point.minute < oldestWritable) {
|
|
137
|
+
break; // ...and everything past here is older still
|
|
138
|
+
}
|
|
139
|
+
await this.store.upsertMinute(name, metric, point.minute, point.value);
|
|
140
|
+
if (point.minute > newest) {
|
|
141
|
+
newest = point.minute;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
this.lastMinute.set(cursorKey, newest);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
exports.MetricsRecorder = MetricsRecorder;
|
|
148
|
+
//# sourceMappingURL=MetricsRecorder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MetricsRecorder.js","sourceRoot":"","sources":["../src/MetricsRecorder.ts"],"names":[],"mappings":";;;AAwEA,4CAaC;AAnFD,6CAAoG;AACpG,+CAAsD;AACtD,iDAA8D;AAC9D,iCAAuD;AACvD,qDAAkD;AAClD,iDAA8C;AAE9C,MAAM,OAAO,GAAkB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACvD,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;;;;GAKG;AACU,QAAA,iBAAiB,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AAqDhF,SAAgB,gBAAgB,CAAC,IAGhC;IACC,MAAM,IAAI,GACR,IAAI,CAAC,aAAa,KAAK,SAAS;QAC9B,CAAC,CAAC,yBAAiB;QACnB,CAAC,CAAC;YACE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC;YAChE,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,IAAI,EAAE,IAAI,CAAC,aAAa;SACzB,CAAC;IACR,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,MAAa,eAAe;IAa1B,YAAY,IAA4B;;QAPvB,eAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAChD,UAAK,GAA0C,IAAI,CAAC;QACpD,YAAO,GAAG,KAAK,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QAKtB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,aAAa,GAAG,OAAO,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;QAC1E,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,kBAAkB,mCAAI,KAAK,CAAC;QACnD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,IAAA,0BAAa,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,IAAI,GAAG,IAAA,kBAAW,EAAC,IAAA,uBAAgB,EAAC,IAAI,CAAC,MAAM,EAAE,IAAA,sBAAS,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,KAAK,GAAG,IAAI,2BAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC;QAC7C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc;YACvC,CAAC,CAAC,IAAI,+BAAc,CAAC;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI;gBACJ,KAAK,EAAE,IAAI,2BAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,iBAAiB,EAAE,IAAI,CAAC,wBAAwB;gBAChD,cAAc,EAAE,IAAI,CAAC,qBAAqB;gBAC1C,OAAO,EAAE,IAAI,CAAC,cAAc;aAC7B,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,4DAA4D;QAC5D,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;IACvB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;gBAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;gBAChD,CAAC;gBACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,WAAW,CACvB,OAAoB,EACpB,IAAY,EACZ,MAAmB;;QAEnB,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,mCAAI,CAAC,CAAC,CAAC;QAEtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,mFAAmF;QACnF,sFAAsF;QACtF,mFAAmF;QACnF,qFAAqF;QACrF,4DAA4D;QAC5D,MAAM,cAAc,GAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,eAAe,CAAC;QAE1F,IAAI,MAAM,GAAG,QAAQ,CAAC;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,CAAC,8DAA8D;YACvE,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBAClC,MAAM,CAAC,6CAA6C;YACtD,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACvE,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;CACF;AArID,0CAqIC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { MetricsHistoryPoint, MetricsHistoryProvider, MetricsHistoryQuery, MetricsLatencyPoint, MetricsLatencyQuery } from '@worker-manager/api/typings/app';
|
|
2
|
+
import { type MetricsConnection } from './connection';
|
|
3
|
+
import { type HistoryStats, type PurgeOptions, type PurgeResult } from './HistoryAdmin';
|
|
4
|
+
import { type Retention } from './HistoryStore';
|
|
5
|
+
export interface RedisMetricsHistoryProviderOptions {
|
|
6
|
+
connection: MetricsConnection;
|
|
7
|
+
/** Must match the recorder's. See `MetricsRecorderOptions.prefix`. */
|
|
8
|
+
prefix?: string;
|
|
9
|
+
/** Should mirror the recorder's retention. Only used to bound the query span. */
|
|
10
|
+
retention?: Partial<Retention>;
|
|
11
|
+
retentionDays?: number;
|
|
12
|
+
}
|
|
13
|
+
export declare class RedisMetricsHistoryProvider implements MetricsHistoryProvider {
|
|
14
|
+
private readonly store;
|
|
15
|
+
private readonly latencyStore;
|
|
16
|
+
private readonly admin;
|
|
17
|
+
private readonly redis;
|
|
18
|
+
private readonly ownsRedis;
|
|
19
|
+
private readonly retentionDays;
|
|
20
|
+
constructor(opts: RedisMetricsHistoryProviderOptions);
|
|
21
|
+
disconnect(): void;
|
|
22
|
+
/** Backs the board's storage panel. See MetricsHistoryAdmin.stats. */
|
|
23
|
+
getUsage(): Promise<HistoryStats>;
|
|
24
|
+
/** Backs the board's "clear history" action. See MetricsHistoryAdmin.purge. */
|
|
25
|
+
purge(options?: PurgeOptions): Promise<PurgeResult>;
|
|
26
|
+
getHistory(query: MetricsHistoryQuery): Promise<MetricsHistoryPoint[]>;
|
|
27
|
+
getLatency(query: MetricsLatencyQuery): Promise<MetricsLatencyPoint[]>;
|
|
28
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisMetricsHistoryProvider = void 0;
|
|
4
|
+
const connection_1 = require("./connection");
|
|
5
|
+
const histogram_1 = require("./histogram");
|
|
6
|
+
const HistoryAdmin_1 = require("./HistoryAdmin");
|
|
7
|
+
const HistoryStore_1 = require("./HistoryStore");
|
|
8
|
+
const keys_1 = require("./keys");
|
|
9
|
+
const LatencyStore_1 = require("./LatencyStore");
|
|
10
|
+
const MetricsRecorder_1 = require("./MetricsRecorder");
|
|
11
|
+
const MS_PER_HOUR = 3600000;
|
|
12
|
+
class RedisMetricsHistoryProvider {
|
|
13
|
+
constructor(opts) {
|
|
14
|
+
const { client, owned } = (0, connection_1.resolveClient)(opts.connection);
|
|
15
|
+
this.redis = client;
|
|
16
|
+
this.ownsRedis = owned;
|
|
17
|
+
const keys = (0, keys_1.metricsKeys)((0, keys_1.resolveNamespace)(opts.prefix, (0, connection_1.isCluster)(client)));
|
|
18
|
+
const retention = (0, MetricsRecorder_1.resolveRetention)(opts);
|
|
19
|
+
this.retentionDays = retention.days;
|
|
20
|
+
this.store = new HistoryStore_1.HistoryStore({ redis: this.redis, keys, retention });
|
|
21
|
+
this.latencyStore = new LatencyStore_1.LatencyStore({ redis: this.redis, keys, retention });
|
|
22
|
+
this.admin = new HistoryAdmin_1.MetricsHistoryAdmin({ connection: this.redis, prefix: opts.prefix });
|
|
23
|
+
}
|
|
24
|
+
disconnect() {
|
|
25
|
+
if (this.ownsRedis) {
|
|
26
|
+
this.redis.disconnect();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** Backs the board's storage panel. See MetricsHistoryAdmin.stats. */
|
|
30
|
+
async getUsage() {
|
|
31
|
+
return this.admin.stats();
|
|
32
|
+
}
|
|
33
|
+
/** Backs the board's "clear history" action. See MetricsHistoryAdmin.purge. */
|
|
34
|
+
async purge(options = {}) {
|
|
35
|
+
return this.admin.purge(options);
|
|
36
|
+
}
|
|
37
|
+
async getHistory(query) {
|
|
38
|
+
var _a, _b;
|
|
39
|
+
const queue = (_a = query.queue) !== null && _a !== void 0 ? _a : keys_1.GLOBAL_QUEUE;
|
|
40
|
+
// Clamp the span to the retention window so an unbounded `from` (e.g. 0) can't make
|
|
41
|
+
// dayRange produce an unbounded number of day buckets -- older data doesn't exist anyway.
|
|
42
|
+
const maxSpanMs = (this.retentionDays + 1) * 86400000;
|
|
43
|
+
const from = Math.max(query.from, query.to - maxSpanMs);
|
|
44
|
+
const days = (0, keys_1.dayRange)(from, query.to);
|
|
45
|
+
if (query.metric === 'queueage') {
|
|
46
|
+
const ages = await this.latencyStore.readQueueAge(queue, query.granularity, days);
|
|
47
|
+
// Day points are stamped at the day's start, so an intraday `from` would drop the day
|
|
48
|
+
// it falls in. Floored, exactly as the counter path below does it.
|
|
49
|
+
const lowerBound = query.granularity === 'day' ? dayFloor(query.from) : query.from;
|
|
50
|
+
return Object.keys(ages)
|
|
51
|
+
.map((key) => ({
|
|
52
|
+
ts: query.granularity === 'day' ? (0, keys_1.dayToStartMs)(key) : Number(key) * MS_PER_HOUR,
|
|
53
|
+
value: ages[key],
|
|
54
|
+
}))
|
|
55
|
+
.filter((p) => p.ts >= lowerBound && p.ts <= query.to)
|
|
56
|
+
.sort((a, b) => a.ts - b.ts);
|
|
57
|
+
}
|
|
58
|
+
if (query.granularity === 'day') {
|
|
59
|
+
const rawTotals = await this.store.readDailyTotalsRaw(queue, query.metric, days);
|
|
60
|
+
// Empty history only when no day in range was ever recorded (all fields missing).
|
|
61
|
+
// A day with a stored '0' still counts as recorded -- otherwise the UI's empty
|
|
62
|
+
// state would be unreachable once any data exists.
|
|
63
|
+
if (rawTotals.every((value) => value == null)) {
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
const totals = {};
|
|
67
|
+
days.forEach((day, i) => {
|
|
68
|
+
totals[day] = Number(rawTotals[i]) || 0;
|
|
69
|
+
});
|
|
70
|
+
return days
|
|
71
|
+
.map((day) => { var _a; return ({ ts: (0, keys_1.dayToStartMs)(day), value: (_a = totals[day]) !== null && _a !== void 0 ? _a : 0 }); })
|
|
72
|
+
.filter((p) => p.ts >= dayFloor(query.from) && p.ts <= query.to);
|
|
73
|
+
}
|
|
74
|
+
const hourBuckets = new Map();
|
|
75
|
+
const dayHours = await Promise.all(days.map((day) => this.store.readDayHours(queue, query.metric, day)));
|
|
76
|
+
for (const hours of dayHours) {
|
|
77
|
+
for (const field of Object.keys(hours)) {
|
|
78
|
+
const ts = Number(field) * MS_PER_HOUR;
|
|
79
|
+
if (ts < query.from || ts > query.to) {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
hourBuckets.set(ts, ((_b = hourBuckets.get(ts)) !== null && _b !== void 0 ? _b : 0) + hours[field]);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return [...hourBuckets.entries()]
|
|
86
|
+
.map(([ts, value]) => ({ ts, value }))
|
|
87
|
+
.sort((a, b) => a.ts - b.ts);
|
|
88
|
+
}
|
|
89
|
+
async getLatency(query) {
|
|
90
|
+
var _a;
|
|
91
|
+
const queue = (_a = query.queue) !== null && _a !== void 0 ? _a : keys_1.GLOBAL_QUEUE;
|
|
92
|
+
const maxSpanMs = (this.retentionDays + 1) * 86400000;
|
|
93
|
+
const from = Math.max(query.from, query.to - maxSpanMs);
|
|
94
|
+
const days = (0, keys_1.dayRange)(from, query.to);
|
|
95
|
+
if (query.granularity === 'range') {
|
|
96
|
+
// Percentiles don't merge: averaging per-day p95s isn't the same number as the p95 of
|
|
97
|
+
// the whole range. Read the day tier's bucket vectors and merge them, then compute each
|
|
98
|
+
// requested percentile once from the summed vector.
|
|
99
|
+
const raw = await this.latencyStore.readRange(queue, query.metric, 'day', days);
|
|
100
|
+
let merged = (0, histogram_1.emptyVector)();
|
|
101
|
+
for (const day of days) {
|
|
102
|
+
const vector = raw[day];
|
|
103
|
+
if (vector) {
|
|
104
|
+
merged = (0, histogram_1.mergeVectors)(merged, vector);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const count = (0, histogram_1.vectorTotal)(merged);
|
|
108
|
+
if (count === 0) {
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
const values = {};
|
|
112
|
+
for (const p of query.percentiles) {
|
|
113
|
+
values[String(p)] = (0, histogram_1.quantile)(merged, p);
|
|
114
|
+
}
|
|
115
|
+
return [{ ts: query.from, count: Math.round(count), values }];
|
|
116
|
+
}
|
|
117
|
+
const raw = await this.latencyStore.readRange(queue, query.metric, query.granularity, days);
|
|
118
|
+
// Same day-start alignment as getHistory: comparing a day bucket against a raw `from`
|
|
119
|
+
// would drop the oldest day and leave this chart one bucket shorter than the throughput
|
|
120
|
+
// chart drawn for the same range.
|
|
121
|
+
const lowerBound = query.granularity === 'day' ? dayFloor(query.from) : query.from;
|
|
122
|
+
const points = [];
|
|
123
|
+
for (const key of Object.keys(raw)) {
|
|
124
|
+
const ts = query.granularity === 'day' ? (0, keys_1.dayToStartMs)(key) : Number(key) * MS_PER_HOUR;
|
|
125
|
+
if (ts < lowerBound || ts > query.to) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
const vector = raw[key];
|
|
129
|
+
const count = (0, histogram_1.vectorTotal)(vector);
|
|
130
|
+
if (count === 0) {
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
const values = {};
|
|
134
|
+
for (const p of query.percentiles) {
|
|
135
|
+
values[String(p)] = (0, histogram_1.quantile)(vector, p);
|
|
136
|
+
}
|
|
137
|
+
points.push({ ts, count: Math.round(count), values });
|
|
138
|
+
}
|
|
139
|
+
return points.sort((a, b) => a.ts - b.ts);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.RedisMetricsHistoryProvider = RedisMetricsHistoryProvider;
|
|
143
|
+
function dayFloor(ms) {
|
|
144
|
+
const d = new Date(ms);
|
|
145
|
+
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate());
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=RedisMetricsHistoryProvider.js.map
|