@worker-manager/metrics 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -6
- package/dist/HistoryAdmin.d.ts +23 -87
- package/dist/HistoryAdmin.js +21 -330
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +9 -9
- package/dist/HistoryStore.js +22 -0
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +27 -32
- package/dist/LatencySampler.js +55 -156
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +8 -3
- package/dist/LatencyStore.js +18 -0
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +34 -10
- package/dist/MetricsRecorder.js +30 -20
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisHistoryAdmin.d.ts +95 -0
- package/dist/RedisHistoryAdmin.js +328 -0
- package/dist/RedisHistoryAdmin.js.map +1 -0
- package/dist/RedisMetricsHistoryProvider.d.ts +7 -17
- package/dist/RedisMetricsHistoryProvider.js +9 -136
- package/dist/RedisMetricsHistoryProvider.js.map +1 -1
- package/dist/RedisMetricsStore.d.ts +27 -0
- package/dist/RedisMetricsStore.js +42 -0
- package/dist/RedisMetricsStore.js.map +1 -0
- package/dist/StoreHistoryProvider.d.ts +27 -0
- package/dist/StoreHistoryProvider.js +127 -0
- package/dist/StoreHistoryProvider.js.map +1 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/jobSources.d.ts +97 -0
- package/dist/jobSources.js +248 -0
- package/dist/jobSources.js.map +1 -0
- package/dist/keys.d.ts +7 -0
- package/dist/keys.js +23 -1
- package/dist/keys.js.map +1 -1
- package/dist/postgres/PostgresMetricsHistoryProvider.d.ts +37 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js +30 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js.map +1 -0
- package/dist/postgres/PostgresMetricsStore.d.ts +65 -0
- package/dist/postgres/PostgresMetricsStore.js +88 -0
- package/dist/postgres/PostgresMetricsStore.js.map +1 -0
- package/dist/postgres/admin.d.ts +31 -0
- package/dist/postgres/admin.js +178 -0
- package/dist/postgres/admin.js.map +1 -0
- package/dist/postgres/connection.d.ts +53 -0
- package/dist/postgres/connection.js +54 -0
- package/dist/postgres/connection.js.map +1 -0
- package/dist/postgres/context.d.ts +29 -0
- package/dist/postgres/context.js +76 -0
- package/dist/postgres/context.js.map +1 -0
- package/dist/postgres/schema.d.ts +26 -0
- package/dist/postgres/schema.js +148 -0
- package/dist/postgres/schema.js.map +1 -0
- package/dist/postgres/stores.d.ts +51 -0
- package/dist/postgres/stores.js +252 -0
- package/dist/postgres/stores.js.map +1 -0
- package/dist/store.d.ts +80 -0
- package/dist/store.js +3 -0
- package/dist/store.js.map +1 -0
- package/package.json +13 -3
package/dist/LatencyStore.js
CHANGED
|
@@ -152,6 +152,10 @@ if newDay then
|
|
|
152
152
|
end
|
|
153
153
|
return 1
|
|
154
154
|
`;
|
|
155
|
+
/** Compare and delete, so a lease that already expired and was retaken is left alone. */
|
|
156
|
+
const RELEASE_LEASE = `if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end
|
|
157
|
+
return 0`;
|
|
158
|
+
/** The Redis `LatencyStorage`: packed vectors in hour and totals hashes, one EVAL per write. */
|
|
155
159
|
class LatencyStore {
|
|
156
160
|
constructor(opts) {
|
|
157
161
|
this.redis = opts.redis;
|
|
@@ -209,6 +213,20 @@ class LatencyStore {
|
|
|
209
213
|
}
|
|
210
214
|
return out;
|
|
211
215
|
}
|
|
216
|
+
async acquireLease(queue, holder, ttlMs) {
|
|
217
|
+
const held = await this.redis.set(this.keys.lease(queue), holder, 'PX', ttlMs, 'NX');
|
|
218
|
+
return held === 'OK';
|
|
219
|
+
}
|
|
220
|
+
async releaseLease(queue, holder) {
|
|
221
|
+
await this.redis.eval(RELEASE_LEASE, 1, this.keys.lease(queue), holder);
|
|
222
|
+
}
|
|
223
|
+
async readWatermark(queue) {
|
|
224
|
+
const raw = await this.redis.get(this.keys.watermark(queue));
|
|
225
|
+
return raw ? Number(raw) : null;
|
|
226
|
+
}
|
|
227
|
+
async writeWatermark(queue, ms, ttlSeconds) {
|
|
228
|
+
await this.redis.set(this.keys.watermark(queue), String(ms), 'EX', ttlSeconds);
|
|
229
|
+
}
|
|
212
230
|
/** Epoch ms for an absolute hour index, for building response timestamps. */
|
|
213
231
|
static hourToMs(hour) {
|
|
214
232
|
return hour * MS_PER_HOUR;
|
package/dist/LatencyStore.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LatencyStore.js","sourceRoot":"","sources":["../src/LatencyStore.ts"],"names":[],"mappings":";;;AACA,2CAAmF;
|
|
1
|
+
{"version":3,"file":"LatencyStore.js","sourceRoot":"","sources":["../src/LatencyStore.ts"],"names":[],"mappings":";;;AACA,2CAAmF;AACnF,iCAA+E;AAKlE,QAAA,gBAAgB,GAAG,UAAU,CAAC;AAE3C,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0DpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CjB,CAAC;AAEF,yFAAyF;AACzF,MAAM,aAAa,GAAG;gBACN,CAAC;AAEjB,gGAAgG;AAChG,MAAa,YAAY;IAKvB,YAAY,IAAuE;QACjF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAa,EACb,MAAqB,EACrB,IAAY,EACZ,MAAgB;QAEhB,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,YAAY,EACZ,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAClC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EACzC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,MAAM,CAAC,EACtC,MAAM,CAAC,IAAI,CAAC,EACZ,GAAG,EACH,IAAA,sBAAU,EAAC,MAAM,CAAC,EAClB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACnC,MAAM,CAAC,wBAAY,CAAC,CACrB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,IAAY,EAAE,EAAU;QAC1D,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,SAAS,EACT,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,wBAAgB,EAAE,GAAG,CAAC,EAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,wBAAgB,CAAC,EACzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAY,EAAE,wBAAgB,EAAE,GAAG,CAAC,EACnD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,mBAAY,EAAE,wBAAgB,CAAC,EAChD,MAAM,CAAC,IAAI,CAAC,EACZ,GAAG,EACH,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EACnC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAa,EACb,MAAqB,EACrB,WAA2B,EAC3B,IAAc;QAEd,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;YACtE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,IAAA,wBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAC1E,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBACrB,CAAC,CAAC,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC,CAAC,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,WAA2B,EAC3B,IAAc;;QAEd,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,wBAAgB,CAAC,CAAC,CAAC;YAChF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,wBAAgB,EAAE,GAAG,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAA,GAAG,CAAC,KAAK,CAAC,mCAAI,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QACrF,OAAO,IAAI,KAAK,IAAI,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc;QAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,EAAU,EAAE,UAAkB;QAChE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACjF,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,QAAQ,CAAC,IAAY;QAC1B,OAAO,IAAI,GAAG,WAAW,CAAC;IAC5B,CAAC;CACF;AApID,oCAoIC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BaseAdapter } from '@worker-manager/api/baseAdapter';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import type { MetricsConnection } from './connection';
|
|
3
|
+
import type { MetricsStore, Retention } from './store';
|
|
4
4
|
/**
|
|
5
5
|
* Minute detail is the expensive tier by two orders of magnitude, so it defaults to a week
|
|
6
6
|
* rather than the full window: long enough to match the recommended `MetricsTime.ONE_WEEK`
|
|
@@ -8,12 +8,8 @@ import { type Retention } from './HistoryStore';
|
|
|
8
8
|
* The hourly and daily rollups are cheap enough to keep for the whole window.
|
|
9
9
|
*/
|
|
10
10
|
export declare const DEFAULT_RETENTION: Retention;
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* A function is resolved on every tick, for a board whose queue set changes while it
|
|
14
|
-
* runs. An array is read once, at construction.
|
|
15
|
-
*/
|
|
16
|
-
queues: BaseAdapter[] | (() => BaseAdapter[]);
|
|
11
|
+
interface RedisOptions {
|
|
12
|
+
/** Redis. Shorthand for `store: new RedisMetricsStore({ connection, prefix })`. */
|
|
17
13
|
connection: MetricsConnection;
|
|
18
14
|
/**
|
|
19
15
|
* Redis key namespace, defaulting to `bull-board:metrics`. Set it to separate two boards
|
|
@@ -25,6 +21,24 @@ export interface MetricsRecorderOptions {
|
|
|
25
21
|
* tag to choose the slot yourself.
|
|
26
22
|
*/
|
|
27
23
|
prefix?: string;
|
|
24
|
+
store?: never;
|
|
25
|
+
}
|
|
26
|
+
interface StoreOptions {
|
|
27
|
+
/**
|
|
28
|
+
* Where history is written, e.g. `new PostgresMetricsStore({ connection, migrate: true })`
|
|
29
|
+
* for a board with no Redis. The recorder never closes a store it was handed.
|
|
30
|
+
*/
|
|
31
|
+
store: MetricsStore;
|
|
32
|
+
connection?: never;
|
|
33
|
+
prefix?: never;
|
|
34
|
+
}
|
|
35
|
+
export type MetricsRecorderOptions = RecorderBaseOptions & (RedisOptions | StoreOptions);
|
|
36
|
+
interface RecorderBaseOptions {
|
|
37
|
+
/**
|
|
38
|
+
* A function is resolved on every tick, for a board whose queue set changes while it
|
|
39
|
+
* runs. An array is read once, at construction.
|
|
40
|
+
*/
|
|
41
|
+
queues: BaseAdapter[] | (() => BaseAdapter[]);
|
|
28
42
|
/** Per-resolution retention in days. Unspecified tiers fall back to the defaults. */
|
|
29
43
|
retention?: Partial<Retention>;
|
|
30
44
|
/**
|
|
@@ -57,6 +71,12 @@ export interface MetricsRecorderOptions {
|
|
|
57
71
|
* wire this to your logger to tell an empty chart from a broken one.
|
|
58
72
|
*/
|
|
59
73
|
onLatencyError?: (error: unknown, queueName: string) => void;
|
|
74
|
+
/**
|
|
75
|
+
* Notified when a scheduled snapshot fails, say because the store is unreachable. The
|
|
76
|
+
* timer keeps running and the next tick retries; without a listener the failure is
|
|
77
|
+
* dropped rather than surfacing as an unhandled rejection.
|
|
78
|
+
*/
|
|
79
|
+
onSnapshotError?: (error: unknown) => void;
|
|
60
80
|
}
|
|
61
81
|
export declare function resolveRetention(opts: {
|
|
62
82
|
retention?: Partial<Retention>;
|
|
@@ -65,8 +85,8 @@ export declare function resolveRetention(opts: {
|
|
|
65
85
|
export declare class MetricsRecorder {
|
|
66
86
|
private readonly resolveQueues;
|
|
67
87
|
private readonly store;
|
|
68
|
-
|
|
69
|
-
private readonly
|
|
88
|
+
/** Set only when the recorder built the store itself, from a `connection`. */
|
|
89
|
+
private readonly ownedStore;
|
|
70
90
|
private readonly intervalMs;
|
|
71
91
|
private readonly lastMinute;
|
|
72
92
|
private timer;
|
|
@@ -74,10 +94,13 @@ export declare class MetricsRecorder {
|
|
|
74
94
|
private stopped;
|
|
75
95
|
readonly latencyEnabled: boolean;
|
|
76
96
|
private readonly latencySampler;
|
|
97
|
+
private readonly onSnapshotError?;
|
|
77
98
|
constructor(opts: MetricsRecorderOptions);
|
|
78
99
|
get retention(): Retention;
|
|
79
100
|
start(): void;
|
|
101
|
+
/** Stops the timer and closes a connection the recorder opened. A store handed in stays open. */
|
|
80
102
|
stop(): void;
|
|
103
|
+
private scheduledSnapshot;
|
|
81
104
|
snapshot(): Promise<void>;
|
|
82
105
|
/**
|
|
83
106
|
* Incrementally copies BullMQ's per-minute ring buffer into long-retention storage.
|
|
@@ -90,3 +113,4 @@ export declare class MetricsRecorder {
|
|
|
90
113
|
*/
|
|
91
114
|
private snapshotOne;
|
|
92
115
|
}
|
|
116
|
+
export {};
|
package/dist/MetricsRecorder.js
CHANGED
|
@@ -2,12 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MetricsRecorder = exports.DEFAULT_RETENTION = void 0;
|
|
4
4
|
exports.resolveRetention = resolveRetention;
|
|
5
|
-
const connection_1 = require("./connection");
|
|
6
5
|
const dataMapping_1 = require("./dataMapping");
|
|
7
|
-
const HistoryStore_1 = require("./HistoryStore");
|
|
8
|
-
const keys_1 = require("./keys");
|
|
9
6
|
const LatencySampler_1 = require("./LatencySampler");
|
|
10
|
-
const
|
|
7
|
+
const RedisMetricsStore_1 = require("./RedisMetricsStore");
|
|
11
8
|
const METRICS = ['completed', 'failed'];
|
|
12
9
|
const MS_PER_MINUTE = 60000;
|
|
13
10
|
const MINUTES_PER_DAY = 1440;
|
|
@@ -30,7 +27,7 @@ function resolveRetention(opts) {
|
|
|
30
27
|
}
|
|
31
28
|
class MetricsRecorder {
|
|
32
29
|
constructor(opts) {
|
|
33
|
-
var _a;
|
|
30
|
+
var _a, _b, _c;
|
|
34
31
|
this.lastMinute = new Map();
|
|
35
32
|
this.timer = null;
|
|
36
33
|
this.running = false;
|
|
@@ -38,17 +35,16 @@ class MetricsRecorder {
|
|
|
38
35
|
const { queues } = opts;
|
|
39
36
|
this.resolveQueues = typeof queues === 'function' ? queues : () => queues;
|
|
40
37
|
this.intervalMs = (_a = opts.snapshotIntervalMs) !== null && _a !== void 0 ? _a : 60000;
|
|
41
|
-
const
|
|
42
|
-
this.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
this.
|
|
38
|
+
const store = (_b = opts.store) !== null && _b !== void 0 ? _b : new RedisMetricsStore_1.RedisMetricsStore({ connection: opts.connection, prefix: opts.prefix });
|
|
39
|
+
this.ownedStore = opts.store ? null : store;
|
|
40
|
+
const retention = resolveRetention(opts);
|
|
41
|
+
this.store = store.counterStore(retention);
|
|
42
|
+
this.onSnapshotError = opts.onSnapshotError;
|
|
46
43
|
this.latencyEnabled = opts.latency !== false;
|
|
47
44
|
this.latencySampler = this.latencyEnabled
|
|
48
45
|
? new LatencySampler_1.LatencySampler({
|
|
49
|
-
redis:
|
|
50
|
-
|
|
51
|
-
store: new LatencyStore_1.LatencyStore({ redis: this.redis, keys, retention: resolveRetention(opts) }),
|
|
46
|
+
redis: (_c = store.jobClient) !== null && _c !== void 0 ? _c : undefined,
|
|
47
|
+
store: store.latencyStore(retention),
|
|
52
48
|
tickMs: this.intervalMs,
|
|
53
49
|
maxSamplesPerTick: opts.maxLatencySamplesPerTick,
|
|
54
50
|
safetyMarginMs: opts.latencySafetyMarginMs,
|
|
@@ -63,25 +59,35 @@ class MetricsRecorder {
|
|
|
63
59
|
if (this.timer) {
|
|
64
60
|
return;
|
|
65
61
|
}
|
|
66
|
-
this.timer = setInterval(() =>
|
|
67
|
-
void this.snapshot();
|
|
68
|
-
}, this.intervalMs);
|
|
62
|
+
this.timer = setInterval(() => this.scheduledSnapshot(), this.intervalMs);
|
|
69
63
|
// Do not keep the event loop alive solely for the recorder.
|
|
70
64
|
if (typeof this.timer.unref === 'function') {
|
|
71
65
|
this.timer.unref();
|
|
72
66
|
}
|
|
73
|
-
|
|
67
|
+
this.scheduledSnapshot();
|
|
74
68
|
}
|
|
69
|
+
/** Stops the timer and closes a connection the recorder opened. A store handed in stays open. */
|
|
75
70
|
stop() {
|
|
76
71
|
if (this.timer) {
|
|
77
72
|
clearInterval(this.timer);
|
|
78
73
|
this.timer = null;
|
|
79
74
|
}
|
|
80
|
-
if (this.
|
|
81
|
-
this.
|
|
75
|
+
if (this.ownedStore && !this.stopped) {
|
|
76
|
+
void this.ownedStore.close();
|
|
82
77
|
}
|
|
83
78
|
this.stopped = true;
|
|
84
79
|
}
|
|
80
|
+
scheduledSnapshot() {
|
|
81
|
+
this.snapshot().catch((error) => {
|
|
82
|
+
var _a;
|
|
83
|
+
try {
|
|
84
|
+
(_a = this.onSnapshotError) === null || _a === void 0 ? void 0 : _a.call(this, error);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// A throwing reporter must not turn a contained failure into an unhandled one.
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
}
|
|
85
91
|
async snapshot() {
|
|
86
92
|
if (this.running) {
|
|
87
93
|
return;
|
|
@@ -129,6 +135,7 @@ class MetricsRecorder {
|
|
|
129
135
|
// it. Nothing is lost that could have been retained anyway.
|
|
130
136
|
const oldestWritable = Math.floor(Date.now() / MS_PER_MINUTE) - this.store.retention.minutes * MINUTES_PER_DAY;
|
|
131
137
|
let newest = seenUpTo;
|
|
138
|
+
const fresh = [];
|
|
132
139
|
for (const point of points) {
|
|
133
140
|
if (point.minute <= seenUpTo) {
|
|
134
141
|
break; // points are newest-first; everything older is already stored
|
|
@@ -136,11 +143,14 @@ class MetricsRecorder {
|
|
|
136
143
|
if (point.minute < oldestWritable) {
|
|
137
144
|
break; // ...and everything past here is older still
|
|
138
145
|
}
|
|
139
|
-
|
|
146
|
+
fresh.push(point);
|
|
140
147
|
if (point.minute > newest) {
|
|
141
148
|
newest = point.minute;
|
|
142
149
|
}
|
|
143
150
|
}
|
|
151
|
+
// One call per queue and metric, so a SQL store can make it one transaction. The
|
|
152
|
+
// watermark only moves once the write has landed.
|
|
153
|
+
await this.store.upsertMinutes(name, metric, fresh);
|
|
144
154
|
this.lastMinute.set(cursorKey, newest);
|
|
145
155
|
}
|
|
146
156
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MetricsRecorder.js","sourceRoot":"","sources":["../src/MetricsRecorder.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"MetricsRecorder.js","sourceRoot":"","sources":["../src/MetricsRecorder.ts"],"names":[],"mappings":";;;AA8FA,4CAaC;AAxGD,+CAAwE;AACxE,qDAAkD;AAClD,2DAAwD;AAGxD,MAAM,OAAO,GAAkB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACvD,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;;;;GAKG;AACU,QAAA,iBAAiB,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AA4EhF,SAAgB,gBAAgB,CAAC,IAGhC;IACC,MAAM,IAAI,GACR,IAAI,CAAC,aAAa,KAAK,SAAS;QAC9B,CAAC,CAAC,yBAAiB;QACnB,CAAC,CAAC;YACE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC;YAChE,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,IAAI,EAAE,IAAI,CAAC,aAAa;SACzB,CAAC;IACR,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,MAAa,eAAe;IAc1B,YAAY,IAA4B;;QARvB,eAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAChD,UAAK,GAA0C,IAAI,CAAC;QACpD,YAAO,GAAG,KAAK,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QAMtB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,aAAa,GAAG,OAAO,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;QAC1E,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,kBAAkB,mCAAI,KAAK,CAAC;QACnD,MAAM,KAAK,GACT,MAAA,IAAI,CAAC,KAAK,mCAAI,IAAI,qCAAiB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5F,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5C,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC;QAC7C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc;YACvC,CAAC,CAAC,IAAI,+BAAc,CAAC;gBACjB,KAAK,EAAE,MAAA,KAAK,CAAC,SAAS,mCAAI,SAAS;gBACnC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,iBAAiB,EAAE,IAAI,CAAC,wBAAwB;gBAChD,cAAc,EAAE,IAAI,CAAC,qBAAqB;gBAC1C,OAAO,EAAE,IAAI,CAAC,cAAc;aAC7B,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1E,4DAA4D;QAC5D,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,iGAAiG;IACjG,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;;YAC9B,IAAI,CAAC;gBACH,MAAA,IAAI,CAAC,eAAe,qDAAG,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,+EAA+E;YACjF,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;gBAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;gBAChD,CAAC;gBACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,WAAW,CACvB,OAAoB,EACpB,IAAY,EACZ,MAAmB;;QAEnB,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,mCAAI,CAAC,CAAC,CAAC;QAEtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,mFAAmF;QACnF,sFAAsF;QACtF,mFAAmF;QACnF,qFAAqF;QACrF,4DAA4D;QAC5D,MAAM,cAAc,GAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,eAAe,CAAC;QAE1F,IAAI,MAAM,GAAG,QAAQ,CAAC;QACtB,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,CAAC,8DAA8D;YACvE,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBAClC,MAAM,CAAC,6CAA6C;YACtD,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;QACH,CAAC;QACD,iFAAiF;QACjF,kDAAkD;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;CACF;AAnJD,0CAmJC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { type MetricsClient } from './connection';
|
|
2
|
+
import type { HistoryStats, HistoryTier, PurgeOptions, PurgeResult } from './HistoryAdmin';
|
|
3
|
+
import { type MetricsKeys } from './keys';
|
|
4
|
+
import type { HistoryAdministration } from './store';
|
|
5
|
+
interface ParsedKey {
|
|
6
|
+
queue: string;
|
|
7
|
+
metric: string;
|
|
8
|
+
tier: HistoryTier;
|
|
9
|
+
/** ISO day the key covers, `null` for the daily totals hash. */
|
|
10
|
+
day: string | null;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Parses the three key shapes from the right, because queue names may themselves contain
|
|
14
|
+
* colons:
|
|
15
|
+
*
|
|
16
|
+
* <ns>:<queue>:<metric>:<day> minute buckets
|
|
17
|
+
* <ns>:<queue>:<metric>:hour:<day> hourly rollup
|
|
18
|
+
* <ns>:<queue>:<metric>:totals daily totals
|
|
19
|
+
*
|
|
20
|
+
* The metric segment is checked against the known set, so a queue named `hour` or one
|
|
21
|
+
* ending in `:completed` still resolves correctly. Anything that doesn't fit returns null
|
|
22
|
+
* and is then reported but never deleted, so a stray key can't be destroyed by accident.
|
|
23
|
+
*/
|
|
24
|
+
export declare function parseHistoryKey(key: string, namespace: string): ParsedKey | null;
|
|
25
|
+
/**
|
|
26
|
+
* Inspection and cleanup for the Redis keys written by `MetricsRecorder`.
|
|
27
|
+
*
|
|
28
|
+
* Every operation is confined to the recorder's namespace and driven by SCAN, so it never
|
|
29
|
+
* blocks Redis and never touches BullMQ's own keys. Deletes use UNLINK.
|
|
30
|
+
*/
|
|
31
|
+
export declare class RedisHistoryAdmin implements HistoryAdministration {
|
|
32
|
+
private readonly redis;
|
|
33
|
+
private readonly keys;
|
|
34
|
+
constructor(opts: {
|
|
35
|
+
redis: MetricsClient;
|
|
36
|
+
keys: MetricsKeys;
|
|
37
|
+
});
|
|
38
|
+
/**
|
|
39
|
+
* Per-queue footprint of the stored history.
|
|
40
|
+
*
|
|
41
|
+
* Every key has to be measured individually, since only `MEMORY USAGE` knows what a hash
|
|
42
|
+
* really costs. Issuing those one at a time would mean a round trip per key, which at a
|
|
43
|
+
* 90-day retention across a dozen queues runs into the thousands, so the measurements go
|
|
44
|
+
* out in pipelined batches instead. Still an ops-scale call rather than a hot path: it
|
|
45
|
+
* reads the whole namespace, so it belongs behind a debug endpoint, not a poll.
|
|
46
|
+
*/
|
|
47
|
+
stats(): Promise<HistoryStats>;
|
|
48
|
+
/**
|
|
49
|
+
* Size and entry count for each key, in pipelined batches so the cost is a handful of
|
|
50
|
+
* round trips rather than one per key. A key that expires between the scan and the
|
|
51
|
+
* measurement simply reads as zero rather than failing the whole call.
|
|
52
|
+
*/
|
|
53
|
+
private measure;
|
|
54
|
+
/**
|
|
55
|
+
* Deletes recorded history. Purging a single queue also subtracts that queue's minutes
|
|
56
|
+
* from the global rollup, so the cross-queue chart stays correct instead of keeping the
|
|
57
|
+
* removed queue's throughput folded into it forever.
|
|
58
|
+
*
|
|
59
|
+
* That correction covers the counter metrics only. The global runtime, waittime and
|
|
60
|
+
* queueage rollups keep the purged queue's contribution until their own retention drops
|
|
61
|
+
* it: a packed bucket vector cannot be decremented field by field, and a max gauge has no
|
|
62
|
+
* record of which queue produced the maximum, so there is nothing to subtract. The
|
|
63
|
+
* per-queue keys are still deleted either way. See SUMMABLE_METRICS.
|
|
64
|
+
*/
|
|
65
|
+
purge(opts?: PurgeOptions): Promise<PurgeResult>;
|
|
66
|
+
/**
|
|
67
|
+
* Removes one queue's buckets from the matching global key, so the cross-queue series
|
|
68
|
+
* reflects the queues that are left rather than keeping the removed queue folded in.
|
|
69
|
+
* Each tier is corrected from its own source key, because the tiers have independent
|
|
70
|
+
* retention and the minute hash may already be gone while the hourly one survives.
|
|
71
|
+
* Fields that drain to zero are dropped: the recorder never writes a zero bucket, so a
|
|
72
|
+
* leftover zero would read as recorded-but-idle instead of not recorded.
|
|
73
|
+
* Returns the number of global keys it deleted.
|
|
74
|
+
*
|
|
75
|
+
* Only the summable metrics are touched; the latency ones are skipped outright rather than
|
|
76
|
+
* silently producing a no-op subtraction of their packed values. See SUMMABLE_METRICS.
|
|
77
|
+
*/
|
|
78
|
+
private subtractDayFromGlobal;
|
|
79
|
+
/**
|
|
80
|
+
* Same idea for the daily rollup: the global totals hash is the sum of the per-queue
|
|
81
|
+
* totals hashes, so it is corrected from those rather than re-derived from day hashes,
|
|
82
|
+
* which may already have expired. Returns the number of global fields it removed.
|
|
83
|
+
*
|
|
84
|
+
* Skips the latency metrics for the same reason as subtractDayFromGlobal.
|
|
85
|
+
*/
|
|
86
|
+
private subtractTotalsFromGlobal;
|
|
87
|
+
/**
|
|
88
|
+
* SCAN over the namespace, once per master: SCAN carries no key, so a cluster client has
|
|
89
|
+
* no slot to route by and would answer from one arbitrary node. SCAN may also hand back the
|
|
90
|
+
* same key on more than one cursor iteration, which would double-count in `stats()`, so
|
|
91
|
+
* emissions are de-duped here. The set is bounded by queues x metrics x retention days.
|
|
92
|
+
*/
|
|
93
|
+
private scan;
|
|
94
|
+
}
|
|
95
|
+
export {};
|