@bull-board/metrics 0.0.0 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -0
- package/dist/HistoryAdmin.d.ts +119 -0
- package/dist/HistoryAdmin.js +325 -0
- package/dist/HistoryAdmin.js.map +1 -0
- package/dist/HistoryStore.d.ts +34 -0
- package/dist/HistoryStore.js +139 -0
- package/dist/HistoryStore.js.map +1 -0
- package/dist/MetricsRecorder.d.ts +51 -0
- package/dist/MetricsRecorder.js +131 -0
- package/dist/MetricsRecorder.js.map +1 -0
- package/dist/RedisMetricsHistoryProvider.d.ts +24 -0
- package/dist/RedisMetricsHistoryProvider.js +83 -0
- package/dist/RedisMetricsHistoryProvider.js.map +1 -0
- package/dist/dataMapping.d.ts +12 -0
- package/dist/dataMapping.js +27 -0
- package/dist/dataMapping.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/keys.d.ts +13 -0
- package/dist/keys.js +59 -0
- package/dist/keys.js.map +1 -0
- package/package.json +46 -4
- package/index.js +0 -1
package/README.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# @bull-board/metrics
|
|
2
|
+
|
|
3
|
+
> Status: Beta. The API and Redis storage layout may still change in a minor release while the feature settles. It is safe to run (opt-in, and it only writes its own namespaced keys), but pin an exact version if you depend on the storage format.
|
|
4
|
+
|
|
5
|
+
Opt-in long-retention historical job metrics for [bull-board](https://github.com/felixmosh/bull-board).
|
|
6
|
+
|
|
7
|
+
Snapshots native BullMQ per-minute metrics into long-retention Redis buckets and exposes a
|
|
8
|
+
`MetricsHistoryProvider` that feeds bull-board's history charts. Everything is opt-in: the core
|
|
9
|
+
`@bull-board/api` stays stateless.
|
|
10
|
+
|
|
11
|
+
## Precondition
|
|
12
|
+
|
|
13
|
+
Your BullMQ workers must have native metrics enabled, with a window large enough to survive any
|
|
14
|
+
recorder downtime, for example:
|
|
15
|
+
|
|
16
|
+
new Worker(name, processor, {
|
|
17
|
+
connection,
|
|
18
|
+
metrics: { maxDataPoints: MetricsTime.ONE_WEEK },
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
import { MetricsRecorder, RedisMetricsHistoryProvider } from '@bull-board/metrics';
|
|
24
|
+
|
|
25
|
+
// In your always-on worker/app process:
|
|
26
|
+
const recorder = new MetricsRecorder({
|
|
27
|
+
queues: [new BullMQAdapter(queue)],
|
|
28
|
+
connection,
|
|
29
|
+
retentionDays: 90,
|
|
30
|
+
});
|
|
31
|
+
recorder.start();
|
|
32
|
+
|
|
33
|
+
// Where you build the board:
|
|
34
|
+
createBullBoard({
|
|
35
|
+
queues,
|
|
36
|
+
serverAdapter,
|
|
37
|
+
options: { historyProvider: new RedisMetricsHistoryProvider({ connection }) },
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
On shutdown, call `recorder.stop()` and `provider.disconnect()`. Both only close the Redis connection if the recorder/provider opened it internally, so it's a safe no-op if you passed in your own `Redis` instance.
|
|
41
|
+
|
|
42
|
+
Timestamps and buckets are UTC.
|
|
43
|
+
|
|
44
|
+
## Storage
|
|
45
|
+
|
|
46
|
+
Every key lives under `bull-board:metrics:`. Each snapshot is written at three resolutions at once, each with its own retention, because they cost very different amounts:
|
|
47
|
+
|
|
48
|
+
| Tier | Default retention | Size per busy day, per queue and metric |
|
|
49
|
+
| --- | --- | --- |
|
|
50
|
+
| Minute | 7 days | ~72 KB |
|
|
51
|
+
| Hour | 90 days | ~0.3 KB |
|
|
52
|
+
| Day | 90 days | ~15 bytes |
|
|
53
|
+
|
|
54
|
+
At the defaults that's roughly 1.1 MB for a queue busy every minute of every day across both metrics, and far less for a bursty one. Minutes with no activity are never written, so the footprint follows how busy a queue is, not how long it has been recording.
|
|
55
|
+
|
|
56
|
+
const recorder = new MetricsRecorder({
|
|
57
|
+
queues,
|
|
58
|
+
connection,
|
|
59
|
+
retention: { minutes: 7, hours: 90, days: 90 },
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
The minute window is the one worth tuning: it holds essentially all the bytes, and it doubles as the recorder's catch-up window after downtime. `retentionDays: N` still works and sets the hourly and daily windows, leaving the minute window at its default.
|
|
63
|
+
|
|
64
|
+
Retention is enforced by Redis. Day-scoped keys expire on their own TTL; the daily totals hashes are trimmed to the window as each new day rolls in.
|
|
65
|
+
|
|
66
|
+
## Inspecting and clearing history
|
|
67
|
+
|
|
68
|
+
import { MetricsHistoryAdmin } from '@bull-board/metrics';
|
|
69
|
+
|
|
70
|
+
const admin = new MetricsHistoryAdmin({ connection });
|
|
71
|
+
|
|
72
|
+
await admin.stats(); // bytes per tier and per queue, day range
|
|
73
|
+
await admin.purge(); // delete everything
|
|
74
|
+
await admin.purge({ queue: 'mailer' }); // delete one queue
|
|
75
|
+
await admin.purge({ before: '2026-06-01' }); // delete anything older than a day
|
|
76
|
+
|
|
77
|
+
Both are `SCAN`-driven and confined to this package's namespace, so they never block Redis and never touch BullMQ's own keys. Purging a single queue also subtracts it from the cross-queue rollup. Call `admin.disconnect()` when done.
|
|
78
|
+
|
|
79
|
+
`RedisMetricsHistoryProvider` exposes the same two operations to the board, which turns them into a storage panel on the Metrics history page with a confirmation before anything is deleted.
|
|
80
|
+
|
|
81
|
+
## Scope
|
|
82
|
+
|
|
83
|
+
The shipped bull-board UI reads daily rollups. `getHistory` also supports hourly granularity for custom consumers (via the core's `/api/metrics/history` endpoint), though the built-in charts don't use it.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { Redis, type RedisOptions } from 'ioredis';
|
|
2
|
+
export interface TierStats {
|
|
3
|
+
keys: number;
|
|
4
|
+
/** Sum of `MEMORY USAGE` over this tier's keys, in bytes. */
|
|
5
|
+
bytes: number;
|
|
6
|
+
}
|
|
7
|
+
export interface HistoryQueueStats {
|
|
8
|
+
/** Queue name, or `__global__` for the cross-queue rollup. */
|
|
9
|
+
queue: string;
|
|
10
|
+
keys: number;
|
|
11
|
+
bytes: number;
|
|
12
|
+
/** Recorded minute buckets, the tier that drives storage size. */
|
|
13
|
+
minutes: number;
|
|
14
|
+
/** Days covered by a minute or hour hash, ascending. */
|
|
15
|
+
days: string[];
|
|
16
|
+
/** Where this queue's bytes actually sit, so a footprint can be diagnosed. */
|
|
17
|
+
tiers: Record<HistoryTier, TierStats>;
|
|
18
|
+
}
|
|
19
|
+
export interface HistoryStats {
|
|
20
|
+
keys: number;
|
|
21
|
+
bytes: number;
|
|
22
|
+
minutes: number;
|
|
23
|
+
oldestDay: string | null;
|
|
24
|
+
newestDay: string | null;
|
|
25
|
+
tiers: Record<HistoryTier, TierStats>;
|
|
26
|
+
queues: HistoryQueueStats[];
|
|
27
|
+
}
|
|
28
|
+
export interface PurgeOptions {
|
|
29
|
+
/** Limit the purge to one queue. Omit to purge every queue plus the global rollup. */
|
|
30
|
+
queue?: string;
|
|
31
|
+
/** Only drop days strictly before this date (UTC). Omit to drop everything in scope. */
|
|
32
|
+
before?: Date | string;
|
|
33
|
+
}
|
|
34
|
+
export interface PurgeResult {
|
|
35
|
+
keysDeleted: number;
|
|
36
|
+
/** Day fields removed from totals hashes. */
|
|
37
|
+
fieldsDeleted: number;
|
|
38
|
+
}
|
|
39
|
+
export interface MetricsHistoryAdminOptions {
|
|
40
|
+
connection: RedisOptions | Redis;
|
|
41
|
+
}
|
|
42
|
+
export type HistoryTier = 'minute' | 'hour' | 'day';
|
|
43
|
+
interface ParsedKey {
|
|
44
|
+
queue: string;
|
|
45
|
+
metric: string;
|
|
46
|
+
tier: HistoryTier;
|
|
47
|
+
/** ISO day the key covers, `null` for the daily totals hash. */
|
|
48
|
+
day: string | null;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Parses the three key shapes from the right, because queue names may themselves contain
|
|
52
|
+
* colons:
|
|
53
|
+
*
|
|
54
|
+
* <ns>:<queue>:<metric>:<day> minute buckets
|
|
55
|
+
* <ns>:<queue>:<metric>:hour:<day> hourly rollup
|
|
56
|
+
* <ns>:<queue>:<metric>:totals daily totals
|
|
57
|
+
*
|
|
58
|
+
* The metric segment is checked against the known set, so a queue named `hour` or one
|
|
59
|
+
* ending in `:completed` still resolves correctly. Anything that doesn't fit returns null
|
|
60
|
+
* and is then reported but never deleted, so a stray key can't be destroyed by accident.
|
|
61
|
+
*/
|
|
62
|
+
export declare function parseHistoryKey(key: string): ParsedKey | null;
|
|
63
|
+
/**
|
|
64
|
+
* Inspection and cleanup for the Redis keys written by `MetricsRecorder`.
|
|
65
|
+
*
|
|
66
|
+
* Every operation is confined to the `bull-board:metrics:` namespace and driven by SCAN,
|
|
67
|
+
* so it never blocks Redis and never touches BullMQ's own keys. Deletes use UNLINK.
|
|
68
|
+
*/
|
|
69
|
+
export declare class MetricsHistoryAdmin {
|
|
70
|
+
private readonly redis;
|
|
71
|
+
private readonly ownsRedis;
|
|
72
|
+
constructor(opts: MetricsHistoryAdminOptions);
|
|
73
|
+
disconnect(): void;
|
|
74
|
+
/**
|
|
75
|
+
* Per-queue footprint of the stored history.
|
|
76
|
+
*
|
|
77
|
+
* Every key has to be measured individually, since only `MEMORY USAGE` knows what a hash
|
|
78
|
+
* really costs. Issuing those one at a time would mean a round trip per key, which at a
|
|
79
|
+
* 90-day retention across a dozen queues runs into the thousands, so the measurements go
|
|
80
|
+
* out in pipelined batches instead. Still an ops-scale call rather than a hot path: it
|
|
81
|
+
* reads the whole namespace, so it belongs behind a debug endpoint, not a poll.
|
|
82
|
+
*/
|
|
83
|
+
stats(): Promise<HistoryStats>;
|
|
84
|
+
/**
|
|
85
|
+
* Size and entry count for each key, in pipelined batches so the cost is a handful of
|
|
86
|
+
* round trips rather than one per key. A key that expires between the scan and the
|
|
87
|
+
* measurement simply reads as zero rather than failing the whole call.
|
|
88
|
+
*/
|
|
89
|
+
private measure;
|
|
90
|
+
/**
|
|
91
|
+
* Deletes recorded history. Purging a single queue also subtracts that queue's minutes
|
|
92
|
+
* from the global rollup, so the cross-queue chart stays correct instead of keeping the
|
|
93
|
+
* removed queue's throughput folded into it forever.
|
|
94
|
+
*/
|
|
95
|
+
purge(opts?: PurgeOptions): Promise<PurgeResult>;
|
|
96
|
+
/**
|
|
97
|
+
* Removes one queue's buckets from the matching global key, so the cross-queue series
|
|
98
|
+
* reflects the queues that are left rather than keeping the removed queue folded in.
|
|
99
|
+
* Each tier is corrected from its own source key, because the tiers have independent
|
|
100
|
+
* retention and the minute hash may already be gone while the hourly one survives.
|
|
101
|
+
* Fields that drain to zero are dropped: the recorder never writes a zero bucket, so a
|
|
102
|
+
* leftover zero would read as recorded-but-idle instead of not recorded.
|
|
103
|
+
* Returns the number of global keys it deleted.
|
|
104
|
+
*/
|
|
105
|
+
private subtractDayFromGlobal;
|
|
106
|
+
/**
|
|
107
|
+
* Same idea for the daily rollup: the global totals hash is the sum of the per-queue
|
|
108
|
+
* totals hashes, so it is corrected from those rather than re-derived from day hashes,
|
|
109
|
+
* which may already have expired. Returns the number of global fields it removed.
|
|
110
|
+
*/
|
|
111
|
+
private subtractTotalsFromGlobal;
|
|
112
|
+
/**
|
|
113
|
+
* SCAN over the namespace. SCAN may hand back the same key on more than one cursor
|
|
114
|
+
* iteration, which would double-count in `stats()`, so emissions are de-duped here.
|
|
115
|
+
* The set is bounded by queues x metrics x retention days.
|
|
116
|
+
*/
|
|
117
|
+
private scan;
|
|
118
|
+
}
|
|
119
|
+
export {};
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MetricsHistoryAdmin = void 0;
|
|
4
|
+
exports.parseHistoryKey = parseHistoryKey;
|
|
5
|
+
const ioredis_1 = require("ioredis");
|
|
6
|
+
const keys_1 = require("./keys");
|
|
7
|
+
const SCAN_COUNT = 500;
|
|
8
|
+
const BATCH = 256;
|
|
9
|
+
/** Keys per pipelined `stats()` batch. Each key costs two commands. */
|
|
10
|
+
const MEASURE_BATCH = 250;
|
|
11
|
+
const DAY_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
|
|
12
|
+
const METRICS = ['completed', 'failed'];
|
|
13
|
+
/**
|
|
14
|
+
* Parses the three key shapes from the right, because queue names may themselves contain
|
|
15
|
+
* colons:
|
|
16
|
+
*
|
|
17
|
+
* <ns>:<queue>:<metric>:<day> minute buckets
|
|
18
|
+
* <ns>:<queue>:<metric>:hour:<day> hourly rollup
|
|
19
|
+
* <ns>:<queue>:<metric>:totals daily totals
|
|
20
|
+
*
|
|
21
|
+
* The metric segment is checked against the known set, so a queue named `hour` or one
|
|
22
|
+
* ending in `:completed` still resolves correctly. Anything that doesn't fit returns null
|
|
23
|
+
* and is then reported but never deleted, so a stray key can't be destroyed by accident.
|
|
24
|
+
*/
|
|
25
|
+
function parseHistoryKey(key) {
|
|
26
|
+
const prefix = `${keys_1.NAMESPACE}:`;
|
|
27
|
+
if (!key.startsWith(prefix)) {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
const parts = key.slice(prefix.length).split(':');
|
|
31
|
+
if (parts.length < 3) {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
const last = parts[parts.length - 1];
|
|
35
|
+
if (last === 'totals' && METRICS.includes(parts[parts.length - 2])) {
|
|
36
|
+
const queue = parts.slice(0, -2).join(':');
|
|
37
|
+
return queue ? { queue, metric: parts[parts.length - 2], tier: 'day', day: null } : null;
|
|
38
|
+
}
|
|
39
|
+
if (!DAY_PATTERN.test(last)) {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
if (parts.length >= 4 && parts[parts.length - 2] === keys_1.HOUR_TIER) {
|
|
43
|
+
const metric = parts[parts.length - 3];
|
|
44
|
+
const queue = parts.slice(0, -3).join(':');
|
|
45
|
+
return queue && METRICS.includes(metric) ? { queue, metric, tier: 'hour', day: last } : null;
|
|
46
|
+
}
|
|
47
|
+
const metric = parts[parts.length - 2];
|
|
48
|
+
const queue = parts.slice(0, -2).join(':');
|
|
49
|
+
return queue && METRICS.includes(metric) ? { queue, metric, tier: 'minute', day: last } : null;
|
|
50
|
+
}
|
|
51
|
+
function emptyTiers() {
|
|
52
|
+
return {
|
|
53
|
+
minute: { keys: 0, bytes: 0 },
|
|
54
|
+
hour: { keys: 0, bytes: 0 },
|
|
55
|
+
day: { keys: 0, bytes: 0 },
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** The global rollup key mirroring a per-queue key, same tier and same day. */
|
|
59
|
+
function globalKeyFor(parsed) {
|
|
60
|
+
if (parsed.day === null) {
|
|
61
|
+
return (0, keys_1.totalsHashKey)(keys_1.GLOBAL_QUEUE, parsed.metric);
|
|
62
|
+
}
|
|
63
|
+
return parsed.tier === 'hour'
|
|
64
|
+
? (0, keys_1.hourHashKey)(keys_1.GLOBAL_QUEUE, parsed.metric, parsed.day)
|
|
65
|
+
: (0, keys_1.dayHashKey)(keys_1.GLOBAL_QUEUE, parsed.metric, parsed.day);
|
|
66
|
+
}
|
|
67
|
+
function toDay(value) {
|
|
68
|
+
if (typeof value === 'string') {
|
|
69
|
+
if (!DAY_PATTERN.test(value)) {
|
|
70
|
+
throw new Error(`Expected a YYYY-MM-DD day or a Date, got "${value}"`);
|
|
71
|
+
}
|
|
72
|
+
return value;
|
|
73
|
+
}
|
|
74
|
+
return value.toISOString().slice(0, 10);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Inspection and cleanup for the Redis keys written by `MetricsRecorder`.
|
|
78
|
+
*
|
|
79
|
+
* Every operation is confined to the `bull-board:metrics:` namespace and driven by SCAN,
|
|
80
|
+
* so it never blocks Redis and never touches BullMQ's own keys. Deletes use UNLINK.
|
|
81
|
+
*/
|
|
82
|
+
class MetricsHistoryAdmin {
|
|
83
|
+
constructor(opts) {
|
|
84
|
+
if (opts.connection instanceof ioredis_1.Redis) {
|
|
85
|
+
this.redis = opts.connection;
|
|
86
|
+
this.ownsRedis = false;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
this.redis = new ioredis_1.Redis(opts.connection);
|
|
90
|
+
this.ownsRedis = true;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
disconnect() {
|
|
94
|
+
if (this.ownsRedis) {
|
|
95
|
+
this.redis.disconnect();
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Per-queue footprint of the stored history.
|
|
100
|
+
*
|
|
101
|
+
* Every key has to be measured individually, since only `MEMORY USAGE` knows what a hash
|
|
102
|
+
* really costs. Issuing those one at a time would mean a round trip per key, which at a
|
|
103
|
+
* 90-day retention across a dozen queues runs into the thousands, so the measurements go
|
|
104
|
+
* out in pipelined batches instead. Still an ops-scale call rather than a hot path: it
|
|
105
|
+
* reads the whole namespace, so it belongs behind a debug endpoint, not a poll.
|
|
106
|
+
*/
|
|
107
|
+
async stats() {
|
|
108
|
+
var _a;
|
|
109
|
+
const byQueue = new Map();
|
|
110
|
+
const tiers = emptyTiers();
|
|
111
|
+
let keys = 0;
|
|
112
|
+
let bytes = 0;
|
|
113
|
+
let minutes = 0;
|
|
114
|
+
let oldestDay = null;
|
|
115
|
+
let newestDay = null;
|
|
116
|
+
const found = [];
|
|
117
|
+
for await (const key of this.scan()) {
|
|
118
|
+
const parsed = parseHistoryKey(key);
|
|
119
|
+
if (parsed) {
|
|
120
|
+
found.push({ key, parsed });
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
const measurements = await this.measure(found.map((item) => item.key));
|
|
124
|
+
for (const [index, { parsed }] of found.entries()) {
|
|
125
|
+
const { size, len } = measurements[index];
|
|
126
|
+
const entry = (_a = byQueue.get(parsed.queue)) !== null && _a !== void 0 ? _a : {
|
|
127
|
+
queue: parsed.queue,
|
|
128
|
+
keys: 0,
|
|
129
|
+
bytes: 0,
|
|
130
|
+
minutes: 0,
|
|
131
|
+
days: [],
|
|
132
|
+
tiers: emptyTiers(),
|
|
133
|
+
};
|
|
134
|
+
entry.keys += 1;
|
|
135
|
+
entry.bytes += size;
|
|
136
|
+
entry.tiers[parsed.tier].keys += 1;
|
|
137
|
+
entry.tiers[parsed.tier].bytes += size;
|
|
138
|
+
keys += 1;
|
|
139
|
+
bytes += size;
|
|
140
|
+
tiers[parsed.tier].keys += 1;
|
|
141
|
+
tiers[parsed.tier].bytes += size;
|
|
142
|
+
if (parsed.tier === 'minute') {
|
|
143
|
+
entry.minutes += len;
|
|
144
|
+
minutes += len;
|
|
145
|
+
}
|
|
146
|
+
if (parsed.day) {
|
|
147
|
+
entry.days.push(parsed.day);
|
|
148
|
+
if (oldestDay === null || parsed.day < oldestDay) {
|
|
149
|
+
oldestDay = parsed.day;
|
|
150
|
+
}
|
|
151
|
+
if (newestDay === null || parsed.day > newestDay) {
|
|
152
|
+
newestDay = parsed.day;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
byQueue.set(parsed.queue, entry);
|
|
156
|
+
}
|
|
157
|
+
const queues = [...byQueue.values()].sort((a, b) => b.bytes - a.bytes);
|
|
158
|
+
for (const queue of queues) {
|
|
159
|
+
queue.days = [...new Set(queue.days)].sort();
|
|
160
|
+
}
|
|
161
|
+
return { keys, bytes, minutes, oldestDay, newestDay, tiers, queues };
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Size and entry count for each key, in pipelined batches so the cost is a handful of
|
|
165
|
+
* round trips rather than one per key. A key that expires between the scan and the
|
|
166
|
+
* measurement simply reads as zero rather than failing the whole call.
|
|
167
|
+
*/
|
|
168
|
+
async measure(keys) {
|
|
169
|
+
var _a, _b, _c, _d;
|
|
170
|
+
const out = [];
|
|
171
|
+
for (let i = 0; i < keys.length; i += MEASURE_BATCH) {
|
|
172
|
+
const chunk = keys.slice(i, i + MEASURE_BATCH);
|
|
173
|
+
const pipeline = this.redis.pipeline();
|
|
174
|
+
for (const key of chunk) {
|
|
175
|
+
pipeline.memory('USAGE', key);
|
|
176
|
+
pipeline.hlen(key);
|
|
177
|
+
}
|
|
178
|
+
const res = await pipeline.exec();
|
|
179
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
180
|
+
out.push({
|
|
181
|
+
size: Number((_b = (_a = res === null || res === void 0 ? void 0 : res[j * 2]) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : 0) || 0,
|
|
182
|
+
len: Number((_d = (_c = res === null || res === void 0 ? void 0 : res[j * 2 + 1]) === null || _c === void 0 ? void 0 : _c[1]) !== null && _d !== void 0 ? _d : 0) || 0,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return out;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Deletes recorded history. Purging a single queue also subtracts that queue's minutes
|
|
190
|
+
* from the global rollup, so the cross-queue chart stays correct instead of keeping the
|
|
191
|
+
* removed queue's throughput folded into it forever.
|
|
192
|
+
*/
|
|
193
|
+
async purge(opts = {}) {
|
|
194
|
+
const before = opts.before === undefined ? null : toDay(opts.before);
|
|
195
|
+
const result = { keysDeleted: 0, fieldsDeleted: 0 };
|
|
196
|
+
const dayKeys = [];
|
|
197
|
+
const totalsKeys = [];
|
|
198
|
+
for await (const key of this.scan()) {
|
|
199
|
+
const parsed = parseHistoryKey(key);
|
|
200
|
+
if (!parsed) {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (opts.queue !== undefined && parsed.queue !== opts.queue) {
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
if (parsed.day === null) {
|
|
207
|
+
totalsKeys.push({ key, parsed });
|
|
208
|
+
}
|
|
209
|
+
else if (before === null || parsed.day < before) {
|
|
210
|
+
dayKeys.push({ key, parsed });
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
// Rewriting the global rollup only makes sense when a single queue is being removed:
|
|
214
|
+
// a full purge drops the global keys outright, along with everything else.
|
|
215
|
+
const adjustGlobal = opts.queue !== undefined && opts.queue !== keys_1.GLOBAL_QUEUE;
|
|
216
|
+
for (const { key, parsed } of dayKeys) {
|
|
217
|
+
if (adjustGlobal) {
|
|
218
|
+
result.keysDeleted += await this.subtractDayFromGlobal(key, parsed);
|
|
219
|
+
}
|
|
220
|
+
result.keysDeleted += await this.redis.unlink(key);
|
|
221
|
+
}
|
|
222
|
+
for (const { key, parsed } of totalsKeys) {
|
|
223
|
+
const stale = (await this.redis.hkeys(key)).filter((day) => before === null || day < before);
|
|
224
|
+
if (adjustGlobal && stale.length > 0) {
|
|
225
|
+
const totals = await this.redis.hmget(key, ...stale);
|
|
226
|
+
result.fieldsDeleted += await this.subtractTotalsFromGlobal(parsed.metric, stale, totals);
|
|
227
|
+
}
|
|
228
|
+
if (before === null) {
|
|
229
|
+
result.keysDeleted += await this.redis.unlink(key);
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
for (let i = 0; i < stale.length; i += BATCH) {
|
|
233
|
+
result.fieldsDeleted += await this.redis.hdel(key, ...stale.slice(i, i + BATCH));
|
|
234
|
+
}
|
|
235
|
+
if ((await this.redis.hlen(key)) === 0) {
|
|
236
|
+
result.keysDeleted += await this.redis.unlink(key);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Removes one queue's buckets from the matching global key, so the cross-queue series
|
|
243
|
+
* reflects the queues that are left rather than keeping the removed queue folded in.
|
|
244
|
+
* Each tier is corrected from its own source key, because the tiers have independent
|
|
245
|
+
* retention and the minute hash may already be gone while the hourly one survives.
|
|
246
|
+
* Fields that drain to zero are dropped: the recorder never writes a zero bucket, so a
|
|
247
|
+
* leftover zero would read as recorded-but-idle instead of not recorded.
|
|
248
|
+
* Returns the number of global keys it deleted.
|
|
249
|
+
*/
|
|
250
|
+
async subtractDayFromGlobal(key, parsed) {
|
|
251
|
+
if (parsed.day === null) {
|
|
252
|
+
return 0;
|
|
253
|
+
}
|
|
254
|
+
const minutes = await this.redis.hgetall(key);
|
|
255
|
+
const globalDay = globalKeyFor(parsed);
|
|
256
|
+
const pipeline = this.redis.multi();
|
|
257
|
+
const touched = [];
|
|
258
|
+
for (const field of Object.keys(minutes)) {
|
|
259
|
+
const value = Number(minutes[field]) || 0;
|
|
260
|
+
if (value === 0) {
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
touched.push(field);
|
|
264
|
+
pipeline.hincrby(globalDay, field, -value);
|
|
265
|
+
}
|
|
266
|
+
if (touched.length === 0) {
|
|
267
|
+
return 0;
|
|
268
|
+
}
|
|
269
|
+
const res = await pipeline.exec();
|
|
270
|
+
const drained = touched.filter((_, i) => { var _a, _b; return Number((_b = (_a = res === null || res === void 0 ? void 0 : res[i]) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : 0) <= 0; });
|
|
271
|
+
for (let i = 0; i < drained.length; i += BATCH) {
|
|
272
|
+
await this.redis.hdel(globalDay, ...drained.slice(i, i + BATCH));
|
|
273
|
+
}
|
|
274
|
+
return (await this.redis.hlen(globalDay)) === 0 ? await this.redis.unlink(globalDay) : 0;
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Same idea for the daily rollup: the global totals hash is the sum of the per-queue
|
|
278
|
+
* totals hashes, so it is corrected from those rather than re-derived from day hashes,
|
|
279
|
+
* which may already have expired. Returns the number of global fields it removed.
|
|
280
|
+
*/
|
|
281
|
+
async subtractTotalsFromGlobal(metric, days, values) {
|
|
282
|
+
const globalTotals = (0, keys_1.totalsHashKey)(keys_1.GLOBAL_QUEUE, metric);
|
|
283
|
+
const pipeline = this.redis.multi();
|
|
284
|
+
const touched = [];
|
|
285
|
+
days.forEach((day, i) => {
|
|
286
|
+
const value = Number(values[i]) || 0;
|
|
287
|
+
if (value === 0) {
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
touched.push(day);
|
|
291
|
+
pipeline.hincrby(globalTotals, day, -value);
|
|
292
|
+
});
|
|
293
|
+
if (touched.length === 0) {
|
|
294
|
+
return 0;
|
|
295
|
+
}
|
|
296
|
+
const res = await pipeline.exec();
|
|
297
|
+
const drained = touched.filter((_, i) => { var _a, _b; return Number((_b = (_a = res === null || res === void 0 ? void 0 : res[i]) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : 0) <= 0; });
|
|
298
|
+
let removed = 0;
|
|
299
|
+
for (let i = 0; i < drained.length; i += BATCH) {
|
|
300
|
+
removed += await this.redis.hdel(globalTotals, ...drained.slice(i, i + BATCH));
|
|
301
|
+
}
|
|
302
|
+
return removed;
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* SCAN over the namespace. SCAN may hand back the same key on more than one cursor
|
|
306
|
+
* iteration, which would double-count in `stats()`, so emissions are de-duped here.
|
|
307
|
+
* The set is bounded by queues x metrics x retention days.
|
|
308
|
+
*/
|
|
309
|
+
async *scan() {
|
|
310
|
+
const seen = new Set();
|
|
311
|
+
let cursor = '0';
|
|
312
|
+
do {
|
|
313
|
+
const [next, batch] = await this.redis.scan(cursor, 'MATCH', `${keys_1.NAMESPACE}:*`, 'COUNT', SCAN_COUNT);
|
|
314
|
+
cursor = next;
|
|
315
|
+
for (const key of batch) {
|
|
316
|
+
if (!seen.has(key)) {
|
|
317
|
+
seen.add(key);
|
|
318
|
+
yield key;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
} while (cursor !== '0');
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
exports.MetricsHistoryAdmin = MetricsHistoryAdmin;
|
|
325
|
+
//# sourceMappingURL=HistoryAdmin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HistoryAdmin.js","sourceRoot":"","sources":["../src/HistoryAdmin.ts"],"names":[],"mappings":";;;AA8EA,0CA0BC;AAxGD,qCAAmD;AACnD,iCAAoG;AAEpG,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,KAAK,GAAG,GAAG,CAAC;AAClB,uEAAuE;AACvE,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,WAAW,GAAG,qBAAqB,CAAC;AAC1C,MAAM,OAAO,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AA0DxC;;;;;;;;;;;GAWG;AACH,SAAgB,eAAe,CAAC,GAAW;IACzC,MAAM,MAAM,GAAG,GAAG,gBAAS,GAAG,CAAC;IAC/B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAErC,IAAI,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3C,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3F,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,gBAAS,EAAE,CAAC;QAC/D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3C,OAAO,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACjG,CAAC;AAED,SAAS,UAAU;IACjB,OAAO;QACL,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC7B,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC3B,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KAC3B,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,SAAS,YAAY,CAAC,MAAiB;IACrC,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;QACxB,OAAO,IAAA,oBAAa,EAAC,mBAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,KAAK,MAAM;QAC3B,CAAC,CAAC,IAAA,kBAAW,EAAC,mBAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC;QACtD,CAAC,CAAC,IAAA,iBAAU,EAAC,mBAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,KAAoB;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,GAAG,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;GAKG;AACH,MAAa,mBAAmB;IAI9B,YAAY,IAAgC;QAC1C,IAAI,IAAI,CAAC,UAAU,YAAY,eAAK,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,UAAU;QACR,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,KAAK;;QACT,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;QACrD,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,SAAS,GAAkB,IAAI,CAAC;QAEpC,MAAM,KAAK,GAAyC,EAAE,CAAC;QACvD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,MAAM,EAAE,CAAC;gBACX,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAEvE,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAE1C,MAAM,KAAK,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,EAAE;gBACR,KAAK,EAAE,UAAU,EAAE;aACpB,CAAC;YACF,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;YAChB,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC;YACpB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YACnC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YACvC,IAAI,IAAI,CAAC,CAAC;YACV,KAAK,IAAI,IAAI,CAAC;YACd,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YAC7B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YAEjC,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC;gBACrB,OAAO,IAAI,GAAG,CAAC;YACjB,CAAC;YACD,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,SAAS,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;oBACjD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;gBACzB,CAAC;gBACD,IAAI,SAAS,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;oBACjD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC;gBACzB,CAAC;YACH,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,OAAO,CAAC,IAAc;;QAClC,MAAM,GAAG,GAAoC,EAAE,CAAC;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,aAAa,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC9B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,GAAG,CAAC,IAAI,CAAC;oBACP,IAAI,EAAE,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,GAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC;oBACzC,GAAG,EAAE,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC;iBAC7C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK,CAAC,OAAqB,EAAE;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrE,MAAM,MAAM,GAAgB,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QACjE,MAAM,OAAO,GAAyC,EAAE,CAAC;QACzD,MAAM,UAAU,GAAyC,EAAE,CAAC;QAE5D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC5D,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;gBACxB,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YACnC,CAAC;iBAAM,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC;gBAClD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,qFAAqF;QACrF,2EAA2E;QAC3E,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,mBAAY,CAAC;QAE7E,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;YACtC,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC;YAC7F,IAAI,YAAY,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACrD,MAAM,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;YAC5F,CAAC;YACD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACnD,SAAS;YACX,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;gBAC7C,MAAM,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,qBAAqB,CAAC,GAAW,EAAE,MAAiB;QAChE,IAAI,MAAM,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,eAAC,OAAA,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,wBAAwB,CACpC,MAAc,EACd,IAAc,EACd,MAAyB;QAEzB,MAAM,YAAY,GAAG,IAAA,oBAAa,EAAC,mBAAY,EAAE,MAAM,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO;YACT,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,eAAC,OAAA,MAAM,CAAC,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,CAAC,CAAC,0CAAG,CAAC,CAAC,mCAAI,CAAC,CAAC,IAAI,CAAC,CAAA,EAAA,CAAC,CAAC;QAC1E,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/C,OAAO,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,CAAC,IAAI;QACjB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,GAAG,CAAC;YACF,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACzC,MAAM,EACN,OAAO,EACP,GAAG,gBAAS,IAAI,EAChB,OAAO,EACP,UAAU,CACX,CAAC;YACF,MAAM,GAAG,IAAI,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACd,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;IAC3B,CAAC;CACF;AA3QD,kDA2QC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Redis } from 'ioredis';
|
|
2
|
+
export interface Retention {
|
|
3
|
+
/** Days of minute-level detail. Doubles as the recorder's catch-up window. */
|
|
4
|
+
minutes: number;
|
|
5
|
+
/** Days of hourly rollup. */
|
|
6
|
+
hours: number;
|
|
7
|
+
/** Days of daily totals, which is what the shipped charts read. */
|
|
8
|
+
days: number;
|
|
9
|
+
}
|
|
10
|
+
export declare class HistoryStore {
|
|
11
|
+
private readonly redis;
|
|
12
|
+
readonly retention: Retention;
|
|
13
|
+
constructor(opts: {
|
|
14
|
+
redis: Redis;
|
|
15
|
+
retention: Retention;
|
|
16
|
+
});
|
|
17
|
+
upsertMinute(queue: string, metric: string, minute: number, value: number): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Raw HMGET of the totals hash: `null` means the day was never recorded, a present
|
|
20
|
+
* string (including `'0'`) means it was. Distinguishing "missing" from "stored zero"
|
|
21
|
+
* matters for callers deciding between empty history vs. a zero-backfilled series.
|
|
22
|
+
*/
|
|
23
|
+
readDailyTotalsRaw(queue: string, metric: string, days: string[]): Promise<(string | null)[]>;
|
|
24
|
+
readDayMinutes(queue: string, metric: string, day: string): Promise<Record<string, number>>;
|
|
25
|
+
/**
|
|
26
|
+
* Hourly buckets for one day, keyed by absolute hour index.
|
|
27
|
+
*
|
|
28
|
+
* Falls back to folding the minute hash when the hourly rollup is absent, which covers
|
|
29
|
+
* days recorded before the rollup existed. Once those minute hashes age out, the days
|
|
30
|
+
* they cover are already outside the minute window anyway.
|
|
31
|
+
*/
|
|
32
|
+
readDayHours(queue: string, metric: string, day: string): Promise<Record<string, number>>;
|
|
33
|
+
private readNumericHash;
|
|
34
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HistoryStore = void 0;
|
|
4
|
+
const keys_1 = require("./keys");
|
|
5
|
+
/**
|
|
6
|
+
* Atomic, idempotent upsert of one minute bucket into all three resolutions at once.
|
|
7
|
+
*
|
|
8
|
+
* KEYS[1] queue minute hash ARGV[1] minute field ARGV[5] minute-tier ttl seconds
|
|
9
|
+
* KEYS[2] queue hour hash ARGV[2] hour field ARGV[6] hour-tier ttl seconds
|
|
10
|
+
* KEYS[3] queue totals hash ARGV[3] day field ARGV[7] day-tier ttl seconds
|
|
11
|
+
* KEYS[4] global minute hash ARGV[4] value ARGV[8] oldest day to keep
|
|
12
|
+
* KEYS[5] global hour hash
|
|
13
|
+
* KEYS[6] global totals hash
|
|
14
|
+
*
|
|
15
|
+
* The minute hash is the ledger the whole thing is built on: `delta` is the difference
|
|
16
|
+
* against the value already stored there, so re-snapshotting an overlapping window (a
|
|
17
|
+
* restart, or a second recorder process) applies a delta of zero and the coarser tiers
|
|
18
|
+
* never double-count. Rolling up on write rather than compacting later keeps that property:
|
|
19
|
+
* one script, one delta, every resolution consistent, nothing to schedule or resume.
|
|
20
|
+
*
|
|
21
|
+
* Retention is per tier. Day-scoped keys fall off on their own, since their TTL is only
|
|
22
|
+
* refreshed while that day is being written, so each dies its tier's retention after the
|
|
23
|
+
* day it holds. The totals hashes are written every day, so their TTL keeps rolling forward
|
|
24
|
+
* and they would otherwise grow a field per day forever; they are trimmed here instead.
|
|
25
|
+
* Day fields are ISO `YYYY-MM-DD`, which sorts lexicographically, so a plain string compare
|
|
26
|
+
* against the cutoff is enough. Trimming only runs on the first write of a new day, which
|
|
27
|
+
* is about once a day per queue.
|
|
28
|
+
*/
|
|
29
|
+
const UPSERT_MINUTE = `
|
|
30
|
+
local function trim(key, cutoff)
|
|
31
|
+
local fields = redis.call('HKEYS', key)
|
|
32
|
+
local stale = {}
|
|
33
|
+
for i = 1, #fields do
|
|
34
|
+
if fields[i] < cutoff then
|
|
35
|
+
stale[#stale + 1] = fields[i]
|
|
36
|
+
if #stale == 256 then
|
|
37
|
+
redis.call('HDEL', key, unpack(stale))
|
|
38
|
+
stale = {}
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
if #stale > 0 then
|
|
43
|
+
redis.call('HDEL', key, unpack(stale))
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
local old = tonumber(redis.call('HGET', KEYS[1], ARGV[1]) or '0')
|
|
48
|
+
local val = tonumber(ARGV[4])
|
|
49
|
+
if val == old then
|
|
50
|
+
return 0
|
|
51
|
+
end
|
|
52
|
+
local delta = val - old
|
|
53
|
+
local newDay = redis.call('HEXISTS', KEYS[3], ARGV[3]) == 0
|
|
54
|
+
redis.call('HSET', KEYS[1], ARGV[1], val)
|
|
55
|
+
redis.call('HINCRBY', KEYS[2], ARGV[2], delta)
|
|
56
|
+
redis.call('HINCRBY', KEYS[3], ARGV[3], delta)
|
|
57
|
+
redis.call('HINCRBY', KEYS[4], ARGV[1], delta)
|
|
58
|
+
redis.call('HINCRBY', KEYS[5], ARGV[2], delta)
|
|
59
|
+
redis.call('HINCRBY', KEYS[6], ARGV[3], delta)
|
|
60
|
+
redis.call('EXPIRE', KEYS[1], ARGV[5])
|
|
61
|
+
redis.call('EXPIRE', KEYS[2], ARGV[6])
|
|
62
|
+
redis.call('EXPIRE', KEYS[3], ARGV[7])
|
|
63
|
+
redis.call('EXPIRE', KEYS[4], ARGV[5])
|
|
64
|
+
redis.call('EXPIRE', KEYS[5], ARGV[6])
|
|
65
|
+
redis.call('EXPIRE', KEYS[6], ARGV[7])
|
|
66
|
+
if newDay then
|
|
67
|
+
trim(KEYS[3], ARGV[8])
|
|
68
|
+
trim(KEYS[6], ARGV[8])
|
|
69
|
+
end
|
|
70
|
+
return delta
|
|
71
|
+
`;
|
|
72
|
+
const SECONDS_PER_DAY = 86400;
|
|
73
|
+
function ttl(days) {
|
|
74
|
+
return String(Math.max(1, Math.floor(days * SECONDS_PER_DAY)));
|
|
75
|
+
}
|
|
76
|
+
class HistoryStore {
|
|
77
|
+
constructor(opts) {
|
|
78
|
+
this.redis = opts.redis;
|
|
79
|
+
this.retention = {
|
|
80
|
+
minutes: Math.max(1, Math.floor(opts.retention.minutes)),
|
|
81
|
+
hours: Math.max(1, Math.floor(opts.retention.hours)),
|
|
82
|
+
days: Math.max(1, Math.floor(opts.retention.days)),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
async upsertMinute(queue, metric, minute, value) {
|
|
86
|
+
const day = (0, keys_1.minuteToDay)(minute);
|
|
87
|
+
await this.redis.eval(UPSERT_MINUTE, 6, (0, keys_1.dayHashKey)(queue, metric, day), (0, keys_1.hourHashKey)(queue, metric, day), (0, keys_1.totalsHashKey)(queue, metric), (0, keys_1.dayHashKey)(keys_1.GLOBAL_QUEUE, metric, day), (0, keys_1.hourHashKey)(keys_1.GLOBAL_QUEUE, metric, day), (0, keys_1.totalsHashKey)(keys_1.GLOBAL_QUEUE, metric), String(minute), String((0, keys_1.minuteToHour)(minute)), day, String(value), ttl(this.retention.minutes), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days));
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Raw HMGET of the totals hash: `null` means the day was never recorded, a present
|
|
91
|
+
* string (including `'0'`) means it was. Distinguishing "missing" from "stored zero"
|
|
92
|
+
* matters for callers deciding between empty history vs. a zero-backfilled series.
|
|
93
|
+
*/
|
|
94
|
+
async readDailyTotalsRaw(queue, metric, days) {
|
|
95
|
+
if (days.length === 0) {
|
|
96
|
+
return [];
|
|
97
|
+
}
|
|
98
|
+
// Pass `days` as a single array (ioredis flattens one level internally) instead of
|
|
99
|
+
// spreading it: spreading blows the JS argument-count limit for large ranges
|
|
100
|
+
// ("Maximum call stack size exceeded"). The cast works around ioredis's types only
|
|
101
|
+
// declaring the spread overload; kept as a method call to preserve `this` binding.
|
|
102
|
+
const redis = this.redis;
|
|
103
|
+
return redis.hmget((0, keys_1.totalsHashKey)(queue, metric), days);
|
|
104
|
+
}
|
|
105
|
+
async readDayMinutes(queue, metric, day) {
|
|
106
|
+
return this.readNumericHash((0, keys_1.dayHashKey)(queue, metric, day));
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Hourly buckets for one day, keyed by absolute hour index.
|
|
110
|
+
*
|
|
111
|
+
* Falls back to folding the minute hash when the hourly rollup is absent, which covers
|
|
112
|
+
* days recorded before the rollup existed. Once those minute hashes age out, the days
|
|
113
|
+
* they cover are already outside the minute window anyway.
|
|
114
|
+
*/
|
|
115
|
+
async readDayHours(queue, metric, day) {
|
|
116
|
+
var _a;
|
|
117
|
+
const hours = await this.readNumericHash((0, keys_1.hourHashKey)(queue, metric, day));
|
|
118
|
+
if (Object.keys(hours).length > 0) {
|
|
119
|
+
return hours;
|
|
120
|
+
}
|
|
121
|
+
const minutes = await this.readDayMinutes(queue, metric, day);
|
|
122
|
+
const folded = {};
|
|
123
|
+
for (const field of Object.keys(minutes)) {
|
|
124
|
+
const hour = String((0, keys_1.minuteToHour)(Number(field)));
|
|
125
|
+
folded[hour] = ((_a = folded[hour]) !== null && _a !== void 0 ? _a : 0) + minutes[field];
|
|
126
|
+
}
|
|
127
|
+
return folded;
|
|
128
|
+
}
|
|
129
|
+
async readNumericHash(key) {
|
|
130
|
+
const raw = await this.redis.hgetall(key);
|
|
131
|
+
const out = {};
|
|
132
|
+
for (const field of Object.keys(raw)) {
|
|
133
|
+
out[field] = Number(raw[field]) || 0;
|
|
134
|
+
}
|
|
135
|
+
return out;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
exports.HistoryStore = HistoryStore;
|
|
139
|
+
//# sourceMappingURL=HistoryStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HistoryStore.js","sourceRoot":"","sources":["../src/HistoryStore.ts"],"names":[],"mappings":";;;AACA,iCAQgB;AAWhB;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0CrB,CAAC;AAEF,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAa,YAAY;IAIvB,YAAY,IAA4C;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACxD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACpD,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,KAAa;QAC7E,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,MAAM,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,aAAa,EACb,CAAC,EACD,IAAA,iBAAU,EAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAC9B,IAAA,kBAAW,EAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAC/B,IAAA,oBAAa,EAAC,KAAK,EAAE,MAAM,CAAC,EAC5B,IAAA,iBAAU,EAAC,mBAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EACrC,IAAA,kBAAW,EAAC,mBAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EACtC,IAAA,oBAAa,EAAC,mBAAY,EAAE,MAAM,CAAC,EACnC,MAAM,CAAC,MAAM,CAAC,EACd,MAAM,CAAC,IAAA,mBAAY,EAAC,MAAM,CAAC,CAAC,EAC5B,GAAG,EACH,MAAM,CAAC,KAAK,CAAC,EACb,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAC3B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB,CACtB,KAAa,EACb,MAAc,EACd,IAAc;QAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,mFAAmF;QACnF,6EAA6E;QAC7E,mFAAmF;QACnF,mFAAmF;QACnF,MAAM,KAAK,GAAG,IAAI,CAAC,KAElB,CAAC;QACF,OAAO,KAAK,CAAC,KAAK,CAAC,IAAA,oBAAa,EAAC,KAAK,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,KAAa,EACb,MAAc,EACd,GAAW;QAEX,OAAO,IAAI,CAAC,eAAe,CAAC,IAAA,iBAAU,EAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,GAAW;;QAC3D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAA,kBAAW,EAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1E,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9D,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAA,mBAAY,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAA,MAAM,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AA/FD,oCA+FC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { BaseAdapter } from '@bull-board/api/baseAdapter';
|
|
2
|
+
import { Redis, type RedisOptions } from 'ioredis';
|
|
3
|
+
import { type Retention } from './HistoryStore';
|
|
4
|
+
/**
|
|
5
|
+
* Minute detail is the expensive tier by two orders of magnitude, so it defaults to a week
|
|
6
|
+
* rather than the full window: long enough to match the recommended `MetricsTime.ONE_WEEK`
|
|
7
|
+
* worker buffer, so the recorder can be down for a week and still catch up completely.
|
|
8
|
+
* The hourly and daily rollups are cheap enough to keep for the whole window.
|
|
9
|
+
*/
|
|
10
|
+
export declare const DEFAULT_RETENTION: Retention;
|
|
11
|
+
export interface MetricsRecorderOptions {
|
|
12
|
+
queues: BaseAdapter[];
|
|
13
|
+
connection: RedisOptions | Redis;
|
|
14
|
+
/** Per-resolution retention in days. Unspecified tiers fall back to the defaults. */
|
|
15
|
+
retention?: Partial<Retention>;
|
|
16
|
+
/**
|
|
17
|
+
* Shorthand that sets the daily and hourly windows. Minute retention stays at its
|
|
18
|
+
* default unless raised explicitly, since that is the tier that drives storage size.
|
|
19
|
+
*/
|
|
20
|
+
retentionDays?: number;
|
|
21
|
+
snapshotIntervalMs?: number;
|
|
22
|
+
}
|
|
23
|
+
export declare function resolveRetention(opts: {
|
|
24
|
+
retention?: Partial<Retention>;
|
|
25
|
+
retentionDays?: number;
|
|
26
|
+
}): Retention;
|
|
27
|
+
export declare class MetricsRecorder {
|
|
28
|
+
private readonly queues;
|
|
29
|
+
private readonly store;
|
|
30
|
+
private readonly redis;
|
|
31
|
+
private readonly ownsRedis;
|
|
32
|
+
private readonly intervalMs;
|
|
33
|
+
private readonly lastMinute;
|
|
34
|
+
private timer;
|
|
35
|
+
private running;
|
|
36
|
+
constructor(opts: MetricsRecorderOptions);
|
|
37
|
+
get retention(): Retention;
|
|
38
|
+
start(): void;
|
|
39
|
+
stop(): void;
|
|
40
|
+
snapshot(): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* Incrementally copies BullMQ's per-minute ring buffer into long-retention storage.
|
|
43
|
+
* `seenUpTo` is a per-(queue, metric) watermark of the newest minute already written.
|
|
44
|
+
* getMetrics() returns points newest-first, so we walk from the newest and stop at the
|
|
45
|
+
* first minute we've already stored: everything past it is older and stored too. Fresh
|
|
46
|
+
* minutes are upserted (safe against overlapping windows across ticks), then the
|
|
47
|
+
* watermark advances. So the first tick backfills the buffer and every later tick only
|
|
48
|
+
* writes the minutes that appeared since.
|
|
49
|
+
*/
|
|
50
|
+
private snapshotOne;
|
|
51
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MetricsRecorder = exports.DEFAULT_RETENTION = void 0;
|
|
4
|
+
exports.resolveRetention = resolveRetention;
|
|
5
|
+
const ioredis_1 = require("ioredis");
|
|
6
|
+
const dataMapping_1 = require("./dataMapping");
|
|
7
|
+
const HistoryStore_1 = require("./HistoryStore");
|
|
8
|
+
const METRICS = ['completed', 'failed'];
|
|
9
|
+
const MS_PER_MINUTE = 60000;
|
|
10
|
+
const MINUTES_PER_DAY = 1440;
|
|
11
|
+
/**
|
|
12
|
+
* Minute detail is the expensive tier by two orders of magnitude, so it defaults to a week
|
|
13
|
+
* rather than the full window: long enough to match the recommended `MetricsTime.ONE_WEEK`
|
|
14
|
+
* worker buffer, so the recorder can be down for a week and still catch up completely.
|
|
15
|
+
* The hourly and daily rollups are cheap enough to keep for the whole window.
|
|
16
|
+
*/
|
|
17
|
+
exports.DEFAULT_RETENTION = { minutes: 7, hours: 90, days: 90 };
|
|
18
|
+
function resolveRetention(opts) {
|
|
19
|
+
const base = opts.retentionDays === undefined
|
|
20
|
+
? exports.DEFAULT_RETENTION
|
|
21
|
+
: {
|
|
22
|
+
minutes: Math.min(exports.DEFAULT_RETENTION.minutes, opts.retentionDays),
|
|
23
|
+
hours: opts.retentionDays,
|
|
24
|
+
days: opts.retentionDays,
|
|
25
|
+
};
|
|
26
|
+
return { ...base, ...opts.retention };
|
|
27
|
+
}
|
|
28
|
+
class MetricsRecorder {
|
|
29
|
+
constructor(opts) {
|
|
30
|
+
var _a;
|
|
31
|
+
this.lastMinute = new Map();
|
|
32
|
+
this.timer = null;
|
|
33
|
+
this.running = false;
|
|
34
|
+
this.queues = opts.queues;
|
|
35
|
+
this.intervalMs = (_a = opts.snapshotIntervalMs) !== null && _a !== void 0 ? _a : 60000;
|
|
36
|
+
if (opts.connection instanceof ioredis_1.Redis) {
|
|
37
|
+
this.redis = opts.connection;
|
|
38
|
+
this.ownsRedis = false;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
this.redis = new ioredis_1.Redis(opts.connection);
|
|
42
|
+
this.ownsRedis = true;
|
|
43
|
+
}
|
|
44
|
+
this.store = new HistoryStore_1.HistoryStore({ redis: this.redis, retention: resolveRetention(opts) });
|
|
45
|
+
}
|
|
46
|
+
get retention() {
|
|
47
|
+
return this.store.retention;
|
|
48
|
+
}
|
|
49
|
+
start() {
|
|
50
|
+
if (this.timer) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
this.timer = setInterval(() => {
|
|
54
|
+
void this.snapshot();
|
|
55
|
+
}, this.intervalMs);
|
|
56
|
+
// Do not keep the event loop alive solely for the recorder.
|
|
57
|
+
if (typeof this.timer.unref === 'function') {
|
|
58
|
+
this.timer.unref();
|
|
59
|
+
}
|
|
60
|
+
void this.snapshot();
|
|
61
|
+
}
|
|
62
|
+
stop() {
|
|
63
|
+
if (this.timer) {
|
|
64
|
+
clearInterval(this.timer);
|
|
65
|
+
this.timer = null;
|
|
66
|
+
}
|
|
67
|
+
if (this.ownsRedis) {
|
|
68
|
+
this.redis.disconnect();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
async snapshot() {
|
|
72
|
+
if (this.running) {
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
this.running = true;
|
|
76
|
+
try {
|
|
77
|
+
for (const adapter of this.queues) {
|
|
78
|
+
const name = adapter.getName();
|
|
79
|
+
for (const metric of METRICS) {
|
|
80
|
+
await this.snapshotOne(adapter, name, metric);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
finally {
|
|
85
|
+
this.running = false;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Incrementally copies BullMQ's per-minute ring buffer into long-retention storage.
|
|
90
|
+
* `seenUpTo` is a per-(queue, metric) watermark of the newest minute already written.
|
|
91
|
+
* getMetrics() returns points newest-first, so we walk from the newest and stop at the
|
|
92
|
+
* first minute we've already stored: everything past it is older and stored too. Fresh
|
|
93
|
+
* minutes are upserted (safe against overlapping windows across ticks), then the
|
|
94
|
+
* watermark advances. So the first tick backfills the buffer and every later tick only
|
|
95
|
+
* writes the minutes that appeared since.
|
|
96
|
+
*/
|
|
97
|
+
async snapshotOne(adapter, name, metric) {
|
|
98
|
+
var _a;
|
|
99
|
+
const cursorKey = `${name}:${metric}`;
|
|
100
|
+
const seenUpTo = (_a = this.lastMinute.get(cursorKey)) !== null && _a !== void 0 ? _a : -1;
|
|
101
|
+
const metrics = await adapter.getMetrics(metric).catch(() => null);
|
|
102
|
+
const points = (0, dataMapping_1.metricsToMinutePoints)(metrics);
|
|
103
|
+
if (points.length === 0) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
// Correctness guard, not an optimization. Idempotency comes from the minute hash
|
|
107
|
+
// holding the previously written value, so a minute whose hash has already expired
|
|
108
|
+
// would look brand new and be added to the hourly and daily rollups a second time.
|
|
109
|
+
// That can only happen when the worker's metrics buffer reaches further back than the
|
|
110
|
+
// minute window (say a two-week buffer against a one-week window) and the recorder
|
|
111
|
+
// restarts, losing its in-memory watermark. Refusing to write past the window closes
|
|
112
|
+
// it. Nothing is lost that could have been retained anyway.
|
|
113
|
+
const oldestWritable = Math.floor(Date.now() / MS_PER_MINUTE) - this.store.retention.minutes * MINUTES_PER_DAY;
|
|
114
|
+
let newest = seenUpTo;
|
|
115
|
+
for (const point of points) {
|
|
116
|
+
if (point.minute <= seenUpTo) {
|
|
117
|
+
break; // points are newest-first; everything older is already stored
|
|
118
|
+
}
|
|
119
|
+
if (point.minute < oldestWritable) {
|
|
120
|
+
break; // ...and everything past here is older still
|
|
121
|
+
}
|
|
122
|
+
await this.store.upsertMinute(name, metric, point.minute, point.value);
|
|
123
|
+
if (point.minute > newest) {
|
|
124
|
+
newest = point.minute;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
this.lastMinute.set(cursorKey, newest);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
exports.MetricsRecorder = MetricsRecorder;
|
|
131
|
+
//# sourceMappingURL=MetricsRecorder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MetricsRecorder.js","sourceRoot":"","sources":["../src/MetricsRecorder.ts"],"names":[],"mappings":";;;AA+BA,4CAaC;AA1CD,qCAAmD;AACnD,+CAAsD;AACtD,iDAA8D;AAE9D,MAAM,OAAO,GAAkB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACvD,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;;;;GAKG;AACU,QAAA,iBAAiB,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AAehF,SAAgB,gBAAgB,CAAC,IAGhC;IACC,MAAM,IAAI,GACR,IAAI,CAAC,aAAa,KAAK,SAAS;QAC9B,CAAC,CAAC,yBAAiB;QACnB,CAAC,CAAC;YACE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC;YAChE,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,IAAI,EAAE,IAAI,CAAC,aAAa;SACzB,CAAC;IACR,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,MAAa,eAAe;IAU1B,YAAY,IAA4B;;QAJvB,eAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAChD,UAAK,GAA0C,IAAI,CAAC;QACpD,YAAO,GAAG,KAAK,CAAC;QAGtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,kBAAkB,mCAAI,KAAK,CAAC;QACnD,IAAI,IAAI,CAAC,UAAU,YAAY,eAAK,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,2BAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,4DAA4D;QAC5D,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;IACvB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,WAAW,CACvB,OAAoB,EACpB,IAAY,EACZ,MAAmB;;QAEnB,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,mCAAI,CAAC,CAAC,CAAC;QAEtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,IAAA,mCAAqB,EAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,mFAAmF;QACnF,sFAAsF;QACtF,mFAAmF;QACnF,qFAAqF;QACrF,4DAA4D;QAC5D,MAAM,cAAc,GAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,eAAe,CAAC;QAE1F,IAAI,MAAM,GAAG,QAAQ,CAAC;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,CAAC,8DAA8D;YACvE,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBAClC,MAAM,CAAC,6CAA6C;YACtD,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACvE,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;CACF;AApHD,0CAoHC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { MetricsHistoryPoint, MetricsHistoryProvider, MetricsHistoryQuery } from '@bull-board/api/typings/app';
|
|
2
|
+
import { Redis, type RedisOptions } from 'ioredis';
|
|
3
|
+
import { type HistoryStats, type PurgeOptions, type PurgeResult } from './HistoryAdmin';
|
|
4
|
+
import { type Retention } from './HistoryStore';
|
|
5
|
+
export interface RedisMetricsHistoryProviderOptions {
|
|
6
|
+
connection: RedisOptions | Redis;
|
|
7
|
+
/** Should mirror the recorder's retention. Only used to bound the query span. */
|
|
8
|
+
retention?: Partial<Retention>;
|
|
9
|
+
retentionDays?: number;
|
|
10
|
+
}
|
|
11
|
+
export declare class RedisMetricsHistoryProvider implements MetricsHistoryProvider {
|
|
12
|
+
private readonly store;
|
|
13
|
+
private readonly admin;
|
|
14
|
+
private readonly redis;
|
|
15
|
+
private readonly ownsRedis;
|
|
16
|
+
private readonly retentionDays;
|
|
17
|
+
constructor(opts: RedisMetricsHistoryProviderOptions);
|
|
18
|
+
disconnect(): void;
|
|
19
|
+
/** Backs the board's storage panel. See MetricsHistoryAdmin.stats. */
|
|
20
|
+
getUsage(): Promise<HistoryStats>;
|
|
21
|
+
/** Backs the board's "clear history" action. See MetricsHistoryAdmin.purge. */
|
|
22
|
+
purge(options?: PurgeOptions): Promise<PurgeResult>;
|
|
23
|
+
getHistory(query: MetricsHistoryQuery): Promise<MetricsHistoryPoint[]>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisMetricsHistoryProvider = void 0;
|
|
4
|
+
const ioredis_1 = require("ioredis");
|
|
5
|
+
const HistoryAdmin_1 = require("./HistoryAdmin");
|
|
6
|
+
const HistoryStore_1 = require("./HistoryStore");
|
|
7
|
+
const keys_1 = require("./keys");
|
|
8
|
+
const MetricsRecorder_1 = require("./MetricsRecorder");
|
|
9
|
+
const MS_PER_HOUR = 3600000;
|
|
10
|
+
class RedisMetricsHistoryProvider {
|
|
11
|
+
constructor(opts) {
|
|
12
|
+
if (opts.connection instanceof ioredis_1.Redis) {
|
|
13
|
+
this.redis = opts.connection;
|
|
14
|
+
this.ownsRedis = false;
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
this.redis = new ioredis_1.Redis(opts.connection);
|
|
18
|
+
this.ownsRedis = true;
|
|
19
|
+
}
|
|
20
|
+
const retention = (0, MetricsRecorder_1.resolveRetention)(opts);
|
|
21
|
+
this.retentionDays = retention.days;
|
|
22
|
+
this.store = new HistoryStore_1.HistoryStore({ redis: this.redis, retention });
|
|
23
|
+
this.admin = new HistoryAdmin_1.MetricsHistoryAdmin({ connection: this.redis });
|
|
24
|
+
}
|
|
25
|
+
disconnect() {
|
|
26
|
+
if (this.ownsRedis) {
|
|
27
|
+
this.redis.disconnect();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Backs the board's storage panel. See MetricsHistoryAdmin.stats. */
|
|
31
|
+
async getUsage() {
|
|
32
|
+
return this.admin.stats();
|
|
33
|
+
}
|
|
34
|
+
/** Backs the board's "clear history" action. See MetricsHistoryAdmin.purge. */
|
|
35
|
+
async purge(options = {}) {
|
|
36
|
+
return this.admin.purge(options);
|
|
37
|
+
}
|
|
38
|
+
async getHistory(query) {
|
|
39
|
+
var _a, _b;
|
|
40
|
+
const queue = (_a = query.queue) !== null && _a !== void 0 ? _a : keys_1.GLOBAL_QUEUE;
|
|
41
|
+
// Clamp the span to the retention window so an unbounded `from` (e.g. 0) can't make
|
|
42
|
+
// dayRange produce an unbounded number of day buckets -- older data doesn't exist anyway.
|
|
43
|
+
const maxSpanMs = (this.retentionDays + 1) * 86400000;
|
|
44
|
+
const from = Math.max(query.from, query.to - maxSpanMs);
|
|
45
|
+
const days = (0, keys_1.dayRange)(from, query.to);
|
|
46
|
+
if (query.granularity === 'day') {
|
|
47
|
+
const rawTotals = await this.store.readDailyTotalsRaw(queue, query.metric, days);
|
|
48
|
+
// Empty history only when no day in range was ever recorded (all fields missing).
|
|
49
|
+
// A day with a stored '0' still counts as recorded -- otherwise the UI's empty
|
|
50
|
+
// state would be unreachable once any data exists.
|
|
51
|
+
if (rawTotals.every((value) => value == null)) {
|
|
52
|
+
return [];
|
|
53
|
+
}
|
|
54
|
+
const totals = {};
|
|
55
|
+
days.forEach((day, i) => {
|
|
56
|
+
totals[day] = Number(rawTotals[i]) || 0;
|
|
57
|
+
});
|
|
58
|
+
return days
|
|
59
|
+
.map((day) => { var _a; return ({ ts: (0, keys_1.dayToStartMs)(day), value: (_a = totals[day]) !== null && _a !== void 0 ? _a : 0 }); })
|
|
60
|
+
.filter((p) => p.ts >= dayFloor(query.from) && p.ts <= query.to);
|
|
61
|
+
}
|
|
62
|
+
const hourBuckets = new Map();
|
|
63
|
+
const dayHours = await Promise.all(days.map((day) => this.store.readDayHours(queue, query.metric, day)));
|
|
64
|
+
for (const hours of dayHours) {
|
|
65
|
+
for (const field of Object.keys(hours)) {
|
|
66
|
+
const ts = Number(field) * MS_PER_HOUR;
|
|
67
|
+
if (ts < query.from || ts > query.to) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
hourBuckets.set(ts, ((_b = hourBuckets.get(ts)) !== null && _b !== void 0 ? _b : 0) + hours[field]);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return [...hourBuckets.entries()]
|
|
74
|
+
.map(([ts, value]) => ({ ts, value }))
|
|
75
|
+
.sort((a, b) => a.ts - b.ts);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
exports.RedisMetricsHistoryProvider = RedisMetricsHistoryProvider;
|
|
79
|
+
function dayFloor(ms) {
|
|
80
|
+
const d = new Date(ms);
|
|
81
|
+
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate());
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=RedisMetricsHistoryProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RedisMetricsHistoryProvider.js","sourceRoot":"","sources":["../src/RedisMetricsHistoryProvider.ts"],"names":[],"mappings":";;;AAKA,qCAAmD;AACnD,iDAKwB;AACxB,iDAA8D;AAC9D,iCAA8D;AAC9D,uDAAqD;AAErD,MAAM,WAAW,GAAG,OAAO,CAAC;AAS5B,MAAa,2BAA2B;IAOtC,YAAY,IAAwC;QAClD,IAAI,IAAI,CAAC,UAAU,YAAY,eAAK,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,MAAM,SAAS,GAAG,IAAA,kCAAgB,EAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,2BAAY,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,KAAK,GAAG,IAAI,kCAAmB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,UAAU;QACR,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,KAAK,CAAC,UAAwB,EAAE;QACpC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAA0B;;QACzC,MAAM,KAAK,GAAG,MAAA,KAAK,CAAC,KAAK,mCAAI,mBAAY,CAAC;QAC1C,oFAAoF;QACpF,0FAA0F;QAC1F,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,IAAA,eAAQ,EAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QAEtC,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACjF,kFAAkF;YAClF,+EAA+E;YAC/E,mDAAmD;YACnD,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC9C,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,MAAM,MAAM,GAA2B,EAAE,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;gBACtB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1C,CAAC,CAAC,CAAC;YACH,OAAO,IAAI;iBACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,CAAC,EAAE,EAAE,EAAE,IAAA,mBAAY,EAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAA,MAAM,CAAC,GAAG,CAAC,mCAAI,CAAC,EAAE,CAAC,CAAA,EAAA,CAAC;iBAClE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CACrE,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC;gBACvC,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,IAAI,EAAE,GAAG,KAAK,CAAC,EAAE,EAAE,CAAC;oBACrC,SAAS;gBACX,CAAC;gBACD,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,MAAA,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,mCAAI,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;aACrC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;CACF;AA/ED,kEA+EC;AAED,SAAS,QAAQ,CAAC,EAAU;IAC1B,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;IACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { QueueMetrics } from '@bull-board/api/typings/app';
|
|
2
|
+
export interface MinutePoint {
|
|
3
|
+
/** Absolute minute index: Math.floor(timestampMs / 60000). */
|
|
4
|
+
minute: number;
|
|
5
|
+
value: number;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Maps BullMQ's getMetrics() data (newest-first) to (minute, value) points.
|
|
9
|
+
* data[i] -> minute index floor(prevTS/60000) - 1 - i. The in-progress current
|
|
10
|
+
* minute is never present in data, so every point returned here is immutable.
|
|
11
|
+
*/
|
|
12
|
+
export declare function metricsToMinutePoints(metrics: QueueMetrics | null | undefined): MinutePoint[];
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.metricsToMinutePoints = metricsToMinutePoints;
|
|
4
|
+
const MS_PER_MINUTE = 60000;
|
|
5
|
+
/**
|
|
6
|
+
* Maps BullMQ's getMetrics() data (newest-first) to (minute, value) points.
|
|
7
|
+
* data[i] -> minute index floor(prevTS/60000) - 1 - i. The in-progress current
|
|
8
|
+
* minute is never present in data, so every point returned here is immutable.
|
|
9
|
+
*/
|
|
10
|
+
function metricsToMinutePoints(metrics) {
|
|
11
|
+
var _a, _b;
|
|
12
|
+
if (!metrics || !metrics.data || metrics.data.length === 0) {
|
|
13
|
+
return [];
|
|
14
|
+
}
|
|
15
|
+
const prevTS = (_b = (_a = metrics.meta) === null || _a === void 0 ? void 0 : _a.prevTS) !== null && _b !== void 0 ? _b : 0;
|
|
16
|
+
const newestMinute = Math.floor(prevTS / MS_PER_MINUTE) - 1;
|
|
17
|
+
const points = [];
|
|
18
|
+
for (let i = 0; i < metrics.data.length; i++) {
|
|
19
|
+
const minute = newestMinute - i;
|
|
20
|
+
if (minute < 0) {
|
|
21
|
+
break;
|
|
22
|
+
}
|
|
23
|
+
points.push({ minute, value: Number(metrics.data[i]) || 0 });
|
|
24
|
+
}
|
|
25
|
+
return points;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=dataMapping.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dataMapping.js","sourceRoot":"","sources":["../src/dataMapping.ts"],"names":[],"mappings":";;AAeA,sDAgBC;AAvBD,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B;;;;GAIG;AACH,SAAgB,qBAAqB,CAAC,OAAwC;;IAC5E,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,MAAM,GAAG,MAAA,MAAA,OAAO,CAAC,IAAI,0CAAE,MAAM,mCAAI,CAAC,CAAC;IACzC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,YAAY,GAAG,CAAC,CAAC;QAChC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,MAAM;QACR,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { MetricsHistoryAdmin } from './HistoryAdmin';
|
|
2
|
+
export type { HistoryQueueStats, HistoryStats, MetricsHistoryAdminOptions, PurgeOptions, PurgeResult, } from './HistoryAdmin';
|
|
3
|
+
export { MetricsRecorder } from './MetricsRecorder';
|
|
4
|
+
export type { MetricsRecorderOptions } from './MetricsRecorder';
|
|
5
|
+
export { RedisMetricsHistoryProvider } from './RedisMetricsHistoryProvider';
|
|
6
|
+
export type { RedisMetricsHistoryProviderOptions } from './RedisMetricsHistoryProvider';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedisMetricsHistoryProvider = exports.MetricsRecorder = exports.MetricsHistoryAdmin = void 0;
|
|
4
|
+
var HistoryAdmin_1 = require("./HistoryAdmin");
|
|
5
|
+
Object.defineProperty(exports, "MetricsHistoryAdmin", { enumerable: true, get: function () { return HistoryAdmin_1.MetricsHistoryAdmin; } });
|
|
6
|
+
var MetricsRecorder_1 = require("./MetricsRecorder");
|
|
7
|
+
Object.defineProperty(exports, "MetricsRecorder", { enumerable: true, get: function () { return MetricsRecorder_1.MetricsRecorder; } });
|
|
8
|
+
var RedisMetricsHistoryProvider_1 = require("./RedisMetricsHistoryProvider");
|
|
9
|
+
Object.defineProperty(exports, "RedisMetricsHistoryProvider", { enumerable: true, get: function () { return RedisMetricsHistoryProvider_1.RedisMetricsHistoryProvider; } });
|
|
10
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,+CAAqD;AAA5C,mHAAA,mBAAmB,OAAA;AAQ5B,qDAAoD;AAA3C,kHAAA,eAAe,OAAA;AAExB,6EAA4E;AAAnE,0IAAA,2BAA2B,OAAA"}
|
package/dist/keys.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const NAMESPACE = "bull-board:metrics";
|
|
2
|
+
export declare const GLOBAL_QUEUE = "__global__";
|
|
3
|
+
/** Marks the hourly rollup key so it can't be mistaken for a minute-level day hash. */
|
|
4
|
+
export declare const HOUR_TIER = "hour";
|
|
5
|
+
export declare function minuteToDay(minute: number): string;
|
|
6
|
+
/** Absolute hour index, the hourly counterpart of the absolute minute index. */
|
|
7
|
+
export declare function minuteToHour(minute: number): number;
|
|
8
|
+
export declare function dayHashKey(queue: string, metric: string, day: string): string;
|
|
9
|
+
export declare function hourHashKey(queue: string, metric: string, day: string): string;
|
|
10
|
+
export declare function totalsHashKey(queue: string, metric: string): string;
|
|
11
|
+
export declare function shiftDay(day: string, offsetDays: number): string;
|
|
12
|
+
export declare function dayToStartMs(day: string): number;
|
|
13
|
+
export declare function dayRange(fromMs: number, toMs: number): string[];
|
package/dist/keys.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HOUR_TIER = exports.GLOBAL_QUEUE = exports.NAMESPACE = void 0;
|
|
4
|
+
exports.minuteToDay = minuteToDay;
|
|
5
|
+
exports.minuteToHour = minuteToHour;
|
|
6
|
+
exports.dayHashKey = dayHashKey;
|
|
7
|
+
exports.hourHashKey = hourHashKey;
|
|
8
|
+
exports.totalsHashKey = totalsHashKey;
|
|
9
|
+
exports.shiftDay = shiftDay;
|
|
10
|
+
exports.dayToStartMs = dayToStartMs;
|
|
11
|
+
exports.dayRange = dayRange;
|
|
12
|
+
exports.NAMESPACE = 'bull-board:metrics';
|
|
13
|
+
exports.GLOBAL_QUEUE = '__global__';
|
|
14
|
+
/** Marks the hourly rollup key so it can't be mistaken for a minute-level day hash. */
|
|
15
|
+
exports.HOUR_TIER = 'hour';
|
|
16
|
+
const MS_PER_MINUTE = 60000;
|
|
17
|
+
const MS_PER_DAY = 86400000;
|
|
18
|
+
const MINUTES_PER_HOUR = 60;
|
|
19
|
+
function pad(n) {
|
|
20
|
+
return n < 10 ? `0${n}` : String(n);
|
|
21
|
+
}
|
|
22
|
+
function msToDay(ms) {
|
|
23
|
+
const d = new Date(ms);
|
|
24
|
+
return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}`;
|
|
25
|
+
}
|
|
26
|
+
function minuteToDay(minute) {
|
|
27
|
+
return msToDay(minute * MS_PER_MINUTE);
|
|
28
|
+
}
|
|
29
|
+
/** Absolute hour index, the hourly counterpart of the absolute minute index. */
|
|
30
|
+
function minuteToHour(minute) {
|
|
31
|
+
return Math.floor(minute / MINUTES_PER_HOUR);
|
|
32
|
+
}
|
|
33
|
+
function dayHashKey(queue, metric, day) {
|
|
34
|
+
return `${exports.NAMESPACE}:${queue}:${metric}:${day}`;
|
|
35
|
+
}
|
|
36
|
+
function hourHashKey(queue, metric, day) {
|
|
37
|
+
return `${exports.NAMESPACE}:${queue}:${metric}:${exports.HOUR_TIER}:${day}`;
|
|
38
|
+
}
|
|
39
|
+
function totalsHashKey(queue, metric) {
|
|
40
|
+
return `${exports.NAMESPACE}:${queue}:${metric}:totals`;
|
|
41
|
+
}
|
|
42
|
+
function shiftDay(day, offsetDays) {
|
|
43
|
+
return msToDay(dayToStartMs(day) + offsetDays * MS_PER_DAY);
|
|
44
|
+
}
|
|
45
|
+
function dayToStartMs(day) {
|
|
46
|
+
const [y, m, d] = day.split('-').map(Number);
|
|
47
|
+
return Date.UTC(y, m - 1, d);
|
|
48
|
+
}
|
|
49
|
+
function dayRange(fromMs, toMs) {
|
|
50
|
+
const days = [];
|
|
51
|
+
let cursor = Date.UTC(new Date(fromMs).getUTCFullYear(), new Date(fromMs).getUTCMonth(), new Date(fromMs).getUTCDate());
|
|
52
|
+
const end = toMs;
|
|
53
|
+
while (cursor <= end) {
|
|
54
|
+
days.push(msToDay(cursor));
|
|
55
|
+
cursor += MS_PER_DAY;
|
|
56
|
+
}
|
|
57
|
+
return days;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=keys.js.map
|
package/dist/keys.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keys.js","sourceRoot":"","sources":["../src/keys.ts"],"names":[],"mappings":";;;AAkBA,kCAEC;AAGD,oCAEC;AAED,gCAEC;AAED,kCAEC;AAED,sCAEC;AAED,4BAEC;AAED,oCAGC;AAED,4BAaC;AA7DY,QAAA,SAAS,GAAG,oBAAoB,CAAC;AACjC,QAAA,YAAY,GAAG,YAAY,CAAC;AACzC,uFAAuF;AAC1E,QAAA,SAAS,GAAG,MAAM,CAAC;AAEhC,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,UAAU,GAAG,QAAQ,CAAC;AAC5B,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,OAAO,CAAC,EAAU;IACzB,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;IACvB,OAAO,GAAG,CAAC,CAAC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;AACpF,CAAC;AAED,SAAgB,WAAW,CAAC,MAAc;IACxC,OAAO,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;AACzC,CAAC;AAED,gFAAgF;AAChF,SAAgB,YAAY,CAAC,MAAc;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,gBAAgB,CAAC,CAAC;AAC/C,CAAC;AAED,SAAgB,UAAU,CAAC,KAAa,EAAE,MAAc,EAAE,GAAW;IACnE,OAAO,GAAG,iBAAS,IAAI,KAAK,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;AAClD,CAAC;AAED,SAAgB,WAAW,CAAC,KAAa,EAAE,MAAc,EAAE,GAAW;IACpE,OAAO,GAAG,iBAAS,IAAI,KAAK,IAAI,MAAM,IAAI,iBAAS,IAAI,GAAG,EAAE,CAAC;AAC/D,CAAC;AAED,SAAgB,aAAa,CAAC,KAAa,EAAE,MAAc;IACzD,OAAO,GAAG,iBAAS,IAAI,KAAK,IAAI,MAAM,SAAS,CAAC;AAClD,CAAC;AAED,SAAgB,QAAQ,CAAC,GAAW,EAAE,UAAkB;IACtD,OAAO,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,UAAU,GAAG,UAAU,CAAC,CAAC;AAC9D,CAAC;AAED,SAAgB,YAAY,CAAC,GAAW;IACtC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,SAAgB,QAAQ,CAAC,MAAc,EAAE,IAAY;IACnD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CACnB,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE,EACjC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAC9B,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,CAAC;IACjB,OAAO,MAAM,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,49 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bull-board/metrics",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "8.3.0",
|
|
4
|
+
"description": "Opt-in long-retention historical job metrics recorder and provider for bull-board.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"bull",
|
|
7
|
+
"bullmq",
|
|
8
|
+
"dashboard",
|
|
9
|
+
"metrics",
|
|
10
|
+
"history",
|
|
11
|
+
"queue",
|
|
12
|
+
"redis"
|
|
13
|
+
],
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"author": "felixmosh",
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git+https://github.com/felixmosh/bull-board.git",
|
|
19
|
+
"directory": "packages/metrics"
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
"dist"
|
|
23
|
+
],
|
|
24
|
+
"main": "dist/index.js",
|
|
25
|
+
"types": "dist/index.d.ts",
|
|
26
|
+
"publishConfig": {
|
|
27
|
+
"access": "public"
|
|
28
|
+
},
|
|
29
|
+
"scripts": {
|
|
30
|
+
"build": "yarn clean && tsc",
|
|
31
|
+
"clean": "rm -rf dist",
|
|
32
|
+
"test": "jest"
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@bull-board/api": "8.3.0"
|
|
36
|
+
},
|
|
37
|
+
"peerDependencies": {
|
|
38
|
+
"ioredis": "^5.0.0"
|
|
39
|
+
},
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"@types/jest": "^30.0.0",
|
|
42
|
+
"@types/node": "^22.20.0",
|
|
43
|
+
"bullmq": "^5.80.9",
|
|
44
|
+
"ioredis": "^5.11.1",
|
|
45
|
+
"jest": "^30.4.2",
|
|
46
|
+
"ts-jest": "^29.4.11",
|
|
47
|
+
"typescript": "^5.9.3"
|
|
48
|
+
}
|
|
7
49
|
}
|
package/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
module.exports = {};
|