@worker-manager/metrics 2.0.1 → 2.1.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/HistoryAdmin.d.ts +11 -0
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +7 -2
- package/dist/HistoryStore.js +18 -4
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +10 -0
- package/dist/LatencySampler.js +47 -25
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +2 -2
- package/dist/LatencyStore.js +4 -4
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +22 -2
- package/dist/MetricsRecorder.js +45 -16
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisHistoryAdmin.js +17 -12
- package/dist/RedisHistoryAdmin.js.map +1 -1
- package/dist/counterSources.d.ts +45 -0
- package/dist/counterSources.js +34 -0
- package/dist/counterSources.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/namespacedHistoryProvider.d.ts +29 -0
- package/dist/namespacedHistoryProvider.js +95 -0
- package/dist/namespacedHistoryProvider.js.map +1 -0
- package/dist/postgres/admin.js +13 -9
- package/dist/postgres/admin.js.map +1 -1
- package/dist/postgres/stores.d.ts +6 -4
- package/dist/postgres/stores.js +17 -14
- package/dist/postgres/stores.js.map +1 -1
- package/dist/store.d.ts +15 -3
- package/package.json +2 -2
package/dist/HistoryAdmin.d.ts
CHANGED
|
@@ -35,6 +35,17 @@ export interface PurgeOptions {
|
|
|
35
35
|
queue?: string;
|
|
36
36
|
/** Only drop days strictly before this date (UTC). Omit to drop everything in scope. */
|
|
37
37
|
before?: Date | string;
|
|
38
|
+
/**
|
|
39
|
+
* The cross-queue series a single-queue purge subtracts the queue's counts from. Defaults to
|
|
40
|
+
* `__global__`; a queue recorded from a `CounterSource` with its own `rollup` (a pg-boss
|
|
41
|
+
* queue) names that rollup here, so purging it does not drain another board's totals.
|
|
42
|
+
*/
|
|
43
|
+
rollup?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Limit the purge to queues whose recorded name starts with this, the rollup included. What
|
|
46
|
+
* `namespacedHistoryProvider` uses so a board's "clear all" stays inside its namespace.
|
|
47
|
+
*/
|
|
48
|
+
queuePrefix?: string;
|
|
38
49
|
}
|
|
39
50
|
export interface PurgeResult {
|
|
40
51
|
keysDeleted: number;
|
package/dist/HistoryAdmin.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HistoryAdmin.js","sourceRoot":"","sources":["../src/HistoryAdmin.ts"],"names":[],"mappings":";;;AACA,2DAAwD;AAGxD,yDAAsD;AAA7C,oHAAA,eAAe,OAAA;
|
|
1
|
+
{"version":3,"file":"HistoryAdmin.js","sourceRoot":"","sources":["../src/HistoryAdmin.ts"],"names":[],"mappings":";;;AACA,2DAAwD;AAGxD,yDAAsD;AAA7C,oHAAA,eAAe,OAAA;AA2ExB;;;GAGG;AACH,MAAa,mBAAmB;IAK9B,YAAY,IAAgC;QAC1C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,IAAI,qCAAiB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1F,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iGAAiG;IACjG,UAAU;;QACR,KAAK,CAAA,MAAA,IAAI,CAAC,UAAU,0CAAE,KAAK,EAAE,CAAA,CAAC;IAChC,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAqB,EAAE;QAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;CACF;AA5BD,kDA4BC"}
|
package/dist/HistoryStore.d.ts
CHANGED
|
@@ -17,9 +17,14 @@ export declare class HistoryStore implements CounterStore {
|
|
|
17
17
|
keys?: MetricsKeys;
|
|
18
18
|
retention: Retention;
|
|
19
19
|
});
|
|
20
|
-
upsertMinute(queue: string, metric: string, minute: number, value: number): Promise<void>;
|
|
20
|
+
upsertMinute(queue: string, metric: string, minute: number, value: number, rollup?: string): Promise<void>;
|
|
21
21
|
/** One EVAL per minute, in order, exactly as a caller looping `upsertMinute` would. */
|
|
22
|
-
upsertMinutes(queue: string, metric: string, points: MinutePoint[]): Promise<void>;
|
|
22
|
+
upsertMinutes(queue: string, metric: string, points: MinutePoint[], rollup?: string): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Walks the minute hashes back from today, one HKEYS per day, and stops at the first day that
|
|
25
|
+
* holds anything. Bounded by the minute window, which is as far back as a minute can exist.
|
|
26
|
+
*/
|
|
27
|
+
latestMinute(queue: string, metric: string): Promise<number | null>;
|
|
23
28
|
readDailyTotals(queue: string, metric: string, days: string[]): Promise<(number | null)[]>;
|
|
24
29
|
readHours(queue: string, metric: string, days: string[]): Promise<Record<string, number>>;
|
|
25
30
|
/**
|
package/dist/HistoryStore.js
CHANGED
|
@@ -89,16 +89,30 @@ class HistoryStore {
|
|
|
89
89
|
days: Math.max(1, Math.floor(opts.retention.days)),
|
|
90
90
|
};
|
|
91
91
|
}
|
|
92
|
-
async upsertMinute(queue, metric, minute, value) {
|
|
92
|
+
async upsertMinute(queue, metric, minute, value, rollup = keys_1.GLOBAL_QUEUE) {
|
|
93
93
|
const day = (0, keys_1.minuteToDay)(minute);
|
|
94
|
-
await this.redis.eval(UPSERT_MINUTE, 6, this.keys.day(queue, metric, day), this.keys.hour(queue, metric, day), this.keys.totals(queue, metric), this.keys.day(
|
|
94
|
+
await this.redis.eval(UPSERT_MINUTE, 6, this.keys.day(queue, metric, day), this.keys.hour(queue, metric, day), this.keys.totals(queue, metric), this.keys.day(rollup, metric, day), this.keys.hour(rollup, metric, day), this.keys.totals(rollup, metric), String(minute), String((0, keys_1.minuteToHour)(minute)), day, String(value), ttl(this.retention.minutes), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days));
|
|
95
95
|
}
|
|
96
96
|
/** One EVAL per minute, in order, exactly as a caller looping `upsertMinute` would. */
|
|
97
|
-
async upsertMinutes(queue, metric, points) {
|
|
97
|
+
async upsertMinutes(queue, metric, points, rollup = keys_1.GLOBAL_QUEUE) {
|
|
98
98
|
for (const point of points) {
|
|
99
|
-
await this.upsertMinute(queue, metric, point.minute, point.value);
|
|
99
|
+
await this.upsertMinute(queue, metric, point.minute, point.value, rollup);
|
|
100
100
|
}
|
|
101
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* Walks the minute hashes back from today, one HKEYS per day, and stops at the first day that
|
|
104
|
+
* holds anything. Bounded by the minute window, which is as far back as a minute can exist.
|
|
105
|
+
*/
|
|
106
|
+
async latestMinute(queue, metric) {
|
|
107
|
+
const today = (0, keys_1.msToDay)(Date.now());
|
|
108
|
+
for (let offset = 0; offset <= this.retention.minutes; offset++) {
|
|
109
|
+
const fields = await this.redis.hkeys(this.keys.day(queue, metric, (0, keys_1.shiftDay)(today, -offset)));
|
|
110
|
+
if (fields.length > 0) {
|
|
111
|
+
return Math.max(...fields.map(Number));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
102
116
|
async readDailyTotals(queue, metric, days) {
|
|
103
117
|
const raw = await this.readDailyTotalsRaw(queue, metric, days);
|
|
104
118
|
return raw.map((value) => (value == null ? null : Number(value) || 0));
|
package/dist/HistoryStore.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HistoryStore.js","sourceRoot":"","sources":["../src/HistoryStore.ts"],"names":[],"mappings":";;;AAEA,
|
|
1
|
+
{"version":3,"file":"HistoryStore.js","sourceRoot":"","sources":["../src/HistoryStore.ts"],"names":[],"mappings":";;;AAEA,iCASgB;AAKhB;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0CrB,CAAC;AAEF,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,2FAA2F;AAC3F,MAAa,YAAY;IAKvB;;;OAGG;IACH,YAAY,IAAwE;;QAClF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,MAAA,IAAI,CAAC,IAAI,mCAAI,IAAA,kBAAW,EAAC,wBAAiB,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACxD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACpD,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,MAAc,EACd,MAAc,EACd,KAAa,EACb,SAAiB,mBAAY;QAE7B,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,MAAM,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,aAAa,EACb,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAClC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAC/B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAClC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EACnC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,MAAM,CAAC,MAAM,CAAC,EACd,MAAM,CAAC,IAAA,mBAAY,EAAC,MAAM,CAAC,CAAC,EAC5B,GAAG,EACH,MAAM,CAAC,KAAK,CAAC,EACb,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAC3B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED,uFAAuF;IACvF,KAAK,CAAC,aAAa,CACjB,KAAa,EACb,MAAc,EACd,MAAqB,EACrB,SAAiB,mBAAY;QAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc;QAC9C,MAAM,KAAK,GAAG,IAAA,cAAO,EAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAClC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;YAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAA,eAAQ,EAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC9F,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc,EAAE,IAAc;QACjE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/D,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,MAAc,EAAE,IAAc;;QAC3D,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAA,GAAG,CAAC,KAAK,CAAC,mCAAI,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB,CACtB,KAAa,EACb,MAAc,EACd,IAAc;QAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,mFAAmF;QACnF,6EAA6E;QAC7E,mFAAmF;QACnF,mFAAmF;QACnF,MAAM,KAAK,GAAG,IAAI,CAAC,KAElB,CAAC;QACF,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,KAAa,EACb,MAAc,EACd,GAAW;QAEX,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,GAAW;;QAC3D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7E,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9D,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAA,mBAAY,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAA,MAAM,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAtJD,oCAsJC"}
|
package/dist/LatencySampler.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { BaseAdapter } from '@worker-manager/api/baseAdapter';
|
|
2
2
|
import type { MetricsClient } from './connection';
|
|
3
|
+
import { type JobSource } from './jobSources';
|
|
3
4
|
import type { MetricsKeys } from './keys';
|
|
4
5
|
import type { LatencyStorage } from './store';
|
|
5
6
|
export interface LatencySamplerOptions {
|
|
@@ -61,6 +62,15 @@ export declare class LatencySampler {
|
|
|
61
62
|
* indistinguishable from a queue with nothing to sample.
|
|
62
63
|
*/
|
|
63
64
|
sample(adapter: BaseAdapter): Promise<void>;
|
|
65
|
+
/**
|
|
66
|
+
* The same tick for a queue that is not an adapter, whose `JobSource` the caller already
|
|
67
|
+
* holds (a `CounterSource`'s). `rollup` names the cross-queue series, `__global__` if omitted.
|
|
68
|
+
* Swallows its errors exactly like `sample`.
|
|
69
|
+
*/
|
|
70
|
+
sampleSource(name: string, source: JobSource | null, rollup?: string): Promise<void>;
|
|
71
|
+
private sampleLeased;
|
|
72
|
+
/** Intentionally swallowed, see `sample`. */
|
|
73
|
+
private report;
|
|
64
74
|
/**
|
|
65
75
|
* PostgreSQL first, because `getQueueKey` answers for a PostgreSQL-backed queue too, with
|
|
66
76
|
* keys no Redis holds: reading those would record a reassuring zero backlog forever. A Redis
|
package/dist/LatencySampler.js
CHANGED
|
@@ -54,7 +54,6 @@ class LatencySampler {
|
|
|
54
54
|
* indistinguishable from a queue with nothing to sample.
|
|
55
55
|
*/
|
|
56
56
|
async sample(adapter) {
|
|
57
|
-
var _a;
|
|
58
57
|
if (!LatencySampler.supports(adapter)) {
|
|
59
58
|
return;
|
|
60
59
|
}
|
|
@@ -64,25 +63,48 @@ class LatencySampler {
|
|
|
64
63
|
if (!source) {
|
|
65
64
|
return;
|
|
66
65
|
}
|
|
67
|
-
|
|
68
|
-
return;
|
|
69
|
-
}
|
|
70
|
-
try {
|
|
71
|
-
await this.sampleDurations(source, name);
|
|
72
|
-
await this.sampleQueueAge(source, name);
|
|
73
|
-
}
|
|
74
|
-
finally {
|
|
75
|
-
await this.store.releaseLease(name, this.id);
|
|
76
|
-
}
|
|
66
|
+
await this.sampleLeased(name, source);
|
|
77
67
|
}
|
|
78
68
|
catch (error) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
69
|
+
this.report(error, name);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The same tick for a queue that is not an adapter, whose `JobSource` the caller already
|
|
74
|
+
* holds (a `CounterSource`'s). `rollup` names the cross-queue series, `__global__` if omitted.
|
|
75
|
+
* Swallows its errors exactly like `sample`.
|
|
76
|
+
*/
|
|
77
|
+
async sampleSource(name, source, rollup) {
|
|
78
|
+
if (!source) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
await this.sampleLeased(name, source, rollup);
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
this.report(error, name);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async sampleLeased(name, source, rollup) {
|
|
89
|
+
if (!(await this.store.acquireLease(name, this.id, this.tickMs * 2))) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
await this.sampleDurations(source, name, rollup);
|
|
94
|
+
await this.sampleQueueAge(source, name, rollup);
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
await this.store.releaseLease(name, this.id);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** Intentionally swallowed, see `sample`. */
|
|
101
|
+
report(error, name) {
|
|
102
|
+
var _a;
|
|
103
|
+
try {
|
|
104
|
+
(_a = this.onError) === null || _a === void 0 ? void 0 : _a.call(this, error, name);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
// A reporter that throws must not resurrect the failure this catch contains.
|
|
86
108
|
}
|
|
87
109
|
}
|
|
88
110
|
/**
|
|
@@ -114,7 +136,7 @@ class LatencySampler {
|
|
|
114
136
|
watermarkTtlSeconds() {
|
|
115
137
|
return Math.max(1, Math.floor(this.store.retention.days * SECONDS_PER_DAY));
|
|
116
138
|
}
|
|
117
|
-
async sampleDurations(source, name) {
|
|
139
|
+
async sampleDurations(source, name, rollup) {
|
|
118
140
|
const stored = await this.store.readWatermark(name);
|
|
119
141
|
// Cold start covers one tick ending at the safety bound rather than backfilling, since a
|
|
120
142
|
// first run against a large completed set would be a surprise fetch storm. Ending at the
|
|
@@ -149,27 +171,27 @@ class LatencySampler {
|
|
|
149
171
|
observe(waitByHour, hour, Math.max(0, job.processedOn - job.timestamp), ratio);
|
|
150
172
|
}
|
|
151
173
|
}
|
|
152
|
-
await this.flush(name, 'runtime', runByHour);
|
|
153
|
-
await this.flush(name, 'waittime', waitByHour);
|
|
174
|
+
await this.flush(name, 'runtime', runByHour, rollup);
|
|
175
|
+
await this.flush(name, 'waittime', waitByHour, rollup);
|
|
154
176
|
// The bound, not the highest score observed. See SAFETY_MARGIN_MS.
|
|
155
177
|
await this.store.writeWatermark(name, upperBound, this.watermarkTtlSeconds());
|
|
156
178
|
}
|
|
157
|
-
async flush(name, metric, byHour) {
|
|
179
|
+
async flush(name, metric, byHour, rollup) {
|
|
158
180
|
for (const [hour, vector] of byHour) {
|
|
159
181
|
// Scaled counts are fractional. Left unrounded, join(',') would write seventeen
|
|
160
182
|
// significant digits per bucket and blow up the packed value the storage design
|
|
161
183
|
// depends on staying small.
|
|
162
|
-
await this.store.addSamples(name, metric, hour, vector.map((v) => Math.round(v)));
|
|
184
|
+
await this.store.addSamples(name, metric, hour, vector.map((v) => Math.round(v)), rollup);
|
|
163
185
|
}
|
|
164
186
|
}
|
|
165
187
|
/** A tick that could not read the backlog cleanly records nothing, see `JobSource`. */
|
|
166
|
-
async sampleQueueAge(source, name) {
|
|
188
|
+
async sampleQueueAge(source, name, rollup) {
|
|
167
189
|
const hour = Math.floor(Date.now() / MS_PER_HOUR);
|
|
168
190
|
const age = await source.oldestWaitingAge(Date.now());
|
|
169
191
|
if (age === null) {
|
|
170
192
|
return;
|
|
171
193
|
}
|
|
172
|
-
await this.store.recordQueueAge(name, hour, age);
|
|
194
|
+
await this.store.recordQueueAge(name, hour, age, rollup);
|
|
173
195
|
}
|
|
174
196
|
}
|
|
175
197
|
exports.LatencySampler = LatencySampler;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LatencySampler.js","sourceRoot":"","sources":["../src/LatencySampler.ts"],"names":[],"mappings":";;;AAEA,2CAAuD;AACvD,6CAAgF;AAIhF,MAAM,WAAW,GAAG,OAAO,CAAC;AAC5B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC;;;;;;;;;;;;GAYG;AACH,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAqC9B;;;;;;;;;;;GAWG;AACH,MAAa,cAAc;IAUzB,YAAY,IAA2B;;QAHtB,OAAE,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,YAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QAG7D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,iBAAiB,mCAAI,mBAAmB,CAAC;QAChE,IAAI,CAAC,cAAc,GAAG,MAAA,IAAI,CAAC,cAAc,mCAAI,gBAAgB,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,OAAoB;QAClC,OAAO,OAAQ,OAAoC,CAAC,WAAW,KAAK,UAAU,CAAC;IACjF,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB
|
|
1
|
+
{"version":3,"file":"LatencySampler.js","sourceRoot":"","sources":["../src/LatencySampler.ts"],"names":[],"mappings":";;;AAEA,2CAAuD;AACvD,6CAAgF;AAIhF,MAAM,WAAW,GAAG,OAAO,CAAC;AAC5B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC;;;;;;;;;;;;GAYG;AACH,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAqC9B;;;;;;;;;;;GAWG;AACH,MAAa,cAAc;IAUzB,YAAY,IAA2B;;QAHtB,OAAE,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,YAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QAG7D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,iBAAiB,mCAAI,mBAAmB,CAAC;QAChE,IAAI,CAAC,cAAc,GAAG,MAAA,IAAI,CAAC,cAAc,mCAAI,gBAAgB,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,OAAoB;QAClC,OAAO,OAAQ,OAAoC,CAAC,WAAW,KAAK,UAAU,CAAC;IACjF,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,OAAoB;QAC/B,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAA0B,EAAE,IAAI,CAAC,CAAC;YACtE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;YACT,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,IAAY,EAAE,MAAwB,EAAE,MAAe;QACxE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAY,EAAE,MAAiB,EAAE,MAAe;QACzE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YACjD,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAClD,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,6CAA6C;IACrC,MAAM,CAAC,KAAc,EAAE,IAAY;;QACzC,IAAI,CAAC;YACH,MAAA,IAAI,CAAC,OAAO,qDAAG,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,6EAA6E;QAC/E,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,SAAS,CAAC,OAAwB,EAAE,IAAY;;QAC5D,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,IAAI,CAAC;QACxC,CAAC;QACD,IAAI,MAAM,GAAqB,IAAA,6BAAgB,EAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;YACvD,MAAM,MAAM,GACV,MAAA,IAAI,CAAC,KAAK,mCAAI,CAAC,OAAO,OAAO,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7F,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,2BAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC/B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACK,mBAAmB;QACzB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,MAAiB,EAAE,IAAY,EAAE,MAAe;QAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACpD,yFAAyF;QACzF,yFAAyF;QACzF,yFAAyF;QACzF,2EAA2E;QAC3E,MAAM,SAAS,GAAG,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC;QAE3E,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC;QACpD,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;YAC5B,OAAO,CAAC,qEAAqE;QAC/E,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CACxD,SAAS,EACT,UAAU,EACV,IAAI,CAAC,UAAU,CAChB,CAAC;QACF,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QAED,qFAAqF;QACrF,sFAAsF;QACtF,qFAAqF;QACrF,6DAA6D;QAC7D,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAE3C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;QAE/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,WAAW,CAAC,CAAC;YAEtD,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YAElE,oFAAoF;YACpF,gFAAgF;YAChF,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAChD,mFAAmF;gBACnF,oFAAoF;gBACpF,qDAAqD;gBACrD,OAAO,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;YACjF,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACvD,mEAAmE;QACnE,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC;IAChF,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,IAAY,EACZ,MAAqB,EACrB,MAA6B,EAC7B,MAAe;QAEf,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACpC,gFAAgF;YAChF,gFAAgF;YAChF,4BAA4B;YAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACzB,IAAI,EACJ,MAAM,EACN,IAAI,EACJ,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAChC,MAAM,CACP,CAAC;QACJ,CAAC;IACH,CAAC;IAED,uFAAuF;IAC/E,KAAK,CAAC,cAAc,CAAC,MAAiB,EAAE,IAAY,EAAE,MAAe;QAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACtD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3D,CAAC;CACF;AAnMD,wCAmMC;AAED,SAAS,OAAO,CACd,MAA6B,EAC7B,IAAY,EACZ,UAAkB,EAClB,KAAa;IAEb,IAAI,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,IAAA,uBAAW,GAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,CAAC,IAAA,uBAAW,EAAC,UAAU,CAAC,CAAC,IAAI,KAAK,CAAC;AAC3C,CAAC"}
|
package/dist/LatencyStore.d.ts
CHANGED
|
@@ -17,8 +17,8 @@ export declare class LatencyStore implements LatencyStorage {
|
|
|
17
17
|
keys?: MetricsKeys;
|
|
18
18
|
retention: Retention;
|
|
19
19
|
});
|
|
20
|
-
addSamples(queue: string, metric: LatencyMetric, hour: number, vector: number[]): Promise<void>;
|
|
21
|
-
recordQueueAge(queue: string, hour: number, ms: number): Promise<void>;
|
|
20
|
+
addSamples(queue: string, metric: LatencyMetric, hour: number, vector: number[], rollup?: string): Promise<void>;
|
|
21
|
+
recordQueueAge(queue: string, hour: number, ms: number, rollup?: string): Promise<void>;
|
|
22
22
|
readRange(queue: string, metric: LatencyMetric, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number[]>>;
|
|
23
23
|
readQueueAge(queue: string, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number>>;
|
|
24
24
|
acquireLease(queue: string, holder: string, ttlMs: number): Promise<boolean>;
|
package/dist/LatencyStore.js
CHANGED
|
@@ -167,13 +167,13 @@ class LatencyStore {
|
|
|
167
167
|
this.keys = (_a = opts.keys) !== null && _a !== void 0 ? _a : (0, keys_1.metricsKeys)(keys_1.DEFAULT_NAMESPACE);
|
|
168
168
|
this.retention = opts.retention;
|
|
169
169
|
}
|
|
170
|
-
async addSamples(queue, metric, hour, vector) {
|
|
170
|
+
async addSamples(queue, metric, hour, vector, rollup = keys_1.GLOBAL_QUEUE) {
|
|
171
171
|
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
172
|
-
await this.redis.eval(MERGE_VECTOR, 4, this.keys.hour(queue, metric, day), this.keys.totals(queue, metric), this.keys.hour(
|
|
172
|
+
await this.redis.eval(MERGE_VECTOR, 4, this.keys.hour(queue, metric, day), this.keys.totals(queue, metric), this.keys.hour(rollup, metric, day), this.keys.totals(rollup, metric), String(hour), day, (0, histogram_1.packVector)(vector), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days), String(histogram_1.BUCKET_COUNT));
|
|
173
173
|
}
|
|
174
|
-
async recordQueueAge(queue, hour, ms) {
|
|
174
|
+
async recordQueueAge(queue, hour, ms, rollup = keys_1.GLOBAL_QUEUE) {
|
|
175
175
|
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
176
|
-
await this.redis.eval(MAX_GAUGE, 4, this.keys.hour(queue, exports.QUEUE_AGE_METRIC, day), this.keys.totals(queue, exports.QUEUE_AGE_METRIC), this.keys.hour(
|
|
176
|
+
await this.redis.eval(MAX_GAUGE, 4, this.keys.hour(queue, exports.QUEUE_AGE_METRIC, day), this.keys.totals(queue, exports.QUEUE_AGE_METRIC), this.keys.hour(rollup, exports.QUEUE_AGE_METRIC, day), this.keys.totals(rollup, exports.QUEUE_AGE_METRIC), String(hour), day, String(Math.max(0, Math.round(ms))), ttl(this.retention.hours), ttl(this.retention.days), (0, keys_1.shiftDay)(day, -this.retention.days));
|
|
177
177
|
}
|
|
178
178
|
async readRange(queue, metric, granularity, days) {
|
|
179
179
|
const out = {};
|
package/dist/LatencyStore.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LatencyStore.js","sourceRoot":"","sources":["../src/LatencyStore.ts"],"names":[],"mappings":";;;AACA,2CAAmF;AACnF,iCAOgB;AAKH,QAAA,gBAAgB,GAAG,UAAU,CAAC;AAE3C,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0DpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CjB,CAAC;AAEF,yFAAyF;AACzF,MAAM,aAAa,GAAG;gBACN,CAAC;AAEjB,gGAAgG;AAChG,MAAa,YAAY;IAKvB;;;OAGG;IACH,YAAY,IAAwE;;QAClF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,MAAA,IAAI,CAAC,IAAI,mCAAI,IAAA,kBAAW,EAAC,wBAAiB,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAa,EACb,MAAqB,EACrB,IAAY,EACZ,MAAgB;
|
|
1
|
+
{"version":3,"file":"LatencyStore.js","sourceRoot":"","sources":["../src/LatencyStore.ts"],"names":[],"mappings":";;;AACA,2CAAmF;AACnF,iCAOgB;AAKH,QAAA,gBAAgB,GAAG,UAAU,CAAC;AAE3C,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0DpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CjB,CAAC;AAEF,yFAAyF;AACzF,MAAM,aAAa,GAAG;gBACN,CAAC;AAEjB,gGAAgG;AAChG,MAAa,YAAY;IAKvB;;;OAGG;IACH,YAAY,IAAwE;;QAClF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,MAAA,IAAI,CAAC,IAAI,mCAAI,IAAA,kBAAW,EAAC,wBAAiB,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAa,EACb,MAAqB,EACrB,IAAY,EACZ,MAAgB,EAChB,SAAiB,mBAAY;QAE7B,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,YAAY,EACZ,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAClC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EACnC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,MAAM,CAAC,IAAI,CAAC,EACZ,GAAG,EACH,IAAA,sBAAU,EAAC,MAAM,CAAC,EAClB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACnC,MAAM,CAAC,wBAAY,CAAC,CACrB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,KAAa,EACb,IAAY,EACZ,EAAU,EACV,SAAiB,mBAAY;QAE7B,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,SAAS,EACT,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,wBAAgB,EAAE,GAAG,CAAC,EAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,wBAAgB,CAAC,EACzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,wBAAgB,EAAE,GAAG,CAAC,EAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,wBAAgB,CAAC,EAC1C,MAAM,CAAC,IAAI,CAAC,EACZ,GAAG,EACH,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EACnC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACzB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACxB,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAa,EACb,MAAqB,EACrB,WAA2B,EAC3B,IAAc;QAEd,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;YACtE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,IAAA,wBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,oFAAoF;QACpF,4EAA4E;QAC5E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAC1E,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBACrB,CAAC,CAAC,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC,CAAC,IAAA,wBAAY,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,WAA2B,EAC3B,IAAc;;QAEd,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,wBAAgB,CAAC,CAAC,CAAC;YAChF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,wBAAgB,EAAE,GAAG,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAA,GAAG,CAAC,KAAK,CAAC,mCAAI,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QACrF,OAAO,IAAI,KAAK,IAAI,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc;QAC9C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,EAAU,EAAE,UAAkB;QAChE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACjF,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,QAAQ,CAAC,IAAY;QAC1B,OAAO,IAAI,GAAG,WAAW,CAAC;IAC5B,CAAC;CACF;AA9ID,oCA8IC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { BaseAdapter } from '@worker-manager/api/baseAdapter';
|
|
2
2
|
import type { MetricsConnection } from './connection';
|
|
3
|
+
import { type CounterSources } from './counterSources';
|
|
3
4
|
import type { MetricsStore, Retention } from './store';
|
|
4
5
|
/**
|
|
5
6
|
* Minute detail is the expensive tier by two orders of magnitude, so it defaults to a week
|
|
@@ -32,13 +33,26 @@ interface StoreOptions {
|
|
|
32
33
|
connection?: never;
|
|
33
34
|
prefix?: never;
|
|
34
35
|
}
|
|
35
|
-
export type MetricsRecorderOptions = RecorderBaseOptions & (RedisOptions | StoreOptions);
|
|
36
|
-
interface
|
|
36
|
+
export type MetricsRecorderOptions = RecorderBaseOptions & (RedisOptions | StoreOptions) & (QueuesOption | SourcesOption);
|
|
37
|
+
interface QueuesOption {
|
|
37
38
|
/**
|
|
38
39
|
* A function is resolved on every tick, for a board whose queue set changes while it
|
|
39
40
|
* runs. An array is read once, at construction.
|
|
40
41
|
*/
|
|
41
42
|
queues: BaseAdapter[] | (() => BaseAdapter[]);
|
|
43
|
+
/** See `SourcesOption.sources`. Either or both may be given. */
|
|
44
|
+
sources?: CounterSources;
|
|
45
|
+
}
|
|
46
|
+
interface SourcesOption {
|
|
47
|
+
queues?: BaseAdapter[] | (() => BaseAdapter[]);
|
|
48
|
+
/**
|
|
49
|
+
* Queues that are not queue adapters, such as `pgBossMetricsSources(engine)` from
|
|
50
|
+
* `@worker-manager/pg-boss`. A function is resolved, and awaited, on every tick; an array is
|
|
51
|
+
* read once. Recorded after `queues`, in the same tick.
|
|
52
|
+
*/
|
|
53
|
+
sources: CounterSources;
|
|
54
|
+
}
|
|
55
|
+
interface RecorderBaseOptions {
|
|
42
56
|
/** Per-resolution retention in days. Unspecified tiers fall back to the defaults. */
|
|
43
57
|
retention?: Partial<Retention>;
|
|
44
58
|
/**
|
|
@@ -84,6 +98,7 @@ export declare function resolveRetention(opts: {
|
|
|
84
98
|
}): Retention;
|
|
85
99
|
export declare class MetricsRecorder {
|
|
86
100
|
private readonly resolveQueues;
|
|
101
|
+
private readonly resolveSources;
|
|
87
102
|
private readonly store;
|
|
88
103
|
/** Set only when the recorder built the store itself, from a `connection`. */
|
|
89
104
|
private readonly ownedStore;
|
|
@@ -110,6 +125,11 @@ export declare class MetricsRecorder {
|
|
|
110
125
|
* minutes are upserted (safe against overlapping windows across ticks), then the
|
|
111
126
|
* watermark advances. So the first tick backfills the buffer and every later tick only
|
|
112
127
|
* writes the minutes that appeared since.
|
|
128
|
+
*
|
|
129
|
+
* A `CounterSource` that is not an adapter is asked for exactly the range past the
|
|
130
|
+
* watermark, and a cold start takes that watermark from the store rather than re-reading the
|
|
131
|
+
* whole minute window: a source that counts rows (pg-boss) can have lost rows it already
|
|
132
|
+
* counted, and writing its smaller count back would subtract them from every rollup.
|
|
113
133
|
*/
|
|
114
134
|
private snapshotOne;
|
|
115
135
|
}
|
package/dist/MetricsRecorder.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MetricsRecorder = exports.DEFAULT_RETENTION = void 0;
|
|
4
4
|
exports.resolveRetention = resolveRetention;
|
|
5
|
-
const
|
|
5
|
+
const counterSources_1 = require("./counterSources");
|
|
6
6
|
const LatencySampler_1 = require("./LatencySampler");
|
|
7
7
|
const RedisMetricsStore_1 = require("./RedisMetricsStore");
|
|
8
8
|
const METRICS = ['completed', 'failed'];
|
|
@@ -32,8 +32,12 @@ class MetricsRecorder {
|
|
|
32
32
|
this.timer = null;
|
|
33
33
|
this.running = false;
|
|
34
34
|
this.stopped = false;
|
|
35
|
-
const { queues } = opts;
|
|
36
|
-
|
|
35
|
+
const { queues, sources } = opts;
|
|
36
|
+
if (!queues && !sources) {
|
|
37
|
+
throw new Error('MetricsRecorder needs `queues`, `sources`, or both.');
|
|
38
|
+
}
|
|
39
|
+
this.resolveQueues = !queues ? null : typeof queues === 'function' ? queues : () => queues;
|
|
40
|
+
this.resolveSources = !sources ? null : typeof sources === 'function' ? sources : () => sources;
|
|
37
41
|
this.intervalMs = (_a = opts.snapshotIntervalMs) !== null && _a !== void 0 ? _a : 60000;
|
|
38
42
|
const store = (_b = opts.store) !== null && _b !== void 0 ? _b : new RedisMetricsStore_1.RedisMetricsStore({ connection: opts.connection, prefix: opts.prefix });
|
|
39
43
|
this.ownedStore = opts.store ? null : store;
|
|
@@ -89,18 +93,26 @@ class MetricsRecorder {
|
|
|
89
93
|
});
|
|
90
94
|
}
|
|
91
95
|
async snapshot() {
|
|
96
|
+
var _a, _b;
|
|
92
97
|
if (this.running) {
|
|
93
98
|
return;
|
|
94
99
|
}
|
|
95
100
|
this.running = true;
|
|
96
101
|
try {
|
|
97
|
-
|
|
98
|
-
|
|
102
|
+
const sources = ((_b = (_a = this.resolveQueues) === null || _a === void 0 ? void 0 : _a.call(this)) !== null && _b !== void 0 ? _b : []).map(counterSources_1.adapterCounterSource);
|
|
103
|
+
if (this.resolveSources) {
|
|
104
|
+
sources.push(...(await this.resolveSources()));
|
|
105
|
+
}
|
|
106
|
+
for (const source of sources) {
|
|
107
|
+
const name = source.name;
|
|
99
108
|
for (const metric of METRICS) {
|
|
100
|
-
await this.snapshotOne(
|
|
109
|
+
await this.snapshotOne(source, name, metric);
|
|
101
110
|
}
|
|
102
111
|
if (this.latencySampler) {
|
|
103
|
-
|
|
112
|
+
const adapter = (0, counterSources_1.adapterOf)(source);
|
|
113
|
+
await (adapter
|
|
114
|
+
? this.latencySampler.sample(adapter)
|
|
115
|
+
: this.latencySampler.sampleSource(name, source.jobSource(), source.rollup));
|
|
104
116
|
}
|
|
105
117
|
}
|
|
106
118
|
}
|
|
@@ -116,13 +128,25 @@ class MetricsRecorder {
|
|
|
116
128
|
* minutes are upserted (safe against overlapping windows across ticks), then the
|
|
117
129
|
* watermark advances. So the first tick backfills the buffer and every later tick only
|
|
118
130
|
* writes the minutes that appeared since.
|
|
131
|
+
*
|
|
132
|
+
* A `CounterSource` that is not an adapter is asked for exactly the range past the
|
|
133
|
+
* watermark, and a cold start takes that watermark from the store rather than re-reading the
|
|
134
|
+
* whole minute window: a source that counts rows (pg-boss) can have lost rows it already
|
|
135
|
+
* counted, and writing its smaller count back would subtract them from every rollup.
|
|
119
136
|
*/
|
|
120
|
-
async snapshotOne(
|
|
137
|
+
async snapshotOne(source, name, metric) {
|
|
121
138
|
var _a;
|
|
122
139
|
const cursorKey = `${name}:${metric}`;
|
|
123
|
-
|
|
124
|
-
const
|
|
125
|
-
const
|
|
140
|
+
let seenUpTo = this.lastMinute.get(cursorKey);
|
|
141
|
+
const currentMinute = Math.floor(Date.now() / MS_PER_MINUTE);
|
|
142
|
+
const isAdapter = (0, counterSources_1.adapterOf)(source) !== null;
|
|
143
|
+
if (seenUpTo === undefined && !isAdapter) {
|
|
144
|
+
seenUpTo = (_a = (await this.store.latestMinute(name, metric))) !== null && _a !== void 0 ? _a : -1;
|
|
145
|
+
this.lastMinute.set(cursorKey, seenUpTo);
|
|
146
|
+
}
|
|
147
|
+
seenUpTo !== null && seenUpTo !== void 0 ? seenUpTo : (seenUpTo = -1);
|
|
148
|
+
const fromMinute = Math.max(seenUpTo + 1, currentMinute - this.store.retention.minutes * MINUTES_PER_DAY);
|
|
149
|
+
const points = await source.readMinutes(metric, fromMinute * MS_PER_MINUTE, currentMinute * MS_PER_MINUTE);
|
|
126
150
|
if (points.length === 0) {
|
|
127
151
|
return;
|
|
128
152
|
}
|
|
@@ -134,14 +158,17 @@ class MetricsRecorder {
|
|
|
134
158
|
// restarts, losing its in-memory watermark. Refusing to write past the window closes
|
|
135
159
|
// it. Nothing is lost that could have been retained anyway.
|
|
136
160
|
const oldestWritable = Math.floor(Date.now() / MS_PER_MINUTE) - this.store.retention.minutes * MINUTES_PER_DAY;
|
|
161
|
+
// A filter rather than an early exit, since only an adapter promises newest-first order.
|
|
162
|
+
// For an adapter the result is the same: its minutes strictly descend, so everything past
|
|
163
|
+
// the first stored or too-old minute is stored or too old as well.
|
|
137
164
|
let newest = seenUpTo;
|
|
138
165
|
const fresh = [];
|
|
139
166
|
for (const point of points) {
|
|
140
|
-
if (point.minute <= seenUpTo) {
|
|
141
|
-
|
|
167
|
+
if (point.minute <= seenUpTo || point.minute < oldestWritable) {
|
|
168
|
+
continue;
|
|
142
169
|
}
|
|
143
|
-
if (point.minute
|
|
144
|
-
|
|
170
|
+
if (!isAdapter && point.minute >= currentMinute) {
|
|
171
|
+
continue; // a source is not trusted with the minute in progress
|
|
145
172
|
}
|
|
146
173
|
fresh.push(point);
|
|
147
174
|
if (point.minute > newest) {
|
|
@@ -150,7 +177,9 @@ class MetricsRecorder {
|
|
|
150
177
|
}
|
|
151
178
|
// One call per queue and metric, so a SQL store can make it one transaction. The
|
|
152
179
|
// watermark only moves once the write has landed.
|
|
153
|
-
await
|
|
180
|
+
await (source.rollup === undefined
|
|
181
|
+
? this.store.upsertMinutes(name, metric, fresh)
|
|
182
|
+
: this.store.upsertMinutes(name, metric, fresh, source.rollup));
|
|
154
183
|
this.lastMinute.set(cursorKey, newest);
|
|
155
184
|
}
|
|
156
185
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MetricsRecorder.js","sourceRoot":"","sources":["../src/MetricsRecorder.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"MetricsRecorder.js","sourceRoot":"","sources":["../src/MetricsRecorder.ts"],"names":[],"mappings":";;;AAqHA,4CAaC;AAhID,qDAM0B;AAE1B,qDAAkD;AAClD,2DAAwD;AAGxD,MAAM,OAAO,GAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACzD,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;;;;GAKG;AACU,QAAA,iBAAiB,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AA6FhF,SAAgB,gBAAgB,CAAC,IAGhC;IACC,MAAM,IAAI,GACR,IAAI,CAAC,aAAa,KAAK,SAAS;QAC9B,CAAC,CAAC,yBAAiB;QACnB,CAAC,CAAC;YACE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC;YAChE,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,IAAI,EAAE,IAAI,CAAC,aAAa;SACzB,CAAC;IACR,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,MAAa,eAAe;IAe1B,YAAY,IAA4B;;QARvB,eAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAChD,UAAK,GAA0C,IAAI,CAAC;QACpD,YAAO,GAAG,KAAK,CAAC;QAChB,YAAO,GAAG,KAAK,CAAC;QAMtB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QACjC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;QAC3F,IAAI,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;QAChG,IAAI,CAAC,UAAU,GAAG,MAAA,IAAI,CAAC,kBAAkB,mCAAI,KAAK,CAAC;QACnD,MAAM,KAAK,GACT,MAAA,IAAI,CAAC,KAAK,mCAAI,IAAI,qCAAiB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5F,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5C,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC;QAC7C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc;YACvC,CAAC,CAAC,IAAI,+BAAc,CAAC;gBACjB,KAAK,EAAE,MAAA,KAAK,CAAC,SAAS,mCAAI,SAAS;gBACnC,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,iBAAiB,EAAE,IAAI,CAAC,wBAAwB;gBAChD,cAAc,EAAE,IAAI,CAAC,qBAAqB;gBAC1C,OAAO,EAAE,IAAI,CAAC,cAAc;aAC7B,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1E,4DAA4D;QAC5D,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,iGAAiG;IACjG,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;;YAC9B,IAAI,CAAC;gBACH,MAAA,IAAI,CAAC,eAAe,qDAAG,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,+EAA+E;YACjF,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,QAAQ;;QACZ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,CAAC,MAAA,MAAA,IAAI,CAAC,aAAa,oDAAI,mCAAI,EAAE,CAAC,CAAC,GAAG,CAAC,qCAAoB,CAAC,CAAC;YACzE,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YACjD,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC/C,CAAC;gBACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBACxB,MAAM,OAAO,GAAG,IAAA,0BAAS,EAAC,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,OAAO;wBACZ,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC;wBACrC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBACjF,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;OAaG;IACK,KAAK,CAAC,WAAW,CACvB,MAAqB,EACrB,IAAY,EACZ,MAAqB;;QAErB,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;QACtC,IAAI,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAA,0BAAS,EAAC,MAAM,CAAC,KAAK,IAAI,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;YACzC,QAAQ,GAAG,MAAA,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,mCAAI,CAAC,CAAC,CAAC;YAC/D,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3C,CAAC;QACD,QAAQ,aAAR,QAAQ,cAAR,QAAQ,IAAR,QAAQ,GAAK,CAAC,CAAC,EAAC;QAEhB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,QAAQ,GAAG,CAAC,EACZ,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,eAAe,CAC/D,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CACrC,MAAM,EACN,UAAU,GAAG,aAAa,EAC1B,aAAa,GAAG,aAAa,CAC9B,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,iFAAiF;QACjF,mFAAmF;QACnF,mFAAmF;QACnF,sFAAsF;QACtF,mFAAmF;QACnF,qFAAqF;QACrF,4DAA4D;QAC5D,MAAM,cAAc,GAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,eAAe,CAAC;QAE1F,yFAAyF;QACzF,0FAA0F;QAC1F,mEAAmE;QACnE,IAAI,MAAM,GAAG,QAAQ,CAAC;QACtB,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;gBAChD,SAAS,CAAC,sDAAsD;YAClE,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;QACH,CAAC;QACD,iFAAiF;QACjF,kDAAkD;QAClD,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS;YAChC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC;YAC/C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;CACF;AAvLD,0CAuLC"}
|
|
@@ -61,14 +61,14 @@ function emptyTiers() {
|
|
|
61
61
|
day: { keys: 0, bytes: 0 },
|
|
62
62
|
};
|
|
63
63
|
}
|
|
64
|
-
/** The
|
|
65
|
-
function globalKeyFor(keys, parsed) {
|
|
64
|
+
/** The rollup key mirroring a per-queue key, same tier and same day. */
|
|
65
|
+
function globalKeyFor(keys, parsed, rollup) {
|
|
66
66
|
if (parsed.day === null) {
|
|
67
|
-
return keys.totals(
|
|
67
|
+
return keys.totals(rollup, parsed.metric);
|
|
68
68
|
}
|
|
69
69
|
return parsed.tier === 'hour'
|
|
70
|
-
? keys.hour(
|
|
71
|
-
: keys.day(
|
|
70
|
+
? keys.hour(rollup, parsed.metric, parsed.day)
|
|
71
|
+
: keys.day(rollup, parsed.metric, parsed.day);
|
|
72
72
|
}
|
|
73
73
|
/**
|
|
74
74
|
* Inspection and cleanup for the Redis keys written by `MetricsRecorder`.
|
|
@@ -183,6 +183,7 @@ class RedisHistoryAdmin {
|
|
|
183
183
|
* per-queue keys are still deleted either way. See SUMMABLE_METRICS.
|
|
184
184
|
*/
|
|
185
185
|
async purge(opts = {}) {
|
|
186
|
+
var _a;
|
|
186
187
|
const before = opts.before === undefined ? null : (0, keys_1.toDay)(opts.before);
|
|
187
188
|
const result = { keysDeleted: 0, fieldsDeleted: 0 };
|
|
188
189
|
const dayKeys = [];
|
|
@@ -195,6 +196,9 @@ class RedisHistoryAdmin {
|
|
|
195
196
|
if (opts.queue !== undefined && parsed.queue !== opts.queue) {
|
|
196
197
|
continue;
|
|
197
198
|
}
|
|
199
|
+
if (opts.queuePrefix !== undefined && !parsed.queue.startsWith(opts.queuePrefix)) {
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
198
202
|
if (parsed.day === null) {
|
|
199
203
|
totalsKeys.push({ key, parsed });
|
|
200
204
|
}
|
|
@@ -204,10 +208,11 @@ class RedisHistoryAdmin {
|
|
|
204
208
|
}
|
|
205
209
|
// Rewriting the global rollup only makes sense when a single queue is being removed:
|
|
206
210
|
// a full purge drops the global keys outright, along with everything else.
|
|
207
|
-
const
|
|
211
|
+
const rollup = (_a = opts.rollup) !== null && _a !== void 0 ? _a : keys_1.GLOBAL_QUEUE;
|
|
212
|
+
const adjustGlobal = opts.queue !== undefined && opts.queue !== keys_1.GLOBAL_QUEUE && opts.queue !== rollup;
|
|
208
213
|
for (const { key, parsed } of dayKeys) {
|
|
209
214
|
if (adjustGlobal) {
|
|
210
|
-
result.keysDeleted += await this.subtractDayFromGlobal(key, parsed);
|
|
215
|
+
result.keysDeleted += await this.subtractDayFromGlobal(key, parsed, rollup);
|
|
211
216
|
}
|
|
212
217
|
result.keysDeleted += await this.redis.unlink(key);
|
|
213
218
|
}
|
|
@@ -215,7 +220,7 @@ class RedisHistoryAdmin {
|
|
|
215
220
|
const stale = (await this.redis.hkeys(key)).filter((day) => before === null || day < before);
|
|
216
221
|
if (adjustGlobal && stale.length > 0) {
|
|
217
222
|
const totals = await this.redis.hmget(key, ...stale);
|
|
218
|
-
result.fieldsDeleted += await this.subtractTotalsFromGlobal(parsed.metric, stale, totals);
|
|
223
|
+
result.fieldsDeleted += await this.subtractTotalsFromGlobal(parsed.metric, stale, totals, rollup);
|
|
219
224
|
}
|
|
220
225
|
if (before === null) {
|
|
221
226
|
result.keysDeleted += await this.redis.unlink(key);
|
|
@@ -242,12 +247,12 @@ class RedisHistoryAdmin {
|
|
|
242
247
|
* Only the summable metrics are touched; the latency ones are skipped outright rather than
|
|
243
248
|
* silently producing a no-op subtraction of their packed values. See SUMMABLE_METRICS.
|
|
244
249
|
*/
|
|
245
|
-
async subtractDayFromGlobal(key, parsed) {
|
|
250
|
+
async subtractDayFromGlobal(key, parsed, rollup) {
|
|
246
251
|
if (parsed.day === null || !SUMMABLE_METRICS.includes(parsed.metric)) {
|
|
247
252
|
return 0;
|
|
248
253
|
}
|
|
249
254
|
const minutes = await this.redis.hgetall(key);
|
|
250
|
-
const globalDay = globalKeyFor(this.keys, parsed);
|
|
255
|
+
const globalDay = globalKeyFor(this.keys, parsed, rollup);
|
|
251
256
|
const pipeline = this.redis.multi();
|
|
252
257
|
const touched = [];
|
|
253
258
|
for (const field of Object.keys(minutes)) {
|
|
@@ -275,11 +280,11 @@ class RedisHistoryAdmin {
|
|
|
275
280
|
*
|
|
276
281
|
* Skips the latency metrics for the same reason as subtractDayFromGlobal.
|
|
277
282
|
*/
|
|
278
|
-
async subtractTotalsFromGlobal(metric, days, values) {
|
|
283
|
+
async subtractTotalsFromGlobal(metric, days, values, rollup) {
|
|
279
284
|
if (!SUMMABLE_METRICS.includes(metric)) {
|
|
280
285
|
return 0;
|
|
281
286
|
}
|
|
282
|
-
const globalTotals = this.keys.totals(
|
|
287
|
+
const globalTotals = this.keys.totals(rollup, metric);
|
|
283
288
|
const pipeline = this.redis.multi();
|
|
284
289
|
const touched = [];
|
|
285
290
|
days.forEach((day, i) => {
|