@worker-manager/metrics 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -6
- package/dist/HistoryAdmin.d.ts +23 -87
- package/dist/HistoryAdmin.js +21 -330
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +9 -9
- package/dist/HistoryStore.js +22 -0
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +27 -32
- package/dist/LatencySampler.js +55 -156
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +8 -3
- package/dist/LatencyStore.js +18 -0
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +34 -10
- package/dist/MetricsRecorder.js +30 -20
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisHistoryAdmin.d.ts +95 -0
- package/dist/RedisHistoryAdmin.js +328 -0
- package/dist/RedisHistoryAdmin.js.map +1 -0
- package/dist/RedisMetricsHistoryProvider.d.ts +7 -17
- package/dist/RedisMetricsHistoryProvider.js +9 -136
- package/dist/RedisMetricsHistoryProvider.js.map +1 -1
- package/dist/RedisMetricsStore.d.ts +27 -0
- package/dist/RedisMetricsStore.js +42 -0
- package/dist/RedisMetricsStore.js.map +1 -0
- package/dist/StoreHistoryProvider.d.ts +27 -0
- package/dist/StoreHistoryProvider.js +127 -0
- package/dist/StoreHistoryProvider.js.map +1 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/jobSources.d.ts +97 -0
- package/dist/jobSources.js +248 -0
- package/dist/jobSources.js.map +1 -0
- package/dist/keys.d.ts +7 -0
- package/dist/keys.js +23 -1
- package/dist/keys.js.map +1 -1
- package/dist/postgres/PostgresMetricsHistoryProvider.d.ts +37 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js +30 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js.map +1 -0
- package/dist/postgres/PostgresMetricsStore.d.ts +65 -0
- package/dist/postgres/PostgresMetricsStore.js +88 -0
- package/dist/postgres/PostgresMetricsStore.js.map +1 -0
- package/dist/postgres/admin.d.ts +31 -0
- package/dist/postgres/admin.js +178 -0
- package/dist/postgres/admin.js.map +1 -0
- package/dist/postgres/connection.d.ts +53 -0
- package/dist/postgres/connection.js +54 -0
- package/dist/postgres/connection.js.map +1 -0
- package/dist/postgres/context.d.ts +29 -0
- package/dist/postgres/context.js +76 -0
- package/dist/postgres/context.js.map +1 -0
- package/dist/postgres/schema.d.ts +26 -0
- package/dist/postgres/schema.js +148 -0
- package/dist/postgres/schema.js.map +1 -0
- package/dist/postgres/stores.d.ts +51 -0
- package/dist/postgres/stores.js +252 -0
- package/dist/postgres/stores.js.map +1 -0
- package/dist/store.d.ts +80 -0
- package/dist/store.js +3 -0
- package/dist/store.js.map +1 -0
- package/package.json +13 -3
package/dist/keys.js
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HOUR_TIER = exports.GLOBAL_QUEUE = exports.DEFAULT_NAMESPACE = void 0;
|
|
3
|
+
exports.DAY_PATTERN = exports.HOUR_TIER = exports.GLOBAL_QUEUE = exports.DEFAULT_NAMESPACE = void 0;
|
|
4
|
+
exports.msToDay = msToDay;
|
|
4
5
|
exports.minuteToDay = minuteToDay;
|
|
5
6
|
exports.minuteToHour = minuteToHour;
|
|
6
7
|
exports.shiftDay = shiftDay;
|
|
7
8
|
exports.dayToStartMs = dayToStartMs;
|
|
9
|
+
exports.dayToIndex = dayToIndex;
|
|
10
|
+
exports.indexToDay = indexToDay;
|
|
8
11
|
exports.dayRange = dayRange;
|
|
9
12
|
exports.resolveNamespace = resolveNamespace;
|
|
10
13
|
exports.metricsKeys = metricsKeys;
|
|
14
|
+
exports.toDay = toDay;
|
|
11
15
|
exports.DEFAULT_NAMESPACE = 'bull-board:metrics';
|
|
12
16
|
exports.GLOBAL_QUEUE = '__global__';
|
|
13
17
|
/** Marks the hourly rollup key so it can't be mistaken for a minute-level day hash. */
|
|
14
18
|
exports.HOUR_TIER = 'hour';
|
|
19
|
+
exports.DAY_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
|
|
15
20
|
const MS_PER_MINUTE = 60000;
|
|
16
21
|
const MS_PER_DAY = 86400000;
|
|
17
22
|
const MINUTES_PER_HOUR = 60;
|
|
@@ -36,6 +41,13 @@ function dayToStartMs(day) {
|
|
|
36
41
|
const [y, m, d] = day.split('-').map(Number);
|
|
37
42
|
return Date.UTC(y, m - 1, d);
|
|
38
43
|
}
|
|
44
|
+
/** Days since the epoch, the integer form of an ISO day that SQL buckets use. */
|
|
45
|
+
function dayToIndex(day) {
|
|
46
|
+
return Math.floor(dayToStartMs(day) / MS_PER_DAY);
|
|
47
|
+
}
|
|
48
|
+
function indexToDay(index) {
|
|
49
|
+
return msToDay(index * MS_PER_DAY);
|
|
50
|
+
}
|
|
39
51
|
function dayRange(fromMs, toMs) {
|
|
40
52
|
const days = [];
|
|
41
53
|
let cursor = Date.UTC(new Date(fromMs).getUTCFullYear(), new Date(fromMs).getUTCMonth(), new Date(fromMs).getUTCDate());
|
|
@@ -73,4 +85,14 @@ function metricsKeys(namespace) {
|
|
|
73
85
|
scanPattern: `${namespace}:*`,
|
|
74
86
|
};
|
|
75
87
|
}
|
|
88
|
+
/** An ISO `YYYY-MM-DD` day, validated, from a day string or a Date (UTC). */
|
|
89
|
+
function toDay(value) {
|
|
90
|
+
if (typeof value === 'string') {
|
|
91
|
+
if (!exports.DAY_PATTERN.test(value)) {
|
|
92
|
+
throw new Error(`Expected a YYYY-MM-DD day or a Date, got "${value}"`);
|
|
93
|
+
}
|
|
94
|
+
return value;
|
|
95
|
+
}
|
|
96
|
+
return value.toISOString().slice(0, 10);
|
|
97
|
+
}
|
|
76
98
|
//# sourceMappingURL=keys.js.map
|
package/dist/keys.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"keys.js","sourceRoot":"","sources":["../src/keys.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"keys.js","sourceRoot":"","sources":["../src/keys.ts"],"names":[],"mappings":";;;AAeA,0BAGC;AAED,kCAEC;AAGD,oCAEC;AAED,4BAEC;AAED,oCAGC;AAGD,gCAEC;AAED,gCAEC;AAED,4BAaC;AAgBD,4CAGC;AAYD,kCAUC;AAGD,sBAQC;AAhHY,QAAA,iBAAiB,GAAG,oBAAoB,CAAC;AACzC,QAAA,YAAY,GAAG,YAAY,CAAC;AACzC,uFAAuF;AAC1E,QAAA,SAAS,GAAG,MAAM,CAAC;AAEnB,QAAA,WAAW,GAAG,qBAAqB,CAAC;AAEjD,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,MAAM,UAAU,GAAG,QAAQ,CAAC;AAC5B,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,SAAgB,OAAO,CAAC,EAAU;IAChC,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;IACvB,OAAO,GAAG,CAAC,CAAC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;AACpF,CAAC;AAED,SAAgB,WAAW,CAAC,MAAc;IACxC,OAAO,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;AACzC,CAAC;AAED,gFAAgF;AAChF,SAAgB,YAAY,CAAC,MAAc;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,gBAAgB,CAAC,CAAC;AAC/C,CAAC;AAED,SAAgB,QAAQ,CAAC,GAAW,EAAE,UAAkB;IACtD,OAAO,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,UAAU,GAAG,UAAU,CAAC,CAAC;AAC9D,CAAC;AAED,SAAgB,YAAY,CAAC,GAAW;IACtC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,iFAAiF;AACjF,SAAgB,UAAU,CAAC,GAAW;IACpC,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;AACpD,CAAC;AAED,SAAgB,UAAU,CAAC,KAAa;IACtC,OAAO,OAAO,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,SAAgB,QAAQ,CAAC,MAAc,EAAE,IAAY;IACnD,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CACnB,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE,EACjC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAC9B,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,CAAC;IACjB,OAAO,MAAM,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,OAAO,GAAG,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;QACzC,GAAG,IAAI,CAAC,CAAC;IACX,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,yFAAyF;AACzF,SAAgB,gBAAgB,CAAC,MAA0B,EAAE,SAAkB;IAC7E,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,yBAAiB,CAAC,IAAI,yBAAiB,CAAC;IAClF,OAAO,SAAS,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC;AAYD,SAAgB,WAAW,CAAC,SAAiB;IAC3C,OAAO;QACL,SAAS;QACT,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,SAAS,IAAI,KAAK,IAAI,MAAM,IAAI,GAAG,EAAE;QACrE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,SAAS,IAAI,KAAK,IAAI,MAAM,IAAI,iBAAS,IAAI,GAAG,EAAE;QACnF,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,SAAS,IAAI,KAAK,IAAI,MAAM,SAAS;QACnE,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS,IAAI,KAAK,gBAAgB;QACvD,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS,IAAI,KAAK,oBAAoB;QAC/D,WAAW,EAAE,GAAG,SAAS,IAAI;KAC9B,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,SAAgB,KAAK,CAAC,KAAoB;IACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,mBAAW,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,GAAG,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { Retention } from '../store';
|
|
2
|
+
import { StoreHistoryProvider } from '../StoreHistoryProvider';
|
|
3
|
+
import type { PostgresConnection } from './connection';
|
|
4
|
+
import { PostgresMetricsStore } from './PostgresMetricsStore';
|
|
5
|
+
interface RetentionOptions {
|
|
6
|
+
/** Should mirror the recorder's retention. Only used to bound the query span. */
|
|
7
|
+
retention?: Partial<Retention>;
|
|
8
|
+
retentionDays?: number;
|
|
9
|
+
}
|
|
10
|
+
export type PostgresMetricsHistoryProviderOptions = RetentionOptions & ({
|
|
11
|
+
/** A `pg.Pool`, a node-postgres pool config, or a connection string. */
|
|
12
|
+
connection: PostgresConnection;
|
|
13
|
+
/** Must match the recorder's store. See `PostgresMetricsStoreOptions`. */
|
|
14
|
+
schema?: string;
|
|
15
|
+
tablePrefix?: string;
|
|
16
|
+
/** See `PostgresMetricsStoreOptions.migrate`. */
|
|
17
|
+
migrate?: boolean;
|
|
18
|
+
onError?: (error: Error) => void;
|
|
19
|
+
store?: never;
|
|
20
|
+
} | {
|
|
21
|
+
/** A store shared with the recorder. Left open on `disconnect()`. */
|
|
22
|
+
store: PostgresMetricsStore;
|
|
23
|
+
connection?: never;
|
|
24
|
+
schema?: never;
|
|
25
|
+
tablePrefix?: never;
|
|
26
|
+
migrate?: never;
|
|
27
|
+
onError?: never;
|
|
28
|
+
});
|
|
29
|
+
/** Serves the board's history charts and storage panel from the PostgreSQL store. */
|
|
30
|
+
export declare class PostgresMetricsHistoryProvider extends StoreHistoryProvider {
|
|
31
|
+
readonly store: PostgresMetricsStore;
|
|
32
|
+
private readonly ownsStore;
|
|
33
|
+
constructor(opts: PostgresMetricsHistoryProviderOptions);
|
|
34
|
+
/** Ends the pool behind a store this provider built. A store or pool handed in is left open. */
|
|
35
|
+
disconnect(): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PostgresMetricsHistoryProvider = void 0;
|
|
4
|
+
const MetricsRecorder_1 = require("../MetricsRecorder");
|
|
5
|
+
const StoreHistoryProvider_1 = require("../StoreHistoryProvider");
|
|
6
|
+
const PostgresMetricsStore_1 = require("./PostgresMetricsStore");
|
|
7
|
+
/** Serves the board's history charts and storage panel from the PostgreSQL store. */
|
|
8
|
+
class PostgresMetricsHistoryProvider extends StoreHistoryProvider_1.StoreHistoryProvider {
|
|
9
|
+
constructor(opts) {
|
|
10
|
+
var _a;
|
|
11
|
+
const store = (_a = opts.store) !== null && _a !== void 0 ? _a : new PostgresMetricsStore_1.PostgresMetricsStore({
|
|
12
|
+
connection: opts.connection,
|
|
13
|
+
schema: opts.schema,
|
|
14
|
+
tablePrefix: opts.tablePrefix,
|
|
15
|
+
migrate: opts.migrate,
|
|
16
|
+
onError: opts.onError,
|
|
17
|
+
});
|
|
18
|
+
super(store, (0, MetricsRecorder_1.resolveRetention)(opts));
|
|
19
|
+
this.store = store;
|
|
20
|
+
this.ownsStore = !opts.store;
|
|
21
|
+
}
|
|
22
|
+
/** Ends the pool behind a store this provider built. A store or pool handed in is left open. */
|
|
23
|
+
async disconnect() {
|
|
24
|
+
if (this.ownsStore) {
|
|
25
|
+
await this.store.close();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.PostgresMetricsHistoryProvider = PostgresMetricsHistoryProvider;
|
|
30
|
+
//# sourceMappingURL=PostgresMetricsHistoryProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PostgresMetricsHistoryProvider.js","sourceRoot":"","sources":["../../src/postgres/PostgresMetricsHistoryProvider.ts"],"names":[],"mappings":";;;AAAA,wDAAsD;AAEtD,kEAA+D;AAE/D,iEAA8D;AAgC9D,qFAAqF;AACrF,MAAa,8BAA+B,SAAQ,2CAAoB;IAItE,YAAY,IAA2C;;QACrD,MAAM,KAAK,GACT,MAAA,IAAI,CAAC,KAAK,mCACV,IAAI,2CAAoB,CAAC;YACvB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACL,KAAK,CAAC,KAAK,EAAE,IAAA,kCAAgB,EAAC,IAAI,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,gGAAgG;IAChG,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AAzBD,wEAyBC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { CounterStore, HistoryAdministration, LatencyStorage, MetricsStore, Retention } from '../store';
|
|
2
|
+
import { type PgPool, type PostgresConnection } from './connection';
|
|
3
|
+
import { type MetricsTables } from './schema';
|
|
4
|
+
export interface PostgresMetricsStoreOptions {
|
|
5
|
+
/** A `pg.Pool`, a node-postgres pool config, or a connection string. */
|
|
6
|
+
connection: PostgresConnection;
|
|
7
|
+
/**
|
|
8
|
+
* Schema holding the tables. Defaults to a `schema` key on a pool config (so the object
|
|
9
|
+
* given to BullMQ's PostgreSQL backend can be reused), then to `public`. Created on
|
|
10
|
+
* migration when missing.
|
|
11
|
+
*/
|
|
12
|
+
schema?: string;
|
|
13
|
+
/** Prepended to every table name. Defaults to `bull_board_metrics_`. */
|
|
14
|
+
tablePrefix?: string;
|
|
15
|
+
/**
|
|
16
|
+
* Create or upgrade the tables before the first query. Off by default, in which case the
|
|
17
|
+
* first query checks the schema version instead and fails with instructions when the tables
|
|
18
|
+
* are missing: in deployments where the application role has no DDL rights, run
|
|
19
|
+
* `migratePostgresMetrics()` from a deploy step.
|
|
20
|
+
*/
|
|
21
|
+
migrate?: boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Errors from idle pooled connections of a pool this store created (a server restart, a
|
|
24
|
+
* dropped connection). Without a listener node-postgres would crash the process on them.
|
|
25
|
+
*/
|
|
26
|
+
onError?: (error: Error) => void;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* History in PostgreSQL, for boards whose queues live on BullMQ's PostgreSQL backend and have
|
|
30
|
+
* no Redis at all. Same tiers, retention, rollup and idempotency as the Redis store; see the
|
|
31
|
+
* package README for the schema and sizing.
|
|
32
|
+
*/
|
|
33
|
+
export declare class PostgresMetricsStore implements MetricsStore {
|
|
34
|
+
readonly pool: PgPool;
|
|
35
|
+
readonly tables: MetricsTables;
|
|
36
|
+
readonly jobClient: null;
|
|
37
|
+
private readonly owned;
|
|
38
|
+
private readonly shouldMigrate;
|
|
39
|
+
private readiness;
|
|
40
|
+
private closed;
|
|
41
|
+
private readonly ctx;
|
|
42
|
+
constructor(opts: PostgresMetricsStoreOptions);
|
|
43
|
+
/** Creates or upgrades the tables now. Idempotent and safe to race. */
|
|
44
|
+
migrate(): Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* Memoized, but not on failure: a database that was unreachable at startup is retried on
|
|
47
|
+
* the next query instead of poisoning the store for the life of the process.
|
|
48
|
+
*/
|
|
49
|
+
ready(): Promise<void>;
|
|
50
|
+
counterStore(retention: Retention): CounterStore;
|
|
51
|
+
latencyStore(retention: Retention): LatencyStorage;
|
|
52
|
+
administration(): HistoryAdministration;
|
|
53
|
+
/** Ends the pool if this store created it. A pool handed in is left to its owner. */
|
|
54
|
+
close(): Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
export interface MigratePostgresMetricsOptions {
|
|
57
|
+
connection: PostgresConnection;
|
|
58
|
+
schema?: string;
|
|
59
|
+
tablePrefix?: string;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Deploy-step form of `migrate: true`: creates or upgrades the metrics tables and returns.
|
|
63
|
+
* A pool handed in is left open; one built from a config or string is ended afterwards.
|
|
64
|
+
*/
|
|
65
|
+
export declare function migratePostgresMetrics(opts: MigratePostgresMetricsOptions): Promise<void>;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PostgresMetricsStore = void 0;
|
|
4
|
+
exports.migratePostgresMetrics = migratePostgresMetrics;
|
|
5
|
+
const admin_1 = require("./admin");
|
|
6
|
+
const connection_1 = require("./connection");
|
|
7
|
+
const schema_1 = require("./schema");
|
|
8
|
+
const stores_1 = require("./stores");
|
|
9
|
+
/**
|
|
10
|
+
* History in PostgreSQL, for boards whose queues live on BullMQ's PostgreSQL backend and have
|
|
11
|
+
* no Redis at all. Same tiers, retention, rollup and idempotency as the Redis store; see the
|
|
12
|
+
* package README for the schema and sizing.
|
|
13
|
+
*/
|
|
14
|
+
class PostgresMetricsStore {
|
|
15
|
+
constructor(opts) {
|
|
16
|
+
var _a, _b, _c;
|
|
17
|
+
this.jobClient = null;
|
|
18
|
+
this.readiness = null;
|
|
19
|
+
this.closed = false;
|
|
20
|
+
const { pool, owned, schema } = (0, connection_1.resolvePool)(opts.connection, opts.onError);
|
|
21
|
+
this.pool = pool;
|
|
22
|
+
this.owned = owned;
|
|
23
|
+
try {
|
|
24
|
+
this.tables = (0, schema_1.metricsTables)((_b = (_a = opts.schema) !== null && _a !== void 0 ? _a : schema) !== null && _b !== void 0 ? _b : schema_1.DEFAULT_SCHEMA, (_c = opts.tablePrefix) !== null && _c !== void 0 ? _c : schema_1.DEFAULT_TABLE_PREFIX);
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
if (owned) {
|
|
28
|
+
void pool.end().catch(() => undefined);
|
|
29
|
+
}
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
this.shouldMigrate = opts.migrate === true;
|
|
33
|
+
this.ctx = { pool: this.pool, tables: this.tables, ready: () => this.ready() };
|
|
34
|
+
}
|
|
35
|
+
/** Creates or upgrades the tables now. Idempotent and safe to race. */
|
|
36
|
+
async migrate() {
|
|
37
|
+
await (0, schema_1.runMigrations)(this.pool, this.tables);
|
|
38
|
+
this.readiness = Promise.resolve();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Memoized, but not on failure: a database that was unreachable at startup is retried on
|
|
42
|
+
* the next query instead of poisoning the store for the life of the process.
|
|
43
|
+
*/
|
|
44
|
+
ready() {
|
|
45
|
+
if (!this.readiness) {
|
|
46
|
+
const attempt = this.shouldMigrate
|
|
47
|
+
? (0, schema_1.runMigrations)(this.pool, this.tables).then(() => undefined)
|
|
48
|
+
: (0, schema_1.assertMigrated)(this.pool, this.tables);
|
|
49
|
+
this.readiness = attempt.catch((error) => {
|
|
50
|
+
this.readiness = null;
|
|
51
|
+
throw error;
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
return this.readiness;
|
|
55
|
+
}
|
|
56
|
+
counterStore(retention) {
|
|
57
|
+
return new stores_1.PostgresCounterStore(this.ctx, retention);
|
|
58
|
+
}
|
|
59
|
+
latencyStore(retention) {
|
|
60
|
+
return new stores_1.PostgresLatencyStore(this.ctx, retention);
|
|
61
|
+
}
|
|
62
|
+
administration() {
|
|
63
|
+
return new admin_1.PostgresHistoryAdmin(this.ctx);
|
|
64
|
+
}
|
|
65
|
+
/** Ends the pool if this store created it. A pool handed in is left to its owner. */
|
|
66
|
+
async close() {
|
|
67
|
+
if (this.owned && !this.closed) {
|
|
68
|
+
this.closed = true;
|
|
69
|
+
await this.pool.end();
|
|
70
|
+
}
|
|
71
|
+
this.closed = true;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.PostgresMetricsStore = PostgresMetricsStore;
|
|
75
|
+
/**
|
|
76
|
+
* Deploy-step form of `migrate: true`: creates or upgrades the metrics tables and returns.
|
|
77
|
+
* A pool handed in is left open; one built from a config or string is ended afterwards.
|
|
78
|
+
*/
|
|
79
|
+
async function migratePostgresMetrics(opts) {
|
|
80
|
+
const store = new PostgresMetricsStore(opts);
|
|
81
|
+
try {
|
|
82
|
+
await store.migrate();
|
|
83
|
+
}
|
|
84
|
+
finally {
|
|
85
|
+
await store.close();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=PostgresMetricsStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PostgresMetricsStore.js","sourceRoot":"","sources":["../../src/postgres/PostgresMetricsStore.ts"],"names":[],"mappings":";;;AAsIA,wDAOC;AAtID,mCAA+C;AAC/C,6CAAiF;AAEjF,qCAOkB;AAClB,qCAAsE;AA2BtE;;;;GAIG;AACH,MAAa,oBAAoB;IAU/B,YAAY,IAAiC;;QAPpC,cAAS,GAAG,IAAI,CAAC;QAGlB,cAAS,GAAyB,IAAI,CAAC;QACvC,WAAM,GAAG,KAAK,CAAC;QAIrB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAA,wBAAW,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,IAAA,sBAAa,EACzB,MAAA,MAAA,IAAI,CAAC,MAAM,mCAAI,MAAM,mCAAI,uBAAc,EACvC,MAAA,IAAI,CAAC,WAAW,mCAAI,6BAAoB,CACzC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC;QAC3C,IAAI,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,OAAO;QACX,MAAM,IAAA,sBAAa,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa;gBAChC,CAAC,CAAC,IAAA,sBAAa,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;gBAC7D,CAAC,CAAC,IAAA,uBAAc,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,OAAO,IAAI,6BAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,OAAO,IAAI,6BAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,4BAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,qFAAqF;IACrF,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;CACF;AAxED,oDAwEC;AAQD;;;GAGG;AACI,KAAK,UAAU,sBAAsB,CAAC,IAAmC;IAC9E,MAAM,KAAK,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { HistoryStats, PurgeOptions, PurgeResult } from '../HistoryAdmin';
|
|
2
|
+
import type { HistoryAdministration } from '../store';
|
|
3
|
+
import { type PgContext } from './context';
|
|
4
|
+
/**
|
|
5
|
+
* `stats()` and `purge()` over the PostgreSQL tables, reported in the same shape as the Redis
|
|
6
|
+
* admin so the board's storage panel reads either one:
|
|
7
|
+
*
|
|
8
|
+
* - `keys` counts rows. A Redis key is a whole hash of buckets; a row here is one bucket, so
|
|
9
|
+
* the numbers are larger for the same history and only comparable within one backend.
|
|
10
|
+
* - `bytes` is the tables' real on-disk footprint, `pg_total_relation_size` (heap, TOAST and
|
|
11
|
+
* indexes), apportioned to queues and tiers by each row's `pg_column_size`. The parts add up
|
|
12
|
+
* to what the tables occupy, rather than to the smaller sum of raw row sizes. Dead tuples
|
|
13
|
+
* awaiting vacuum count until autovacuum reclaims them, as they do on disk.
|
|
14
|
+
* - `minutes` counts minute-tier rows, which is the same number as the Redis minute fields.
|
|
15
|
+
*/
|
|
16
|
+
export declare class PostgresHistoryAdmin implements HistoryAdministration {
|
|
17
|
+
private readonly ctx;
|
|
18
|
+
constructor(ctx: PgContext);
|
|
19
|
+
stats(): Promise<HistoryStats>;
|
|
20
|
+
/**
|
|
21
|
+
* One transaction. Purging a single queue first subtracts its completed and failed rows
|
|
22
|
+
* from the matching `__global__` rows, tier by tier, and drops the global rows that drain to
|
|
23
|
+
* zero or below, exactly as the Redis admin does; the latency rollups keep the queue's share
|
|
24
|
+
* until retention drops it, for the same reason (see `SUMMABLE_METRICS` there).
|
|
25
|
+
*
|
|
26
|
+
* `keysDeleted` counts minute and hour rows, the rows a Redis day-scoped key holds;
|
|
27
|
+
* `fieldsDeleted` counts day rows, the Redis totals-hash fields. Both include drained global
|
|
28
|
+
* rows.
|
|
29
|
+
*/
|
|
30
|
+
purge(opts?: PurgeOptions): Promise<PurgeResult>;
|
|
31
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PostgresHistoryAdmin = void 0;
|
|
4
|
+
const keys_1 = require("../keys");
|
|
5
|
+
const context_1 = require("./context");
|
|
6
|
+
/** Same reasoning as the Redis admin: only plain sums can have one queue's share taken out. */
|
|
7
|
+
const SUMMABLE_METRICS = ['completed', 'failed'];
|
|
8
|
+
function emptyTiers() {
|
|
9
|
+
return {
|
|
10
|
+
minute: { keys: 0, bytes: 0 },
|
|
11
|
+
hour: { keys: 0, bytes: 0 },
|
|
12
|
+
day: { keys: 0, bytes: 0 },
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* The day a row covers, as a SQL expression over its `tier` and `bucket` columns. `null` for
|
|
17
|
+
* the day tier, matching the Redis admin, where only minute and hour hashes carry a day.
|
|
18
|
+
*/
|
|
19
|
+
const ROW_DAY = `CASE tier WHEN 'minute' THEN bucket / 1440 WHEN 'hour' THEN bucket / 24 END`;
|
|
20
|
+
/**
|
|
21
|
+
* `stats()` and `purge()` over the PostgreSQL tables, reported in the same shape as the Redis
|
|
22
|
+
* admin so the board's storage panel reads either one:
|
|
23
|
+
*
|
|
24
|
+
* - `keys` counts rows. A Redis key is a whole hash of buckets; a row here is one bucket, so
|
|
25
|
+
* the numbers are larger for the same history and only comparable within one backend.
|
|
26
|
+
* - `bytes` is the tables' real on-disk footprint, `pg_total_relation_size` (heap, TOAST and
|
|
27
|
+
* indexes), apportioned to queues and tiers by each row's `pg_column_size`. The parts add up
|
|
28
|
+
* to what the tables occupy, rather than to the smaller sum of raw row sizes. Dead tuples
|
|
29
|
+
* awaiting vacuum count until autovacuum reclaims them, as they do on disk.
|
|
30
|
+
* - `minutes` counts minute-tier rows, which is the same number as the Redis minute fields.
|
|
31
|
+
*/
|
|
32
|
+
class PostgresHistoryAdmin {
|
|
33
|
+
constructor(ctx) {
|
|
34
|
+
this.ctx = ctx;
|
|
35
|
+
}
|
|
36
|
+
async stats() {
|
|
37
|
+
await this.ctx.ready();
|
|
38
|
+
const { counters, histograms } = this.ctx.tables;
|
|
39
|
+
const [groups, days] = await Promise.all([
|
|
40
|
+
this.ctx.pool.query(`WITH sized AS (
|
|
41
|
+
SELECT 'counters' AS tbl, queue, tier,
|
|
42
|
+
count(*) AS rows, sum(pg_column_size(c.*))::float8 AS size
|
|
43
|
+
FROM ${counters} c GROUP BY queue, tier
|
|
44
|
+
UNION ALL
|
|
45
|
+
SELECT 'histograms', queue, tier,
|
|
46
|
+
count(*), sum(pg_column_size(h.*))::float8
|
|
47
|
+
FROM ${histograms} h GROUP BY queue, tier
|
|
48
|
+
)
|
|
49
|
+
SELECT s.queue, s.tier, s.rows,
|
|
50
|
+
CASE WHEN t.size > 0
|
|
51
|
+
THEN round(s.size / t.size * t.bytes)
|
|
52
|
+
ELSE 0 END AS bytes
|
|
53
|
+
FROM sized s
|
|
54
|
+
JOIN (SELECT tbl, sum(size) AS size,
|
|
55
|
+
CASE tbl WHEN 'counters' THEN pg_total_relation_size($1::regclass)
|
|
56
|
+
ELSE pg_total_relation_size($2::regclass) END AS bytes
|
|
57
|
+
FROM sized GROUP BY tbl) t ON t.tbl = s.tbl`, [counters, histograms]),
|
|
58
|
+
this.ctx.pool.query(`SELECT DISTINCT queue, ${ROW_DAY} AS day FROM ${counters} WHERE tier <> 'day'
|
|
59
|
+
UNION
|
|
60
|
+
SELECT DISTINCT queue, ${ROW_DAY} FROM ${histograms} WHERE tier <> 'day'`),
|
|
61
|
+
]);
|
|
62
|
+
const byQueue = new Map();
|
|
63
|
+
const entry = (queue) => {
|
|
64
|
+
let found = byQueue.get(queue);
|
|
65
|
+
if (!found) {
|
|
66
|
+
found = { queue, keys: 0, bytes: 0, minutes: 0, days: [], tiers: emptyTiers() };
|
|
67
|
+
byQueue.set(queue, found);
|
|
68
|
+
}
|
|
69
|
+
return found;
|
|
70
|
+
};
|
|
71
|
+
const tiers = emptyTiers();
|
|
72
|
+
let keys = 0;
|
|
73
|
+
let bytes = 0;
|
|
74
|
+
let minutes = 0;
|
|
75
|
+
for (const row of groups.rows) {
|
|
76
|
+
const tier = row.tier;
|
|
77
|
+
const rows = Number(row.rows);
|
|
78
|
+
const size = Number(row.bytes);
|
|
79
|
+
const queue = entry(row.queue);
|
|
80
|
+
queue.keys += rows;
|
|
81
|
+
queue.bytes += size;
|
|
82
|
+
queue.tiers[tier].keys += rows;
|
|
83
|
+
queue.tiers[tier].bytes += size;
|
|
84
|
+
tiers[tier].keys += rows;
|
|
85
|
+
tiers[tier].bytes += size;
|
|
86
|
+
keys += rows;
|
|
87
|
+
bytes += size;
|
|
88
|
+
if (tier === 'minute') {
|
|
89
|
+
queue.minutes += rows;
|
|
90
|
+
minutes += rows;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
let oldestDay = null;
|
|
94
|
+
let newestDay = null;
|
|
95
|
+
for (const row of days.rows) {
|
|
96
|
+
const day = (0, keys_1.indexToDay)(Number(row.day));
|
|
97
|
+
entry(row.queue).days.push(day);
|
|
98
|
+
if (oldestDay === null || day < oldestDay) {
|
|
99
|
+
oldestDay = day;
|
|
100
|
+
}
|
|
101
|
+
if (newestDay === null || day > newestDay) {
|
|
102
|
+
newestDay = day;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const queues = [...byQueue.values()].sort((a, b) => b.bytes - a.bytes);
|
|
106
|
+
for (const queue of queues) {
|
|
107
|
+
queue.days = [...new Set(queue.days)].sort();
|
|
108
|
+
}
|
|
109
|
+
return { keys, bytes, minutes, oldestDay, newestDay, tiers, queues };
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* One transaction. Purging a single queue first subtracts its completed and failed rows
|
|
113
|
+
* from the matching `__global__` rows, tier by tier, and drops the global rows that drain to
|
|
114
|
+
* zero or below, exactly as the Redis admin does; the latency rollups keep the queue's share
|
|
115
|
+
* until retention drops it, for the same reason (see `SUMMABLE_METRICS` there).
|
|
116
|
+
*
|
|
117
|
+
* `keysDeleted` counts minute and hour rows, the rows a Redis day-scoped key holds;
|
|
118
|
+
* `fieldsDeleted` counts day rows, the Redis totals-hash fields. Both include drained global
|
|
119
|
+
* rows.
|
|
120
|
+
*/
|
|
121
|
+
async purge(opts = {}) {
|
|
122
|
+
var _a;
|
|
123
|
+
const before = opts.before === undefined ? null : (0, keys_1.dayToIndex)((0, keys_1.toDay)(opts.before));
|
|
124
|
+
const queue = (_a = opts.queue) !== null && _a !== void 0 ? _a : null;
|
|
125
|
+
const adjustGlobal = queue !== null && queue !== keys_1.GLOBAL_QUEUE;
|
|
126
|
+
const { counters, histograms } = this.ctx.tables;
|
|
127
|
+
const scope = `($1::text IS NULL OR queue = $1)
|
|
128
|
+
AND ($2::bigint IS NULL OR coalesce(${ROW_DAY}, bucket) < $2)`;
|
|
129
|
+
return (0, context_1.inTransaction)(this.ctx, async (client) => {
|
|
130
|
+
const result = { keysDeleted: 0, fieldsDeleted: 0 };
|
|
131
|
+
const tally = (tier, count) => {
|
|
132
|
+
if (tier === 'day') {
|
|
133
|
+
result.fieldsDeleted += count;
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
result.keysDeleted += count;
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
const { rows: removed } = await client.query(`WITH doomed AS (
|
|
140
|
+
DELETE FROM ${counters} WHERE ${scope}
|
|
141
|
+
RETURNING metric, tier, bucket, value
|
|
142
|
+
),
|
|
143
|
+
drained AS (
|
|
144
|
+
UPDATE ${counters} g SET value = g.value - d.value
|
|
145
|
+
FROM doomed d
|
|
146
|
+
WHERE $3 AND g.queue = $4 AND g.metric = d.metric AND g.tier = d.tier
|
|
147
|
+
AND g.bucket = d.bucket AND d.metric = ANY($5::text[]) AND d.value <> 0
|
|
148
|
+
RETURNING g.metric, g.tier, g.bucket, g.value
|
|
149
|
+
)
|
|
150
|
+
SELECT 'doomed' AS kind, tier, count(*) AS n, NULL::text[] AS metrics,
|
|
151
|
+
NULL::bigint[] AS buckets
|
|
152
|
+
FROM doomed GROUP BY tier
|
|
153
|
+
UNION ALL
|
|
154
|
+
SELECT 'drained', tier, count(*), array_agg(metric), array_agg(bucket)
|
|
155
|
+
FROM drained WHERE value <= 0 GROUP BY tier`, [queue, before, adjustGlobal, keys_1.GLOBAL_QUEUE, SUMMABLE_METRICS]);
|
|
156
|
+
for (const row of removed) {
|
|
157
|
+
if (row.kind === 'doomed') {
|
|
158
|
+
tally(row.tier, Number(row.n));
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
// The recorder never writes a zero bucket, so a leftover zero would read as
|
|
162
|
+
// recorded-but-idle instead of not recorded.
|
|
163
|
+
const { rowCount } = await client.query(`DELETE FROM ${counters} g
|
|
164
|
+
USING unnest($2::text[], $3::bigint[]) AS d(metric, bucket)
|
|
165
|
+
WHERE g.queue = $1 AND g.tier = $4 AND g.metric = d.metric AND g.bucket = d.bucket`, [keys_1.GLOBAL_QUEUE, row.metrics, row.buckets, row.tier]);
|
|
166
|
+
tally(row.tier, rowCount !== null && rowCount !== void 0 ? rowCount : 0);
|
|
167
|
+
}
|
|
168
|
+
const { rows: latency } = await client.query(`WITH doomed AS (DELETE FROM ${histograms} WHERE ${scope} RETURNING tier)
|
|
169
|
+
SELECT tier, count(*) AS n FROM doomed GROUP BY tier`, [queue, before]);
|
|
170
|
+
for (const row of latency) {
|
|
171
|
+
tally(row.tier, Number(row.n));
|
|
172
|
+
}
|
|
173
|
+
return result;
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
exports.PostgresHistoryAdmin = PostgresHistoryAdmin;
|
|
178
|
+
//# sourceMappingURL=admin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"admin.js","sourceRoot":"","sources":["../../src/postgres/admin.ts"],"names":[],"mappings":";;;AAQA,kCAAsE;AAEtE,uCAA0D;AAE1D,+FAA+F;AAC/F,MAAM,gBAAgB,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AAEjD,SAAS,UAAU;IACjB,OAAO;QACL,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC7B,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC3B,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KAC3B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,GAAG,6EAA6E,CAAC;AAE9F;;;;;;;;;;;GAWG;AACH,MAAa,oBAAoB;IAC/B,YAA6B,GAAc;QAAd,QAAG,GAAH,GAAG,CAAW;IAAG,CAAC;IAE/C,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACjD,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACjB;;;oBAGY,QAAQ;;;;oBAIR,UAAU;;;;;;;;;;+DAUiC,EACvD,CAAC,QAAQ,EAAE,UAAU,CAAC,CACvB;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACjB,0BAA0B,OAAO,gBAAgB,QAAQ;;kCAE/B,OAAO,SAAS,UAAU,sBAAsB,CAC3E;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;QACrD,MAAM,KAAK,GAAG,CAAC,KAAa,EAAqB,EAAE;YACjD,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,KAAK,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;gBAChF,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAC5B,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAmB,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC;YACnB,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC;YACpB,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC;YAC/B,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YAC1B,IAAI,IAAI,IAAI,CAAC;YACb,KAAK,IAAI,IAAI,CAAC;YACd,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC;gBACtB,OAAO,IAAI,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,SAAS,KAAK,IAAI,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;gBAC1C,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;YACD,IAAI,SAAS,KAAK,IAAI,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;gBAC1C,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,KAAK,CAAC,OAAqB,EAAE;;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,iBAAU,EAAC,IAAA,YAAK,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACjF,MAAM,KAAK,GAAG,MAAA,IAAI,CAAC,KAAK,mCAAI,IAAI,CAAC;QACjC,MAAM,YAAY,GAAG,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,mBAAY,CAAC;QAC9D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACjD,MAAM,KAAK,GAAG;4CAC0B,OAAO,iBAAiB,CAAC;QAEjE,OAAO,IAAA,uBAAa,EAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAgB,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;gBAC5C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;oBACnB,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC;gBAC9B,CAAC;YACH,CAAC,CAAC;YAEF,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC1C;yBACiB,QAAQ,UAAU,KAAK;;;;oBAI5B,QAAQ;;;;;;;;;;;uDAW2B,EAC/C,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,mBAAY,EAAE,gBAAgB,CAAC,CAC9D,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC1B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,SAAS;gBACX,CAAC;gBACD,4EAA4E;gBAC5E,6CAA6C;gBAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACrC,eAAe,QAAQ;;+FAE8D,EACrF,CAAC,mBAAY,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CACnD,CAAC;gBACF,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,CAAC,CAAC,CAAC;YACjC,CAAC;YAED,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC1C,+BAA+B,UAAU,UAAU,KAAK;8DACF,EACtD,CAAC,KAAK,EAAE,MAAM,CAAC,CAChB,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AApKD,oDAoKC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slice of node-postgres this package uses, declared structurally so that the published
|
|
3
|
+
* types never name `pg`: a Redis-only install has neither `pg` nor `@types/pg`, and must not
|
|
4
|
+
* need them to compile.
|
|
5
|
+
*/
|
|
6
|
+
export interface PgQueryResult {
|
|
7
|
+
rows: Record<string, any>[];
|
|
8
|
+
rowCount?: number | null;
|
|
9
|
+
}
|
|
10
|
+
export interface PgQueryable {
|
|
11
|
+
query(text: string, values?: unknown[]): Promise<PgQueryResult>;
|
|
12
|
+
}
|
|
13
|
+
export interface PgPoolClient extends PgQueryable {
|
|
14
|
+
release(error?: Error | boolean): void;
|
|
15
|
+
}
|
|
16
|
+
/** A `pg.Pool`, or anything shaped like one. */
|
|
17
|
+
export interface PgPool extends PgQueryable {
|
|
18
|
+
connect(): Promise<PgPoolClient>;
|
|
19
|
+
end(): Promise<void>;
|
|
20
|
+
on?(event: 'error', listener: (error: Error) => void): unknown;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* A node-postgres pool config (`connectionString`, `host`, `max`, `ssl`, ...). A `schema` key
|
|
24
|
+
* is accepted too, so the object handed to BullMQ's PostgreSQL backend can be reused as is;
|
|
25
|
+
* it is lifted out rather than passed to the pool.
|
|
26
|
+
*/
|
|
27
|
+
export interface PostgresPoolConfig {
|
|
28
|
+
connectionString?: string;
|
|
29
|
+
host?: string;
|
|
30
|
+
port?: number;
|
|
31
|
+
user?: string;
|
|
32
|
+
password?: string | (() => string | Promise<string>);
|
|
33
|
+
database?: string;
|
|
34
|
+
max?: number;
|
|
35
|
+
schema?: string;
|
|
36
|
+
[option: string]: unknown;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Mirrors the ergonomics of BullMQ v6's PostgreSQL connection option: an existing pool, a
|
|
40
|
+
* pool config, or a connection string.
|
|
41
|
+
*/
|
|
42
|
+
export type PostgresConnection = PgPool | PostgresPoolConfig | string;
|
|
43
|
+
/**
|
|
44
|
+
* Schema names and table prefixes end up inside DDL, where they cannot be bound as
|
|
45
|
+
* parameters, so they are held to plain identifiers and always quoted.
|
|
46
|
+
*/
|
|
47
|
+
export declare function quoteIdentifier(name: string): string;
|
|
48
|
+
export declare function isPool(connection: unknown): connection is PgPool;
|
|
49
|
+
export declare function resolvePool(connection: PostgresConnection, onError?: (error: Error) => void): {
|
|
50
|
+
pool: PgPool;
|
|
51
|
+
owned: boolean;
|
|
52
|
+
schema: string | undefined;
|
|
53
|
+
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.quoteIdentifier = quoteIdentifier;
|
|
4
|
+
exports.isPool = isPool;
|
|
5
|
+
exports.resolvePool = resolvePool;
|
|
6
|
+
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
7
|
+
/**
|
|
8
|
+
* Schema names and table prefixes end up inside DDL, where they cannot be bound as
|
|
9
|
+
* parameters, so they are held to plain identifiers and always quoted.
|
|
10
|
+
*/
|
|
11
|
+
function quoteIdentifier(name) {
|
|
12
|
+
if (!IDENTIFIER.test(name) || name.length > 63) {
|
|
13
|
+
throw new Error(`Invalid PostgreSQL identifier ${JSON.stringify(name)}: use letters, digits and ` +
|
|
14
|
+
'underscores, starting with a letter or underscore, at most 63 characters.');
|
|
15
|
+
}
|
|
16
|
+
return `"${name}"`;
|
|
17
|
+
}
|
|
18
|
+
function isPool(connection) {
|
|
19
|
+
const candidate = connection;
|
|
20
|
+
return (!!candidate &&
|
|
21
|
+
typeof candidate.connect === 'function' &&
|
|
22
|
+
typeof candidate.query === 'function' &&
|
|
23
|
+
typeof candidate.end === 'function');
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* `pg` is an optional peer dependency, so it is required only here, on the first PostgreSQL
|
|
27
|
+
* store that has to build its own pool. Handing in a pool never loads it.
|
|
28
|
+
*/
|
|
29
|
+
function loadPg() {
|
|
30
|
+
try {
|
|
31
|
+
// oxlint-disable-next-line typescript/no-require-imports
|
|
32
|
+
return require('pg');
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
throw new Error("PostgreSQL metrics storage needs the optional 'pg' package. Install it with " +
|
|
36
|
+
'`npm install pg`, or pass an existing pg.Pool as `connection`.');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function resolvePool(connection, onError) {
|
|
40
|
+
var _a;
|
|
41
|
+
if (isPool(connection)) {
|
|
42
|
+
return { pool: connection, owned: false, schema: undefined };
|
|
43
|
+
}
|
|
44
|
+
const { schema, ...config } = typeof connection === 'string'
|
|
45
|
+
? { connectionString: connection }
|
|
46
|
+
: connection;
|
|
47
|
+
const { Pool } = loadPg();
|
|
48
|
+
const pool = new Pool(config);
|
|
49
|
+
// node-postgres emits `error` for a pooled client that dies while idle (a server restart, a
|
|
50
|
+
// dropped connection). Unhandled, that event takes the whole process down with it.
|
|
51
|
+
(_a = pool.on) === null || _a === void 0 ? void 0 : _a.call(pool, 'error', (error) => onError === null || onError === void 0 ? void 0 : onError(error));
|
|
52
|
+
return { pool, owned: true, schema };
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=connection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/postgres/connection.ts"],"names":[],"mappings":";;AAsDA,0CAQC;AAED,wBAQC;AAkBD,kCAiBC;AA3DD,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAE9C;;;GAGG;AACH,SAAgB,eAAe,CAAC,IAAY;IAC1C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iCAAiC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B;YAC/E,2EAA2E,CAC9E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,IAAI,GAAG,CAAC;AACrB,CAAC;AAED,SAAgB,MAAM,CAAC,UAAmB;IACxC,MAAM,SAAS,GAAG,UAAoC,CAAC;IACvD,OAAO,CACL,CAAC,CAAC,SAAS;QACX,OAAO,SAAS,CAAC,OAAO,KAAK,UAAU;QACvC,OAAO,SAAS,CAAC,KAAK,KAAK,UAAU;QACrC,OAAO,SAAS,CAAC,GAAG,KAAK,UAAU,CACpC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,MAAM;IACb,IAAI,CAAC;QACH,yDAAyD;QACzD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,8EAA8E;YAC5E,gEAAgE,CACnE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAgB,WAAW,CACzB,UAA8B,EAC9B,OAAgC;;IAEhC,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/D,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GACzB,OAAO,UAAU,KAAK,QAAQ;QAC5B,CAAC,CAAE,EAAE,gBAAgB,EAAE,UAAU,EAAyB;QAC1D,CAAC,CAAC,UAAU,CAAC;IACjB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,4FAA4F;IAC5F,mFAAmF;IACnF,MAAA,IAAI,CAAC,EAAE,qDAAG,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,KAAK,CAAC,CAAC,CAAC;IAChD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC"}
|