@worker-manager/metrics 1.1.0 → 9.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -86
- package/dist/HistoryAdmin.d.ts +87 -23
- package/dist/HistoryAdmin.js +330 -21
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +9 -9
- package/dist/HistoryStore.js +0 -22
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +32 -27
- package/dist/LatencySampler.js +156 -55
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +3 -8
- package/dist/LatencyStore.js +0 -18
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +10 -34
- package/dist/MetricsRecorder.js +20 -30
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisMetricsHistoryProvider.d.ts +17 -7
- package/dist/RedisMetricsHistoryProvider.js +136 -9
- package/dist/RedisMetricsHistoryProvider.js.map +1 -1
- package/dist/index.d.ts +2 -10
- package/dist/index.js +1 -10
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +0 -7
- package/dist/keys.js +1 -23
- package/dist/keys.js.map +1 -1
- package/package.json +4 -14
- package/dist/RedisHistoryAdmin.d.ts +0 -95
- package/dist/RedisHistoryAdmin.js +0 -328
- package/dist/RedisHistoryAdmin.js.map +0 -1
- package/dist/RedisMetricsStore.d.ts +0 -27
- package/dist/RedisMetricsStore.js +0 -42
- package/dist/RedisMetricsStore.js.map +0 -1
- package/dist/StoreHistoryProvider.d.ts +0 -27
- package/dist/StoreHistoryProvider.js +0 -127
- package/dist/StoreHistoryProvider.js.map +0 -1
- package/dist/jobSources.d.ts +0 -97
- package/dist/jobSources.js +0 -248
- package/dist/jobSources.js.map +0 -1
- package/dist/postgres/PostgresMetricsHistoryProvider.d.ts +0 -37
- package/dist/postgres/PostgresMetricsHistoryProvider.js +0 -30
- package/dist/postgres/PostgresMetricsHistoryProvider.js.map +0 -1
- package/dist/postgres/PostgresMetricsStore.d.ts +0 -65
- package/dist/postgres/PostgresMetricsStore.js +0 -88
- package/dist/postgres/PostgresMetricsStore.js.map +0 -1
- package/dist/postgres/admin.d.ts +0 -31
- package/dist/postgres/admin.js +0 -178
- package/dist/postgres/admin.js.map +0 -1
- package/dist/postgres/connection.d.ts +0 -53
- package/dist/postgres/connection.js +0 -54
- package/dist/postgres/connection.js.map +0 -1
- package/dist/postgres/context.d.ts +0 -29
- package/dist/postgres/context.js +0 -76
- package/dist/postgres/context.js.map +0 -1
- package/dist/postgres/schema.d.ts +0 -26
- package/dist/postgres/schema.js +0 -148
- package/dist/postgres/schema.js.map +0 -1
- package/dist/postgres/stores.d.ts +0 -51
- package/dist/postgres/stores.js +0 -252
- package/dist/postgres/stores.js.map +0 -1
- package/dist/store.d.ts +0 -80
- package/dist/store.js +0 -3
- package/dist/store.js.map +0 -1
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PostgresMetricsStore = void 0;
|
|
4
|
-
exports.migratePostgresMetrics = migratePostgresMetrics;
|
|
5
|
-
const admin_1 = require("./admin");
|
|
6
|
-
const connection_1 = require("./connection");
|
|
7
|
-
const schema_1 = require("./schema");
|
|
8
|
-
const stores_1 = require("./stores");
|
|
9
|
-
/**
|
|
10
|
-
* History in PostgreSQL, for boards whose queues live on BullMQ's PostgreSQL backend and have
|
|
11
|
-
* no Redis at all. Same tiers, retention, rollup and idempotency as the Redis store; see the
|
|
12
|
-
* package README for the schema and sizing.
|
|
13
|
-
*/
|
|
14
|
-
class PostgresMetricsStore {
|
|
15
|
-
constructor(opts) {
|
|
16
|
-
var _a, _b, _c;
|
|
17
|
-
this.jobClient = null;
|
|
18
|
-
this.readiness = null;
|
|
19
|
-
this.closed = false;
|
|
20
|
-
const { pool, owned, schema } = (0, connection_1.resolvePool)(opts.connection, opts.onError);
|
|
21
|
-
this.pool = pool;
|
|
22
|
-
this.owned = owned;
|
|
23
|
-
try {
|
|
24
|
-
this.tables = (0, schema_1.metricsTables)((_b = (_a = opts.schema) !== null && _a !== void 0 ? _a : schema) !== null && _b !== void 0 ? _b : schema_1.DEFAULT_SCHEMA, (_c = opts.tablePrefix) !== null && _c !== void 0 ? _c : schema_1.DEFAULT_TABLE_PREFIX);
|
|
25
|
-
}
|
|
26
|
-
catch (error) {
|
|
27
|
-
if (owned) {
|
|
28
|
-
void pool.end().catch(() => undefined);
|
|
29
|
-
}
|
|
30
|
-
throw error;
|
|
31
|
-
}
|
|
32
|
-
this.shouldMigrate = opts.migrate === true;
|
|
33
|
-
this.ctx = { pool: this.pool, tables: this.tables, ready: () => this.ready() };
|
|
34
|
-
}
|
|
35
|
-
/** Creates or upgrades the tables now. Idempotent and safe to race. */
|
|
36
|
-
async migrate() {
|
|
37
|
-
await (0, schema_1.runMigrations)(this.pool, this.tables);
|
|
38
|
-
this.readiness = Promise.resolve();
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Memoized, but not on failure: a database that was unreachable at startup is retried on
|
|
42
|
-
* the next query instead of poisoning the store for the life of the process.
|
|
43
|
-
*/
|
|
44
|
-
ready() {
|
|
45
|
-
if (!this.readiness) {
|
|
46
|
-
const attempt = this.shouldMigrate
|
|
47
|
-
? (0, schema_1.runMigrations)(this.pool, this.tables).then(() => undefined)
|
|
48
|
-
: (0, schema_1.assertMigrated)(this.pool, this.tables);
|
|
49
|
-
this.readiness = attempt.catch((error) => {
|
|
50
|
-
this.readiness = null;
|
|
51
|
-
throw error;
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
return this.readiness;
|
|
55
|
-
}
|
|
56
|
-
counterStore(retention) {
|
|
57
|
-
return new stores_1.PostgresCounterStore(this.ctx, retention);
|
|
58
|
-
}
|
|
59
|
-
latencyStore(retention) {
|
|
60
|
-
return new stores_1.PostgresLatencyStore(this.ctx, retention);
|
|
61
|
-
}
|
|
62
|
-
administration() {
|
|
63
|
-
return new admin_1.PostgresHistoryAdmin(this.ctx);
|
|
64
|
-
}
|
|
65
|
-
/** Ends the pool if this store created it. A pool handed in is left to its owner. */
|
|
66
|
-
async close() {
|
|
67
|
-
if (this.owned && !this.closed) {
|
|
68
|
-
this.closed = true;
|
|
69
|
-
await this.pool.end();
|
|
70
|
-
}
|
|
71
|
-
this.closed = true;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
exports.PostgresMetricsStore = PostgresMetricsStore;
|
|
75
|
-
/**
|
|
76
|
-
* Deploy-step form of `migrate: true`: creates or upgrades the metrics tables and returns.
|
|
77
|
-
* A pool handed in is left open; one built from a config or string is ended afterwards.
|
|
78
|
-
*/
|
|
79
|
-
async function migratePostgresMetrics(opts) {
|
|
80
|
-
const store = new PostgresMetricsStore(opts);
|
|
81
|
-
try {
|
|
82
|
-
await store.migrate();
|
|
83
|
-
}
|
|
84
|
-
finally {
|
|
85
|
-
await store.close();
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
//# sourceMappingURL=PostgresMetricsStore.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresMetricsStore.js","sourceRoot":"","sources":["../../src/postgres/PostgresMetricsStore.ts"],"names":[],"mappings":";;;AAsIA,wDAOC;AAtID,mCAA+C;AAC/C,6CAAiF;AAEjF,qCAOkB;AAClB,qCAAsE;AA2BtE;;;;GAIG;AACH,MAAa,oBAAoB;IAU/B,YAAY,IAAiC;;QAPpC,cAAS,GAAG,IAAI,CAAC;QAGlB,cAAS,GAAyB,IAAI,CAAC;QACvC,WAAM,GAAG,KAAK,CAAC;QAIrB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAA,wBAAW,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,IAAA,sBAAa,EACzB,MAAA,MAAA,IAAI,CAAC,MAAM,mCAAI,MAAM,mCAAI,uBAAc,EACvC,MAAA,IAAI,CAAC,WAAW,mCAAI,6BAAoB,CACzC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC;QAC3C,IAAI,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,OAAO;QACX,MAAM,IAAA,sBAAa,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa;gBAChC,CAAC,CAAC,IAAA,sBAAa,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;gBAC7D,CAAC,CAAC,IAAA,uBAAc,EAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,OAAO,IAAI,6BAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,YAAY,CAAC,SAAoB;QAC/B,OAAO,IAAI,6BAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,4BAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,qFAAqF;IACrF,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;CACF;AAxED,oDAwEC;AAQD;;;GAGG;AACI,KAAK,UAAU,sBAAsB,CAAC,IAAmC;IAC9E,MAAM,KAAK,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;AACH,CAAC"}
|
package/dist/postgres/admin.d.ts
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import type { HistoryStats, PurgeOptions, PurgeResult } from '../HistoryAdmin';
|
|
2
|
-
import type { HistoryAdministration } from '../store';
|
|
3
|
-
import { type PgContext } from './context';
|
|
4
|
-
/**
|
|
5
|
-
* `stats()` and `purge()` over the PostgreSQL tables, reported in the same shape as the Redis
|
|
6
|
-
* admin so the board's storage panel reads either one:
|
|
7
|
-
*
|
|
8
|
-
* - `keys` counts rows. A Redis key is a whole hash of buckets; a row here is one bucket, so
|
|
9
|
-
* the numbers are larger for the same history and only comparable within one backend.
|
|
10
|
-
* - `bytes` is the tables' real on-disk footprint, `pg_total_relation_size` (heap, TOAST and
|
|
11
|
-
* indexes), apportioned to queues and tiers by each row's `pg_column_size`. The parts add up
|
|
12
|
-
* to what the tables occupy, rather than to the smaller sum of raw row sizes. Dead tuples
|
|
13
|
-
* awaiting vacuum count until autovacuum reclaims them, as they do on disk.
|
|
14
|
-
* - `minutes` counts minute-tier rows, which is the same number as the Redis minute fields.
|
|
15
|
-
*/
|
|
16
|
-
export declare class PostgresHistoryAdmin implements HistoryAdministration {
|
|
17
|
-
private readonly ctx;
|
|
18
|
-
constructor(ctx: PgContext);
|
|
19
|
-
stats(): Promise<HistoryStats>;
|
|
20
|
-
/**
|
|
21
|
-
* One transaction. Purging a single queue first subtracts its completed and failed rows
|
|
22
|
-
* from the matching `__global__` rows, tier by tier, and drops the global rows that drain to
|
|
23
|
-
* zero or below, exactly as the Redis admin does; the latency rollups keep the queue's share
|
|
24
|
-
* until retention drops it, for the same reason (see `SUMMABLE_METRICS` there).
|
|
25
|
-
*
|
|
26
|
-
* `keysDeleted` counts minute and hour rows, the rows a Redis day-scoped key holds;
|
|
27
|
-
* `fieldsDeleted` counts day rows, the Redis totals-hash fields. Both include drained global
|
|
28
|
-
* rows.
|
|
29
|
-
*/
|
|
30
|
-
purge(opts?: PurgeOptions): Promise<PurgeResult>;
|
|
31
|
-
}
|
package/dist/postgres/admin.js
DELETED
|
@@ -1,178 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PostgresHistoryAdmin = void 0;
|
|
4
|
-
const keys_1 = require("../keys");
|
|
5
|
-
const context_1 = require("./context");
|
|
6
|
-
/** Same reasoning as the Redis admin: only plain sums can have one queue's share taken out. */
|
|
7
|
-
const SUMMABLE_METRICS = ['completed', 'failed'];
|
|
8
|
-
function emptyTiers() {
|
|
9
|
-
return {
|
|
10
|
-
minute: { keys: 0, bytes: 0 },
|
|
11
|
-
hour: { keys: 0, bytes: 0 },
|
|
12
|
-
day: { keys: 0, bytes: 0 },
|
|
13
|
-
};
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* The day a row covers, as a SQL expression over its `tier` and `bucket` columns. `null` for
|
|
17
|
-
* the day tier, matching the Redis admin, where only minute and hour hashes carry a day.
|
|
18
|
-
*/
|
|
19
|
-
const ROW_DAY = `CASE tier WHEN 'minute' THEN bucket / 1440 WHEN 'hour' THEN bucket / 24 END`;
|
|
20
|
-
/**
|
|
21
|
-
* `stats()` and `purge()` over the PostgreSQL tables, reported in the same shape as the Redis
|
|
22
|
-
* admin so the board's storage panel reads either one:
|
|
23
|
-
*
|
|
24
|
-
* - `keys` counts rows. A Redis key is a whole hash of buckets; a row here is one bucket, so
|
|
25
|
-
* the numbers are larger for the same history and only comparable within one backend.
|
|
26
|
-
* - `bytes` is the tables' real on-disk footprint, `pg_total_relation_size` (heap, TOAST and
|
|
27
|
-
* indexes), apportioned to queues and tiers by each row's `pg_column_size`. The parts add up
|
|
28
|
-
* to what the tables occupy, rather than to the smaller sum of raw row sizes. Dead tuples
|
|
29
|
-
* awaiting vacuum count until autovacuum reclaims them, as they do on disk.
|
|
30
|
-
* - `minutes` counts minute-tier rows, which is the same number as the Redis minute fields.
|
|
31
|
-
*/
|
|
32
|
-
class PostgresHistoryAdmin {
|
|
33
|
-
constructor(ctx) {
|
|
34
|
-
this.ctx = ctx;
|
|
35
|
-
}
|
|
36
|
-
async stats() {
|
|
37
|
-
await this.ctx.ready();
|
|
38
|
-
const { counters, histograms } = this.ctx.tables;
|
|
39
|
-
const [groups, days] = await Promise.all([
|
|
40
|
-
this.ctx.pool.query(`WITH sized AS (
|
|
41
|
-
SELECT 'counters' AS tbl, queue, tier,
|
|
42
|
-
count(*) AS rows, sum(pg_column_size(c.*))::float8 AS size
|
|
43
|
-
FROM ${counters} c GROUP BY queue, tier
|
|
44
|
-
UNION ALL
|
|
45
|
-
SELECT 'histograms', queue, tier,
|
|
46
|
-
count(*), sum(pg_column_size(h.*))::float8
|
|
47
|
-
FROM ${histograms} h GROUP BY queue, tier
|
|
48
|
-
)
|
|
49
|
-
SELECT s.queue, s.tier, s.rows,
|
|
50
|
-
CASE WHEN t.size > 0
|
|
51
|
-
THEN round(s.size / t.size * t.bytes)
|
|
52
|
-
ELSE 0 END AS bytes
|
|
53
|
-
FROM sized s
|
|
54
|
-
JOIN (SELECT tbl, sum(size) AS size,
|
|
55
|
-
CASE tbl WHEN 'counters' THEN pg_total_relation_size($1::regclass)
|
|
56
|
-
ELSE pg_total_relation_size($2::regclass) END AS bytes
|
|
57
|
-
FROM sized GROUP BY tbl) t ON t.tbl = s.tbl`, [counters, histograms]),
|
|
58
|
-
this.ctx.pool.query(`SELECT DISTINCT queue, ${ROW_DAY} AS day FROM ${counters} WHERE tier <> 'day'
|
|
59
|
-
UNION
|
|
60
|
-
SELECT DISTINCT queue, ${ROW_DAY} FROM ${histograms} WHERE tier <> 'day'`),
|
|
61
|
-
]);
|
|
62
|
-
const byQueue = new Map();
|
|
63
|
-
const entry = (queue) => {
|
|
64
|
-
let found = byQueue.get(queue);
|
|
65
|
-
if (!found) {
|
|
66
|
-
found = { queue, keys: 0, bytes: 0, minutes: 0, days: [], tiers: emptyTiers() };
|
|
67
|
-
byQueue.set(queue, found);
|
|
68
|
-
}
|
|
69
|
-
return found;
|
|
70
|
-
};
|
|
71
|
-
const tiers = emptyTiers();
|
|
72
|
-
let keys = 0;
|
|
73
|
-
let bytes = 0;
|
|
74
|
-
let minutes = 0;
|
|
75
|
-
for (const row of groups.rows) {
|
|
76
|
-
const tier = row.tier;
|
|
77
|
-
const rows = Number(row.rows);
|
|
78
|
-
const size = Number(row.bytes);
|
|
79
|
-
const queue = entry(row.queue);
|
|
80
|
-
queue.keys += rows;
|
|
81
|
-
queue.bytes += size;
|
|
82
|
-
queue.tiers[tier].keys += rows;
|
|
83
|
-
queue.tiers[tier].bytes += size;
|
|
84
|
-
tiers[tier].keys += rows;
|
|
85
|
-
tiers[tier].bytes += size;
|
|
86
|
-
keys += rows;
|
|
87
|
-
bytes += size;
|
|
88
|
-
if (tier === 'minute') {
|
|
89
|
-
queue.minutes += rows;
|
|
90
|
-
minutes += rows;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
let oldestDay = null;
|
|
94
|
-
let newestDay = null;
|
|
95
|
-
for (const row of days.rows) {
|
|
96
|
-
const day = (0, keys_1.indexToDay)(Number(row.day));
|
|
97
|
-
entry(row.queue).days.push(day);
|
|
98
|
-
if (oldestDay === null || day < oldestDay) {
|
|
99
|
-
oldestDay = day;
|
|
100
|
-
}
|
|
101
|
-
if (newestDay === null || day > newestDay) {
|
|
102
|
-
newestDay = day;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
const queues = [...byQueue.values()].sort((a, b) => b.bytes - a.bytes);
|
|
106
|
-
for (const queue of queues) {
|
|
107
|
-
queue.days = [...new Set(queue.days)].sort();
|
|
108
|
-
}
|
|
109
|
-
return { keys, bytes, minutes, oldestDay, newestDay, tiers, queues };
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* One transaction. Purging a single queue first subtracts its completed and failed rows
|
|
113
|
-
* from the matching `__global__` rows, tier by tier, and drops the global rows that drain to
|
|
114
|
-
* zero or below, exactly as the Redis admin does; the latency rollups keep the queue's share
|
|
115
|
-
* until retention drops it, for the same reason (see `SUMMABLE_METRICS` there).
|
|
116
|
-
*
|
|
117
|
-
* `keysDeleted` counts minute and hour rows, the rows a Redis day-scoped key holds;
|
|
118
|
-
* `fieldsDeleted` counts day rows, the Redis totals-hash fields. Both include drained global
|
|
119
|
-
* rows.
|
|
120
|
-
*/
|
|
121
|
-
async purge(opts = {}) {
|
|
122
|
-
var _a;
|
|
123
|
-
const before = opts.before === undefined ? null : (0, keys_1.dayToIndex)((0, keys_1.toDay)(opts.before));
|
|
124
|
-
const queue = (_a = opts.queue) !== null && _a !== void 0 ? _a : null;
|
|
125
|
-
const adjustGlobal = queue !== null && queue !== keys_1.GLOBAL_QUEUE;
|
|
126
|
-
const { counters, histograms } = this.ctx.tables;
|
|
127
|
-
const scope = `($1::text IS NULL OR queue = $1)
|
|
128
|
-
AND ($2::bigint IS NULL OR coalesce(${ROW_DAY}, bucket) < $2)`;
|
|
129
|
-
return (0, context_1.inTransaction)(this.ctx, async (client) => {
|
|
130
|
-
const result = { keysDeleted: 0, fieldsDeleted: 0 };
|
|
131
|
-
const tally = (tier, count) => {
|
|
132
|
-
if (tier === 'day') {
|
|
133
|
-
result.fieldsDeleted += count;
|
|
134
|
-
}
|
|
135
|
-
else {
|
|
136
|
-
result.keysDeleted += count;
|
|
137
|
-
}
|
|
138
|
-
};
|
|
139
|
-
const { rows: removed } = await client.query(`WITH doomed AS (
|
|
140
|
-
DELETE FROM ${counters} WHERE ${scope}
|
|
141
|
-
RETURNING metric, tier, bucket, value
|
|
142
|
-
),
|
|
143
|
-
drained AS (
|
|
144
|
-
UPDATE ${counters} g SET value = g.value - d.value
|
|
145
|
-
FROM doomed d
|
|
146
|
-
WHERE $3 AND g.queue = $4 AND g.metric = d.metric AND g.tier = d.tier
|
|
147
|
-
AND g.bucket = d.bucket AND d.metric = ANY($5::text[]) AND d.value <> 0
|
|
148
|
-
RETURNING g.metric, g.tier, g.bucket, g.value
|
|
149
|
-
)
|
|
150
|
-
SELECT 'doomed' AS kind, tier, count(*) AS n, NULL::text[] AS metrics,
|
|
151
|
-
NULL::bigint[] AS buckets
|
|
152
|
-
FROM doomed GROUP BY tier
|
|
153
|
-
UNION ALL
|
|
154
|
-
SELECT 'drained', tier, count(*), array_agg(metric), array_agg(bucket)
|
|
155
|
-
FROM drained WHERE value <= 0 GROUP BY tier`, [queue, before, adjustGlobal, keys_1.GLOBAL_QUEUE, SUMMABLE_METRICS]);
|
|
156
|
-
for (const row of removed) {
|
|
157
|
-
if (row.kind === 'doomed') {
|
|
158
|
-
tally(row.tier, Number(row.n));
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
// The recorder never writes a zero bucket, so a leftover zero would read as
|
|
162
|
-
// recorded-but-idle instead of not recorded.
|
|
163
|
-
const { rowCount } = await client.query(`DELETE FROM ${counters} g
|
|
164
|
-
USING unnest($2::text[], $3::bigint[]) AS d(metric, bucket)
|
|
165
|
-
WHERE g.queue = $1 AND g.tier = $4 AND g.metric = d.metric AND g.bucket = d.bucket`, [keys_1.GLOBAL_QUEUE, row.metrics, row.buckets, row.tier]);
|
|
166
|
-
tally(row.tier, rowCount !== null && rowCount !== void 0 ? rowCount : 0);
|
|
167
|
-
}
|
|
168
|
-
const { rows: latency } = await client.query(`WITH doomed AS (DELETE FROM ${histograms} WHERE ${scope} RETURNING tier)
|
|
169
|
-
SELECT tier, count(*) AS n FROM doomed GROUP BY tier`, [queue, before]);
|
|
170
|
-
for (const row of latency) {
|
|
171
|
-
tally(row.tier, Number(row.n));
|
|
172
|
-
}
|
|
173
|
-
return result;
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
exports.PostgresHistoryAdmin = PostgresHistoryAdmin;
|
|
178
|
-
//# sourceMappingURL=admin.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"admin.js","sourceRoot":"","sources":["../../src/postgres/admin.ts"],"names":[],"mappings":";;;AAQA,kCAAsE;AAEtE,uCAA0D;AAE1D,+FAA+F;AAC/F,MAAM,gBAAgB,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AAEjD,SAAS,UAAU;IACjB,OAAO;QACL,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC7B,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;QAC3B,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KAC3B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,GAAG,6EAA6E,CAAC;AAE9F;;;;;;;;;;;GAWG;AACH,MAAa,oBAAoB;IAC/B,YAA6B,GAAc;QAAd,QAAG,GAAH,GAAG,CAAW;IAAG,CAAC;IAE/C,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACjD,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACjB;;;oBAGY,QAAQ;;;;oBAIR,UAAU;;;;;;;;;;+DAUiC,EACvD,CAAC,QAAQ,EAAE,UAAU,CAAC,CACvB;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACjB,0BAA0B,OAAO,gBAAgB,QAAQ;;kCAE/B,OAAO,SAAS,UAAU,sBAAsB,CAC3E;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;QACrD,MAAM,KAAK,GAAG,CAAC,KAAa,EAAqB,EAAE;YACjD,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,KAAK,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;gBAChF,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAC5B,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAmB,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC;YACnB,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC;YACpB,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC;YAC/B,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;YAC1B,IAAI,IAAI,IAAI,CAAC;YACb,KAAK,IAAI,IAAI,CAAC;YACd,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC;gBACtB,OAAO,IAAI,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,SAAS,KAAK,IAAI,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;gBAC1C,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;YACD,IAAI,SAAS,KAAK,IAAI,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;gBAC1C,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACvE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,KAAK,CAAC,OAAqB,EAAE;;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,iBAAU,EAAC,IAAA,YAAK,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACjF,MAAM,KAAK,GAAG,MAAA,IAAI,CAAC,KAAK,mCAAI,IAAI,CAAC;QACjC,MAAM,YAAY,GAAG,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,mBAAY,CAAC;QAC9D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACjD,MAAM,KAAK,GAAG;4CAC0B,OAAO,iBAAiB,CAAC;QAEjE,OAAO,IAAA,uBAAa,EAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAgB,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;gBAC5C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;oBACnB,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC;gBAC9B,CAAC;YACH,CAAC,CAAC;YAEF,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC1C;yBACiB,QAAQ,UAAU,KAAK;;;;oBAI5B,QAAQ;;;;;;;;;;;uDAW2B,EAC/C,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,mBAAY,EAAE,gBAAgB,CAAC,CAC9D,CAAC;YAEF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC1B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/B,SAAS;gBACX,CAAC;gBACD,4EAA4E;gBAC5E,6CAA6C;gBAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACrC,eAAe,QAAQ;;+FAE8D,EACrF,CAAC,mBAAY,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CACnD,CAAC;gBACF,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,CAAC,CAAC,CAAC;YACjC,CAAC;YAED,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC1C,+BAA+B,UAAU,UAAU,KAAK;8DACF,EACtD,CAAC,KAAK,EAAE,MAAM,CAAC,CAChB,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AApKD,oDAoKC"}
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The slice of node-postgres this package uses, declared structurally so that the published
|
|
3
|
-
* types never name `pg`: a Redis-only install has neither `pg` nor `@types/pg`, and must not
|
|
4
|
-
* need them to compile.
|
|
5
|
-
*/
|
|
6
|
-
export interface PgQueryResult {
|
|
7
|
-
rows: Record<string, any>[];
|
|
8
|
-
rowCount?: number | null;
|
|
9
|
-
}
|
|
10
|
-
export interface PgQueryable {
|
|
11
|
-
query(text: string, values?: unknown[]): Promise<PgQueryResult>;
|
|
12
|
-
}
|
|
13
|
-
export interface PgPoolClient extends PgQueryable {
|
|
14
|
-
release(error?: Error | boolean): void;
|
|
15
|
-
}
|
|
16
|
-
/** A `pg.Pool`, or anything shaped like one. */
|
|
17
|
-
export interface PgPool extends PgQueryable {
|
|
18
|
-
connect(): Promise<PgPoolClient>;
|
|
19
|
-
end(): Promise<void>;
|
|
20
|
-
on?(event: 'error', listener: (error: Error) => void): unknown;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* A node-postgres pool config (`connectionString`, `host`, `max`, `ssl`, ...). A `schema` key
|
|
24
|
-
* is accepted too, so the object handed to BullMQ's PostgreSQL backend can be reused as is;
|
|
25
|
-
* it is lifted out rather than passed to the pool.
|
|
26
|
-
*/
|
|
27
|
-
export interface PostgresPoolConfig {
|
|
28
|
-
connectionString?: string;
|
|
29
|
-
host?: string;
|
|
30
|
-
port?: number;
|
|
31
|
-
user?: string;
|
|
32
|
-
password?: string | (() => string | Promise<string>);
|
|
33
|
-
database?: string;
|
|
34
|
-
max?: number;
|
|
35
|
-
schema?: string;
|
|
36
|
-
[option: string]: unknown;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Mirrors the ergonomics of BullMQ v6's PostgreSQL connection option: an existing pool, a
|
|
40
|
-
* pool config, or a connection string.
|
|
41
|
-
*/
|
|
42
|
-
export type PostgresConnection = PgPool | PostgresPoolConfig | string;
|
|
43
|
-
/**
|
|
44
|
-
* Schema names and table prefixes end up inside DDL, where they cannot be bound as
|
|
45
|
-
* parameters, so they are held to plain identifiers and always quoted.
|
|
46
|
-
*/
|
|
47
|
-
export declare function quoteIdentifier(name: string): string;
|
|
48
|
-
export declare function isPool(connection: unknown): connection is PgPool;
|
|
49
|
-
export declare function resolvePool(connection: PostgresConnection, onError?: (error: Error) => void): {
|
|
50
|
-
pool: PgPool;
|
|
51
|
-
owned: boolean;
|
|
52
|
-
schema: string | undefined;
|
|
53
|
-
};
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.quoteIdentifier = quoteIdentifier;
|
|
4
|
-
exports.isPool = isPool;
|
|
5
|
-
exports.resolvePool = resolvePool;
|
|
6
|
-
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
7
|
-
/**
|
|
8
|
-
* Schema names and table prefixes end up inside DDL, where they cannot be bound as
|
|
9
|
-
* parameters, so they are held to plain identifiers and always quoted.
|
|
10
|
-
*/
|
|
11
|
-
function quoteIdentifier(name) {
|
|
12
|
-
if (!IDENTIFIER.test(name) || name.length > 63) {
|
|
13
|
-
throw new Error(`Invalid PostgreSQL identifier ${JSON.stringify(name)}: use letters, digits and ` +
|
|
14
|
-
'underscores, starting with a letter or underscore, at most 63 characters.');
|
|
15
|
-
}
|
|
16
|
-
return `"${name}"`;
|
|
17
|
-
}
|
|
18
|
-
function isPool(connection) {
|
|
19
|
-
const candidate = connection;
|
|
20
|
-
return (!!candidate &&
|
|
21
|
-
typeof candidate.connect === 'function' &&
|
|
22
|
-
typeof candidate.query === 'function' &&
|
|
23
|
-
typeof candidate.end === 'function');
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* `pg` is an optional peer dependency, so it is required only here, on the first PostgreSQL
|
|
27
|
-
* store that has to build its own pool. Handing in a pool never loads it.
|
|
28
|
-
*/
|
|
29
|
-
function loadPg() {
|
|
30
|
-
try {
|
|
31
|
-
// oxlint-disable-next-line typescript/no-require-imports
|
|
32
|
-
return require('pg');
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
throw new Error("PostgreSQL metrics storage needs the optional 'pg' package. Install it with " +
|
|
36
|
-
'`npm install pg`, or pass an existing pg.Pool as `connection`.');
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
function resolvePool(connection, onError) {
|
|
40
|
-
var _a;
|
|
41
|
-
if (isPool(connection)) {
|
|
42
|
-
return { pool: connection, owned: false, schema: undefined };
|
|
43
|
-
}
|
|
44
|
-
const { schema, ...config } = typeof connection === 'string'
|
|
45
|
-
? { connectionString: connection }
|
|
46
|
-
: connection;
|
|
47
|
-
const { Pool } = loadPg();
|
|
48
|
-
const pool = new Pool(config);
|
|
49
|
-
// node-postgres emits `error` for a pooled client that dies while idle (a server restart, a
|
|
50
|
-
// dropped connection). Unhandled, that event takes the whole process down with it.
|
|
51
|
-
(_a = pool.on) === null || _a === void 0 ? void 0 : _a.call(pool, 'error', (error) => onError === null || onError === void 0 ? void 0 : onError(error));
|
|
52
|
-
return { pool, owned: true, schema };
|
|
53
|
-
}
|
|
54
|
-
//# sourceMappingURL=connection.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/postgres/connection.ts"],"names":[],"mappings":";;AAsDA,0CAQC;AAED,wBAQC;AAkBD,kCAiBC;AA3DD,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAE9C;;;GAGG;AACH,SAAgB,eAAe,CAAC,IAAY;IAC1C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iCAAiC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B;YAC/E,2EAA2E,CAC9E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,IAAI,GAAG,CAAC;AACrB,CAAC;AAED,SAAgB,MAAM,CAAC,UAAmB;IACxC,MAAM,SAAS,GAAG,UAAoC,CAAC;IACvD,OAAO,CACL,CAAC,CAAC,SAAS;QACX,OAAO,SAAS,CAAC,OAAO,KAAK,UAAU;QACvC,OAAO,SAAS,CAAC,KAAK,KAAK,UAAU;QACrC,OAAO,SAAS,CAAC,GAAG,KAAK,UAAU,CACpC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,MAAM;IACb,IAAI,CAAC;QACH,yDAAyD;QACzD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,8EAA8E;YAC5E,gEAAgE,CACnE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAgB,WAAW,CACzB,UAA8B,EAC9B,OAAgC;;IAEhC,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC/D,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GACzB,OAAO,UAAU,KAAK,QAAQ;QAC5B,CAAC,CAAE,EAAE,gBAAgB,EAAE,UAAU,EAAyB;QAC1D,CAAC,CAAC,UAAU,CAAC;IACjB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,4FAA4F;IAC5F,mFAAmF;IACnF,MAAA,IAAI,CAAC,EAAE,qDAAG,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,KAAK,CAAC,CAAC,CAAC;IAChD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC"}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import type { Retention } from '../store';
|
|
2
|
-
import type { PgPool, PgPoolClient } from './connection';
|
|
3
|
-
import type { MetricsTables } from './schema';
|
|
4
|
-
/** What the PostgreSQL stores share: one pool, one set of tables, one readiness gate. */
|
|
5
|
-
export interface PgContext {
|
|
6
|
-
pool: PgPool;
|
|
7
|
-
tables: MetricsTables;
|
|
8
|
-
/** Resolves once the tables are there, migrating first when the store was asked to. */
|
|
9
|
-
ready(): Promise<void>;
|
|
10
|
-
}
|
|
11
|
-
export declare function inTransaction<T>(ctx: PgContext, work: (client: PgPoolClient) => Promise<T>): Promise<T>;
|
|
12
|
-
export declare function normalizeRetention(retention: Retention): Retention;
|
|
13
|
-
/**
|
|
14
|
-
* The SQL counterpart of the Redis TTLs and totals trimming: every tier drops the days older
|
|
15
|
-
* than its own retention, counted back from `day`.
|
|
16
|
-
*
|
|
17
|
-
* Anchored on the day being written rather than on the wall clock, exactly as the Redis
|
|
18
|
-
* scripts trim against the day of the write, and run by each writer on its first write and
|
|
19
|
-
* then on the first write of every new day, so about once a day per process. The deletes
|
|
20
|
-
* are range scans on the `(tier, bucket)` indexes. Sampler state past its expiry goes too:
|
|
21
|
-
* a watermark outliving its queue is the unbounded-storage failure this package avoids.
|
|
22
|
-
*/
|
|
23
|
-
export declare class RetentionPruner {
|
|
24
|
-
private readonly ctx;
|
|
25
|
-
private readonly retention;
|
|
26
|
-
private prunedFor;
|
|
27
|
-
constructor(ctx: PgContext, retention: Retention);
|
|
28
|
-
afterWrite(day: string): Promise<void>;
|
|
29
|
-
}
|
package/dist/postgres/context.js
DELETED
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RetentionPruner = void 0;
|
|
4
|
-
exports.inTransaction = inTransaction;
|
|
5
|
-
exports.normalizeRetention = normalizeRetention;
|
|
6
|
-
const keys_1 = require("../keys");
|
|
7
|
-
const MINUTES_PER_DAY = 1440;
|
|
8
|
-
const HOURS_PER_DAY = 24;
|
|
9
|
-
async function inTransaction(ctx, work) {
|
|
10
|
-
await ctx.ready();
|
|
11
|
-
const client = await ctx.pool.connect();
|
|
12
|
-
try {
|
|
13
|
-
await client.query('BEGIN');
|
|
14
|
-
try {
|
|
15
|
-
const result = await work(client);
|
|
16
|
-
await client.query('COMMIT');
|
|
17
|
-
return result;
|
|
18
|
-
}
|
|
19
|
-
catch (error) {
|
|
20
|
-
await client.query('ROLLBACK').catch(() => undefined);
|
|
21
|
-
throw error;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
finally {
|
|
25
|
-
client.release();
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
function normalizeRetention(retention) {
|
|
29
|
-
return {
|
|
30
|
-
minutes: Math.max(1, Math.floor(retention.minutes)),
|
|
31
|
-
hours: Math.max(1, Math.floor(retention.hours)),
|
|
32
|
-
days: Math.max(1, Math.floor(retention.days)),
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* The SQL counterpart of the Redis TTLs and totals trimming: every tier drops the days older
|
|
37
|
-
* than its own retention, counted back from `day`.
|
|
38
|
-
*
|
|
39
|
-
* Anchored on the day being written rather than on the wall clock, exactly as the Redis
|
|
40
|
-
* scripts trim against the day of the write, and run by each writer on its first write and
|
|
41
|
-
* then on the first write of every new day, so about once a day per process. The deletes
|
|
42
|
-
* are range scans on the `(tier, bucket)` indexes. Sampler state past its expiry goes too:
|
|
43
|
-
* a watermark outliving its queue is the unbounded-storage failure this package avoids.
|
|
44
|
-
*/
|
|
45
|
-
class RetentionPruner {
|
|
46
|
-
constructor(ctx, retention) {
|
|
47
|
-
this.ctx = ctx;
|
|
48
|
-
this.retention = retention;
|
|
49
|
-
this.prunedFor = null;
|
|
50
|
-
}
|
|
51
|
-
async afterWrite(day) {
|
|
52
|
-
if (this.prunedFor !== null && day <= this.prunedFor) {
|
|
53
|
-
return;
|
|
54
|
-
}
|
|
55
|
-
this.prunedFor = day;
|
|
56
|
-
const { counters, histograms, samplerState } = this.ctx.tables;
|
|
57
|
-
const minuteCutoff = (0, keys_1.dayToIndex)((0, keys_1.shiftDay)(day, -this.retention.minutes)) * MINUTES_PER_DAY;
|
|
58
|
-
const hourCutoff = (0, keys_1.dayToIndex)((0, keys_1.shiftDay)(day, -this.retention.hours)) * HOURS_PER_DAY;
|
|
59
|
-
const dayCutoff = (0, keys_1.dayToIndex)((0, keys_1.shiftDay)(day, -this.retention.days));
|
|
60
|
-
try {
|
|
61
|
-
await this.ctx.pool.query(`DELETE FROM ${counters}
|
|
62
|
-
WHERE (tier = 'minute' AND bucket < $1)
|
|
63
|
-
OR (tier = 'hour' AND bucket < $2)
|
|
64
|
-
OR (tier = 'day' AND bucket < $3)`, [minuteCutoff, hourCutoff, dayCutoff]);
|
|
65
|
-
await this.ctx.pool.query(`DELETE FROM ${histograms}
|
|
66
|
-
WHERE (tier = 'hour' AND bucket < $1) OR (tier = 'day' AND bucket < $2)`, [hourCutoff, dayCutoff]);
|
|
67
|
-
await this.ctx.pool.query(`DELETE FROM ${samplerState} WHERE expires_at <= now()`);
|
|
68
|
-
}
|
|
69
|
-
catch (error) {
|
|
70
|
-
this.prunedFor = null; // retried on the next write instead of waiting a day
|
|
71
|
-
throw error;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
exports.RetentionPruner = RetentionPruner;
|
|
76
|
-
//# sourceMappingURL=context.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/postgres/context.ts"],"names":[],"mappings":";;;AAgBA,sCAmBC;AAED,gDAMC;AA3CD,kCAA+C;AAK/C,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,aAAa,GAAG,EAAE,CAAC;AAUlB,KAAK,UAAU,aAAa,CACjC,GAAc,EACd,IAA0C;IAE1C,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB,CAAC,SAAoB;IACrD,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACnD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAa,eAAe;IAG1B,YACmB,GAAc,EACd,SAAoB;QADpB,QAAG,GAAH,GAAG,CAAW;QACd,cAAS,GAAT,SAAS,CAAW;QAJ/B,cAAS,GAAkB,IAAI,CAAC;IAKrC,CAAC;IAEJ,KAAK,CAAC,UAAU,CAAC,GAAW;QAC1B,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC;QACrB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QAC/D,MAAM,YAAY,GAAG,IAAA,iBAAU,EAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,eAAe,CAAC;QAC1F,MAAM,UAAU,GAAG,IAAA,iBAAU,EAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,aAAa,CAAC;QACpF,MAAM,SAAS,GAAG,IAAA,iBAAU,EAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,QAAQ;;;+CAGgB,EACvC,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,CAAC,CACtC,CAAC;YACF,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,UAAU;kFACiD,EAC1E,CAAC,UAAU,EAAE,SAAS,CAAC,CACxB,CAAC;YACF,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,YAAY,4BAA4B,CAAC,CAAC;QACrF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,qDAAqD;YAC5E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF;AApCD,0CAoCC"}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { type PgPool } from './connection';
|
|
2
|
-
export declare const DEFAULT_SCHEMA = "public";
|
|
3
|
-
/** The SQL twin of the Redis namespace `bull-board:metrics`. */
|
|
4
|
-
export declare const DEFAULT_TABLE_PREFIX = "bull_board_metrics_";
|
|
5
|
-
export declare const SCHEMA_VERSION = 1;
|
|
6
|
-
export interface MetricsTables {
|
|
7
|
-
schema: string;
|
|
8
|
-
prefix: string;
|
|
9
|
-
/** Schema-qualified, quoted names, ready to splice into SQL or cast to `regclass`. */
|
|
10
|
-
meta: string;
|
|
11
|
-
counters: string;
|
|
12
|
-
histograms: string;
|
|
13
|
-
samplerState: string;
|
|
14
|
-
}
|
|
15
|
-
export declare function metricsTables(schema: string, prefix: string): MetricsTables;
|
|
16
|
-
/**
|
|
17
|
-
* Creates or upgrades the metrics tables. Idempotent, and safe to run from several processes
|
|
18
|
-
* at once: the whole upgrade is one transaction behind an advisory lock scoped to this
|
|
19
|
-
* schema and prefix, so a second migrator waits and then finds nothing left to do.
|
|
20
|
-
*
|
|
21
|
-
* The schema is created only when it does not exist yet, since `CREATE SCHEMA IF NOT EXISTS`
|
|
22
|
-
* still demands the database-level CREATE privilege that an application role often lacks.
|
|
23
|
-
*/
|
|
24
|
-
export declare function runMigrations(pool: PgPool, tables: MetricsTables): Promise<number>;
|
|
25
|
-
/** Throws unless the tables exist at the version this build writes. */
|
|
26
|
-
export declare function assertMigrated(pool: PgPool, tables: MetricsTables): Promise<void>;
|