@worker-manager/metrics 1.1.0 → 9.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -86
- package/dist/HistoryAdmin.d.ts +87 -23
- package/dist/HistoryAdmin.js +330 -21
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +9 -9
- package/dist/HistoryStore.js +0 -22
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +32 -27
- package/dist/LatencySampler.js +156 -55
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +3 -8
- package/dist/LatencyStore.js +0 -18
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +10 -34
- package/dist/MetricsRecorder.js +20 -30
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisMetricsHistoryProvider.d.ts +17 -7
- package/dist/RedisMetricsHistoryProvider.js +136 -9
- package/dist/RedisMetricsHistoryProvider.js.map +1 -1
- package/dist/index.d.ts +2 -10
- package/dist/index.js +1 -10
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +0 -7
- package/dist/keys.js +1 -23
- package/dist/keys.js.map +1 -1
- package/package.json +4 -14
- package/dist/RedisHistoryAdmin.d.ts +0 -95
- package/dist/RedisHistoryAdmin.js +0 -328
- package/dist/RedisHistoryAdmin.js.map +0 -1
- package/dist/RedisMetricsStore.d.ts +0 -27
- package/dist/RedisMetricsStore.js +0 -42
- package/dist/RedisMetricsStore.js.map +0 -1
- package/dist/StoreHistoryProvider.d.ts +0 -27
- package/dist/StoreHistoryProvider.js +0 -127
- package/dist/StoreHistoryProvider.js.map +0 -1
- package/dist/jobSources.d.ts +0 -97
- package/dist/jobSources.js +0 -248
- package/dist/jobSources.js.map +0 -1
- package/dist/postgres/PostgresMetricsHistoryProvider.d.ts +0 -37
- package/dist/postgres/PostgresMetricsHistoryProvider.js +0 -30
- package/dist/postgres/PostgresMetricsHistoryProvider.js.map +0 -1
- package/dist/postgres/PostgresMetricsStore.d.ts +0 -65
- package/dist/postgres/PostgresMetricsStore.js +0 -88
- package/dist/postgres/PostgresMetricsStore.js.map +0 -1
- package/dist/postgres/admin.d.ts +0 -31
- package/dist/postgres/admin.js +0 -178
- package/dist/postgres/admin.js.map +0 -1
- package/dist/postgres/connection.d.ts +0 -53
- package/dist/postgres/connection.js +0 -54
- package/dist/postgres/connection.js.map +0 -1
- package/dist/postgres/context.d.ts +0 -29
- package/dist/postgres/context.js +0 -76
- package/dist/postgres/context.js.map +0 -1
- package/dist/postgres/schema.d.ts +0 -26
- package/dist/postgres/schema.js +0 -148
- package/dist/postgres/schema.js.map +0 -1
- package/dist/postgres/stores.d.ts +0 -51
- package/dist/postgres/stores.js +0 -252
- package/dist/postgres/stores.js.map +0 -1
- package/dist/store.d.ts +0 -80
- package/dist/store.js +0 -3
- package/dist/store.js.map +0 -1
package/dist/postgres/schema.js
DELETED
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SCHEMA_VERSION = exports.DEFAULT_TABLE_PREFIX = exports.DEFAULT_SCHEMA = void 0;
|
|
4
|
-
exports.metricsTables = metricsTables;
|
|
5
|
-
exports.runMigrations = runMigrations;
|
|
6
|
-
exports.assertMigrated = assertMigrated;
|
|
7
|
-
const connection_1 = require("./connection");
|
|
8
|
-
exports.DEFAULT_SCHEMA = 'public';
|
|
9
|
-
/** The SQL twin of the Redis namespace `bull-board:metrics`. */
|
|
10
|
-
exports.DEFAULT_TABLE_PREFIX = 'bull_board_metrics_';
|
|
11
|
-
/** Keeps the longest derived name, an index, inside PostgreSQL's 63-character limit. */
|
|
12
|
-
const MAX_PREFIX_LENGTH = 32;
|
|
13
|
-
exports.SCHEMA_VERSION = 1;
|
|
14
|
-
function metricsTables(schema, prefix) {
|
|
15
|
-
(0, connection_1.quoteIdentifier)(schema);
|
|
16
|
-
if (prefix.length > MAX_PREFIX_LENGTH) {
|
|
17
|
-
throw new Error(`tablePrefix must be at most ${MAX_PREFIX_LENGTH} characters.`);
|
|
18
|
-
}
|
|
19
|
-
const table = (name) => `${(0, connection_1.quoteIdentifier)(schema)}.${(0, connection_1.quoteIdentifier)(prefix + name)}`;
|
|
20
|
-
return {
|
|
21
|
-
schema,
|
|
22
|
-
prefix,
|
|
23
|
-
meta: table('meta'),
|
|
24
|
-
counters: table('counters'),
|
|
25
|
-
histograms: table('histograms'),
|
|
26
|
-
samplerState: table('sampler_state'),
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Ordered, append-only. A released migration is never edited: a change ships as the next
|
|
31
|
-
* version, and `migrate()` applies whatever a database is missing.
|
|
32
|
-
*/
|
|
33
|
-
const MIGRATIONS = [
|
|
34
|
-
{
|
|
35
|
-
version: 1,
|
|
36
|
-
sql: (t) => {
|
|
37
|
-
const index = (name) => (0, connection_1.quoteIdentifier)(`${t.prefix}${name}`);
|
|
38
|
-
return `
|
|
39
|
-
-- completed / failed sums at minute, hour and day resolution, and the queue-age gauge
|
|
40
|
-
-- (a max) at hour and day resolution. bucket is an absolute minute, hour or day index
|
|
41
|
-
-- since the epoch. queue = '__global__' is the cross-queue rollup.
|
|
42
|
-
CREATE TABLE IF NOT EXISTS ${t.counters} (
|
|
43
|
-
queue text NOT NULL,
|
|
44
|
-
metric text NOT NULL,
|
|
45
|
-
tier text NOT NULL CHECK (tier IN ('minute', 'hour', 'day')),
|
|
46
|
-
bucket bigint NOT NULL,
|
|
47
|
-
value bigint NOT NULL,
|
|
48
|
-
PRIMARY KEY (queue, metric, tier, bucket)
|
|
49
|
-
);
|
|
50
|
-
CREATE INDEX IF NOT EXISTS ${index('counters_retention_idx')}
|
|
51
|
-
ON ${t.counters} (tier, bucket);
|
|
52
|
-
|
|
53
|
-
-- runtime / waittime latency histograms: one count per fixed bucket bound, merged
|
|
54
|
-
-- element-wise.
|
|
55
|
-
CREATE TABLE IF NOT EXISTS ${t.histograms} (
|
|
56
|
-
queue text NOT NULL,
|
|
57
|
-
metric text NOT NULL,
|
|
58
|
-
tier text NOT NULL CHECK (tier IN ('hour', 'day')),
|
|
59
|
-
bucket bigint NOT NULL,
|
|
60
|
-
counts bigint[] NOT NULL,
|
|
61
|
-
PRIMARY KEY (queue, metric, tier, bucket)
|
|
62
|
-
);
|
|
63
|
-
CREATE INDEX IF NOT EXISTS ${index('histograms_retention_idx')}
|
|
64
|
-
ON ${t.histograms} (tier, bucket);
|
|
65
|
-
|
|
66
|
-
-- The sampler's per-queue lease and finish-time watermark. Rows past expires_at are
|
|
67
|
-
-- treated as absent and pruned with the history.
|
|
68
|
-
CREATE TABLE IF NOT EXISTS ${t.samplerState} (
|
|
69
|
-
queue text NOT NULL,
|
|
70
|
-
kind text NOT NULL CHECK (kind IN ('lease', 'watermark')),
|
|
71
|
-
value text NOT NULL,
|
|
72
|
-
expires_at timestamptz NOT NULL,
|
|
73
|
-
PRIMARY KEY (queue, kind)
|
|
74
|
-
);
|
|
75
|
-
`;
|
|
76
|
-
},
|
|
77
|
-
},
|
|
78
|
-
];
|
|
79
|
-
/**
|
|
80
|
-
* Creates or upgrades the metrics tables. Idempotent, and safe to run from several processes
|
|
81
|
-
* at once: the whole upgrade is one transaction behind an advisory lock scoped to this
|
|
82
|
-
* schema and prefix, so a second migrator waits and then finds nothing left to do.
|
|
83
|
-
*
|
|
84
|
-
* The schema is created only when it does not exist yet, since `CREATE SCHEMA IF NOT EXISTS`
|
|
85
|
-
* still demands the database-level CREATE privilege that an application role often lacks.
|
|
86
|
-
*/
|
|
87
|
-
async function runMigrations(pool, tables) {
|
|
88
|
-
var _a;
|
|
89
|
-
const client = await pool.connect();
|
|
90
|
-
try {
|
|
91
|
-
await client.query('BEGIN');
|
|
92
|
-
try {
|
|
93
|
-
await client.query('SELECT pg_advisory_xact_lock(hashtext($1))', [
|
|
94
|
-
`worker-manager-metrics:${tables.schema}.${tables.prefix}`,
|
|
95
|
-
]);
|
|
96
|
-
const { rows: found } = await client.query('SELECT to_regnamespace($1) AS oid', [
|
|
97
|
-
tables.schema,
|
|
98
|
-
]);
|
|
99
|
-
if (!((_a = found[0]) === null || _a === void 0 ? void 0 : _a.oid)) {
|
|
100
|
-
await client.query(`CREATE SCHEMA ${(0, connection_1.quoteIdentifier)(tables.schema)}`);
|
|
101
|
-
}
|
|
102
|
-
await client.query(`CREATE TABLE IF NOT EXISTS ${tables.meta} (name text PRIMARY KEY, value text NOT NULL)`);
|
|
103
|
-
const current = await readVersion(client, tables);
|
|
104
|
-
if (current > exports.SCHEMA_VERSION) {
|
|
105
|
-
throw newerSchemaError(current);
|
|
106
|
-
}
|
|
107
|
-
for (const migration of MIGRATIONS) {
|
|
108
|
-
if (migration.version > current) {
|
|
109
|
-
await client.query(migration.sql(tables));
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
await client.query(`INSERT INTO ${tables.meta} (name, value) VALUES ('schema_version', $1)
|
|
113
|
-
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value`, [String(exports.SCHEMA_VERSION)]);
|
|
114
|
-
await client.query('COMMIT');
|
|
115
|
-
return exports.SCHEMA_VERSION;
|
|
116
|
-
}
|
|
117
|
-
catch (error) {
|
|
118
|
-
await client.query('ROLLBACK').catch(() => undefined);
|
|
119
|
-
throw error;
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
finally {
|
|
123
|
-
client.release();
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
/** Throws unless the tables exist at the version this build writes. */
|
|
127
|
-
async function assertMigrated(pool, tables) {
|
|
128
|
-
var _a;
|
|
129
|
-
const { rows } = await pool.query('SELECT to_regclass($1) AS oid', [tables.meta]);
|
|
130
|
-
const current = ((_a = rows[0]) === null || _a === void 0 ? void 0 : _a.oid) ? await readVersion(pool, tables) : 0;
|
|
131
|
-
if (current > exports.SCHEMA_VERSION) {
|
|
132
|
-
throw newerSchemaError(current);
|
|
133
|
-
}
|
|
134
|
-
if (current < exports.SCHEMA_VERSION) {
|
|
135
|
-
throw new Error(`The metrics tables in schema "${tables.schema}" (prefix "${tables.prefix}") are ` +
|
|
136
|
-
`missing or at version ${current}, and this build needs version ${exports.SCHEMA_VERSION}. ` +
|
|
137
|
-
'Pass `migrate: true` to the store, or run `migratePostgresMetrics()` as a deploy step.');
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
async function readVersion(client, tables) {
|
|
141
|
-
const { rows } = await client.query(`SELECT value FROM ${tables.meta} WHERE name = 'schema_version'`);
|
|
142
|
-
return rows[0] ? Number(rows[0].value) || 0 : 0;
|
|
143
|
-
}
|
|
144
|
-
function newerSchemaError(current) {
|
|
145
|
-
return new Error(`The metrics tables are at schema version ${current}, newer than the ${exports.SCHEMA_VERSION} ` +
|
|
146
|
-
'this build of @worker-manager/metrics understands. Upgrade the package.');
|
|
147
|
-
}
|
|
148
|
-
//# sourceMappingURL=schema.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/postgres/schema.ts"],"names":[],"mappings":";;;AAoBA,sCAcC;AA6DD,sCAwCC;AAGD,wCAaC;AAvJD,6CAA4D;AAE/C,QAAA,cAAc,GAAG,QAAQ,CAAC;AACvC,gEAAgE;AACnD,QAAA,oBAAoB,GAAG,qBAAqB,CAAC;AAC1D,wFAAwF;AACxF,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEhB,QAAA,cAAc,GAAG,CAAC,CAAC;AAYhC,SAAgB,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,IAAA,4BAAe,EAAC,MAAM,CAAC,CAAC;IACxB,IAAI,MAAM,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,+BAA+B,iBAAiB,cAAc,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,IAAA,4BAAe,EAAC,MAAM,CAAC,IAAI,IAAA,4BAAe,EAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;IAC/F,OAAO;QACL,MAAM;QACN,MAAM;QACN,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;QACnB,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,UAAU,EAAE,KAAK,CAAC,YAAY,CAAC;QAC/B,YAAY,EAAE,KAAK,CAAC,eAAe,CAAC;KACrC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,GAA6D;IAC3E;QACE,OAAO,EAAE,CAAC;QACV,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAA,4BAAe,EAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;YACtE,OAAO;;;;qCAIwB,CAAC,CAAC,QAAQ;;;;;;;;qCAQV,KAAK,CAAC,wBAAwB,CAAC;eACrD,CAAC,CAAC,QAAQ;;;;qCAIY,CAAC,CAAC,UAAU;;;;;;;;qCAQZ,KAAK,CAAC,0BAA0B,CAAC;eACvD,CAAC,CAAC,UAAU;;;;qCAIU,CAAC,CAAC,YAAY;;;;;;;OAO5C,CAAC;QACJ,CAAC;KACF;CACF,CAAC;AAEF;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,MAAqB;;IACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE;gBAC/D,0BAA0B,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE;aAC3D,CAAC,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE;gBAC9E,MAAM,CAAC,MAAM;aACd,CAAC,CAAC;YACH,IAAI,CAAC,CAAA,MAAA,KAAK,CAAC,CAAC,CAAC,0CAAE,GAAG,CAAA,EAAE,CAAC;gBACnB,MAAM,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAA,4BAAe,EAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAChB,8BAA8B,MAAM,CAAC,IAAI,+CAA+C,CACzF,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,OAAO,GAAG,sBAAc,EAAE,CAAC;gBAC7B,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YACD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,IAAI,SAAS,CAAC,OAAO,GAAG,OAAO,EAAE,CAAC;oBAChC,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAChB,eAAe,MAAM,CAAC,IAAI;iEAC+B,EACzD,CAAC,MAAM,CAAC,sBAAc,CAAC,CAAC,CACzB,CAAC;YACF,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,sBAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,uEAAuE;AAChE,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,MAAqB;;IACtE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,CAAA,MAAA,IAAI,CAAC,CAAC,CAAC,0CAAE,GAAG,EAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,GAAG,sBAAc,EAAE,CAAC;QAC7B,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,OAAO,GAAG,sBAAc,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,MAAM,cAAc,MAAM,CAAC,MAAM,SAAS;YAChF,yBAAyB,OAAO,kCAAkC,sBAAc,IAAI;YACpF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAkC,EAClC,MAAqB;IAErB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACjC,qBAAqB,MAAM,CAAC,IAAI,gCAAgC,CACjE,CAAC;IACF,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,OAAO,IAAI,KAAK,CACd,4CAA4C,OAAO,oBAAoB,sBAAc,GAAG;QACtF,yEAAyE,CAC5E,CAAC;AACJ,CAAC"}
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import type { MinutePoint } from '../dataMapping';
|
|
2
|
-
import type { CounterStore, LatencyMetric, LatencyStorage, Retention } from '../store';
|
|
3
|
-
import { type PgContext } from './context';
|
|
4
|
-
/**
|
|
5
|
-
* The PostgreSQL `CounterStore`. One transaction per snapshot of a queue's metric: a
|
|
6
|
-
* transaction-scoped advisory lock on (tables, queue, metric), then one statement that
|
|
7
|
-
* diffs the incoming minutes against the minute ledger and applies only the differences to
|
|
8
|
-
* the hour and day tiers of the queue and of `__global__`.
|
|
9
|
-
*
|
|
10
|
-
* The lock is what makes that diff safe across processes, the way running inside one EVAL
|
|
11
|
-
* does in Redis: a second recorder snapshotting the same window waits, then reads the ledger
|
|
12
|
-
* the first one committed and finds nothing left to add. The rollup rows are written in a
|
|
13
|
-
* fixed order, so two queues contending for the same `__global__` rows cannot deadlock.
|
|
14
|
-
*/
|
|
15
|
-
export declare class PostgresCounterStore implements CounterStore {
|
|
16
|
-
private readonly ctx;
|
|
17
|
-
readonly retention: Retention;
|
|
18
|
-
private readonly pruner;
|
|
19
|
-
constructor(ctx: PgContext, retention: Retention);
|
|
20
|
-
upsertMinute(queue: string, metric: string, minute: number, value: number): Promise<void>;
|
|
21
|
-
upsertMinutes(queue: string, metric: string, points: MinutePoint[]): Promise<void>;
|
|
22
|
-
readDailyTotals(queue: string, metric: string, days: string[]): Promise<(number | null)[]>;
|
|
23
|
-
readHours(queue: string, metric: string, days: string[]): Promise<Record<string, number>>;
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* The PostgreSQL `LatencyStorage`. Each write is one multi-row upsert covering the queue's
|
|
27
|
-
* hour and day rows and the `__global__` rollup of both, so it is atomic on its own. The rows
|
|
28
|
-
* go in a fixed order (queue, then global; day, then hour) for the same deadlock reason as
|
|
29
|
-
* the counters.
|
|
30
|
-
*
|
|
31
|
-
* Histograms merge element-wise in SQL, keeping bucket order through `WITH ORDINALITY`; the
|
|
32
|
-
* queue-age gauge keeps the GREATEST value, since an hour holds the worst backlog seen in it.
|
|
33
|
-
*/
|
|
34
|
-
export declare class PostgresLatencyStore implements LatencyStorage {
|
|
35
|
-
private readonly ctx;
|
|
36
|
-
readonly retention: Retention;
|
|
37
|
-
private readonly pruner;
|
|
38
|
-
constructor(ctx: PgContext, retention: Retention);
|
|
39
|
-
addSamples(queue: string, metric: LatencyMetric, hour: number, vector: number[]): Promise<void>;
|
|
40
|
-
recordQueueAge(queue: string, hour: number, ms: number): Promise<void>;
|
|
41
|
-
readRange(queue: string, metric: LatencyMetric, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number[]>>;
|
|
42
|
-
readQueueAge(queue: string, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number>>;
|
|
43
|
-
/**
|
|
44
|
-
* Set-if-absent on the database clock: the upsert only overwrites a lease that has already
|
|
45
|
-
* expired, and returns a row only when it wrote one.
|
|
46
|
-
*/
|
|
47
|
-
acquireLease(queue: string, holder: string, ttlMs: number): Promise<boolean>;
|
|
48
|
-
releaseLease(queue: string, holder: string): Promise<void>;
|
|
49
|
-
readWatermark(queue: string): Promise<number | null>;
|
|
50
|
-
writeWatermark(queue: string, ms: number, ttlSeconds: number): Promise<void>;
|
|
51
|
-
}
|
package/dist/postgres/stores.js
DELETED
|
@@ -1,252 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PostgresLatencyStore = exports.PostgresCounterStore = void 0;
|
|
4
|
-
const histogram_1 = require("../histogram");
|
|
5
|
-
const keys_1 = require("../keys");
|
|
6
|
-
const LatencyStore_1 = require("../LatencyStore");
|
|
7
|
-
const context_1 = require("./context");
|
|
8
|
-
const HOURS_PER_DAY = 24;
|
|
9
|
-
/** Contiguous `[first, last]` day-index bounds of a day list, for an indexed range read. */
|
|
10
|
-
function dayBounds(days) {
|
|
11
|
-
const indices = days.map(keys_1.dayToIndex);
|
|
12
|
-
return [Math.min(...indices), Math.max(...indices)];
|
|
13
|
-
}
|
|
14
|
-
function toVector(raw) {
|
|
15
|
-
const out = (0, histogram_1.emptyVector)();
|
|
16
|
-
if (!Array.isArray(raw)) {
|
|
17
|
-
return out;
|
|
18
|
-
}
|
|
19
|
-
for (let i = 0; i < histogram_1.BUCKET_COUNT && i < raw.length; i++) {
|
|
20
|
-
out[i] = Number(raw[i]) || 0;
|
|
21
|
-
}
|
|
22
|
-
return out;
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* The PostgreSQL `CounterStore`. One transaction per snapshot of a queue's metric: a
|
|
26
|
-
* transaction-scoped advisory lock on (tables, queue, metric), then one statement that
|
|
27
|
-
* diffs the incoming minutes against the minute ledger and applies only the differences to
|
|
28
|
-
* the hour and day tiers of the queue and of `__global__`.
|
|
29
|
-
*
|
|
30
|
-
* The lock is what makes that diff safe across processes, the way running inside one EVAL
|
|
31
|
-
* does in Redis: a second recorder snapshotting the same window waits, then reads the ledger
|
|
32
|
-
* the first one committed and finds nothing left to add. The rollup rows are written in a
|
|
33
|
-
* fixed order, so two queues contending for the same `__global__` rows cannot deadlock.
|
|
34
|
-
*/
|
|
35
|
-
class PostgresCounterStore {
|
|
36
|
-
constructor(ctx, retention) {
|
|
37
|
-
this.ctx = ctx;
|
|
38
|
-
this.retention = (0, context_1.normalizeRetention)(retention);
|
|
39
|
-
this.pruner = new context_1.RetentionPruner(ctx, this.retention);
|
|
40
|
-
}
|
|
41
|
-
async upsertMinute(queue, metric, minute, value) {
|
|
42
|
-
await this.upsertMinutes(queue, metric, [{ minute, value }]);
|
|
43
|
-
}
|
|
44
|
-
async upsertMinutes(queue, metric, points) {
|
|
45
|
-
if (points.length === 0) {
|
|
46
|
-
return;
|
|
47
|
-
}
|
|
48
|
-
const { counters, prefix, schema } = this.ctx.tables;
|
|
49
|
-
await (0, context_1.inTransaction)(this.ctx, async (client) => {
|
|
50
|
-
await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [
|
|
51
|
-
`${schema}.${prefix}:${queue}:${metric}`,
|
|
52
|
-
]);
|
|
53
|
-
await client.query(`WITH incoming AS (
|
|
54
|
-
SELECT DISTINCT ON (minute) minute, value
|
|
55
|
-
FROM unnest($3::bigint[], $4::bigint[]) AS t(minute, value)
|
|
56
|
-
ORDER BY minute
|
|
57
|
-
),
|
|
58
|
-
changed AS (
|
|
59
|
-
SELECT i.minute, i.value, i.value - coalesce(c.value, 0) AS delta
|
|
60
|
-
FROM incoming i
|
|
61
|
-
LEFT JOIN ${counters} c
|
|
62
|
-
ON c.queue = $1 AND c.metric = $2 AND c.tier = 'minute' AND c.bucket = i.minute
|
|
63
|
-
WHERE i.value <> coalesce(c.value, 0)
|
|
64
|
-
),
|
|
65
|
-
ledger AS (
|
|
66
|
-
INSERT INTO ${counters} (queue, metric, tier, bucket, value)
|
|
67
|
-
SELECT $1, $2, 'minute', minute, value FROM changed
|
|
68
|
-
ON CONFLICT (queue, metric, tier, bucket) DO UPDATE SET value = EXCLUDED.value
|
|
69
|
-
),
|
|
70
|
-
rollup AS (
|
|
71
|
-
SELECT r.queue, r.tier, r.bucket, sum(c.delta) AS delta
|
|
72
|
-
FROM changed c
|
|
73
|
-
CROSS JOIN LATERAL (VALUES
|
|
74
|
-
($1::text, 'hour', c.minute / 60),
|
|
75
|
-
($1::text, 'day', c.minute / 1440),
|
|
76
|
-
($5::text, 'minute', c.minute),
|
|
77
|
-
($5::text, 'hour', c.minute / 60),
|
|
78
|
-
($5::text, 'day', c.minute / 1440)
|
|
79
|
-
) AS r(queue, tier, bucket)
|
|
80
|
-
GROUP BY r.queue, r.tier, r.bucket
|
|
81
|
-
)
|
|
82
|
-
INSERT INTO ${counters} (queue, metric, tier, bucket, value)
|
|
83
|
-
SELECT queue, $2, tier, bucket, delta FROM rollup ORDER BY queue, tier, bucket
|
|
84
|
-
ON CONFLICT (queue, metric, tier, bucket)
|
|
85
|
-
DO UPDATE SET value = ${counters}.value + EXCLUDED.value`, [
|
|
86
|
-
queue,
|
|
87
|
-
metric,
|
|
88
|
-
points.map((p) => p.minute),
|
|
89
|
-
points.map((p) => Math.round(p.value)),
|
|
90
|
-
keys_1.GLOBAL_QUEUE,
|
|
91
|
-
]);
|
|
92
|
-
});
|
|
93
|
-
const newest = Math.max(...points.map((p) => p.minute));
|
|
94
|
-
await this.pruner.afterWrite((0, keys_1.minuteToDay)(newest));
|
|
95
|
-
}
|
|
96
|
-
async readDailyTotals(queue, metric, days) {
|
|
97
|
-
if (days.length === 0) {
|
|
98
|
-
return [];
|
|
99
|
-
}
|
|
100
|
-
await this.ctx.ready();
|
|
101
|
-
const { rows } = await this.ctx.pool.query(`SELECT bucket, value FROM ${this.ctx.tables.counters}
|
|
102
|
-
WHERE queue = $1 AND metric = $2 AND tier = 'day' AND bucket = ANY($3::bigint[])`, [queue, metric, days.map(keys_1.dayToIndex)]);
|
|
103
|
-
const byDay = new Map(rows.map((row) => [(0, keys_1.indexToDay)(Number(row.bucket)), Number(row.value)]));
|
|
104
|
-
return days.map((day) => { var _a; return (_a = byDay.get(day)) !== null && _a !== void 0 ? _a : null; });
|
|
105
|
-
}
|
|
106
|
-
async readHours(queue, metric, days) {
|
|
107
|
-
return readHourScalars(this.ctx, queue, metric, days);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
exports.PostgresCounterStore = PostgresCounterStore;
|
|
111
|
-
async function readHourScalars(ctx, queue, metric, days) {
|
|
112
|
-
const out = {};
|
|
113
|
-
if (days.length === 0) {
|
|
114
|
-
return out;
|
|
115
|
-
}
|
|
116
|
-
await ctx.ready();
|
|
117
|
-
const [first, last] = dayBounds(days);
|
|
118
|
-
const wanted = new Set(days);
|
|
119
|
-
const { rows } = await ctx.pool.query(`SELECT bucket, value FROM ${ctx.tables.counters}
|
|
120
|
-
WHERE queue = $1 AND metric = $2 AND tier = 'hour' AND bucket >= $3 AND bucket < $4`, [queue, metric, first * HOURS_PER_DAY, (last + 1) * HOURS_PER_DAY]);
|
|
121
|
-
for (const row of rows) {
|
|
122
|
-
const hour = Number(row.bucket);
|
|
123
|
-
if (wanted.has((0, keys_1.indexToDay)(Math.floor(hour / HOURS_PER_DAY)))) {
|
|
124
|
-
out[String(hour)] = Number(row.value);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
return out;
|
|
128
|
-
}
|
|
129
|
-
/**
|
|
130
|
-
* The PostgreSQL `LatencyStorage`. Each write is one multi-row upsert covering the queue's
|
|
131
|
-
* hour and day rows and the `__global__` rollup of both, so it is atomic on its own. The rows
|
|
132
|
-
* go in a fixed order (queue, then global; day, then hour) for the same deadlock reason as
|
|
133
|
-
* the counters.
|
|
134
|
-
*
|
|
135
|
-
* Histograms merge element-wise in SQL, keeping bucket order through `WITH ORDINALITY`; the
|
|
136
|
-
* queue-age gauge keeps the GREATEST value, since an hour holds the worst backlog seen in it.
|
|
137
|
-
*/
|
|
138
|
-
class PostgresLatencyStore {
|
|
139
|
-
constructor(ctx, retention) {
|
|
140
|
-
this.ctx = ctx;
|
|
141
|
-
this.retention = (0, context_1.normalizeRetention)(retention);
|
|
142
|
-
this.pruner = new context_1.RetentionPruner(ctx, this.retention);
|
|
143
|
-
}
|
|
144
|
-
async addSamples(queue, metric, hour, vector) {
|
|
145
|
-
await this.ctx.ready();
|
|
146
|
-
const { histograms } = this.ctx.tables;
|
|
147
|
-
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
148
|
-
const counts = toVector(vector).map((v) => Math.round(v));
|
|
149
|
-
await this.ctx.pool.query(`INSERT INTO ${histograms} (queue, metric, tier, bucket, counts) VALUES
|
|
150
|
-
($1, $2, 'day', $4, $5::bigint[]),
|
|
151
|
-
($1, $2, 'hour', $3, $5::bigint[]),
|
|
152
|
-
($6, $2, 'day', $4, $5::bigint[]),
|
|
153
|
-
($6, $2, 'hour', $3, $5::bigint[])
|
|
154
|
-
ON CONFLICT (queue, metric, tier, bucket) DO UPDATE SET counts = ARRAY(
|
|
155
|
-
SELECT coalesce(a, 0) + coalesce(b, 0)
|
|
156
|
-
FROM unnest(${histograms}.counts, EXCLUDED.counts) WITH ORDINALITY AS t(a, b, i)
|
|
157
|
-
ORDER BY i
|
|
158
|
-
)`, [queue, metric, hour, (0, keys_1.dayToIndex)(day), counts, keys_1.GLOBAL_QUEUE]);
|
|
159
|
-
await this.pruner.afterWrite(day);
|
|
160
|
-
}
|
|
161
|
-
async recordQueueAge(queue, hour, ms) {
|
|
162
|
-
await this.ctx.ready();
|
|
163
|
-
const { counters } = this.ctx.tables;
|
|
164
|
-
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
165
|
-
await this.ctx.pool.query(`INSERT INTO ${counters} (queue, metric, tier, bucket, value) VALUES
|
|
166
|
-
($1, $2, 'day', $4, $5),
|
|
167
|
-
($1, $2, 'hour', $3, $5),
|
|
168
|
-
($6, $2, 'day', $4, $5),
|
|
169
|
-
($6, $2, 'hour', $3, $5)
|
|
170
|
-
ON CONFLICT (queue, metric, tier, bucket)
|
|
171
|
-
DO UPDATE SET value = GREATEST(${counters}.value, EXCLUDED.value)`, [queue, LatencyStore_1.QUEUE_AGE_METRIC, hour, (0, keys_1.dayToIndex)(day), Math.max(0, Math.round(ms)), keys_1.GLOBAL_QUEUE]);
|
|
172
|
-
await this.pruner.afterWrite(day);
|
|
173
|
-
}
|
|
174
|
-
async readRange(queue, metric, granularity, days) {
|
|
175
|
-
const out = {};
|
|
176
|
-
if (days.length === 0) {
|
|
177
|
-
return out;
|
|
178
|
-
}
|
|
179
|
-
await this.ctx.ready();
|
|
180
|
-
const { histograms } = this.ctx.tables;
|
|
181
|
-
if (granularity === 'day') {
|
|
182
|
-
const { rows } = await this.ctx.pool.query(`SELECT bucket, counts FROM ${histograms}
|
|
183
|
-
WHERE queue = $1 AND metric = $2 AND tier = 'day' AND bucket = ANY($3::bigint[])`, [queue, metric, days.map(keys_1.dayToIndex)]);
|
|
184
|
-
for (const row of rows) {
|
|
185
|
-
out[(0, keys_1.indexToDay)(Number(row.bucket))] = toVector(row.counts);
|
|
186
|
-
}
|
|
187
|
-
return out;
|
|
188
|
-
}
|
|
189
|
-
const [first, last] = dayBounds(days);
|
|
190
|
-
const wanted = new Set(days);
|
|
191
|
-
const { rows } = await this.ctx.pool.query(`SELECT bucket, counts FROM ${histograms}
|
|
192
|
-
WHERE queue = $1 AND metric = $2 AND tier = 'hour' AND bucket >= $3 AND bucket < $4`, [queue, metric, first * HOURS_PER_DAY, (last + 1) * HOURS_PER_DAY]);
|
|
193
|
-
for (const row of rows) {
|
|
194
|
-
const hour = Number(row.bucket);
|
|
195
|
-
if (wanted.has((0, keys_1.indexToDay)(Math.floor(hour / HOURS_PER_DAY)))) {
|
|
196
|
-
out[String(hour)] = toVector(row.counts);
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
return out;
|
|
200
|
-
}
|
|
201
|
-
async readQueueAge(queue, granularity, days) {
|
|
202
|
-
if (granularity === 'hour') {
|
|
203
|
-
return readHourScalars(this.ctx, queue, LatencyStore_1.QUEUE_AGE_METRIC, days);
|
|
204
|
-
}
|
|
205
|
-
const out = {};
|
|
206
|
-
if (days.length === 0) {
|
|
207
|
-
return out;
|
|
208
|
-
}
|
|
209
|
-
await this.ctx.ready();
|
|
210
|
-
const { rows } = await this.ctx.pool.query(`SELECT bucket, value FROM ${this.ctx.tables.counters}
|
|
211
|
-
WHERE queue = $1 AND metric = $2 AND tier = 'day' AND bucket = ANY($3::bigint[])`, [queue, LatencyStore_1.QUEUE_AGE_METRIC, days.map(keys_1.dayToIndex)]);
|
|
212
|
-
for (const row of rows) {
|
|
213
|
-
out[(0, keys_1.indexToDay)(Number(row.bucket))] = Number(row.value);
|
|
214
|
-
}
|
|
215
|
-
return out;
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Set-if-absent on the database clock: the upsert only overwrites a lease that has already
|
|
219
|
-
* expired, and returns a row only when it wrote one.
|
|
220
|
-
*/
|
|
221
|
-
async acquireLease(queue, holder, ttlMs) {
|
|
222
|
-
await this.ctx.ready();
|
|
223
|
-
const { samplerState } = this.ctx.tables;
|
|
224
|
-
const { rows } = await this.ctx.pool.query(`INSERT INTO ${samplerState} (queue, kind, value, expires_at)
|
|
225
|
-
VALUES ($1, 'lease', $2, now() + $3 * interval '1 millisecond')
|
|
226
|
-
ON CONFLICT (queue, kind) DO UPDATE
|
|
227
|
-
SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at
|
|
228
|
-
WHERE ${samplerState}.expires_at <= now()
|
|
229
|
-
RETURNING value`, [queue, holder, Math.max(1, Math.round(ttlMs))]);
|
|
230
|
-
return rows.length > 0;
|
|
231
|
-
}
|
|
232
|
-
async releaseLease(queue, holder) {
|
|
233
|
-
await this.ctx.ready();
|
|
234
|
-
await this.ctx.pool.query(`DELETE FROM ${this.ctx.tables.samplerState}
|
|
235
|
-
WHERE queue = $1 AND kind = 'lease' AND value = $2`, [queue, holder]);
|
|
236
|
-
}
|
|
237
|
-
async readWatermark(queue) {
|
|
238
|
-
await this.ctx.ready();
|
|
239
|
-
const { rows } = await this.ctx.pool.query(`SELECT value FROM ${this.ctx.tables.samplerState}
|
|
240
|
-
WHERE queue = $1 AND kind = 'watermark' AND expires_at > now()`, [queue]);
|
|
241
|
-
return rows[0] ? Number(rows[0].value) : null;
|
|
242
|
-
}
|
|
243
|
-
async writeWatermark(queue, ms, ttlSeconds) {
|
|
244
|
-
await this.ctx.ready();
|
|
245
|
-
await this.ctx.pool.query(`INSERT INTO ${this.ctx.tables.samplerState} (queue, kind, value, expires_at)
|
|
246
|
-
VALUES ($1, 'watermark', $2, now() + $3 * interval '1 second')
|
|
247
|
-
ON CONFLICT (queue, kind) DO UPDATE
|
|
248
|
-
SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at`, [queue, String(ms), Math.max(1, Math.round(ttlSeconds))]);
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
exports.PostgresLatencyStore = PostgresLatencyStore;
|
|
252
|
-
//# sourceMappingURL=stores.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"stores.js","sourceRoot":"","sources":["../../src/postgres/stores.ts"],"names":[],"mappings":";;;AACA,4CAAyD;AACzD,kCAA4E;AAC5E,kDAAmD;AAEnD,uCAA+F;AAE/F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,4FAA4F;AAC5F,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,QAAQ,CAAC,GAAY;IAC5B,MAAM,GAAG,GAAG,IAAA,uBAAW,GAAE,CAAC;IAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAY,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxD,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAa,oBAAoB;IAI/B,YACmB,GAAc,EAC/B,SAAoB;QADH,QAAG,GAAH,GAAG,CAAW;QAG/B,IAAI,CAAC,SAAS,GAAG,IAAA,4BAAkB,EAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAe,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,KAAa;QAC7E,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,MAAc,EAAE,MAAqB;QACtE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACrD,MAAM,IAAA,uBAAa,EAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAC7C,MAAM,MAAM,CAAC,KAAK,CAAC,uDAAuD,EAAE;gBAC1E,GAAG,MAAM,IAAI,MAAM,IAAI,KAAK,IAAI,MAAM,EAAE;aACzC,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAChB;;;;;;;;yBAQiB,QAAQ;;;;;yBAKR,QAAQ;;;;;;;;;;;;;;;;uBAgBV,QAAQ;;;mCAGI,QAAQ,yBAAyB,EAC5D;gBACE,KAAK;gBACL,MAAM;gBACN,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,mBAAY;aACb,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAA,kBAAW,EAAC,MAAM,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc,EAAE,IAAc;QACjE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,6BAA6B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ;yFAC8B,EACnF,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC,CACtC,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,MAAA,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,mCAAI,IAAI,CAAA,EAAA,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,MAAc,EAAE,IAAc;QAC3D,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC;CACF;AAzFD,oDAyFC;AAED,KAAK,UAAU,eAAe,CAC5B,GAAc,EACd,KAAa,EACb,MAAc,EACd,IAAc;IAEd,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;IAClB,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CACnC,6BAA6B,GAAG,CAAC,MAAM,CAAC,QAAQ;0FACsC,EACtF,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,aAAa,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CACnE,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAA,iBAAU,EAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAa,oBAAoB;IAI/B,YACmB,GAAc,EAC/B,SAAoB;QADH,QAAG,GAAH,GAAG,CAAW;QAG/B,IAAI,CAAC,SAAS,GAAG,IAAA,4BAAkB,EAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAe,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAa,EACb,MAAqB,EACrB,IAAY,EACZ,MAAgB;QAEhB,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACvC,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,UAAU;;;;;;;yBAON,UAAU;;SAE1B,EACH,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAA,iBAAU,EAAC,GAAG,CAAC,EAAE,MAAM,EAAE,mBAAY,CAAC,CAC7D,CAAC;QACF,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,IAAY,EAAE,EAAU;QAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACrC,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,QAAQ;;;;;;0CAMa,QAAQ,yBAAyB,EACrE,CAAC,KAAK,EAAE,+BAAgB,EAAE,IAAI,EAAE,IAAA,iBAAU,EAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,mBAAY,CAAC,CAC5F,CAAC;QACF,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAa,EACb,MAAqB,EACrB,WAA2B,EAC3B,IAAc;QAEd,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACvC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,8BAA8B,UAAU;2FAC2C,EACnF,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC,CACtC,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,GAAG,CAAC,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7D,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,8BAA8B,UAAU;4FAC8C,EACtF,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,aAAa,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CACnE,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAA,iBAAU,EAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7D,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,WAA2B,EAC3B,IAAc;QAEd,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,+BAAgB,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,6BAA6B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ;yFAC8B,EACnF,CAAC,KAAK,EAAE,+BAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC,CAChD,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QAC7D,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,eAAe,YAAY;;;;iBAIhB,YAAY;uBACN,EACjB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc;QAC9C,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY;2DACU,EACrD,CAAC,KAAK,EAAE,MAAM,CAAC,CAChB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,qBAAqB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY;uEACgB,EACjE,CAAC,KAAK,CAAC,CACR,CAAC;QACF,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,EAAU,EAAE,UAAkB;QAChE,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY;;;sEAGqB,EAChE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;CACF;AAtKD,oDAsKC"}
|
package/dist/store.d.ts
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
import type { MetricsClient } from './connection';
|
|
2
|
-
import type { MinutePoint } from './dataMapping';
|
|
3
|
-
import type { HistoryStats, PurgeOptions, PurgeResult } from './HistoryAdmin';
|
|
4
|
-
export interface Retention {
|
|
5
|
-
/** Days of minute-level detail. Doubles as the recorder's catch-up window. */
|
|
6
|
-
minutes: number;
|
|
7
|
-
/** Days of hourly rollup. */
|
|
8
|
-
hours: number;
|
|
9
|
-
/** Days of daily totals, which is what the shipped charts read. */
|
|
10
|
-
days: number;
|
|
11
|
-
}
|
|
12
|
-
export type LatencyMetric = 'runtime' | 'waittime';
|
|
13
|
-
/**
|
|
14
|
-
* Completed and failed throughput, stored at minute, hour and day resolution per queue plus
|
|
15
|
-
* the `__global__` rollup.
|
|
16
|
-
*
|
|
17
|
-
* Writes are absolute per-minute values, never increments: the minute tier is a ledger, and
|
|
18
|
-
* each write applies only its difference against what the ledger already holds, to every
|
|
19
|
-
* coarser tier. Re-writing a minute that is already stored is therefore a no-op, which is what
|
|
20
|
-
* lets a restarted recorder, or a second one, re-snapshot an overlapping window safely.
|
|
21
|
-
*/
|
|
22
|
-
export interface CounterStore {
|
|
23
|
-
readonly retention: Retention;
|
|
24
|
-
upsertMinutes(queue: string, metric: string, points: MinutePoint[]): Promise<void>;
|
|
25
|
-
/** One entry per requested day: `null` when never recorded, a number (maybe 0) when it was. */
|
|
26
|
-
readDailyTotals(queue: string, metric: string, days: string[]): Promise<(number | null)[]>;
|
|
27
|
-
/** Hourly buckets over the given days, keyed by absolute hour index. */
|
|
28
|
-
readHours(queue: string, metric: string, days: string[]): Promise<Record<string, number>>;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Latency histograms (runtime, waittime) and the queue-age gauge, at hour and day resolution,
|
|
32
|
-
* plus the small amount of coordination state the sampler keeps next to them.
|
|
33
|
-
*
|
|
34
|
-
* Histograms are merged by increment, so unlike the counters a double write double-counts.
|
|
35
|
-
* The lease is what prevents that: only its holder scans a queue on a given tick.
|
|
36
|
-
*/
|
|
37
|
-
export interface LatencyStorage {
|
|
38
|
-
readonly retention: Retention;
|
|
39
|
-
addSamples(queue: string, metric: LatencyMetric, hour: number, vector: number[]): Promise<void>;
|
|
40
|
-
recordQueueAge(queue: string, hour: number, ms: number): Promise<void>;
|
|
41
|
-
/** Keyed by ISO day for `'day'`, by absolute hour index for `'hour'`. */
|
|
42
|
-
readRange(queue: string, metric: LatencyMetric, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number[]>>;
|
|
43
|
-
readQueueAge(queue: string, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number>>;
|
|
44
|
-
/** Set-if-absent with a crash-ceiling TTL. `true` when `holder` now owns the queue's lease. */
|
|
45
|
-
acquireLease(queue: string, holder: string, ttlMs: number): Promise<boolean>;
|
|
46
|
-
/** Compare and delete: a lease that expired and was retaken by someone else is left alone. */
|
|
47
|
-
releaseLease(queue: string, holder: string): Promise<void>;
|
|
48
|
-
/** Finish-time bound of the last scan, epoch ms, or `null` for a cold start. */
|
|
49
|
-
readWatermark(queue: string): Promise<number | null>;
|
|
50
|
-
writeWatermark(queue: string, ms: number, ttlSeconds: number): Promise<void>;
|
|
51
|
-
}
|
|
52
|
-
/** Footprint and cleanup, backing the board's storage panel. */
|
|
53
|
-
export interface HistoryAdministration {
|
|
54
|
-
stats(): Promise<HistoryStats>;
|
|
55
|
-
purge(options?: PurgeOptions): Promise<PurgeResult>;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Where recorded history lives. `RedisMetricsStore` and `PostgresMetricsStore` are the two
|
|
59
|
-
* implementations; hand either to `MetricsRecorder`, a history provider or
|
|
60
|
-
* `MetricsHistoryAdmin` as `store`.
|
|
61
|
-
*
|
|
62
|
-
* The members are the seam those classes are built on rather than an API to call directly.
|
|
63
|
-
*/
|
|
64
|
-
export interface MetricsStore {
|
|
65
|
-
/** @internal */
|
|
66
|
-
counterStore(retention: Retention): CounterStore;
|
|
67
|
-
/** @internal */
|
|
68
|
-
latencyStore(retention: Retention): LatencyStorage;
|
|
69
|
-
/** @internal */
|
|
70
|
-
administration(): HistoryAdministration;
|
|
71
|
-
/**
|
|
72
|
-
* @internal
|
|
73
|
-
* The Redis the sampler reads Redis-backed queues through. A Redis store answers with its
|
|
74
|
-
* own client, which is the queues' Redis in the classic single-Redis setup; `null` makes the
|
|
75
|
-
* sampler ask each adapter for its own client instead.
|
|
76
|
-
*/
|
|
77
|
-
readonly jobClient: MetricsClient | null;
|
|
78
|
-
/** Releases what the store opened itself. A connection handed in is left to its owner. */
|
|
79
|
-
close(): Promise<void>;
|
|
80
|
-
}
|
package/dist/store.js
DELETED
package/dist/store.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":""}
|