@worker-manager/metrics 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -6
- package/dist/HistoryAdmin.d.ts +23 -87
- package/dist/HistoryAdmin.js +21 -330
- package/dist/HistoryAdmin.js.map +1 -1
- package/dist/HistoryStore.d.ts +9 -9
- package/dist/HistoryStore.js +22 -0
- package/dist/HistoryStore.js.map +1 -1
- package/dist/LatencySampler.d.ts +27 -32
- package/dist/LatencySampler.js +55 -156
- package/dist/LatencySampler.js.map +1 -1
- package/dist/LatencyStore.d.ts +8 -3
- package/dist/LatencyStore.js +18 -0
- package/dist/LatencyStore.js.map +1 -1
- package/dist/MetricsRecorder.d.ts +34 -10
- package/dist/MetricsRecorder.js +30 -20
- package/dist/MetricsRecorder.js.map +1 -1
- package/dist/RedisHistoryAdmin.d.ts +95 -0
- package/dist/RedisHistoryAdmin.js +328 -0
- package/dist/RedisHistoryAdmin.js.map +1 -0
- package/dist/RedisMetricsHistoryProvider.d.ts +7 -17
- package/dist/RedisMetricsHistoryProvider.js +9 -136
- package/dist/RedisMetricsHistoryProvider.js.map +1 -1
- package/dist/RedisMetricsStore.d.ts +27 -0
- package/dist/RedisMetricsStore.js +42 -0
- package/dist/RedisMetricsStore.js.map +1 -0
- package/dist/StoreHistoryProvider.d.ts +27 -0
- package/dist/StoreHistoryProvider.js +127 -0
- package/dist/StoreHistoryProvider.js.map +1 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/jobSources.d.ts +97 -0
- package/dist/jobSources.js +248 -0
- package/dist/jobSources.js.map +1 -0
- package/dist/keys.d.ts +7 -0
- package/dist/keys.js +23 -1
- package/dist/keys.js.map +1 -1
- package/dist/postgres/PostgresMetricsHistoryProvider.d.ts +37 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js +30 -0
- package/dist/postgres/PostgresMetricsHistoryProvider.js.map +1 -0
- package/dist/postgres/PostgresMetricsStore.d.ts +65 -0
- package/dist/postgres/PostgresMetricsStore.js +88 -0
- package/dist/postgres/PostgresMetricsStore.js.map +1 -0
- package/dist/postgres/admin.d.ts +31 -0
- package/dist/postgres/admin.js +178 -0
- package/dist/postgres/admin.js.map +1 -0
- package/dist/postgres/connection.d.ts +53 -0
- package/dist/postgres/connection.js +54 -0
- package/dist/postgres/connection.js.map +1 -0
- package/dist/postgres/context.d.ts +29 -0
- package/dist/postgres/context.js +76 -0
- package/dist/postgres/context.js.map +1 -0
- package/dist/postgres/schema.d.ts +26 -0
- package/dist/postgres/schema.js +148 -0
- package/dist/postgres/schema.js.map +1 -0
- package/dist/postgres/stores.d.ts +51 -0
- package/dist/postgres/stores.js +252 -0
- package/dist/postgres/stores.js.map +1 -0
- package/dist/store.d.ts +80 -0
- package/dist/store.js +3 -0
- package/dist/store.js.map +1 -0
- package/package.json +13 -3
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Retention } from '../store';
|
|
2
|
+
import type { PgPool, PgPoolClient } from './connection';
|
|
3
|
+
import type { MetricsTables } from './schema';
|
|
4
|
+
/** What the PostgreSQL stores share: one pool, one set of tables, one readiness gate. */
|
|
5
|
+
export interface PgContext {
|
|
6
|
+
pool: PgPool;
|
|
7
|
+
tables: MetricsTables;
|
|
8
|
+
/** Resolves once the tables are there, migrating first when the store was asked to. */
|
|
9
|
+
ready(): Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
export declare function inTransaction<T>(ctx: PgContext, work: (client: PgPoolClient) => Promise<T>): Promise<T>;
|
|
12
|
+
export declare function normalizeRetention(retention: Retention): Retention;
|
|
13
|
+
/**
|
|
14
|
+
* The SQL counterpart of the Redis TTLs and totals trimming: every tier drops the days older
|
|
15
|
+
* than its own retention, counted back from `day`.
|
|
16
|
+
*
|
|
17
|
+
* Anchored on the day being written rather than on the wall clock, exactly as the Redis
|
|
18
|
+
* scripts trim against the day of the write, and run by each writer on its first write and
|
|
19
|
+
* then on the first write of every new day, so about once a day per process. The deletes
|
|
20
|
+
* are range scans on the `(tier, bucket)` indexes. Sampler state past its expiry goes too:
|
|
21
|
+
* a watermark outliving its queue is the unbounded-storage failure this package avoids.
|
|
22
|
+
*/
|
|
23
|
+
export declare class RetentionPruner {
|
|
24
|
+
private readonly ctx;
|
|
25
|
+
private readonly retention;
|
|
26
|
+
private prunedFor;
|
|
27
|
+
constructor(ctx: PgContext, retention: Retention);
|
|
28
|
+
afterWrite(day: string): Promise<void>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RetentionPruner = void 0;
|
|
4
|
+
exports.inTransaction = inTransaction;
|
|
5
|
+
exports.normalizeRetention = normalizeRetention;
|
|
6
|
+
const keys_1 = require("../keys");
|
|
7
|
+
const MINUTES_PER_DAY = 1440;
|
|
8
|
+
const HOURS_PER_DAY = 24;
|
|
9
|
+
async function inTransaction(ctx, work) {
|
|
10
|
+
await ctx.ready();
|
|
11
|
+
const client = await ctx.pool.connect();
|
|
12
|
+
try {
|
|
13
|
+
await client.query('BEGIN');
|
|
14
|
+
try {
|
|
15
|
+
const result = await work(client);
|
|
16
|
+
await client.query('COMMIT');
|
|
17
|
+
return result;
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
21
|
+
throw error;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
client.release();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function normalizeRetention(retention) {
|
|
29
|
+
return {
|
|
30
|
+
minutes: Math.max(1, Math.floor(retention.minutes)),
|
|
31
|
+
hours: Math.max(1, Math.floor(retention.hours)),
|
|
32
|
+
days: Math.max(1, Math.floor(retention.days)),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The SQL counterpart of the Redis TTLs and totals trimming: every tier drops the days older
|
|
37
|
+
* than its own retention, counted back from `day`.
|
|
38
|
+
*
|
|
39
|
+
* Anchored on the day being written rather than on the wall clock, exactly as the Redis
|
|
40
|
+
* scripts trim against the day of the write, and run by each writer on its first write and
|
|
41
|
+
* then on the first write of every new day, so about once a day per process. The deletes
|
|
42
|
+
* are range scans on the `(tier, bucket)` indexes. Sampler state past its expiry goes too:
|
|
43
|
+
* a watermark outliving its queue is the unbounded-storage failure this package avoids.
|
|
44
|
+
*/
|
|
45
|
+
class RetentionPruner {
|
|
46
|
+
constructor(ctx, retention) {
|
|
47
|
+
this.ctx = ctx;
|
|
48
|
+
this.retention = retention;
|
|
49
|
+
this.prunedFor = null;
|
|
50
|
+
}
|
|
51
|
+
async afterWrite(day) {
|
|
52
|
+
if (this.prunedFor !== null && day <= this.prunedFor) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
this.prunedFor = day;
|
|
56
|
+
const { counters, histograms, samplerState } = this.ctx.tables;
|
|
57
|
+
const minuteCutoff = (0, keys_1.dayToIndex)((0, keys_1.shiftDay)(day, -this.retention.minutes)) * MINUTES_PER_DAY;
|
|
58
|
+
const hourCutoff = (0, keys_1.dayToIndex)((0, keys_1.shiftDay)(day, -this.retention.hours)) * HOURS_PER_DAY;
|
|
59
|
+
const dayCutoff = (0, keys_1.dayToIndex)((0, keys_1.shiftDay)(day, -this.retention.days));
|
|
60
|
+
try {
|
|
61
|
+
await this.ctx.pool.query(`DELETE FROM ${counters}
|
|
62
|
+
WHERE (tier = 'minute' AND bucket < $1)
|
|
63
|
+
OR (tier = 'hour' AND bucket < $2)
|
|
64
|
+
OR (tier = 'day' AND bucket < $3)`, [minuteCutoff, hourCutoff, dayCutoff]);
|
|
65
|
+
await this.ctx.pool.query(`DELETE FROM ${histograms}
|
|
66
|
+
WHERE (tier = 'hour' AND bucket < $1) OR (tier = 'day' AND bucket < $2)`, [hourCutoff, dayCutoff]);
|
|
67
|
+
await this.ctx.pool.query(`DELETE FROM ${samplerState} WHERE expires_at <= now()`);
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
this.prunedFor = null; // retried on the next write instead of waiting a day
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.RetentionPruner = RetentionPruner;
|
|
76
|
+
//# sourceMappingURL=context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/postgres/context.ts"],"names":[],"mappings":";;;AAgBA,sCAmBC;AAED,gDAMC;AA3CD,kCAA+C;AAK/C,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,aAAa,GAAG,EAAE,CAAC;AAUlB,KAAK,UAAU,aAAa,CACjC,GAAc,EACd,IAA0C;IAE1C,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB,CAAC,SAAoB;IACrD,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACnD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAa,eAAe;IAG1B,YACmB,GAAc,EACd,SAAoB;QADpB,QAAG,GAAH,GAAG,CAAW;QACd,cAAS,GAAT,SAAS,CAAW;QAJ/B,cAAS,GAAkB,IAAI,CAAC;IAKrC,CAAC;IAEJ,KAAK,CAAC,UAAU,CAAC,GAAW;QAC1B,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC;QACrB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QAC/D,MAAM,YAAY,GAAG,IAAA,iBAAU,EAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,eAAe,CAAC;QAC1F,MAAM,UAAU,GAAG,IAAA,iBAAU,EAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,aAAa,CAAC;QACpF,MAAM,SAAS,GAAG,IAAA,iBAAU,EAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,QAAQ;;;+CAGgB,EACvC,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,CAAC,CACtC,CAAC;YACF,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,UAAU;kFACiD,EAC1E,CAAC,UAAU,EAAE,SAAS,CAAC,CACxB,CAAC;YACF,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,YAAY,4BAA4B,CAAC,CAAC;QACrF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,qDAAqD;YAC5E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF;AApCD,0CAoCC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type PgPool } from './connection';
|
|
2
|
+
export declare const DEFAULT_SCHEMA = "public";
|
|
3
|
+
/** The SQL twin of the Redis namespace `bull-board:metrics`. */
|
|
4
|
+
export declare const DEFAULT_TABLE_PREFIX = "bull_board_metrics_";
|
|
5
|
+
export declare const SCHEMA_VERSION = 1;
|
|
6
|
+
export interface MetricsTables {
|
|
7
|
+
schema: string;
|
|
8
|
+
prefix: string;
|
|
9
|
+
/** Schema-qualified, quoted names, ready to splice into SQL or cast to `regclass`. */
|
|
10
|
+
meta: string;
|
|
11
|
+
counters: string;
|
|
12
|
+
histograms: string;
|
|
13
|
+
samplerState: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function metricsTables(schema: string, prefix: string): MetricsTables;
|
|
16
|
+
/**
|
|
17
|
+
* Creates or upgrades the metrics tables. Idempotent, and safe to run from several processes
|
|
18
|
+
* at once: the whole upgrade is one transaction behind an advisory lock scoped to this
|
|
19
|
+
* schema and prefix, so a second migrator waits and then finds nothing left to do.
|
|
20
|
+
*
|
|
21
|
+
* The schema is created only when it does not exist yet, since `CREATE SCHEMA IF NOT EXISTS`
|
|
22
|
+
* still demands the database-level CREATE privilege that an application role often lacks.
|
|
23
|
+
*/
|
|
24
|
+
export declare function runMigrations(pool: PgPool, tables: MetricsTables): Promise<number>;
|
|
25
|
+
/** Throws unless the tables exist at the version this build writes. */
|
|
26
|
+
export declare function assertMigrated(pool: PgPool, tables: MetricsTables): Promise<void>;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SCHEMA_VERSION = exports.DEFAULT_TABLE_PREFIX = exports.DEFAULT_SCHEMA = void 0;
|
|
4
|
+
exports.metricsTables = metricsTables;
|
|
5
|
+
exports.runMigrations = runMigrations;
|
|
6
|
+
exports.assertMigrated = assertMigrated;
|
|
7
|
+
const connection_1 = require("./connection");
|
|
8
|
+
exports.DEFAULT_SCHEMA = 'public';
|
|
9
|
+
/** The SQL twin of the Redis namespace `bull-board:metrics`. */
|
|
10
|
+
exports.DEFAULT_TABLE_PREFIX = 'bull_board_metrics_';
|
|
11
|
+
/** Keeps the longest derived name, an index, inside PostgreSQL's 63-character limit. */
|
|
12
|
+
const MAX_PREFIX_LENGTH = 32;
|
|
13
|
+
exports.SCHEMA_VERSION = 1;
|
|
14
|
+
function metricsTables(schema, prefix) {
|
|
15
|
+
(0, connection_1.quoteIdentifier)(schema);
|
|
16
|
+
if (prefix.length > MAX_PREFIX_LENGTH) {
|
|
17
|
+
throw new Error(`tablePrefix must be at most ${MAX_PREFIX_LENGTH} characters.`);
|
|
18
|
+
}
|
|
19
|
+
const table = (name) => `${(0, connection_1.quoteIdentifier)(schema)}.${(0, connection_1.quoteIdentifier)(prefix + name)}`;
|
|
20
|
+
return {
|
|
21
|
+
schema,
|
|
22
|
+
prefix,
|
|
23
|
+
meta: table('meta'),
|
|
24
|
+
counters: table('counters'),
|
|
25
|
+
histograms: table('histograms'),
|
|
26
|
+
samplerState: table('sampler_state'),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Ordered, append-only. A released migration is never edited: a change ships as the next
|
|
31
|
+
* version, and `migrate()` applies whatever a database is missing.
|
|
32
|
+
*/
|
|
33
|
+
const MIGRATIONS = [
|
|
34
|
+
{
|
|
35
|
+
version: 1,
|
|
36
|
+
sql: (t) => {
|
|
37
|
+
const index = (name) => (0, connection_1.quoteIdentifier)(`${t.prefix}${name}`);
|
|
38
|
+
return `
|
|
39
|
+
-- completed / failed sums at minute, hour and day resolution, and the queue-age gauge
|
|
40
|
+
-- (a max) at hour and day resolution. bucket is an absolute minute, hour or day index
|
|
41
|
+
-- since the epoch. queue = '__global__' is the cross-queue rollup.
|
|
42
|
+
CREATE TABLE IF NOT EXISTS ${t.counters} (
|
|
43
|
+
queue text NOT NULL,
|
|
44
|
+
metric text NOT NULL,
|
|
45
|
+
tier text NOT NULL CHECK (tier IN ('minute', 'hour', 'day')),
|
|
46
|
+
bucket bigint NOT NULL,
|
|
47
|
+
value bigint NOT NULL,
|
|
48
|
+
PRIMARY KEY (queue, metric, tier, bucket)
|
|
49
|
+
);
|
|
50
|
+
CREATE INDEX IF NOT EXISTS ${index('counters_retention_idx')}
|
|
51
|
+
ON ${t.counters} (tier, bucket);
|
|
52
|
+
|
|
53
|
+
-- runtime / waittime latency histograms: one count per fixed bucket bound, merged
|
|
54
|
+
-- element-wise.
|
|
55
|
+
CREATE TABLE IF NOT EXISTS ${t.histograms} (
|
|
56
|
+
queue text NOT NULL,
|
|
57
|
+
metric text NOT NULL,
|
|
58
|
+
tier text NOT NULL CHECK (tier IN ('hour', 'day')),
|
|
59
|
+
bucket bigint NOT NULL,
|
|
60
|
+
counts bigint[] NOT NULL,
|
|
61
|
+
PRIMARY KEY (queue, metric, tier, bucket)
|
|
62
|
+
);
|
|
63
|
+
CREATE INDEX IF NOT EXISTS ${index('histograms_retention_idx')}
|
|
64
|
+
ON ${t.histograms} (tier, bucket);
|
|
65
|
+
|
|
66
|
+
-- The sampler's per-queue lease and finish-time watermark. Rows past expires_at are
|
|
67
|
+
-- treated as absent and pruned with the history.
|
|
68
|
+
CREATE TABLE IF NOT EXISTS ${t.samplerState} (
|
|
69
|
+
queue text NOT NULL,
|
|
70
|
+
kind text NOT NULL CHECK (kind IN ('lease', 'watermark')),
|
|
71
|
+
value text NOT NULL,
|
|
72
|
+
expires_at timestamptz NOT NULL,
|
|
73
|
+
PRIMARY KEY (queue, kind)
|
|
74
|
+
);
|
|
75
|
+
`;
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
/**
|
|
80
|
+
* Creates or upgrades the metrics tables. Idempotent, and safe to run from several processes
|
|
81
|
+
* at once: the whole upgrade is one transaction behind an advisory lock scoped to this
|
|
82
|
+
* schema and prefix, so a second migrator waits and then finds nothing left to do.
|
|
83
|
+
*
|
|
84
|
+
* The schema is created only when it does not exist yet, since `CREATE SCHEMA IF NOT EXISTS`
|
|
85
|
+
* still demands the database-level CREATE privilege that an application role often lacks.
|
|
86
|
+
*/
|
|
87
|
+
async function runMigrations(pool, tables) {
|
|
88
|
+
var _a;
|
|
89
|
+
const client = await pool.connect();
|
|
90
|
+
try {
|
|
91
|
+
await client.query('BEGIN');
|
|
92
|
+
try {
|
|
93
|
+
await client.query('SELECT pg_advisory_xact_lock(hashtext($1))', [
|
|
94
|
+
`worker-manager-metrics:${tables.schema}.${tables.prefix}`,
|
|
95
|
+
]);
|
|
96
|
+
const { rows: found } = await client.query('SELECT to_regnamespace($1) AS oid', [
|
|
97
|
+
tables.schema,
|
|
98
|
+
]);
|
|
99
|
+
if (!((_a = found[0]) === null || _a === void 0 ? void 0 : _a.oid)) {
|
|
100
|
+
await client.query(`CREATE SCHEMA ${(0, connection_1.quoteIdentifier)(tables.schema)}`);
|
|
101
|
+
}
|
|
102
|
+
await client.query(`CREATE TABLE IF NOT EXISTS ${tables.meta} (name text PRIMARY KEY, value text NOT NULL)`);
|
|
103
|
+
const current = await readVersion(client, tables);
|
|
104
|
+
if (current > exports.SCHEMA_VERSION) {
|
|
105
|
+
throw newerSchemaError(current);
|
|
106
|
+
}
|
|
107
|
+
for (const migration of MIGRATIONS) {
|
|
108
|
+
if (migration.version > current) {
|
|
109
|
+
await client.query(migration.sql(tables));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
await client.query(`INSERT INTO ${tables.meta} (name, value) VALUES ('schema_version', $1)
|
|
113
|
+
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value`, [String(exports.SCHEMA_VERSION)]);
|
|
114
|
+
await client.query('COMMIT');
|
|
115
|
+
return exports.SCHEMA_VERSION;
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
await client.query('ROLLBACK').catch(() => undefined);
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
client.release();
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/** Throws unless the tables exist at the version this build writes. */
|
|
127
|
+
async function assertMigrated(pool, tables) {
|
|
128
|
+
var _a;
|
|
129
|
+
const { rows } = await pool.query('SELECT to_regclass($1) AS oid', [tables.meta]);
|
|
130
|
+
const current = ((_a = rows[0]) === null || _a === void 0 ? void 0 : _a.oid) ? await readVersion(pool, tables) : 0;
|
|
131
|
+
if (current > exports.SCHEMA_VERSION) {
|
|
132
|
+
throw newerSchemaError(current);
|
|
133
|
+
}
|
|
134
|
+
if (current < exports.SCHEMA_VERSION) {
|
|
135
|
+
throw new Error(`The metrics tables in schema "${tables.schema}" (prefix "${tables.prefix}") are ` +
|
|
136
|
+
`missing or at version ${current}, and this build needs version ${exports.SCHEMA_VERSION}. ` +
|
|
137
|
+
'Pass `migrate: true` to the store, or run `migratePostgresMetrics()` as a deploy step.');
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async function readVersion(client, tables) {
|
|
141
|
+
const { rows } = await client.query(`SELECT value FROM ${tables.meta} WHERE name = 'schema_version'`);
|
|
142
|
+
return rows[0] ? Number(rows[0].value) || 0 : 0;
|
|
143
|
+
}
|
|
144
|
+
function newerSchemaError(current) {
|
|
145
|
+
return new Error(`The metrics tables are at schema version ${current}, newer than the ${exports.SCHEMA_VERSION} ` +
|
|
146
|
+
'this build of @worker-manager/metrics understands. Upgrade the package.');
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/postgres/schema.ts"],"names":[],"mappings":";;;AAoBA,sCAcC;AA6DD,sCAwCC;AAGD,wCAaC;AAvJD,6CAA4D;AAE/C,QAAA,cAAc,GAAG,QAAQ,CAAC;AACvC,gEAAgE;AACnD,QAAA,oBAAoB,GAAG,qBAAqB,CAAC;AAC1D,wFAAwF;AACxF,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEhB,QAAA,cAAc,GAAG,CAAC,CAAC;AAYhC,SAAgB,aAAa,CAAC,MAAc,EAAE,MAAc;IAC1D,IAAA,4BAAe,EAAC,MAAM,CAAC,CAAC;IACxB,IAAI,MAAM,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,+BAA+B,iBAAiB,cAAc,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,IAAA,4BAAe,EAAC,MAAM,CAAC,IAAI,IAAA,4BAAe,EAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;IAC/F,OAAO;QACL,MAAM;QACN,MAAM;QACN,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;QACnB,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,UAAU,EAAE,KAAK,CAAC,YAAY,CAAC;QAC/B,YAAY,EAAE,KAAK,CAAC,eAAe,CAAC;KACrC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,GAA6D;IAC3E;QACE,OAAO,EAAE,CAAC;QACV,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAA,4BAAe,EAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;YACtE,OAAO;;;;qCAIwB,CAAC,CAAC,QAAQ;;;;;;;;qCAQV,KAAK,CAAC,wBAAwB,CAAC;eACrD,CAAC,CAAC,QAAQ;;;;qCAIY,CAAC,CAAC,UAAU;;;;;;;;qCAQZ,KAAK,CAAC,0BAA0B,CAAC;eACvD,CAAC,CAAC,UAAU;;;;qCAIU,CAAC,CAAC,YAAY;;;;;;;OAO5C,CAAC;QACJ,CAAC;KACF;CACF,CAAC;AAEF;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,MAAqB;;IACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE;gBAC/D,0BAA0B,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE;aAC3D,CAAC,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE;gBAC9E,MAAM,CAAC,MAAM;aACd,CAAC,CAAC;YACH,IAAI,CAAC,CAAA,MAAA,KAAK,CAAC,CAAC,CAAC,0CAAE,GAAG,CAAA,EAAE,CAAC;gBACnB,MAAM,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAA,4BAAe,EAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAChB,8BAA8B,MAAM,CAAC,IAAI,+CAA+C,CACzF,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,OAAO,GAAG,sBAAc,EAAE,CAAC;gBAC7B,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YACD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,IAAI,SAAS,CAAC,OAAO,GAAG,OAAO,EAAE,CAAC;oBAChC,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAChB,eAAe,MAAM,CAAC,IAAI;iEAC+B,EACzD,CAAC,MAAM,CAAC,sBAAc,CAAC,CAAC,CACzB,CAAC;YACF,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,sBAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,uEAAuE;AAChE,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,MAAqB;;IACtE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAClF,MAAM,OAAO,GAAG,CAAA,MAAA,IAAI,CAAC,CAAC,CAAC,0CAAE,GAAG,EAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,GAAG,sBAAc,EAAE,CAAC;QAC7B,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,OAAO,GAAG,sBAAc,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,MAAM,cAAc,MAAM,CAAC,MAAM,SAAS;YAChF,yBAAyB,OAAO,kCAAkC,sBAAc,IAAI;YACpF,wFAAwF,CAC3F,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAkC,EAClC,MAAqB;IAErB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACjC,qBAAqB,MAAM,CAAC,IAAI,gCAAgC,CACjE,CAAC;IACF,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,OAAO,IAAI,KAAK,CACd,4CAA4C,OAAO,oBAAoB,sBAAc,GAAG;QACtF,yEAAyE,CAC5E,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { MinutePoint } from '../dataMapping';
|
|
2
|
+
import type { CounterStore, LatencyMetric, LatencyStorage, Retention } from '../store';
|
|
3
|
+
import { type PgContext } from './context';
|
|
4
|
+
/**
|
|
5
|
+
* The PostgreSQL `CounterStore`. One transaction per snapshot of a queue's metric: a
|
|
6
|
+
* transaction-scoped advisory lock on (tables, queue, metric), then one statement that
|
|
7
|
+
* diffs the incoming minutes against the minute ledger and applies only the differences to
|
|
8
|
+
* the hour and day tiers of the queue and of `__global__`.
|
|
9
|
+
*
|
|
10
|
+
* The lock is what makes that diff safe across processes, the way running inside one EVAL
|
|
11
|
+
* does in Redis: a second recorder snapshotting the same window waits, then reads the ledger
|
|
12
|
+
* the first one committed and finds nothing left to add. The rollup rows are written in a
|
|
13
|
+
* fixed order, so two queues contending for the same `__global__` rows cannot deadlock.
|
|
14
|
+
*/
|
|
15
|
+
export declare class PostgresCounterStore implements CounterStore {
|
|
16
|
+
private readonly ctx;
|
|
17
|
+
readonly retention: Retention;
|
|
18
|
+
private readonly pruner;
|
|
19
|
+
constructor(ctx: PgContext, retention: Retention);
|
|
20
|
+
upsertMinute(queue: string, metric: string, minute: number, value: number): Promise<void>;
|
|
21
|
+
upsertMinutes(queue: string, metric: string, points: MinutePoint[]): Promise<void>;
|
|
22
|
+
readDailyTotals(queue: string, metric: string, days: string[]): Promise<(number | null)[]>;
|
|
23
|
+
readHours(queue: string, metric: string, days: string[]): Promise<Record<string, number>>;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The PostgreSQL `LatencyStorage`. Each write is one multi-row upsert covering the queue's
|
|
27
|
+
* hour and day rows and the `__global__` rollup of both, so it is atomic on its own. The rows
|
|
28
|
+
* go in a fixed order (queue, then global; day, then hour) for the same deadlock reason as
|
|
29
|
+
* the counters.
|
|
30
|
+
*
|
|
31
|
+
* Histograms merge element-wise in SQL, keeping bucket order through `WITH ORDINALITY`; the
|
|
32
|
+
* queue-age gauge keeps the GREATEST value, since an hour holds the worst backlog seen in it.
|
|
33
|
+
*/
|
|
34
|
+
export declare class PostgresLatencyStore implements LatencyStorage {
|
|
35
|
+
private readonly ctx;
|
|
36
|
+
readonly retention: Retention;
|
|
37
|
+
private readonly pruner;
|
|
38
|
+
constructor(ctx: PgContext, retention: Retention);
|
|
39
|
+
addSamples(queue: string, metric: LatencyMetric, hour: number, vector: number[]): Promise<void>;
|
|
40
|
+
recordQueueAge(queue: string, hour: number, ms: number): Promise<void>;
|
|
41
|
+
readRange(queue: string, metric: LatencyMetric, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number[]>>;
|
|
42
|
+
readQueueAge(queue: string, granularity: 'hour' | 'day', days: string[]): Promise<Record<string, number>>;
|
|
43
|
+
/**
|
|
44
|
+
* Set-if-absent on the database clock: the upsert only overwrites a lease that has already
|
|
45
|
+
* expired, and returns a row only when it wrote one.
|
|
46
|
+
*/
|
|
47
|
+
acquireLease(queue: string, holder: string, ttlMs: number): Promise<boolean>;
|
|
48
|
+
releaseLease(queue: string, holder: string): Promise<void>;
|
|
49
|
+
readWatermark(queue: string): Promise<number | null>;
|
|
50
|
+
writeWatermark(queue: string, ms: number, ttlSeconds: number): Promise<void>;
|
|
51
|
+
}
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PostgresLatencyStore = exports.PostgresCounterStore = void 0;
|
|
4
|
+
const histogram_1 = require("../histogram");
|
|
5
|
+
const keys_1 = require("../keys");
|
|
6
|
+
const LatencyStore_1 = require("../LatencyStore");
|
|
7
|
+
const context_1 = require("./context");
|
|
8
|
+
const HOURS_PER_DAY = 24;
|
|
9
|
+
/** Contiguous `[first, last]` day-index bounds of a day list, for an indexed range read. */
|
|
10
|
+
function dayBounds(days) {
|
|
11
|
+
const indices = days.map(keys_1.dayToIndex);
|
|
12
|
+
return [Math.min(...indices), Math.max(...indices)];
|
|
13
|
+
}
|
|
14
|
+
function toVector(raw) {
|
|
15
|
+
const out = (0, histogram_1.emptyVector)();
|
|
16
|
+
if (!Array.isArray(raw)) {
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
for (let i = 0; i < histogram_1.BUCKET_COUNT && i < raw.length; i++) {
|
|
20
|
+
out[i] = Number(raw[i]) || 0;
|
|
21
|
+
}
|
|
22
|
+
return out;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The PostgreSQL `CounterStore`. One transaction per snapshot of a queue's metric: a
|
|
26
|
+
* transaction-scoped advisory lock on (tables, queue, metric), then one statement that
|
|
27
|
+
* diffs the incoming minutes against the minute ledger and applies only the differences to
|
|
28
|
+
* the hour and day tiers of the queue and of `__global__`.
|
|
29
|
+
*
|
|
30
|
+
* The lock is what makes that diff safe across processes, the way running inside one EVAL
|
|
31
|
+
* does in Redis: a second recorder snapshotting the same window waits, then reads the ledger
|
|
32
|
+
* the first one committed and finds nothing left to add. The rollup rows are written in a
|
|
33
|
+
* fixed order, so two queues contending for the same `__global__` rows cannot deadlock.
|
|
34
|
+
*/
|
|
35
|
+
class PostgresCounterStore {
|
|
36
|
+
constructor(ctx, retention) {
|
|
37
|
+
this.ctx = ctx;
|
|
38
|
+
this.retention = (0, context_1.normalizeRetention)(retention);
|
|
39
|
+
this.pruner = new context_1.RetentionPruner(ctx, this.retention);
|
|
40
|
+
}
|
|
41
|
+
async upsertMinute(queue, metric, minute, value) {
|
|
42
|
+
await this.upsertMinutes(queue, metric, [{ minute, value }]);
|
|
43
|
+
}
|
|
44
|
+
async upsertMinutes(queue, metric, points) {
|
|
45
|
+
if (points.length === 0) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const { counters, prefix, schema } = this.ctx.tables;
|
|
49
|
+
await (0, context_1.inTransaction)(this.ctx, async (client) => {
|
|
50
|
+
await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [
|
|
51
|
+
`${schema}.${prefix}:${queue}:${metric}`,
|
|
52
|
+
]);
|
|
53
|
+
await client.query(`WITH incoming AS (
|
|
54
|
+
SELECT DISTINCT ON (minute) minute, value
|
|
55
|
+
FROM unnest($3::bigint[], $4::bigint[]) AS t(minute, value)
|
|
56
|
+
ORDER BY minute
|
|
57
|
+
),
|
|
58
|
+
changed AS (
|
|
59
|
+
SELECT i.minute, i.value, i.value - coalesce(c.value, 0) AS delta
|
|
60
|
+
FROM incoming i
|
|
61
|
+
LEFT JOIN ${counters} c
|
|
62
|
+
ON c.queue = $1 AND c.metric = $2 AND c.tier = 'minute' AND c.bucket = i.minute
|
|
63
|
+
WHERE i.value <> coalesce(c.value, 0)
|
|
64
|
+
),
|
|
65
|
+
ledger AS (
|
|
66
|
+
INSERT INTO ${counters} (queue, metric, tier, bucket, value)
|
|
67
|
+
SELECT $1, $2, 'minute', minute, value FROM changed
|
|
68
|
+
ON CONFLICT (queue, metric, tier, bucket) DO UPDATE SET value = EXCLUDED.value
|
|
69
|
+
),
|
|
70
|
+
rollup AS (
|
|
71
|
+
SELECT r.queue, r.tier, r.bucket, sum(c.delta) AS delta
|
|
72
|
+
FROM changed c
|
|
73
|
+
CROSS JOIN LATERAL (VALUES
|
|
74
|
+
($1::text, 'hour', c.minute / 60),
|
|
75
|
+
($1::text, 'day', c.minute / 1440),
|
|
76
|
+
($5::text, 'minute', c.minute),
|
|
77
|
+
($5::text, 'hour', c.minute / 60),
|
|
78
|
+
($5::text, 'day', c.minute / 1440)
|
|
79
|
+
) AS r(queue, tier, bucket)
|
|
80
|
+
GROUP BY r.queue, r.tier, r.bucket
|
|
81
|
+
)
|
|
82
|
+
INSERT INTO ${counters} (queue, metric, tier, bucket, value)
|
|
83
|
+
SELECT queue, $2, tier, bucket, delta FROM rollup ORDER BY queue, tier, bucket
|
|
84
|
+
ON CONFLICT (queue, metric, tier, bucket)
|
|
85
|
+
DO UPDATE SET value = ${counters}.value + EXCLUDED.value`, [
|
|
86
|
+
queue,
|
|
87
|
+
metric,
|
|
88
|
+
points.map((p) => p.minute),
|
|
89
|
+
points.map((p) => Math.round(p.value)),
|
|
90
|
+
keys_1.GLOBAL_QUEUE,
|
|
91
|
+
]);
|
|
92
|
+
});
|
|
93
|
+
const newest = Math.max(...points.map((p) => p.minute));
|
|
94
|
+
await this.pruner.afterWrite((0, keys_1.minuteToDay)(newest));
|
|
95
|
+
}
|
|
96
|
+
async readDailyTotals(queue, metric, days) {
|
|
97
|
+
if (days.length === 0) {
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
await this.ctx.ready();
|
|
101
|
+
const { rows } = await this.ctx.pool.query(`SELECT bucket, value FROM ${this.ctx.tables.counters}
|
|
102
|
+
WHERE queue = $1 AND metric = $2 AND tier = 'day' AND bucket = ANY($3::bigint[])`, [queue, metric, days.map(keys_1.dayToIndex)]);
|
|
103
|
+
const byDay = new Map(rows.map((row) => [(0, keys_1.indexToDay)(Number(row.bucket)), Number(row.value)]));
|
|
104
|
+
return days.map((day) => { var _a; return (_a = byDay.get(day)) !== null && _a !== void 0 ? _a : null; });
|
|
105
|
+
}
|
|
106
|
+
async readHours(queue, metric, days) {
|
|
107
|
+
return readHourScalars(this.ctx, queue, metric, days);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
exports.PostgresCounterStore = PostgresCounterStore;
|
|
111
|
+
async function readHourScalars(ctx, queue, metric, days) {
|
|
112
|
+
const out = {};
|
|
113
|
+
if (days.length === 0) {
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
await ctx.ready();
|
|
117
|
+
const [first, last] = dayBounds(days);
|
|
118
|
+
const wanted = new Set(days);
|
|
119
|
+
const { rows } = await ctx.pool.query(`SELECT bucket, value FROM ${ctx.tables.counters}
|
|
120
|
+
WHERE queue = $1 AND metric = $2 AND tier = 'hour' AND bucket >= $3 AND bucket < $4`, [queue, metric, first * HOURS_PER_DAY, (last + 1) * HOURS_PER_DAY]);
|
|
121
|
+
for (const row of rows) {
|
|
122
|
+
const hour = Number(row.bucket);
|
|
123
|
+
if (wanted.has((0, keys_1.indexToDay)(Math.floor(hour / HOURS_PER_DAY)))) {
|
|
124
|
+
out[String(hour)] = Number(row.value);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return out;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* The PostgreSQL `LatencyStorage`. Each write is one multi-row upsert covering the queue's
|
|
131
|
+
* hour and day rows and the `__global__` rollup of both, so it is atomic on its own. The rows
|
|
132
|
+
* go in a fixed order (queue, then global; day, then hour) for the same deadlock reason as
|
|
133
|
+
* the counters.
|
|
134
|
+
*
|
|
135
|
+
* Histograms merge element-wise in SQL, keeping bucket order through `WITH ORDINALITY`; the
|
|
136
|
+
* queue-age gauge keeps the GREATEST value, since an hour holds the worst backlog seen in it.
|
|
137
|
+
*/
|
|
138
|
+
class PostgresLatencyStore {
|
|
139
|
+
constructor(ctx, retention) {
|
|
140
|
+
this.ctx = ctx;
|
|
141
|
+
this.retention = (0, context_1.normalizeRetention)(retention);
|
|
142
|
+
this.pruner = new context_1.RetentionPruner(ctx, this.retention);
|
|
143
|
+
}
|
|
144
|
+
async addSamples(queue, metric, hour, vector) {
|
|
145
|
+
await this.ctx.ready();
|
|
146
|
+
const { histograms } = this.ctx.tables;
|
|
147
|
+
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
148
|
+
const counts = toVector(vector).map((v) => Math.round(v));
|
|
149
|
+
await this.ctx.pool.query(`INSERT INTO ${histograms} (queue, metric, tier, bucket, counts) VALUES
|
|
150
|
+
($1, $2, 'day', $4, $5::bigint[]),
|
|
151
|
+
($1, $2, 'hour', $3, $5::bigint[]),
|
|
152
|
+
($6, $2, 'day', $4, $5::bigint[]),
|
|
153
|
+
($6, $2, 'hour', $3, $5::bigint[])
|
|
154
|
+
ON CONFLICT (queue, metric, tier, bucket) DO UPDATE SET counts = ARRAY(
|
|
155
|
+
SELECT coalesce(a, 0) + coalesce(b, 0)
|
|
156
|
+
FROM unnest(${histograms}.counts, EXCLUDED.counts) WITH ORDINALITY AS t(a, b, i)
|
|
157
|
+
ORDER BY i
|
|
158
|
+
)`, [queue, metric, hour, (0, keys_1.dayToIndex)(day), counts, keys_1.GLOBAL_QUEUE]);
|
|
159
|
+
await this.pruner.afterWrite(day);
|
|
160
|
+
}
|
|
161
|
+
async recordQueueAge(queue, hour, ms) {
|
|
162
|
+
await this.ctx.ready();
|
|
163
|
+
const { counters } = this.ctx.tables;
|
|
164
|
+
const day = (0, keys_1.minuteToDay)(hour * 60);
|
|
165
|
+
await this.ctx.pool.query(`INSERT INTO ${counters} (queue, metric, tier, bucket, value) VALUES
|
|
166
|
+
($1, $2, 'day', $4, $5),
|
|
167
|
+
($1, $2, 'hour', $3, $5),
|
|
168
|
+
($6, $2, 'day', $4, $5),
|
|
169
|
+
($6, $2, 'hour', $3, $5)
|
|
170
|
+
ON CONFLICT (queue, metric, tier, bucket)
|
|
171
|
+
DO UPDATE SET value = GREATEST(${counters}.value, EXCLUDED.value)`, [queue, LatencyStore_1.QUEUE_AGE_METRIC, hour, (0, keys_1.dayToIndex)(day), Math.max(0, Math.round(ms)), keys_1.GLOBAL_QUEUE]);
|
|
172
|
+
await this.pruner.afterWrite(day);
|
|
173
|
+
}
|
|
174
|
+
async readRange(queue, metric, granularity, days) {
|
|
175
|
+
const out = {};
|
|
176
|
+
if (days.length === 0) {
|
|
177
|
+
return out;
|
|
178
|
+
}
|
|
179
|
+
await this.ctx.ready();
|
|
180
|
+
const { histograms } = this.ctx.tables;
|
|
181
|
+
if (granularity === 'day') {
|
|
182
|
+
const { rows } = await this.ctx.pool.query(`SELECT bucket, counts FROM ${histograms}
|
|
183
|
+
WHERE queue = $1 AND metric = $2 AND tier = 'day' AND bucket = ANY($3::bigint[])`, [queue, metric, days.map(keys_1.dayToIndex)]);
|
|
184
|
+
for (const row of rows) {
|
|
185
|
+
out[(0, keys_1.indexToDay)(Number(row.bucket))] = toVector(row.counts);
|
|
186
|
+
}
|
|
187
|
+
return out;
|
|
188
|
+
}
|
|
189
|
+
const [first, last] = dayBounds(days);
|
|
190
|
+
const wanted = new Set(days);
|
|
191
|
+
const { rows } = await this.ctx.pool.query(`SELECT bucket, counts FROM ${histograms}
|
|
192
|
+
WHERE queue = $1 AND metric = $2 AND tier = 'hour' AND bucket >= $3 AND bucket < $4`, [queue, metric, first * HOURS_PER_DAY, (last + 1) * HOURS_PER_DAY]);
|
|
193
|
+
for (const row of rows) {
|
|
194
|
+
const hour = Number(row.bucket);
|
|
195
|
+
if (wanted.has((0, keys_1.indexToDay)(Math.floor(hour / HOURS_PER_DAY)))) {
|
|
196
|
+
out[String(hour)] = toVector(row.counts);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
async readQueueAge(queue, granularity, days) {
|
|
202
|
+
if (granularity === 'hour') {
|
|
203
|
+
return readHourScalars(this.ctx, queue, LatencyStore_1.QUEUE_AGE_METRIC, days);
|
|
204
|
+
}
|
|
205
|
+
const out = {};
|
|
206
|
+
if (days.length === 0) {
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
await this.ctx.ready();
|
|
210
|
+
const { rows } = await this.ctx.pool.query(`SELECT bucket, value FROM ${this.ctx.tables.counters}
|
|
211
|
+
WHERE queue = $1 AND metric = $2 AND tier = 'day' AND bucket = ANY($3::bigint[])`, [queue, LatencyStore_1.QUEUE_AGE_METRIC, days.map(keys_1.dayToIndex)]);
|
|
212
|
+
for (const row of rows) {
|
|
213
|
+
out[(0, keys_1.indexToDay)(Number(row.bucket))] = Number(row.value);
|
|
214
|
+
}
|
|
215
|
+
return out;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Set-if-absent on the database clock: the upsert only overwrites a lease that has already
|
|
219
|
+
* expired, and returns a row only when it wrote one.
|
|
220
|
+
*/
|
|
221
|
+
async acquireLease(queue, holder, ttlMs) {
|
|
222
|
+
await this.ctx.ready();
|
|
223
|
+
const { samplerState } = this.ctx.tables;
|
|
224
|
+
const { rows } = await this.ctx.pool.query(`INSERT INTO ${samplerState} (queue, kind, value, expires_at)
|
|
225
|
+
VALUES ($1, 'lease', $2, now() + $3 * interval '1 millisecond')
|
|
226
|
+
ON CONFLICT (queue, kind) DO UPDATE
|
|
227
|
+
SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at
|
|
228
|
+
WHERE ${samplerState}.expires_at <= now()
|
|
229
|
+
RETURNING value`, [queue, holder, Math.max(1, Math.round(ttlMs))]);
|
|
230
|
+
return rows.length > 0;
|
|
231
|
+
}
|
|
232
|
+
async releaseLease(queue, holder) {
|
|
233
|
+
await this.ctx.ready();
|
|
234
|
+
await this.ctx.pool.query(`DELETE FROM ${this.ctx.tables.samplerState}
|
|
235
|
+
WHERE queue = $1 AND kind = 'lease' AND value = $2`, [queue, holder]);
|
|
236
|
+
}
|
|
237
|
+
async readWatermark(queue) {
|
|
238
|
+
await this.ctx.ready();
|
|
239
|
+
const { rows } = await this.ctx.pool.query(`SELECT value FROM ${this.ctx.tables.samplerState}
|
|
240
|
+
WHERE queue = $1 AND kind = 'watermark' AND expires_at > now()`, [queue]);
|
|
241
|
+
return rows[0] ? Number(rows[0].value) : null;
|
|
242
|
+
}
|
|
243
|
+
async writeWatermark(queue, ms, ttlSeconds) {
|
|
244
|
+
await this.ctx.ready();
|
|
245
|
+
await this.ctx.pool.query(`INSERT INTO ${this.ctx.tables.samplerState} (queue, kind, value, expires_at)
|
|
246
|
+
VALUES ($1, 'watermark', $2, now() + $3 * interval '1 second')
|
|
247
|
+
ON CONFLICT (queue, kind) DO UPDATE
|
|
248
|
+
SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at`, [queue, String(ms), Math.max(1, Math.round(ttlSeconds))]);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
exports.PostgresLatencyStore = PostgresLatencyStore;
|
|
252
|
+
//# sourceMappingURL=stores.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stores.js","sourceRoot":"","sources":["../../src/postgres/stores.ts"],"names":[],"mappings":";;;AACA,4CAAyD;AACzD,kCAA4E;AAC5E,kDAAmD;AAEnD,uCAA+F;AAE/F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,4FAA4F;AAC5F,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,QAAQ,CAAC,GAAY;IAC5B,MAAM,GAAG,GAAG,IAAA,uBAAW,GAAE,CAAC;IAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAY,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxD,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAa,oBAAoB;IAI/B,YACmB,GAAc,EAC/B,SAAoB;QADH,QAAG,GAAH,GAAG,CAAW;QAG/B,IAAI,CAAC,SAAS,GAAG,IAAA,4BAAkB,EAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAe,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,KAAa;QAC7E,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,MAAc,EAAE,MAAqB;QACtE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACrD,MAAM,IAAA,uBAAa,EAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAC7C,MAAM,MAAM,CAAC,KAAK,CAAC,uDAAuD,EAAE;gBAC1E,GAAG,MAAM,IAAI,MAAM,IAAI,KAAK,IAAI,MAAM,EAAE;aACzC,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAChB;;;;;;;;yBAQiB,QAAQ;;;;;yBAKR,QAAQ;;;;;;;;;;;;;;;;uBAgBV,QAAQ;;;mCAGI,QAAQ,yBAAyB,EAC5D;gBACE,KAAK;gBACL,MAAM;gBACN,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtC,mBAAY;aACb,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAA,kBAAW,EAAC,MAAM,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAc,EAAE,IAAc;QACjE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,6BAA6B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ;yFAC8B,EACnF,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC,CACtC,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAC,OAAA,MAAA,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,mCAAI,IAAI,CAAA,EAAA,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,MAAc,EAAE,IAAc;QAC3D,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC;CACF;AAzFD,oDAyFC;AAED,KAAK,UAAU,eAAe,CAC5B,GAAc,EACd,KAAa,EACb,MAAc,EACd,IAAc;IAEd,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;IAClB,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CACnC,6BAA6B,GAAG,CAAC,MAAM,CAAC,QAAQ;0FACsC,EACtF,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,aAAa,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CACnE,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAA,iBAAU,EAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAa,oBAAoB;IAI/B,YACmB,GAAc,EAC/B,SAAoB;QADH,QAAG,GAAH,GAAG,CAAW;QAG/B,IAAI,CAAC,SAAS,GAAG,IAAA,4BAAkB,EAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAe,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAa,EACb,MAAqB,EACrB,IAAY,EACZ,MAAgB;QAEhB,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACvC,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,UAAU;;;;;;;yBAON,UAAU;;SAE1B,EACH,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAA,iBAAU,EAAC,GAAG,CAAC,EAAE,MAAM,EAAE,mBAAY,CAAC,CAC7D,CAAC;QACF,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,IAAY,EAAE,EAAU;QAC1D,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACrC,MAAM,GAAG,GAAG,IAAA,kBAAW,EAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,QAAQ;;;;;;0CAMa,QAAQ,yBAAyB,EACrE,CAAC,KAAK,EAAE,+BAAgB,EAAE,IAAI,EAAE,IAAA,iBAAU,EAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,mBAAY,CAAC,CAC5F,CAAC;QACF,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAa,EACb,MAAqB,EACrB,WAA2B,EAC3B,IAAc;QAEd,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACvC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,8BAA8B,UAAU;2FAC2C,EACnF,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC,CACtC,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,GAAG,CAAC,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7D,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,8BAA8B,UAAU;4FAC8C,EACtF,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAG,aAAa,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CACnE,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAA,iBAAU,EAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7D,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,WAA2B,EAC3B,IAAc;QAEd,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,+BAAgB,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,6BAA6B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ;yFAC8B,EACnF,CAAC,KAAK,EAAE,+BAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAU,CAAC,CAAC,CAChD,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,IAAA,iBAAU,EAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc,EAAE,KAAa;QAC7D,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;QACzC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,eAAe,YAAY;;;;iBAIhB,YAAY;uBACN,EACjB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAChD,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAc;QAC9C,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY;2DACU,EACrD,CAAC,KAAK,EAAE,MAAM,CAAC,CAChB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACxC,qBAAqB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY;uEACgB,EACjE,CAAC,KAAK,CAAC,CACR,CAAC;QACF,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,EAAU,EAAE,UAAkB;QAChE,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CACvB,eAAe,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY;;;sEAGqB,EAChE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;CACF;AAtKD,oDAsKC"}
|