@x12i/countex 1.8.5 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +867 -860
- package/dist/cli.js +6 -6
- package/dist/core/countex.d.ts +61 -1
- package/dist/core/countex.d.ts.map +1 -1
- package/dist/core/countex.js +241 -0
- package/dist/core/countex.js.map +1 -1
- package/dist/core/managed.d.ts +200 -0
- package/dist/core/managed.d.ts.map +1 -0
- package/dist/core/managed.js +431 -0
- package/dist/core/managed.js.map +1 -0
- package/dist/core/timeframe.d.ts +5 -0
- package/dist/core/timeframe.d.ts.map +1 -1
- package/dist/core/timeframe.js +36 -0
- package/dist/core/timeframe.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/insights/advisors/resource-advisor.d.ts.map +1 -1
- package/dist/insights/advisors/resource-advisor.js +5 -1
- package/dist/insights/advisors/resource-advisor.js.map +1 -1
- package/dist/insights/types.d.ts +4 -0
- package/dist/insights/types.d.ts.map +1 -1
- package/dist/storage/managed-memory.d.ts +33 -0
- package/dist/storage/managed-memory.d.ts.map +1 -0
- package/dist/storage/managed-memory.js +211 -0
- package/dist/storage/managed-memory.js.map +1 -0
- package/dist/storage/memory.d.ts +7 -4
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js.map +1 -1
- package/dist/storage/postgres.d.ts +123 -0
- package/dist/storage/postgres.d.ts.map +1 -0
- package/dist/storage/postgres.js +1325 -0
- package/dist/storage/postgres.js.map +1 -0
- package/dist/storage/redis.d.ts +6 -0
- package/dist/storage/redis.d.ts.map +1 -1
- package/dist/storage/redis.js +21 -0
- package/dist/storage/redis.js.map +1 -1
- package/dist/types/index.d.ts +63 -5
- package/dist/types/index.d.ts.map +1 -1
- package/dist/utils/errors.d.ts +12 -0
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +22 -1
- package/dist/utils/errors.js.map +1 -1
- package/docs/DEPLOYMENT.md +399 -375
- package/docs/END_TO_END_EXAMPLE.md +316 -316
- package/docs/INSIGHTS.md +59 -59
- package/docs/INSIGHTS_ANOMALY_DETECTION.md +33 -33
- package/docs/INSIGHTS_LLM_COST_AND_TOKENS.md +26 -26
- package/docs/INSIGHTS_RESOURCE_ADVISOR.md +33 -31
- package/docs/countex_gap_analysis_complete_checklist.md +1340 -1340
- package/docs/guides/README.md +37 -34
- package/docs/guides/adapter-memory.md +60 -60
- package/docs/guides/adapter-mongo.md +64 -62
- package/docs/guides/adapter-postgres.md +92 -0
- package/docs/guides/adapter-redis.md +89 -89
- package/docs/guides/backup-export.md +135 -135
- package/docs/guides/cardinality.md +50 -50
- package/docs/guides/cli.md +53 -53
- package/docs/guides/client-events.md +65 -65
- package/docs/guides/counter-maps.md +160 -160
- package/docs/guides/idempotency.md +39 -39
- package/docs/guides/ingestion-and-flush.md +67 -67
- package/docs/guides/insights-advisors-and-optimization.md +39 -39
- package/docs/guides/insights-anomalies-and-rules.md +48 -48
- package/docs/guides/insights-comparison-and-trends.md +47 -47
- package/docs/guides/insights-forecasting-and-budget.md +61 -61
- package/docs/guides/insights-overview.md +82 -82
- package/docs/guides/insights-ranking.md +81 -81
- package/docs/guides/managed-commit.md +119 -0
- package/docs/guides/measures.md +72 -71
- package/docs/guides/migrating-mongo-to-postgres.md +127 -0
- package/docs/guides/preview.md +92 -92
- package/docs/guides/querying.md +178 -178
- package/docs/guides/simulation-profiles.md +33 -33
- package/docs/guides/simulation-resource-estimation.md +31 -30
- package/docs/guides/simulation.md +36 -36
- package/docs/guides/timeframes-and-bucketing.md +55 -55
- package/docs/internals/README.md +21 -21
- package/docs/internals/internal-architecture-overview.md +65 -65
- package/docs/internals/internal-backup-stream.md +23 -23
- package/docs/internals/internal-buffer-merge-and-flush.md +51 -51
- package/docs/internals/internal-cardinality.md +30 -30
- package/docs/internals/internal-dimension-hash.md +34 -34
- package/docs/internals/internal-idempotency.md +36 -36
- package/docs/internals/internal-ingest-pipeline.md +52 -52
- package/docs/internals/internal-insights-layer.md +36 -36
- package/docs/internals/internal-mapping-validation-and-extraction.md +59 -59
- package/docs/internals/internal-measures-and-scales.md +35 -35
- package/docs/internals/internal-query-path.md +46 -46
- package/docs/internals/internal-storage-memory.md +13 -13
- package/docs/internals/internal-storage-mongo.md +44 -44
- package/docs/internals/internal-storage-postgres.md +32 -0
- package/docs/internals/internal-storage-redis.md +63 -63
- package/docs/internals/internal-timeframe-bucketing.md +33 -33
- package/docs/use-cases/README.md +56 -56
- package/docs/use-cases/audit-security-log-counts.md +550 -550
- package/docs/use-cases/llm-cost-and-token-metering.md +255 -255
- package/docs/use-cases/multi-tenant-isolation.md +203 -203
- package/docs/use-cases/ops-dashboard-daily-rollups.md +255 -255
- package/docs/use-cases/processing-workflow-activity.md +165 -165
- package/docs/use-cases/product-feature-usage.md +240 -240
- package/docs/use-cases/webhook-idempotent-ingestion.md +188 -188
- package/package.json +190 -170
|
@@ -0,0 +1,1325 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* PostgreSQL durable adapter.
|
|
4
|
+
*
|
|
5
|
+
* Same cell, dictionary, sequence, and evidence semantics as MongoDurable.
|
|
6
|
+
* Writes are batched `INSERT … ON CONFLICT DO UPDATE` increments so concurrent
|
|
7
|
+
* flushes add counts instead of replacing them. Measure math runs in SQL.
|
|
8
|
+
*
|
|
9
|
+
* The host owns the `pg` Pool (connect, TLS, shutdown), same as MongoClient.
|
|
10
|
+
* Requires PostgreSQL 15+ (unique constraints on partitioned tables).
|
|
11
|
+
*
|
|
12
|
+
* Layout (one schema, default `countex`):
|
|
13
|
+
* cells — list-partitioned by timeframe_type (minute, hour, default)
|
|
14
|
+
* dict — (scope, kind, qualifier, source_key) → internal_id
|
|
15
|
+
* seq — per-scope dictionary id block
|
|
16
|
+
* evidence — exemplar payloads trimmed to maxSamplesAtCapture
|
|
17
|
+
*
|
|
18
|
+
* Identity columns are not updated on increment, so counter updates can be
|
|
19
|
+
* HOT. fillfactor and a tighter autovacuum scale factor are set on the
|
|
20
|
+
* minute/hour/default partitions for that reason.
|
|
21
|
+
*/
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.PostgresDurable = exports.POSTGRES_CELL_SCHEMA_VERSION = void 0;
|
|
24
|
+
const managed_js_1 = require("../core/managed.js");
|
|
25
|
+
const index_js_1 = require("../types/index.js");
|
|
26
|
+
const snapshot_encoding_js_1 = require("../backup/snapshot-encoding.js");
|
|
27
|
+
const timeframe_js_1 = require("../core/timeframe.js");
|
|
28
|
+
const errors_js_1 = require("../utils/errors.js");
|
|
29
|
+
const hash_js_1 = require("../utils/hash.js");
|
|
30
|
+
/** Persisted on cell rows. Aligns with COUNTEX_CELL_SCHEMA_VERSION. */
|
|
31
|
+
exports.POSTGRES_CELL_SCHEMA_VERSION = 1;
|
|
32
|
+
/** Default rows per INSERT. Keeps a flush to a handful of round-trips. */
|
|
33
|
+
const DEFAULT_BATCH_SIZE = 1000;
|
|
34
|
+
const IDENT = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
35
|
+
class PostgresDurable {
|
|
36
|
+
opts;
|
|
37
|
+
ready = new Map();
|
|
38
|
+
/** Test-only. The next managed commit writes, then rolls the transaction back. */
|
|
39
|
+
failNextCommit = false;
|
|
40
|
+
constructor(opts) {
|
|
41
|
+
const batch = opts.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
42
|
+
this.opts = {
|
|
43
|
+
pool: opts.pool,
|
|
44
|
+
schema: assertIdent(opts.schema ?? "countex", "schema"),
|
|
45
|
+
schemaResolver: opts.schemaResolver,
|
|
46
|
+
cellsTable: assertIdent(opts.cellsTable ?? "cells", "cellsTable"),
|
|
47
|
+
dictTable: assertIdent(opts.dictTable ?? "dict", "dictTable"),
|
|
48
|
+
seqTable: assertIdent(opts.seqTable ?? "seq", "seqTable"),
|
|
49
|
+
evidenceTable: assertIdent(opts.evidenceTable ?? "evidence", "evidenceTable"),
|
|
50
|
+
skipIndexInit: opts.skipIndexInit ?? false,
|
|
51
|
+
batchSize: Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : DEFAULT_BATCH_SIZE,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
async init() {
|
|
55
|
+
if (this.opts.schemaResolver)
|
|
56
|
+
return;
|
|
57
|
+
await this.ensureSchema(this.opts.schema);
|
|
58
|
+
}
|
|
59
|
+
async close() {
|
|
60
|
+
// The host owns the pool.
|
|
61
|
+
}
|
|
62
|
+
async applyDeltas(deltas, opts) {
|
|
63
|
+
if (deltas.length === 0)
|
|
64
|
+
return;
|
|
65
|
+
throwIfAborted(opts?.signal, "applyDeltas aborted");
|
|
66
|
+
const collapsed = collapseDeltas(deltas).sort((a, b) => (0, hash_js_1.cellKey)(a.identity).localeCompare((0, hash_js_1.cellKey)(b.identity)));
|
|
67
|
+
for (const delta of collapsed) {
|
|
68
|
+
await this.ensureSchema(this.schemaFor(delta.identity.scopeId));
|
|
69
|
+
}
|
|
70
|
+
const client = await this.opts.pool.connect();
|
|
71
|
+
try {
|
|
72
|
+
await client.query("BEGIN");
|
|
73
|
+
await this.writeIncrements(client, collapsed, opts?.signal);
|
|
74
|
+
await client.query("COMMIT");
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
await rollback(client);
|
|
78
|
+
throw asStorageError("Postgres applyDeltas failed", err);
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
client.release();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async registerManagedGeneration(row) {
|
|
85
|
+
await this.ensureSchema(this.opts.schema);
|
|
86
|
+
const table = qual(this.opts.schema, "map_generations");
|
|
87
|
+
try {
|
|
88
|
+
const inserted = await this.opts.pool.query(`INSERT INTO ${table} (generation, counter_id, map_hash, map_json)
|
|
89
|
+
VALUES ($1, $2, $3, $4::jsonb)
|
|
90
|
+
ON CONFLICT (generation) DO NOTHING
|
|
91
|
+
RETURNING generation`, [row.generation, row.counterId, row.mapHash, JSON.stringify(row.map)]);
|
|
92
|
+
if (inserted.rows.length > 0)
|
|
93
|
+
return row;
|
|
94
|
+
const existing = await this.readManagedGeneration(row.generation);
|
|
95
|
+
if (!existing || existing.mapHash !== row.mapHash || existing.counterId !== row.counterId) {
|
|
96
|
+
throw new errors_js_1.StorageError("map generation is immutable", { generation: row.generation });
|
|
97
|
+
}
|
|
98
|
+
return existing;
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
throw asStorageError("Postgres registerManagedGeneration failed", err);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async readManagedGeneration(generation) {
|
|
105
|
+
await this.ensureSchema(this.opts.schema);
|
|
106
|
+
const table = qual(this.opts.schema, "map_generations");
|
|
107
|
+
const res = await this.opts.pool.query(`SELECT generation, counter_id, map_hash, map_json FROM ${table} WHERE generation = $1`, [generation]);
|
|
108
|
+
const row = res.rows[0];
|
|
109
|
+
if (!row)
|
|
110
|
+
return null;
|
|
111
|
+
const map = typeof row.map_json === "string" ? JSON.parse(row.map_json) : row.map_json;
|
|
112
|
+
return {
|
|
113
|
+
generation: row.generation,
|
|
114
|
+
counterId: row.counter_id,
|
|
115
|
+
mapHash: row.map_hash,
|
|
116
|
+
map,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
async readManagedCommit(key) {
|
|
120
|
+
const schema = this.schemaFor(key.scopeId);
|
|
121
|
+
await this.ensureSchema(schema);
|
|
122
|
+
const res = await this.opts.pool.query(`SELECT scope_id, counter_id, generation, event_id, batch_id, projection_hash, effects
|
|
123
|
+
FROM ${qual(schema, "commits")}
|
|
124
|
+
WHERE scope_id = $1 AND counter_id = $2 AND generation = $3 AND event_id = $4`, [key.scopeId, key.counterId, key.generation, key.eventId]);
|
|
125
|
+
const row = res.rows[0];
|
|
126
|
+
return row ? outcomeFromRow(row, "committed") : null;
|
|
127
|
+
}
|
|
128
|
+
async commitManaged(req) {
|
|
129
|
+
throwIfAborted(req.signal, "managed commit aborted");
|
|
130
|
+
const schemas = new Set();
|
|
131
|
+
for (const event of req.events)
|
|
132
|
+
schemas.add(this.schemaFor(event.scopeId));
|
|
133
|
+
if (schemas.size === 0)
|
|
134
|
+
schemas.add(this.opts.schema);
|
|
135
|
+
for (const schema of schemas)
|
|
136
|
+
await this.ensureSchema(schema);
|
|
137
|
+
const client = await this.opts.pool.connect();
|
|
138
|
+
const fail = this.failNextCommit;
|
|
139
|
+
this.failNextCommit = false;
|
|
140
|
+
try {
|
|
141
|
+
await client.query("BEGIN");
|
|
142
|
+
const timeout = Math.max(1, Math.floor(req.statementTimeoutMs));
|
|
143
|
+
await client.query(`SET LOCAL statement_timeout = ${timeout}`);
|
|
144
|
+
const outcomes = [];
|
|
145
|
+
const fresh = [];
|
|
146
|
+
for (const event of req.events) {
|
|
147
|
+
throwIfAborted(req.signal, "managed commit aborted");
|
|
148
|
+
const schema = this.schemaFor(event.scopeId);
|
|
149
|
+
const table = qual(schema, "commits");
|
|
150
|
+
const inserted = await client.query(`INSERT INTO ${table} (
|
|
151
|
+
scope_id, counter_id, generation, event_id, batch_id, projection_hash, effects
|
|
152
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
|
|
153
|
+
ON CONFLICT (scope_id, counter_id, generation, event_id) DO NOTHING
|
|
154
|
+
RETURNING event_id`, [
|
|
155
|
+
event.scopeId,
|
|
156
|
+
event.counterId,
|
|
157
|
+
event.generation,
|
|
158
|
+
event.eventId,
|
|
159
|
+
req.batchId,
|
|
160
|
+
event.projectionHash,
|
|
161
|
+
JSON.stringify(event.effects),
|
|
162
|
+
]);
|
|
163
|
+
if (inserted.rows.length === 0) {
|
|
164
|
+
const existing = await client.query(`SELECT scope_id, counter_id, generation, event_id, batch_id, projection_hash, effects
|
|
165
|
+
FROM ${table}
|
|
166
|
+
WHERE scope_id = $1 AND counter_id = $2 AND generation = $3 AND event_id = $4`, [event.scopeId, event.counterId, event.generation, event.eventId]);
|
|
167
|
+
const row = existing.rows[0];
|
|
168
|
+
if (!row)
|
|
169
|
+
throw new errors_js_1.StorageError("commit identity missing after conflict", {});
|
|
170
|
+
outcomes.push(outcomeFromRow(row, "duplicate"));
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
fresh.push(event);
|
|
174
|
+
outcomes.push(outcomeFromPrepared(event, req.batchId, "committed"));
|
|
175
|
+
}
|
|
176
|
+
const applied = [];
|
|
177
|
+
for (const event of fresh) {
|
|
178
|
+
const ids = await this.resolveIdsWith(client, (0, managed_js_1.dictionaryLookups)(event.increments));
|
|
179
|
+
applied.push(...(0, managed_js_1.incrementsToDeltas)(event.increments, ids, event.measureScales));
|
|
180
|
+
}
|
|
181
|
+
if (applied.length > 0)
|
|
182
|
+
await this.writeIncrements(client, applied, req.signal);
|
|
183
|
+
if (fail)
|
|
184
|
+
throw new errors_js_1.StorageError("managed commit aborted before durable commit", {});
|
|
185
|
+
await client.query("COMMIT");
|
|
186
|
+
return { outcomes, appliedDeltas: applied };
|
|
187
|
+
}
|
|
188
|
+
catch (err) {
|
|
189
|
+
await rollback(client);
|
|
190
|
+
throw asStorageError("Postgres commitManaged failed", err);
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
client.release();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async exportRecoverySet(filter) {
|
|
197
|
+
const schema = this.schemaFor(filter.scopeId);
|
|
198
|
+
await this.ensureSchema(schema);
|
|
199
|
+
await this.ensureSchema(this.opts.schema);
|
|
200
|
+
const cellRows = await this.opts.pool.query(`SELECT * FROM ${qual(schema, this.opts.cellsTable)} WHERE scope_id = $1 AND counter_id = $2`, [filter.scopeId, filter.counterId]);
|
|
201
|
+
const cells = cellRows.rows.map(rowToCell);
|
|
202
|
+
const dict = qual(schema, this.opts.dictTable);
|
|
203
|
+
const dictRows = await this.opts.pool.query(`SELECT kind, qualifier, source_key, internal_id FROM ${dict} WHERE scope_id = $1`, [filter.scopeId]);
|
|
204
|
+
const commits = qual(schema, "commits");
|
|
205
|
+
const commitRows = await this.opts.pool.query(`SELECT scope_id, counter_id, generation, event_id, batch_id, projection_hash, effects
|
|
206
|
+
FROM ${commits} WHERE scope_id = $1 AND counter_id = $2`, [filter.scopeId, filter.counterId]);
|
|
207
|
+
const generations = qual(this.opts.schema, "map_generations");
|
|
208
|
+
const genRows = await this.opts.pool.query(`SELECT generation, counter_id, map_hash, map_json FROM ${generations} WHERE counter_id = $1`, [
|
|
209
|
+
filter.counterId,
|
|
210
|
+
]);
|
|
211
|
+
const draft = {
|
|
212
|
+
contractVersion: managed_js_1.MANAGED_CONTRACT_VERSION,
|
|
213
|
+
scopeId: filter.scopeId,
|
|
214
|
+
counterId: filter.counterId,
|
|
215
|
+
generations: genRows.rows.map((row) => ({
|
|
216
|
+
generation: row.generation,
|
|
217
|
+
counterId: row.counter_id,
|
|
218
|
+
mapHash: row.map_hash,
|
|
219
|
+
map: typeof row.map_json === "string" ? JSON.parse(row.map_json) : row.map_json,
|
|
220
|
+
})),
|
|
221
|
+
commits: commitRows.rows.map((row) => outcomeFromRow(row, "committed")),
|
|
222
|
+
dictionary: dictRows.rows.map((row) => ({
|
|
223
|
+
kind: row.kind,
|
|
224
|
+
qualifier: row.qualifier,
|
|
225
|
+
sourceKey: row.source_key,
|
|
226
|
+
internalId: asNumber(row.internal_id),
|
|
227
|
+
})),
|
|
228
|
+
cells,
|
|
229
|
+
};
|
|
230
|
+
return { ...draft, fingerprint: (0, managed_js_1.fingerprintRecovery)(draft) };
|
|
231
|
+
}
|
|
232
|
+
async importRecoverySet(set, mode) {
|
|
233
|
+
const fingerprint = (0, managed_js_1.fingerprintRecovery)(set);
|
|
234
|
+
if (fingerprint !== set.fingerprint) {
|
|
235
|
+
throw new errors_js_1.StorageError("recovery fingerprint mismatch", {});
|
|
236
|
+
}
|
|
237
|
+
if (mode === "validate-only") {
|
|
238
|
+
return { mode, fingerprint, cellCount: set.cells.length, commitCount: set.commits.length };
|
|
239
|
+
}
|
|
240
|
+
const schema = this.schemaFor(set.scopeId);
|
|
241
|
+
await this.ensureSchema(schema);
|
|
242
|
+
const client = await this.opts.pool.connect();
|
|
243
|
+
try {
|
|
244
|
+
await client.query("BEGIN");
|
|
245
|
+
await client.query(`DELETE FROM ${qual(schema, this.opts.cellsTable)} WHERE scope_id = $1 AND counter_id = $2`, [set.scopeId, set.counterId]);
|
|
246
|
+
await client.query(`DELETE FROM ${qual(schema, "commits")} WHERE scope_id = $1 AND counter_id = $2`, [
|
|
247
|
+
set.scopeId,
|
|
248
|
+
set.counterId,
|
|
249
|
+
]);
|
|
250
|
+
await client.query(`DELETE FROM ${qual(schema, this.opts.dictTable)} WHERE scope_id = $1`, [set.scopeId]);
|
|
251
|
+
if (set.dictionary.length > 0) {
|
|
252
|
+
await this.insertDict(schema, set.dictionary.map((row) => ({
|
|
253
|
+
lookup: {
|
|
254
|
+
scopeId: set.scopeId,
|
|
255
|
+
kind: row.kind,
|
|
256
|
+
qualifier: row.qualifier,
|
|
257
|
+
sourceKey: row.sourceKey,
|
|
258
|
+
},
|
|
259
|
+
internalId: row.internalId,
|
|
260
|
+
})), client);
|
|
261
|
+
const maxId = set.dictionary.reduce((m, row) => Math.max(m, row.internalId), 0);
|
|
262
|
+
await client.query(`INSERT INTO ${qual(schema, this.opts.seqTable)} AS id_seq (scope_id, seq)
|
|
263
|
+
VALUES ($1, $2)
|
|
264
|
+
ON CONFLICT (scope_id) DO UPDATE SET seq = GREATEST(id_seq.seq, EXCLUDED.seq)`, [set.scopeId, maxId]);
|
|
265
|
+
}
|
|
266
|
+
for (const row of set.generations) {
|
|
267
|
+
await client.query(`INSERT INTO ${qual(schema, "map_generations")} (generation, counter_id, map_hash, map_json)
|
|
268
|
+
VALUES ($1, $2, $3, $4::jsonb)
|
|
269
|
+
ON CONFLICT (generation) DO UPDATE SET
|
|
270
|
+
counter_id = EXCLUDED.counter_id,
|
|
271
|
+
map_hash = EXCLUDED.map_hash,
|
|
272
|
+
map_json = EXCLUDED.map_json`, [row.generation, row.counterId, row.mapHash, JSON.stringify(row.map)]);
|
|
273
|
+
}
|
|
274
|
+
for (const row of set.commits) {
|
|
275
|
+
await client.query(`INSERT INTO ${qual(schema, "commits")} (
|
|
276
|
+
scope_id, counter_id, generation, event_id, batch_id, projection_hash, effects
|
|
277
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)`, [
|
|
278
|
+
row.scopeId,
|
|
279
|
+
row.counterId,
|
|
280
|
+
row.generation,
|
|
281
|
+
row.eventId,
|
|
282
|
+
row.batchId,
|
|
283
|
+
row.projectionHash,
|
|
284
|
+
JSON.stringify(row.effects),
|
|
285
|
+
]);
|
|
286
|
+
}
|
|
287
|
+
for (const batch of chunk(set.cells, this.opts.batchSize)) {
|
|
288
|
+
if (batch.length === 0)
|
|
289
|
+
continue;
|
|
290
|
+
await client.query(replaceSql(schema, this.opts.cellsTable), replaceParams(batch));
|
|
291
|
+
}
|
|
292
|
+
await client.query("COMMIT");
|
|
293
|
+
}
|
|
294
|
+
catch (err) {
|
|
295
|
+
await rollback(client);
|
|
296
|
+
throw asStorageError("Postgres importRecoverySet failed", err);
|
|
297
|
+
}
|
|
298
|
+
finally {
|
|
299
|
+
client.release();
|
|
300
|
+
}
|
|
301
|
+
return { mode, fingerprint, cellCount: set.cells.length, commitCount: set.commits.length };
|
|
302
|
+
}
|
|
303
|
+
async readCells(spec, opts) {
|
|
304
|
+
throwIfAborted(opts?.signal, "readCells aborted");
|
|
305
|
+
const schema = this.schemaFor(spec.scope);
|
|
306
|
+
await this.ensureSchema(schema);
|
|
307
|
+
const filter = cellFilter(spec);
|
|
308
|
+
const limit = spec.limit && !spec.groupBy?.length ? spec.limit : undefined;
|
|
309
|
+
const params = filter.params;
|
|
310
|
+
let sql = `SELECT * FROM ${qual(schema, this.opts.cellsTable)} WHERE ${filter.where}`;
|
|
311
|
+
if (limit !== undefined) {
|
|
312
|
+
params.push(limit);
|
|
313
|
+
sql += ` LIMIT $${params.length}`;
|
|
314
|
+
}
|
|
315
|
+
try {
|
|
316
|
+
const res = await this.opts.pool.query(sql, params);
|
|
317
|
+
return res.rows.map(rowToCell);
|
|
318
|
+
}
|
|
319
|
+
catch (err) {
|
|
320
|
+
throw asStorageError("Postgres readCells failed", err);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
async importSnapshotCells(cells, opts) {
|
|
324
|
+
if (cells.length === 0)
|
|
325
|
+
return;
|
|
326
|
+
throwIfAborted(opts?.signal, "importSnapshotCells aborted");
|
|
327
|
+
const collapsed = collapseSnapshots(cells).sort((a, b) => (0, hash_js_1.cellKey)(a).localeCompare((0, hash_js_1.cellKey)(b)));
|
|
328
|
+
const bySchema = new Map();
|
|
329
|
+
for (const cell of collapsed) {
|
|
330
|
+
const schema = this.schemaFor(cell.scopeId);
|
|
331
|
+
const arr = bySchema.get(schema) ?? [];
|
|
332
|
+
arr.push(cell);
|
|
333
|
+
bySchema.set(schema, arr);
|
|
334
|
+
}
|
|
335
|
+
for (const schema of bySchema.keys()) {
|
|
336
|
+
await this.ensureSchema(schema);
|
|
337
|
+
}
|
|
338
|
+
const client = await this.opts.pool.connect();
|
|
339
|
+
try {
|
|
340
|
+
await client.query("BEGIN");
|
|
341
|
+
for (const [schema, group] of bySchema) {
|
|
342
|
+
for (const batch of chunk(group, this.opts.batchSize)) {
|
|
343
|
+
throwIfAborted(opts?.signal, "importSnapshotCells aborted");
|
|
344
|
+
await client.query(replaceSql(schema, this.opts.cellsTable), replaceParams(batch));
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
await client.query("COMMIT");
|
|
348
|
+
}
|
|
349
|
+
catch (err) {
|
|
350
|
+
await rollback(client);
|
|
351
|
+
throw asStorageError("Postgres importSnapshotCells failed", err);
|
|
352
|
+
}
|
|
353
|
+
finally {
|
|
354
|
+
client.release();
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
async resolveIds(lookups, opts) {
|
|
358
|
+
const out = new Map();
|
|
359
|
+
if (lookups.length === 0)
|
|
360
|
+
return out;
|
|
361
|
+
throwIfAborted(opts?.signal, "resolveIds aborted");
|
|
362
|
+
const bySchema = groupLookups(lookups, (l) => this.schemaFor(l.scopeId));
|
|
363
|
+
for (const [schema, group] of bySchema) {
|
|
364
|
+
await this.ensureSchema(schema);
|
|
365
|
+
const have = await this.readDict(schema, group);
|
|
366
|
+
const missing = group.filter((l) => !have.has(dictKey(l)));
|
|
367
|
+
if (missing.length > 0) {
|
|
368
|
+
const byScope = groupLookups(missing, (l) => l.scopeId);
|
|
369
|
+
for (const [scope, scopeMissing] of byScope) {
|
|
370
|
+
throwIfAborted(opts?.signal, "resolveIds aborted");
|
|
371
|
+
const startId = await this.allocateIds(schema, scope, scopeMissing.length);
|
|
372
|
+
await this.insertDict(schema, scopeMissing.map((l, i) => ({ lookup: l, internalId: startId + i })));
|
|
373
|
+
}
|
|
374
|
+
const reread = await this.readDict(schema, missing);
|
|
375
|
+
for (const [key, id] of reread)
|
|
376
|
+
have.set(key, id);
|
|
377
|
+
for (const l of missing) {
|
|
378
|
+
if (!have.has(dictKey(l))) {
|
|
379
|
+
throw new errors_js_1.StorageError("Failed to allocate dict id", { lookup: l });
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
for (const l of group) {
|
|
384
|
+
const id = have.get(dictKey(l));
|
|
385
|
+
if (id !== undefined)
|
|
386
|
+
out.set(dictKey(l), id);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
return out;
|
|
390
|
+
}
|
|
391
|
+
async lookupIds(lookups, opts) {
|
|
392
|
+
const out = new Map();
|
|
393
|
+
if (lookups.length === 0)
|
|
394
|
+
return out;
|
|
395
|
+
throwIfAborted(opts?.signal, "lookupIds aborted");
|
|
396
|
+
const bySchema = groupLookups(lookups, (l) => this.schemaFor(l.scopeId));
|
|
397
|
+
for (const [schema, group] of bySchema) {
|
|
398
|
+
await this.ensureSchema(schema);
|
|
399
|
+
const have = await this.readDict(schema, group);
|
|
400
|
+
for (const l of group)
|
|
401
|
+
out.set(dictKey(l), have.get(dictKey(l)));
|
|
402
|
+
}
|
|
403
|
+
return out;
|
|
404
|
+
}
|
|
405
|
+
async reverseLookupIds(requests, opts) {
|
|
406
|
+
const out = new Map();
|
|
407
|
+
if (requests.length === 0)
|
|
408
|
+
return out;
|
|
409
|
+
throwIfAborted(opts?.signal, "reverseLookupIds aborted");
|
|
410
|
+
const bySchema = new Map();
|
|
411
|
+
for (const r of requests) {
|
|
412
|
+
const schema = this.schemaFor(r.scopeId);
|
|
413
|
+
const arr = bySchema.get(schema) ?? [];
|
|
414
|
+
arr.push(r);
|
|
415
|
+
bySchema.set(schema, arr);
|
|
416
|
+
}
|
|
417
|
+
for (const [schema, group] of bySchema) {
|
|
418
|
+
await this.ensureSchema(schema);
|
|
419
|
+
const dict = qual(schema, this.opts.dictTable);
|
|
420
|
+
try {
|
|
421
|
+
const res = await this.opts.pool.query(`SELECT scope_id, kind, qualifier, source_key, internal_id
|
|
422
|
+
FROM ${dict}
|
|
423
|
+
WHERE (scope_id, kind, qualifier, internal_id) IN (
|
|
424
|
+
SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::bigint[])
|
|
425
|
+
)`, [
|
|
426
|
+
group.map((r) => r.scopeId),
|
|
427
|
+
group.map((r) => r.kind),
|
|
428
|
+
group.map((r) => r.qualifier ?? ""),
|
|
429
|
+
group.map((r) => r.internalId),
|
|
430
|
+
]);
|
|
431
|
+
for (const row of res.rows) {
|
|
432
|
+
out.set((0, index_js_1.reverseDictResultKey)({
|
|
433
|
+
scopeId: row.scope_id,
|
|
434
|
+
kind: row.kind,
|
|
435
|
+
qualifier: row.qualifier,
|
|
436
|
+
internalId: asNumber(row.internal_id),
|
|
437
|
+
}), row.source_key);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
catch (err) {
|
|
441
|
+
throw asStorageError("Postgres reverseLookupIds failed", err);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return out;
|
|
445
|
+
}
|
|
446
|
+
async *streamCells(filter, opts) {
|
|
447
|
+
throwIfAborted(opts?.signal, "streamCells aborted");
|
|
448
|
+
const schema = this.schemaFor(filter.scope);
|
|
449
|
+
await this.ensureSchema(schema);
|
|
450
|
+
const spec = {
|
|
451
|
+
scope: filter.scope,
|
|
452
|
+
counterId: filter.counterId,
|
|
453
|
+
timeframe: filter.timeframe,
|
|
454
|
+
};
|
|
455
|
+
const base = cellFilter(spec);
|
|
456
|
+
const table = qual(schema, this.opts.cellsTable);
|
|
457
|
+
const order = "timeframe_key, subject_type, subject_id, metric_id, dimensions_hash";
|
|
458
|
+
let cursor = null;
|
|
459
|
+
for (;;) {
|
|
460
|
+
throwIfAborted(opts?.signal, "streamCells aborted");
|
|
461
|
+
const params = [...base.params];
|
|
462
|
+
let where = base.where;
|
|
463
|
+
if (cursor) {
|
|
464
|
+
params.push(cursor.timeframeKey, cursor.subjectType, cursor.subjectId, cursor.metricId, cursor.dimensionsHash);
|
|
465
|
+
const n = base.params.length;
|
|
466
|
+
where += ` AND (${order}) > ($${n + 1}, $${n + 2}, $${n + 3}::float8, $${n + 4}::float8, $${n + 5})`;
|
|
467
|
+
}
|
|
468
|
+
params.push(500);
|
|
469
|
+
const sql = `SELECT * FROM ${table} WHERE ${where} ORDER BY ${order} LIMIT $${params.length}`;
|
|
470
|
+
let res;
|
|
471
|
+
try {
|
|
472
|
+
res = await this.opts.pool.query(sql, params);
|
|
473
|
+
}
|
|
474
|
+
catch (err) {
|
|
475
|
+
throw asStorageError("Postgres streamCells failed", err);
|
|
476
|
+
}
|
|
477
|
+
if (res.rows.length === 0)
|
|
478
|
+
return;
|
|
479
|
+
for (const row of res.rows)
|
|
480
|
+
yield rowToCell(row);
|
|
481
|
+
const last = res.rows[res.rows.length - 1];
|
|
482
|
+
cursor = {
|
|
483
|
+
timeframeKey: last.timeframe_key,
|
|
484
|
+
subjectType: last.subject_type,
|
|
485
|
+
subjectId: asNumber(last.subject_id),
|
|
486
|
+
metricId: asNumber(last.metric_id),
|
|
487
|
+
dimensionsHash: last.dimensions_hash,
|
|
488
|
+
};
|
|
489
|
+
if (res.rows.length < 500)
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
async appendEvidence(records, opts) {
|
|
494
|
+
if (records.length === 0)
|
|
495
|
+
return;
|
|
496
|
+
throwIfAborted(opts?.signal, "appendEvidence aborted");
|
|
497
|
+
const bySchema = new Map();
|
|
498
|
+
for (const record of records) {
|
|
499
|
+
const schema = this.schemaFor(record.identity.scopeId);
|
|
500
|
+
const arr = bySchema.get(schema) ?? [];
|
|
501
|
+
arr.push(record);
|
|
502
|
+
bySchema.set(schema, arr);
|
|
503
|
+
}
|
|
504
|
+
for (const schema of bySchema.keys()) {
|
|
505
|
+
await this.ensureSchema(schema);
|
|
506
|
+
}
|
|
507
|
+
const client = await this.opts.pool.connect();
|
|
508
|
+
try {
|
|
509
|
+
await client.query("BEGIN");
|
|
510
|
+
for (const [schema, group] of bySchema) {
|
|
511
|
+
throwIfAborted(opts?.signal, "appendEvidence aborted");
|
|
512
|
+
const table = qual(schema, this.opts.evidenceTable);
|
|
513
|
+
await client.query(`INSERT INTO ${table} (
|
|
514
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
515
|
+
subject_type, subject_id, metric_id, dimensions_hash,
|
|
516
|
+
dimensions, payload, source_event_id, captured_at, max_samples_at_capture
|
|
517
|
+
)
|
|
518
|
+
SELECT
|
|
519
|
+
d.scope_id, d.counter_id, d.timeframe_type, d.timeframe_key,
|
|
520
|
+
d.subject_type, d.subject_id, d.metric_id, d.dimensions_hash,
|
|
521
|
+
d.dimensions::jsonb, d.payload::jsonb, NULLIF(d.source_event_id, ''),
|
|
522
|
+
d.captured_at::timestamptz, d.max_samples
|
|
523
|
+
FROM unnest(
|
|
524
|
+
$1::text[], $2::text[], $3::text[], $4::text[],
|
|
525
|
+
$5::text[], $6::float8[], $7::float8[], $8::text[],
|
|
526
|
+
$9::text[], $10::text[], $11::text[], $12::text[], $13::int[]
|
|
527
|
+
) AS d(
|
|
528
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
529
|
+
subject_type, subject_id, metric_id, dimensions_hash,
|
|
530
|
+
dimensions, payload, source_event_id, captured_at, max_samples
|
|
531
|
+
)`, [
|
|
532
|
+
group.map((r) => r.identity.scopeId),
|
|
533
|
+
group.map((r) => r.identity.counterId),
|
|
534
|
+
group.map((r) => r.identity.timeframeType),
|
|
535
|
+
group.map((r) => r.identity.timeframeKey),
|
|
536
|
+
group.map((r) => r.identity.subjectType),
|
|
537
|
+
group.map((r) => r.identity.subjectId),
|
|
538
|
+
group.map((r) => r.identity.metricId),
|
|
539
|
+
group.map((r) => r.identity.dimensionsHash),
|
|
540
|
+
group.map((r) => JSON.stringify(r.identity.dimensions ?? {})),
|
|
541
|
+
group.map((r) => JSON.stringify(r.payload ?? {})),
|
|
542
|
+
group.map((r) => r.sourceEventId ?? ""),
|
|
543
|
+
group.map((r) => r.capturedAt.toISOString()),
|
|
544
|
+
group.map((r) => r.maxSamplesAtCapture),
|
|
545
|
+
]);
|
|
546
|
+
const unique = new Map();
|
|
547
|
+
for (const record of group) {
|
|
548
|
+
const key = (0, hash_js_1.cellKey)(record.identity);
|
|
549
|
+
if (!unique.has(key)) {
|
|
550
|
+
unique.set(key, { identity: record.identity, maxSamples: record.maxSamplesAtCapture });
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
for (const { identity, maxSamples } of unique.values()) {
|
|
554
|
+
const count = await client.query(`SELECT COUNT(*)::int AS n FROM ${table}
|
|
555
|
+
WHERE scope_id = $1 AND counter_id = $2 AND timeframe_type = $3 AND timeframe_key = $4
|
|
556
|
+
AND subject_type = $5 AND subject_id = $6 AND metric_id = $7 AND dimensions_hash = $8`, identityParams(identity));
|
|
557
|
+
const total = count.rows[0]?.n ?? 0;
|
|
558
|
+
if (total > maxSamples) {
|
|
559
|
+
await client.query(`DELETE FROM ${table}
|
|
560
|
+
WHERE id IN (
|
|
561
|
+
SELECT id FROM ${table}
|
|
562
|
+
WHERE scope_id = $1 AND counter_id = $2 AND timeframe_type = $3 AND timeframe_key = $4
|
|
563
|
+
AND subject_type = $5 AND subject_id = $6 AND metric_id = $7 AND dimensions_hash = $8
|
|
564
|
+
ORDER BY captured_at ASC, id ASC
|
|
565
|
+
LIMIT $9
|
|
566
|
+
)`, [...identityParams(identity), total - maxSamples]);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
await client.query("COMMIT");
|
|
571
|
+
}
|
|
572
|
+
catch (err) {
|
|
573
|
+
await rollback(client);
|
|
574
|
+
throw asStorageError("Postgres appendEvidence failed", err);
|
|
575
|
+
}
|
|
576
|
+
finally {
|
|
577
|
+
client.release();
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
async writeIncrements(db, deltas, signal) {
|
|
581
|
+
const collapsed = collapseDeltas(deltas).sort((a, b) => (0, hash_js_1.cellKey)(a.identity).localeCompare((0, hash_js_1.cellKey)(b.identity)));
|
|
582
|
+
const bySchema = new Map();
|
|
583
|
+
for (const delta of collapsed) {
|
|
584
|
+
const schema = this.schemaFor(delta.identity.scopeId);
|
|
585
|
+
const arr = bySchema.get(schema) ?? [];
|
|
586
|
+
arr.push(delta);
|
|
587
|
+
bySchema.set(schema, arr);
|
|
588
|
+
}
|
|
589
|
+
for (const [schema, group] of bySchema) {
|
|
590
|
+
for (const batch of chunk(group, this.opts.batchSize)) {
|
|
591
|
+
throwIfAborted(signal, "applyDeltas aborted");
|
|
592
|
+
await db.query(incrementSql(schema, this.opts.cellsTable), incrementParams(batch));
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
async resolveIdsWith(db, lookups) {
|
|
597
|
+
const out = new Map();
|
|
598
|
+
if (lookups.length === 0)
|
|
599
|
+
return out;
|
|
600
|
+
const bySchema = groupLookups(lookups, (l) => this.schemaFor(l.scopeId));
|
|
601
|
+
for (const [schema, group] of bySchema) {
|
|
602
|
+
const have = await this.readDict(schema, group, db);
|
|
603
|
+
const missing = group.filter((l) => !have.has(dictKey(l)));
|
|
604
|
+
if (missing.length > 0) {
|
|
605
|
+
const byScope = groupLookups(missing, (l) => l.scopeId);
|
|
606
|
+
for (const [scope, scopeMissing] of byScope) {
|
|
607
|
+
const startId = await this.allocateIds(schema, scope, scopeMissing.length, db);
|
|
608
|
+
await this.insertDict(schema, scopeMissing.map((l, i) => ({ lookup: l, internalId: startId + i })), db);
|
|
609
|
+
}
|
|
610
|
+
const reread = await this.readDict(schema, missing, db);
|
|
611
|
+
for (const [key, id] of reread)
|
|
612
|
+
have.set(key, id);
|
|
613
|
+
}
|
|
614
|
+
for (const l of group) {
|
|
615
|
+
const id = have.get(dictKey(l));
|
|
616
|
+
if (id !== undefined)
|
|
617
|
+
out.set(dictKey(l), id);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
return out;
|
|
621
|
+
}
|
|
622
|
+
schemaFor(scope) {
|
|
623
|
+
const name = this.opts.schemaResolver?.({ scope }) ?? this.opts.schema;
|
|
624
|
+
return assertIdent(name, "schema");
|
|
625
|
+
}
|
|
626
|
+
ensureSchema(schema) {
|
|
627
|
+
if (this.opts.skipIndexInit)
|
|
628
|
+
return Promise.resolve();
|
|
629
|
+
const existing = this.ready.get(schema);
|
|
630
|
+
if (existing)
|
|
631
|
+
return existing;
|
|
632
|
+
const job = this.createSchema(schema).catch((err) => {
|
|
633
|
+
this.ready.delete(schema);
|
|
634
|
+
throw err;
|
|
635
|
+
});
|
|
636
|
+
this.ready.set(schema, job);
|
|
637
|
+
return job;
|
|
638
|
+
}
|
|
639
|
+
async createSchema(schema) {
|
|
640
|
+
const cells = qual(schema, this.opts.cellsTable);
|
|
641
|
+
const dict = qual(schema, this.opts.dictTable);
|
|
642
|
+
const seq = qual(schema, this.opts.seqTable);
|
|
643
|
+
const evidence = qual(schema, this.opts.evidenceTable);
|
|
644
|
+
const s = quoteIdent(schema);
|
|
645
|
+
const partitionOpts = "fillfactor = 85, autovacuum_vacuum_scale_factor = 0.05, autovacuum_vacuum_threshold = 1000, autovacuum_analyze_scale_factor = 0.02";
|
|
646
|
+
const sql = `
|
|
647
|
+
CREATE SCHEMA IF NOT EXISTS ${s};
|
|
648
|
+
${mergeMeasuresFunctionSql(schema)}
|
|
649
|
+
CREATE TABLE IF NOT EXISTS ${cells} (
|
|
650
|
+
scope_id text NOT NULL,
|
|
651
|
+
counter_id text NOT NULL,
|
|
652
|
+
timeframe_type text NOT NULL,
|
|
653
|
+
timeframe_key text NOT NULL,
|
|
654
|
+
subject_type text NOT NULL,
|
|
655
|
+
subject_id double precision NOT NULL,
|
|
656
|
+
metric_id double precision NOT NULL,
|
|
657
|
+
dimensions_hash text NOT NULL,
|
|
658
|
+
dimensions jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
659
|
+
count double precision NOT NULL,
|
|
660
|
+
measures jsonb,
|
|
661
|
+
schema_version integer NOT NULL DEFAULT ${exports.POSTGRES_CELL_SCHEMA_VERSION},
|
|
662
|
+
created_at timestamptz NOT NULL,
|
|
663
|
+
updated_at timestamptz NOT NULL,
|
|
664
|
+
CONSTRAINT ${quoteIdent(`uniq_${this.opts.cellsTable}_identity`)} UNIQUE (
|
|
665
|
+
counter_id, scope_id, timeframe_type, timeframe_key,
|
|
666
|
+
subject_type, subject_id, metric_id, dimensions_hash
|
|
667
|
+
)
|
|
668
|
+
) PARTITION BY LIST (timeframe_type);
|
|
669
|
+
CREATE TABLE IF NOT EXISTS ${qual(schema, `${this.opts.cellsTable}_minute`)}
|
|
670
|
+
PARTITION OF ${cells} FOR VALUES IN ('minute')
|
|
671
|
+
WITH (${partitionOpts});
|
|
672
|
+
CREATE TABLE IF NOT EXISTS ${qual(schema, `${this.opts.cellsTable}_hour`)}
|
|
673
|
+
PARTITION OF ${cells} FOR VALUES IN ('hour')
|
|
674
|
+
WITH (${partitionOpts});
|
|
675
|
+
CREATE TABLE IF NOT EXISTS ${qual(schema, `${this.opts.cellsTable}_default`)}
|
|
676
|
+
PARTITION OF ${cells} DEFAULT
|
|
677
|
+
WITH (${partitionOpts});
|
|
678
|
+
CREATE INDEX IF NOT EXISTS ${quoteIdent(`${this.opts.cellsTable}_by_subject`)}
|
|
679
|
+
ON ${cells} (counter_id, scope_id, subject_id, timeframe_type, timeframe_key);
|
|
680
|
+
CREATE INDEX IF NOT EXISTS ${quoteIdent(`${this.opts.cellsTable}_by_metric`)}
|
|
681
|
+
ON ${cells} (counter_id, scope_id, metric_id, timeframe_type, timeframe_key);
|
|
682
|
+
CREATE TABLE IF NOT EXISTS ${dict} (
|
|
683
|
+
scope_id text NOT NULL,
|
|
684
|
+
kind text NOT NULL,
|
|
685
|
+
qualifier text NOT NULL DEFAULT '',
|
|
686
|
+
source_key text NOT NULL,
|
|
687
|
+
internal_id bigint NOT NULL,
|
|
688
|
+
CONSTRAINT ${quoteIdent(`uniq_${this.opts.dictTable}_key`)} UNIQUE (scope_id, kind, qualifier, source_key)
|
|
689
|
+
);
|
|
690
|
+
CREATE INDEX IF NOT EXISTS ${quoteIdent(`${this.opts.dictTable}_reverse_lookup`)}
|
|
691
|
+
ON ${dict} (scope_id, kind, qualifier, internal_id);
|
|
692
|
+
CREATE TABLE IF NOT EXISTS ${seq} (
|
|
693
|
+
scope_id text PRIMARY KEY,
|
|
694
|
+
seq bigint NOT NULL
|
|
695
|
+
);
|
|
696
|
+
CREATE TABLE IF NOT EXISTS ${evidence} (
|
|
697
|
+
id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
|
698
|
+
scope_id text NOT NULL,
|
|
699
|
+
counter_id text NOT NULL,
|
|
700
|
+
timeframe_type text NOT NULL,
|
|
701
|
+
timeframe_key text NOT NULL,
|
|
702
|
+
subject_type text NOT NULL,
|
|
703
|
+
subject_id double precision NOT NULL,
|
|
704
|
+
metric_id double precision NOT NULL,
|
|
705
|
+
dimensions_hash text NOT NULL,
|
|
706
|
+
dimensions jsonb NOT NULL,
|
|
707
|
+
payload jsonb NOT NULL,
|
|
708
|
+
source_event_id text,
|
|
709
|
+
captured_at timestamptz NOT NULL,
|
|
710
|
+
max_samples_at_capture integer NOT NULL
|
|
711
|
+
);
|
|
712
|
+
CREATE INDEX IF NOT EXISTS ${quoteIdent(`${this.opts.evidenceTable}_cell_time`)}
|
|
713
|
+
ON ${evidence} (
|
|
714
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
715
|
+
subject_type, subject_id, metric_id, dimensions_hash, captured_at
|
|
716
|
+
);
|
|
717
|
+
CREATE TABLE IF NOT EXISTS ${qual(schema, "map_generations")} (
|
|
718
|
+
generation text PRIMARY KEY,
|
|
719
|
+
counter_id text NOT NULL,
|
|
720
|
+
map_hash text NOT NULL,
|
|
721
|
+
map_json jsonb NOT NULL,
|
|
722
|
+
created_at timestamptz NOT NULL DEFAULT now()
|
|
723
|
+
);
|
|
724
|
+
CREATE TABLE IF NOT EXISTS ${qual(schema, "commits")} (
|
|
725
|
+
scope_id text NOT NULL,
|
|
726
|
+
counter_id text NOT NULL,
|
|
727
|
+
generation text NOT NULL,
|
|
728
|
+
event_id text NOT NULL,
|
|
729
|
+
batch_id text,
|
|
730
|
+
projection_hash text NOT NULL,
|
|
731
|
+
effects jsonb NOT NULL,
|
|
732
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
733
|
+
PRIMARY KEY (scope_id, counter_id, generation, event_id)
|
|
734
|
+
);
|
|
735
|
+
`;
|
|
736
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
737
|
+
try {
|
|
738
|
+
await this.opts.pool.query(sql);
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
catch (err) {
|
|
742
|
+
const code = err.code;
|
|
743
|
+
if (attempt === 0 && code && ["42P06", "42P07", "23505", "42710"].includes(code)) {
|
|
744
|
+
continue;
|
|
745
|
+
}
|
|
746
|
+
throw asStorageError("Postgres schema init failed", err);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
async readDict(schema, lookups, db = this.opts.pool) {
|
|
751
|
+
const have = new Map();
|
|
752
|
+
if (lookups.length === 0)
|
|
753
|
+
return have;
|
|
754
|
+
const dict = qual(schema, this.opts.dictTable);
|
|
755
|
+
try {
|
|
756
|
+
const res = await db.query(`SELECT scope_id, kind, qualifier, source_key, internal_id
|
|
757
|
+
FROM ${dict}
|
|
758
|
+
WHERE (scope_id, kind, qualifier, source_key) IN (
|
|
759
|
+
SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[])
|
|
760
|
+
)`, [
|
|
761
|
+
lookups.map((l) => l.scopeId),
|
|
762
|
+
lookups.map((l) => l.kind),
|
|
763
|
+
lookups.map((l) => l.qualifier ?? ""),
|
|
764
|
+
lookups.map((l) => l.sourceKey),
|
|
765
|
+
]);
|
|
766
|
+
for (const row of res.rows) {
|
|
767
|
+
have.set((0, index_js_1.dictionaryLookupKey)({
|
|
768
|
+
scopeId: row.scope_id,
|
|
769
|
+
kind: row.kind,
|
|
770
|
+
qualifier: row.qualifier,
|
|
771
|
+
sourceKey: row.source_key,
|
|
772
|
+
}), asNumber(row.internal_id));
|
|
773
|
+
}
|
|
774
|
+
return have;
|
|
775
|
+
}
|
|
776
|
+
catch (err) {
|
|
777
|
+
throw asStorageError("Postgres dict read failed", err);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
async allocateIds(schema, scope, count, db = this.opts.pool) {
|
|
781
|
+
const seq = qual(schema, this.opts.seqTable);
|
|
782
|
+
try {
|
|
783
|
+
const res = await db.query(`INSERT INTO ${seq} AS id_seq (scope_id, seq) VALUES ($1, $2)
|
|
784
|
+
ON CONFLICT (scope_id) DO UPDATE SET seq = id_seq.seq + EXCLUDED.seq
|
|
785
|
+
RETURNING seq`, [scope, count]);
|
|
786
|
+
const value = res.rows[0]?.seq;
|
|
787
|
+
if (value === undefined)
|
|
788
|
+
throw new errors_js_1.StorageError("Failed to allocate sequence", { scope });
|
|
789
|
+
const after = asNumber(value);
|
|
790
|
+
return after - count + 1;
|
|
791
|
+
}
|
|
792
|
+
catch (err) {
|
|
793
|
+
throw asStorageError("Postgres allocateIds failed", err);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
async insertDict(schema, rows, db = this.opts.pool) {
|
|
797
|
+
if (rows.length === 0)
|
|
798
|
+
return;
|
|
799
|
+
const dict = qual(schema, this.opts.dictTable);
|
|
800
|
+
try {
|
|
801
|
+
await db.query(`INSERT INTO ${dict} (scope_id, kind, qualifier, source_key, internal_id)
|
|
802
|
+
SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[])
|
|
803
|
+
ON CONFLICT (scope_id, kind, qualifier, source_key) DO NOTHING`, [
|
|
804
|
+
rows.map((r) => r.lookup.scopeId),
|
|
805
|
+
rows.map((r) => r.lookup.kind),
|
|
806
|
+
rows.map((r) => r.lookup.qualifier ?? ""),
|
|
807
|
+
rows.map((r) => r.lookup.sourceKey),
|
|
808
|
+
rows.map((r) => r.internalId),
|
|
809
|
+
]);
|
|
810
|
+
}
|
|
811
|
+
catch (err) {
|
|
812
|
+
throw asStorageError("Postgres dict insert failed", err);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
exports.PostgresDurable = PostgresDurable;
|
|
817
|
+
function incrementSql(schema, cellsTable) {
|
|
818
|
+
const cells = qual(schema, cellsTable);
|
|
819
|
+
const fn = `${quoteIdent(schema)}.merge_measures`;
|
|
820
|
+
return `
|
|
821
|
+
INSERT INTO ${cells} AS c (
|
|
822
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
823
|
+
subject_type, subject_id, metric_id, dimensions_hash,
|
|
824
|
+
dimensions, count, measures, schema_version, created_at, updated_at
|
|
825
|
+
)
|
|
826
|
+
SELECT
|
|
827
|
+
d.scope_id, d.counter_id, d.timeframe_type, d.timeframe_key,
|
|
828
|
+
d.subject_type, d.subject_id, d.metric_id, d.dimensions_hash,
|
|
829
|
+
d.dimensions::jsonb, d.delta,
|
|
830
|
+
${fn}('{}'::jsonb, d.measures::jsonb, d.scales::jsonb, true),
|
|
831
|
+
${exports.POSTGRES_CELL_SCHEMA_VERSION}, now(), now()
|
|
832
|
+
FROM unnest(
|
|
833
|
+
$1::text[], $2::text[], $3::text[], $4::text[],
|
|
834
|
+
$5::text[], $6::float8[], $7::float8[], $8::text[],
|
|
835
|
+
$9::text[], $10::float8[], $11::text[], $12::text[]
|
|
836
|
+
) AS d(
|
|
837
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
838
|
+
subject_type, subject_id, metric_id, dimensions_hash,
|
|
839
|
+
dimensions, delta, measures, scales
|
|
840
|
+
)
|
|
841
|
+
ON CONFLICT (counter_id, scope_id, timeframe_type, timeframe_key, subject_type, subject_id, metric_id, dimensions_hash)
|
|
842
|
+
DO UPDATE SET
|
|
843
|
+
count = c.count + EXCLUDED.count,
|
|
844
|
+
measures = ${fn}(c.measures, EXCLUDED.measures, '{}'::jsonb, false),
|
|
845
|
+
updated_at = now()
|
|
846
|
+
`;
|
|
847
|
+
}
|
|
848
|
+
function incrementParams(batch) {
|
|
849
|
+
return [
|
|
850
|
+
batch.map((d) => d.identity.scopeId),
|
|
851
|
+
batch.map((d) => d.identity.counterId),
|
|
852
|
+
batch.map((d) => d.identity.timeframeType),
|
|
853
|
+
batch.map((d) => d.identity.timeframeKey),
|
|
854
|
+
batch.map((d) => d.identity.subjectType),
|
|
855
|
+
batch.map((d) => d.identity.subjectId),
|
|
856
|
+
batch.map((d) => d.identity.metricId),
|
|
857
|
+
batch.map((d) => d.identity.dimensionsHash),
|
|
858
|
+
batch.map((d) => JSON.stringify(d.identity.dimensions ?? {})),
|
|
859
|
+
batch.map((d) => d.delta),
|
|
860
|
+
batch.map((d) => measureDeltaJson(d.measures)),
|
|
861
|
+
batch.map((d) => JSON.stringify(d.measureScales ?? {})),
|
|
862
|
+
];
|
|
863
|
+
}
|
|
864
|
+
function replaceSql(schema, cellsTable) {
|
|
865
|
+
const cells = qual(schema, cellsTable);
|
|
866
|
+
return `
|
|
867
|
+
INSERT INTO ${cells} AS c (
|
|
868
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
869
|
+
subject_type, subject_id, metric_id, dimensions_hash,
|
|
870
|
+
dimensions, count, measures, schema_version, created_at, updated_at
|
|
871
|
+
)
|
|
872
|
+
SELECT
|
|
873
|
+
d.scope_id, d.counter_id, d.timeframe_type, d.timeframe_key,
|
|
874
|
+
d.subject_type, d.subject_id, d.metric_id, d.dimensions_hash,
|
|
875
|
+
d.dimensions::jsonb, d.count, d.measures::jsonb,
|
|
876
|
+
${exports.POSTGRES_CELL_SCHEMA_VERSION}, d.created_at::timestamptz, d.updated_at::timestamptz
|
|
877
|
+
FROM unnest(
|
|
878
|
+
$1::text[], $2::text[], $3::text[], $4::text[],
|
|
879
|
+
$5::text[], $6::float8[], $7::float8[], $8::text[],
|
|
880
|
+
$9::text[], $10::float8[], $11::text[], $12::text[], $13::text[]
|
|
881
|
+
) AS d(
|
|
882
|
+
scope_id, counter_id, timeframe_type, timeframe_key,
|
|
883
|
+
subject_type, subject_id, metric_id, dimensions_hash,
|
|
884
|
+
dimensions, count, measures, created_at, updated_at
|
|
885
|
+
)
|
|
886
|
+
ON CONFLICT (counter_id, scope_id, timeframe_type, timeframe_key, subject_type, subject_id, metric_id, dimensions_hash)
|
|
887
|
+
DO UPDATE SET
|
|
888
|
+
dimensions = EXCLUDED.dimensions,
|
|
889
|
+
count = EXCLUDED.count,
|
|
890
|
+
measures = EXCLUDED.measures,
|
|
891
|
+
schema_version = EXCLUDED.schema_version,
|
|
892
|
+
created_at = EXCLUDED.created_at,
|
|
893
|
+
updated_at = EXCLUDED.updated_at
|
|
894
|
+
`;
|
|
895
|
+
}
|
|
896
|
+
function replaceParams(batch) {
|
|
897
|
+
const now = new Date().toISOString();
|
|
898
|
+
return [
|
|
899
|
+
batch.map((c) => c.scopeId),
|
|
900
|
+
batch.map((c) => c.counterId),
|
|
901
|
+
batch.map((c) => c.timeframeType),
|
|
902
|
+
batch.map((c) => c.timeframeKey),
|
|
903
|
+
batch.map((c) => c.subjectType),
|
|
904
|
+
batch.map((c) => c.subjectId),
|
|
905
|
+
batch.map((c) => c.metricId),
|
|
906
|
+
batch.map((c) => c.dimensionsHash),
|
|
907
|
+
batch.map((c) => JSON.stringify((0, snapshot_encoding_js_1.presentationToStorageCell)(c).dimensions ?? {})),
|
|
908
|
+
batch.map((c) => (0, snapshot_encoding_js_1.presentationToStorageCell)(c).count),
|
|
909
|
+
batch.map((c) => storageMeasuresJson(c)),
|
|
910
|
+
batch.map((c) => (c.createdAt ?? new Date()).toISOString()),
|
|
911
|
+
batch.map(() => now),
|
|
912
|
+
];
|
|
913
|
+
}
|
|
914
|
+
function cellFilter(spec) {
|
|
915
|
+
const buckets = (0, timeframe_js_1.bucketRange)(spec.timeframe.type, spec.timeframe.from, spec.timeframe.to);
|
|
916
|
+
const params = [spec.counterId, spec.scope, spec.timeframe.type, buckets];
|
|
917
|
+
const where = [
|
|
918
|
+
"counter_id = $1",
|
|
919
|
+
"scope_id = $2",
|
|
920
|
+
"timeframe_type = $3",
|
|
921
|
+
"timeframe_key = ANY($4::text[])",
|
|
922
|
+
];
|
|
923
|
+
if (spec.subject?.type) {
|
|
924
|
+
params.push(spec.subject.type);
|
|
925
|
+
where.push(`subject_type = $${params.length}`);
|
|
926
|
+
}
|
|
927
|
+
if (spec.subject?.ids !== undefined) {
|
|
928
|
+
params.push(spec.subject.ids.map((id) => Number(id)));
|
|
929
|
+
where.push(`subject_id = ANY($${params.length}::float8[])`);
|
|
930
|
+
}
|
|
931
|
+
if (spec.metrics !== undefined) {
|
|
932
|
+
if (spec.metrics.length === 0) {
|
|
933
|
+
where.push("FALSE");
|
|
934
|
+
}
|
|
935
|
+
else {
|
|
936
|
+
const ids = spec.metrics.map((id) => Number(id)).filter((n) => Number.isFinite(n));
|
|
937
|
+
if (ids.length > 0) {
|
|
938
|
+
params.push(ids);
|
|
939
|
+
where.push(`metric_id = ANY($${params.length}::float8[])`);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
if (spec.dimensions) {
|
|
944
|
+
for (const [name, allowed] of Object.entries(spec.dimensions)) {
|
|
945
|
+
params.push(name);
|
|
946
|
+
const nameIdx = params.length;
|
|
947
|
+
params.push(JSON.stringify(allowed));
|
|
948
|
+
const valuesIdx = params.length;
|
|
949
|
+
where.push(`(
|
|
950
|
+
dimensions -> $${nameIdx} IN (SELECT jsonb_array_elements($${valuesIdx}::jsonb))
|
|
951
|
+
OR (
|
|
952
|
+
$${valuesIdx}::jsonb @> '[null]'::jsonb
|
|
953
|
+
AND NOT (dimensions ? $${nameIdx})
|
|
954
|
+
)
|
|
955
|
+
)`);
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
return { where: where.join(" AND "), params };
|
|
959
|
+
}
|
|
960
|
+
function identityParams(id) {
|
|
961
|
+
return [
|
|
962
|
+
id.scopeId,
|
|
963
|
+
id.counterId,
|
|
964
|
+
id.timeframeType,
|
|
965
|
+
id.timeframeKey,
|
|
966
|
+
id.subjectType,
|
|
967
|
+
id.subjectId,
|
|
968
|
+
id.metricId,
|
|
969
|
+
id.dimensionsHash,
|
|
970
|
+
];
|
|
971
|
+
}
|
|
972
|
+
function mergeMeasuresFunctionSql(schema) {
|
|
973
|
+
return `
|
|
974
|
+
CREATE OR REPLACE FUNCTION ${quoteIdent(schema)}.merge_measures(
|
|
975
|
+
existing jsonb,
|
|
976
|
+
incoming jsonb,
|
|
977
|
+
scales jsonb,
|
|
978
|
+
apply_scale boolean
|
|
979
|
+
) RETURNS jsonb
|
|
980
|
+
LANGUAGE plpgsql
|
|
981
|
+
IMMUTABLE
|
|
982
|
+
SET search_path = pg_temp
|
|
983
|
+
AS $fn$
|
|
984
|
+
DECLARE
|
|
985
|
+
result jsonb := '{}'::jsonb;
|
|
986
|
+
k text;
|
|
987
|
+
e jsonb;
|
|
988
|
+
i jsonb;
|
|
989
|
+
sum_v numeric;
|
|
990
|
+
count_v numeric;
|
|
991
|
+
min_v numeric;
|
|
992
|
+
max_v numeric;
|
|
993
|
+
scale_v numeric;
|
|
994
|
+
obj jsonb;
|
|
995
|
+
BEGIN
|
|
996
|
+
existing := COALESCE(existing, '{}'::jsonb);
|
|
997
|
+
incoming := COALESCE(incoming, '{}'::jsonb);
|
|
998
|
+
scales := COALESCE(scales, '{}'::jsonb);
|
|
999
|
+
|
|
1000
|
+
FOR k IN
|
|
1001
|
+
SELECT DISTINCT key FROM (
|
|
1002
|
+
SELECT jsonb_object_keys(existing) AS key
|
|
1003
|
+
UNION
|
|
1004
|
+
SELECT jsonb_object_keys(incoming)
|
|
1005
|
+
UNION
|
|
1006
|
+
SELECT jsonb_object_keys(scales)
|
|
1007
|
+
) AS keys
|
|
1008
|
+
LOOP
|
|
1009
|
+
e := existing -> k;
|
|
1010
|
+
i := incoming -> k;
|
|
1011
|
+
|
|
1012
|
+
IF i IS NULL AND (NOT apply_scale OR scales -> k IS NULL) THEN
|
|
1013
|
+
IF e IS NOT NULL THEN
|
|
1014
|
+
result := result || jsonb_build_object(k, e);
|
|
1015
|
+
END IF;
|
|
1016
|
+
CONTINUE;
|
|
1017
|
+
END IF;
|
|
1018
|
+
|
|
1019
|
+
IF (i IS NULL OR NOT (i ? 'sum')) AND (e IS NULL OR NOT (e ? 'sum')) THEN
|
|
1020
|
+
sum_v := NULL;
|
|
1021
|
+
ELSIF i IS NULL OR NOT (i ? 'sum') THEN
|
|
1022
|
+
sum_v := (e ->> 'sum')::numeric;
|
|
1023
|
+
ELSIF e IS NULL OR NOT (e ? 'sum') THEN
|
|
1024
|
+
sum_v := (i ->> 'sum')::numeric;
|
|
1025
|
+
ELSE
|
|
1026
|
+
sum_v := (e ->> 'sum')::numeric + (i ->> 'sum')::numeric;
|
|
1027
|
+
END IF;
|
|
1028
|
+
|
|
1029
|
+
IF (i IS NULL OR NOT (i ? 'count')) AND (e IS NULL OR NOT (e ? 'count')) THEN
|
|
1030
|
+
count_v := NULL;
|
|
1031
|
+
ELSIF i IS NULL OR NOT (i ? 'count') THEN
|
|
1032
|
+
count_v := (e ->> 'count')::numeric;
|
|
1033
|
+
ELSIF e IS NULL OR NOT (e ? 'count') THEN
|
|
1034
|
+
count_v := (i ->> 'count')::numeric;
|
|
1035
|
+
ELSE
|
|
1036
|
+
count_v := (e ->> 'count')::numeric + (i ->> 'count')::numeric;
|
|
1037
|
+
END IF;
|
|
1038
|
+
|
|
1039
|
+
IF i IS NULL OR NOT (i ? 'min') THEN
|
|
1040
|
+
IF e IS NULL OR NOT (e ? 'min') THEN min_v := NULL;
|
|
1041
|
+
ELSE min_v := (e ->> 'min')::numeric;
|
|
1042
|
+
END IF;
|
|
1043
|
+
ELSIF e IS NULL OR NOT (e ? 'min') THEN
|
|
1044
|
+
min_v := (i ->> 'min')::numeric;
|
|
1045
|
+
ELSE
|
|
1046
|
+
min_v := LEAST((e ->> 'min')::numeric, (i ->> 'min')::numeric);
|
|
1047
|
+
END IF;
|
|
1048
|
+
|
|
1049
|
+
IF i IS NULL OR NOT (i ? 'max') THEN
|
|
1050
|
+
IF e IS NULL OR NOT (e ? 'max') THEN max_v := NULL;
|
|
1051
|
+
ELSE max_v := (e ->> 'max')::numeric;
|
|
1052
|
+
END IF;
|
|
1053
|
+
ELSIF e IS NULL OR NOT (e ? 'max') THEN
|
|
1054
|
+
max_v := (i ->> 'max')::numeric;
|
|
1055
|
+
ELSE
|
|
1056
|
+
max_v := GREATEST((e ->> 'max')::numeric, (i ->> 'max')::numeric);
|
|
1057
|
+
END IF;
|
|
1058
|
+
|
|
1059
|
+
IF apply_scale THEN
|
|
1060
|
+
IF e IS NOT NULL AND (e ? 'scale') THEN
|
|
1061
|
+
scale_v := (e ->> 'scale')::numeric;
|
|
1062
|
+
ELSIF scales ? k THEN
|
|
1063
|
+
scale_v := (scales ->> k)::numeric;
|
|
1064
|
+
ELSIF i IS NOT NULL AND (i ? 'scale') THEN
|
|
1065
|
+
scale_v := (i ->> 'scale')::numeric;
|
|
1066
|
+
ELSE
|
|
1067
|
+
scale_v := NULL;
|
|
1068
|
+
END IF;
|
|
1069
|
+
ELSE
|
|
1070
|
+
IF e IS NOT NULL AND (e ? 'scale') THEN
|
|
1071
|
+
scale_v := (e ->> 'scale')::numeric;
|
|
1072
|
+
ELSE
|
|
1073
|
+
scale_v := NULL;
|
|
1074
|
+
END IF;
|
|
1075
|
+
END IF;
|
|
1076
|
+
|
|
1077
|
+
obj := '{}'::jsonb;
|
|
1078
|
+
IF sum_v IS NOT NULL THEN obj := obj || jsonb_build_object('sum', sum_v); END IF;
|
|
1079
|
+
IF count_v IS NOT NULL THEN obj := obj || jsonb_build_object('count', count_v); END IF;
|
|
1080
|
+
IF min_v IS NOT NULL THEN obj := obj || jsonb_build_object('min', min_v); END IF;
|
|
1081
|
+
IF max_v IS NOT NULL THEN obj := obj || jsonb_build_object('max', max_v); END IF;
|
|
1082
|
+
IF scale_v IS NOT NULL THEN obj := obj || jsonb_build_object('scale', scale_v); END IF;
|
|
1083
|
+
IF obj <> '{}'::jsonb THEN
|
|
1084
|
+
result := result || jsonb_build_object(k, obj);
|
|
1085
|
+
END IF;
|
|
1086
|
+
END LOOP;
|
|
1087
|
+
|
|
1088
|
+
RETURN result;
|
|
1089
|
+
END;
|
|
1090
|
+
$fn$;
|
|
1091
|
+
`;
|
|
1092
|
+
}
|
|
1093
|
+
function collapseDeltas(deltas) {
|
|
1094
|
+
const map = new Map();
|
|
1095
|
+
for (const delta of deltas) {
|
|
1096
|
+
const key = (0, hash_js_1.cellKey)(delta.identity);
|
|
1097
|
+
const cur = map.get(key);
|
|
1098
|
+
if (!cur) {
|
|
1099
|
+
map.set(key, {
|
|
1100
|
+
identity: delta.identity,
|
|
1101
|
+
delta: delta.delta,
|
|
1102
|
+
measures: delta.measures ? mergeMeasureMaps(undefined, delta.measures) : undefined,
|
|
1103
|
+
measureScales: delta.measureScales ? { ...delta.measureScales } : undefined,
|
|
1104
|
+
});
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
cur.delta += delta.delta;
|
|
1108
|
+
if (delta.measures)
|
|
1109
|
+
cur.measures = mergeMeasureMaps(cur.measures, delta.measures);
|
|
1110
|
+
cur.measureScales = mergeScales(cur.measureScales, delta.measureScales);
|
|
1111
|
+
}
|
|
1112
|
+
return [...map.values()];
|
|
1113
|
+
}
|
|
1114
|
+
function mergeMeasureDelta(cur, inc) {
|
|
1115
|
+
const out = { ...cur };
|
|
1116
|
+
if (inc.sum !== undefined)
|
|
1117
|
+
out.sum = (out.sum ?? 0) + inc.sum;
|
|
1118
|
+
if (inc.count !== undefined)
|
|
1119
|
+
out.count = (out.count ?? 0) + inc.count;
|
|
1120
|
+
if (inc.min !== undefined)
|
|
1121
|
+
out.min = out.min !== undefined ? Math.min(out.min, inc.min) : inc.min;
|
|
1122
|
+
if (inc.max !== undefined)
|
|
1123
|
+
out.max = out.max !== undefined ? Math.max(out.max, inc.max) : inc.max;
|
|
1124
|
+
return out;
|
|
1125
|
+
}
|
|
1126
|
+
function mergeMeasureMaps(cur, inc) {
|
|
1127
|
+
const out = cur ? { ...cur } : {};
|
|
1128
|
+
for (const [name, delta] of Object.entries(inc)) {
|
|
1129
|
+
out[name] = mergeMeasureDelta(out[name], delta);
|
|
1130
|
+
}
|
|
1131
|
+
return out;
|
|
1132
|
+
}
|
|
1133
|
+
function mergeScales(cur, inc) {
|
|
1134
|
+
if (!inc || Object.keys(inc).length === 0)
|
|
1135
|
+
return cur;
|
|
1136
|
+
const out = { ...cur };
|
|
1137
|
+
for (const [key, value] of Object.entries(inc)) {
|
|
1138
|
+
if (out[key] === undefined)
|
|
1139
|
+
out[key] = value;
|
|
1140
|
+
}
|
|
1141
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
1142
|
+
}
|
|
1143
|
+
function collapseSnapshots(cells) {
|
|
1144
|
+
const map = new Map();
|
|
1145
|
+
for (const cell of cells)
|
|
1146
|
+
map.set((0, hash_js_1.cellKey)(cell), cell);
|
|
1147
|
+
return [...map.values()];
|
|
1148
|
+
}
|
|
1149
|
+
function measureDeltaJson(measures) {
|
|
1150
|
+
if (!measures)
|
|
1151
|
+
return "{}";
|
|
1152
|
+
const out = {};
|
|
1153
|
+
for (const [name, delta] of Object.entries(measures)) {
|
|
1154
|
+
const row = {};
|
|
1155
|
+
if (delta.sum !== undefined)
|
|
1156
|
+
row.sum = delta.sum;
|
|
1157
|
+
if (delta.count !== undefined)
|
|
1158
|
+
row.count = delta.count;
|
|
1159
|
+
if (delta.min !== undefined)
|
|
1160
|
+
row.min = delta.min;
|
|
1161
|
+
if (delta.max !== undefined)
|
|
1162
|
+
row.max = delta.max;
|
|
1163
|
+
if (Object.keys(row).length > 0)
|
|
1164
|
+
out[name] = row;
|
|
1165
|
+
}
|
|
1166
|
+
return JSON.stringify(out);
|
|
1167
|
+
}
|
|
1168
|
+
function storageMeasuresJson(cell) {
|
|
1169
|
+
const stored = (0, snapshot_encoding_js_1.presentationToStorageCell)(cell);
|
|
1170
|
+
if (!stored.measures || Object.keys(stored.measures).length === 0)
|
|
1171
|
+
return "null";
|
|
1172
|
+
const out = {};
|
|
1173
|
+
for (const [name, value] of Object.entries(stored.measures)) {
|
|
1174
|
+
const row = {};
|
|
1175
|
+
if (value.sum !== undefined)
|
|
1176
|
+
row.sum = value.sum;
|
|
1177
|
+
if (value.count !== undefined)
|
|
1178
|
+
row.count = value.count;
|
|
1179
|
+
if (value.min !== undefined)
|
|
1180
|
+
row.min = value.min;
|
|
1181
|
+
if (value.max !== undefined)
|
|
1182
|
+
row.max = value.max;
|
|
1183
|
+
if (value.scale !== undefined)
|
|
1184
|
+
row.scale = value.scale;
|
|
1185
|
+
out[name] = row;
|
|
1186
|
+
}
|
|
1187
|
+
return JSON.stringify(out);
|
|
1188
|
+
}
|
|
1189
|
+
function rowToCell(row) {
|
|
1190
|
+
const dimensions = typeof row.dimensions === "string" ? JSON.parse(row.dimensions) : row.dimensions;
|
|
1191
|
+
return {
|
|
1192
|
+
scopeId: row.scope_id,
|
|
1193
|
+
counterId: row.counter_id,
|
|
1194
|
+
timeframeType: row.timeframe_type,
|
|
1195
|
+
timeframeKey: row.timeframe_key,
|
|
1196
|
+
subjectType: row.subject_type,
|
|
1197
|
+
subjectId: asNumber(row.subject_id),
|
|
1198
|
+
metricId: asNumber(row.metric_id),
|
|
1199
|
+
dimensionsHash: row.dimensions_hash,
|
|
1200
|
+
dimensions: dimensions ?? {},
|
|
1201
|
+
count: asNumber(row.count),
|
|
1202
|
+
measures: unscaleMeasures(row.measures),
|
|
1203
|
+
createdAt: asDate(row.created_at),
|
|
1204
|
+
updatedAt: asDate(row.updated_at),
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
function unscaleMeasures(raw) {
|
|
1208
|
+
if (raw == null)
|
|
1209
|
+
return undefined;
|
|
1210
|
+
const parsed = typeof raw === "string" ? JSON.parse(raw) : raw;
|
|
1211
|
+
if (!parsed || Object.keys(parsed).length === 0)
|
|
1212
|
+
return undefined;
|
|
1213
|
+
const out = {};
|
|
1214
|
+
for (const [name, value] of Object.entries(parsed)) {
|
|
1215
|
+
const scale = asOptionalNumber(value.scale);
|
|
1216
|
+
const div = scale !== undefined && scale > 0 ? scale : 1;
|
|
1217
|
+
const sum = asOptionalNumber(value.sum);
|
|
1218
|
+
const min = asOptionalNumber(value.min);
|
|
1219
|
+
const max = asOptionalNumber(value.max);
|
|
1220
|
+
out[name] = {
|
|
1221
|
+
sum: sum !== undefined ? sum / div : undefined,
|
|
1222
|
+
count: asOptionalNumber(value.count),
|
|
1223
|
+
min: min !== undefined ? min / div : undefined,
|
|
1224
|
+
max: max !== undefined ? max / div : undefined,
|
|
1225
|
+
scale,
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1228
|
+
return out;
|
|
1229
|
+
}
|
|
1230
|
+
function dictKey(lookup) {
|
|
1231
|
+
return (0, index_js_1.dictionaryLookupKey)({ ...lookup, qualifier: lookup.qualifier ?? "" });
|
|
1232
|
+
}
|
|
1233
|
+
function groupLookups(lookups, keyOf) {
|
|
1234
|
+
const out = new Map();
|
|
1235
|
+
for (const lookup of lookups) {
|
|
1236
|
+
const key = keyOf(lookup);
|
|
1237
|
+
const arr = out.get(key) ?? [];
|
|
1238
|
+
arr.push(lookup);
|
|
1239
|
+
out.set(key, arr);
|
|
1240
|
+
}
|
|
1241
|
+
return out;
|
|
1242
|
+
}
|
|
1243
|
+
function chunk(items, size) {
|
|
1244
|
+
const out = [];
|
|
1245
|
+
for (let i = 0; i < items.length; i += size)
|
|
1246
|
+
out.push(items.slice(i, i + size));
|
|
1247
|
+
return out;
|
|
1248
|
+
}
|
|
1249
|
+
function assertIdent(name, label) {
|
|
1250
|
+
if (!IDENT.test(name)) {
|
|
1251
|
+
throw new errors_js_1.StorageError(`Invalid Postgres ${label}`, { name });
|
|
1252
|
+
}
|
|
1253
|
+
return name;
|
|
1254
|
+
}
|
|
1255
|
+
function quoteIdent(name) {
|
|
1256
|
+
return `"${name}"`;
|
|
1257
|
+
}
|
|
1258
|
+
function qual(schema, table) {
|
|
1259
|
+
return `${quoteIdent(schema)}.${quoteIdent(table)}`;
|
|
1260
|
+
}
|
|
1261
|
+
function asNumber(value) {
|
|
1262
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
1263
|
+
return n;
|
|
1264
|
+
}
|
|
1265
|
+
function asOptionalNumber(value) {
|
|
1266
|
+
if (value === undefined || value === null)
|
|
1267
|
+
return undefined;
|
|
1268
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
1269
|
+
return Number.isFinite(n) ? n : undefined;
|
|
1270
|
+
}
|
|
1271
|
+
function asDate(value) {
|
|
1272
|
+
if (value == null)
|
|
1273
|
+
return undefined;
|
|
1274
|
+
return value instanceof Date ? value : new Date(value);
|
|
1275
|
+
}
|
|
1276
|
+
function throwIfAborted(signal, message) {
|
|
1277
|
+
if (signal?.aborted)
|
|
1278
|
+
throw new errors_js_1.StorageError(message, {});
|
|
1279
|
+
}
|
|
1280
|
+
function asStorageError(message, err) {
|
|
1281
|
+
if (err instanceof errors_js_1.StorageError)
|
|
1282
|
+
return err;
|
|
1283
|
+
return new errors_js_1.StorageError(message, { cause: err instanceof Error ? err.message : String(err) });
|
|
1284
|
+
}
|
|
1285
|
+
function outcomeFromPrepared(event, batchId, status) {
|
|
1286
|
+
return {
|
|
1287
|
+
status,
|
|
1288
|
+
commitId: (0, managed_js_1.commitIdOf)(event),
|
|
1289
|
+
scopeId: event.scopeId,
|
|
1290
|
+
counterId: event.counterId,
|
|
1291
|
+
generation: event.generation,
|
|
1292
|
+
eventId: event.eventId,
|
|
1293
|
+
batchId,
|
|
1294
|
+
projectionHash: event.projectionHash,
|
|
1295
|
+
effects: event.effects,
|
|
1296
|
+
};
|
|
1297
|
+
}
|
|
1298
|
+
function outcomeFromRow(row, status) {
|
|
1299
|
+
const effects = typeof row.effects === "string" ? JSON.parse(row.effects) : row.effects;
|
|
1300
|
+
return {
|
|
1301
|
+
status,
|
|
1302
|
+
commitId: (0, managed_js_1.commitIdOf)({
|
|
1303
|
+
counterId: row.counter_id,
|
|
1304
|
+
generation: row.generation,
|
|
1305
|
+
scopeId: row.scope_id,
|
|
1306
|
+
eventId: row.event_id,
|
|
1307
|
+
}),
|
|
1308
|
+
scopeId: row.scope_id,
|
|
1309
|
+
counterId: row.counter_id,
|
|
1310
|
+
generation: row.generation,
|
|
1311
|
+
eventId: row.event_id,
|
|
1312
|
+
batchId: row.batch_id,
|
|
1313
|
+
projectionHash: row.projection_hash,
|
|
1314
|
+
effects,
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
async function rollback(client) {
|
|
1318
|
+
try {
|
|
1319
|
+
await client.query("ROLLBACK");
|
|
1320
|
+
}
|
|
1321
|
+
catch {
|
|
1322
|
+
// The connection is released by the caller either way.
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
//# sourceMappingURL=postgres.js.map
|