@noego/telemetry 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -0
- package/dist/browser/index.cjs +1346 -0
- package/dist/browser/index.cjs.map +1 -0
- package/dist/browser/index.d.cts +5 -0
- package/dist/browser/index.d.ts +5 -0
- package/dist/browser/index.js +1318 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/browser_telemetry_client-BzBPQm4m.d.cts +29 -0
- package/dist/browser_telemetry_client-Ch_VzwEa.d.ts +29 -0
- package/dist/cli/telemetry_bin.cjs +5531 -0
- package/dist/cli/telemetry_bin.cjs.map +1 -0
- package/dist/cli/telemetry_bin.d.cts +1 -0
- package/dist/cli/telemetry_bin.d.ts +1 -0
- package/dist/cli/telemetry_bin.js +5517 -0
- package/dist/cli/telemetry_bin.js.map +1 -0
- package/dist/cli/telemetry_cli.cjs +5523 -0
- package/dist/cli/telemetry_cli.cjs.map +1 -0
- package/dist/cli/telemetry_cli.d.cts +92 -0
- package/dist/cli/telemetry_cli.d.ts +92 -0
- package/dist/cli/telemetry_cli.js +5497 -0
- package/dist/cli/telemetry_cli.js.map +1 -0
- package/dist/client_defaults-B5tC-TgG.d.ts +8 -0
- package/dist/client_defaults-C9LnSX4j.d.cts +8 -0
- package/dist/core/index.cjs +2354 -0
- package/dist/core/index.cjs.map +1 -0
- package/dist/core/index.d.cts +168 -0
- package/dist/core/index.d.ts +168 -0
- package/dist/core/index.js +2318 -0
- package/dist/core/index.js.map +1 -0
- package/dist/electron/index.cjs +1379 -0
- package/dist/electron/index.cjs.map +1 -0
- package/dist/electron/index.d.cts +31 -0
- package/dist/electron/index.d.ts +31 -0
- package/dist/electron/index.js +1353 -0
- package/dist/electron/index.js.map +1 -0
- package/dist/envelope.types-rOLpWXRC.d.cts +42 -0
- package/dist/envelope.types-rOLpWXRC.d.ts +42 -0
- package/dist/in_memory_batch_transport-DKW9EZnU.d.cts +16 -0
- package/dist/in_memory_batch_transport-oOwNxXnA.d.ts +16 -0
- package/dist/index.cjs +3325 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +66 -0
- package/dist/index.d.ts +66 -0
- package/dist/index.js +3285 -0
- package/dist/index.js.map +1 -0
- package/dist/maintenance_coordinator.types-B-bZUNvw.d.cts +127 -0
- package/dist/maintenance_coordinator.types-CBA5y74b.d.ts +127 -0
- package/dist/metric_definition_contract-BR6a_GzF.d.cts +12 -0
- package/dist/metric_definition_contract-l4i4SaAp.d.ts +12 -0
- package/dist/node/index.cjs +7157 -0
- package/dist/node/index.cjs.map +1 -0
- package/dist/node/index.d.cts +74 -0
- package/dist/node/index.d.ts +74 -0
- package/dist/node/index.js +7122 -0
- package/dist/node/index.js.map +1 -0
- package/dist/node/producer/index.cjs +1322 -0
- package/dist/node/producer/index.cjs.map +1 -0
- package/dist/node/producer/index.d.cts +21 -0
- package/dist/node/producer/index.d.ts +21 -0
- package/dist/node/producer/index.js +1294 -0
- package/dist/node/producer/index.js.map +1 -0
- package/dist/query.types-Bntwh6KF.d.cts +186 -0
- package/dist/query.types-DySoqYop.d.ts +186 -0
- package/dist/result.types-171Huujb.d.cts +9 -0
- package/dist/result.types-171Huujb.d.ts +9 -0
- package/dist/sqlite/index.cjs +3409 -0
- package/dist/sqlite/index.cjs.map +1 -0
- package/dist/sqlite/index.d.cts +77 -0
- package/dist/sqlite/index.d.ts +77 -0
- package/dist/sqlite/index.js +3382 -0
- package/dist/sqlite/index.js.map +1 -0
- package/dist/storage.types-BTCGVxOM.d.ts +76 -0
- package/dist/storage.types-DjuUMinz.d.cts +76 -0
- package/dist/telemetry_client-BXw8WH_F.d.ts +141 -0
- package/dist/telemetry_client-CYaPYfOl.d.cts +141 -0
- package/dist/telemetry_configuration_parser-C7Et_gwG.d.ts +39 -0
- package/dist/telemetry_configuration_parser-R-niZQRZ.d.cts +39 -0
- package/dist/testing/index.cjs +215 -0
- package/dist/testing/index.cjs.map +1 -0
- package/dist/testing/index.d.cts +32 -0
- package/dist/testing/index.d.ts +32 -0
- package/dist/testing/index.js +187 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/trace-adapter/index.cjs +1034 -0
- package/dist/trace-adapter/index.cjs.map +1 -0
- package/dist/trace-adapter/index.d.cts +72 -0
- package/dist/trace-adapter/index.d.ts +72 -0
- package/dist/trace-adapter/index.js +1007 -0
- package/dist/trace-adapter/index.js.map +1 -0
- package/dist/transport.types-BAl_TRtU.d.cts +99 -0
- package/dist/transport.types-DDnZoKWi.d.ts +99 -0
- package/migrations/1783695629569_raw_events.down.sql +1 -0
- package/migrations/1783695629569_raw_events.up.sql +30 -0
- package/migrations/1783695630053_batch_acknowledgements.down.sql +1 -0
- package/migrations/1783695630053_batch_acknowledgements.up.sql +13 -0
- package/migrations/1783695630960_ingest_watermarks.down.sql +1 -0
- package/migrations/1783695630960_ingest_watermarks.up.sql +9 -0
- package/migrations/1783695631522_metric_definitions.down.sql +1 -0
- package/migrations/1783695631522_metric_definitions.up.sql +10 -0
- package/migrations/1783695631966_metric_samples.down.sql +1 -0
- package/migrations/1783695631966_metric_samples.up.sql +15 -0
- package/migrations/1783695632417_metric_rollups.down.sql +1 -0
- package/migrations/1783695632417_metric_rollups.up.sql +22 -0
- package/migrations/1783695632860_rollup_completions.down.sql +1 -0
- package/migrations/1783695632860_rollup_completions.up.sql +17 -0
- package/migrations/1783695633302_extraction_failures.down.sql +1 -0
- package/migrations/1783695633302_extraction_failures.up.sql +10 -0
- package/migrations/1783695633748_maintenance_runs.down.sql +1 -0
- package/migrations/1783695633748_maintenance_runs.up.sql +12 -0
- package/migrations/1783695634221_reader_leases.down.sql +1 -0
- package/migrations/1783695634221_reader_leases.up.sql +9 -0
- package/migrations/1783702703104_committed_drop_counters.down.sql +1 -0
- package/migrations/1783702703104_committed_drop_counters.up.sql +11 -0
- package/migrations/1783705542684_rollup_completion_state.down.sql +2 -0
- package/migrations/1783705542684_rollup_completion_state.up.sql +6 -0
- package/migrations/1784309799539_metric_samples_event_index.down.sql +1 -0
- package/migrations/1784309799539_metric_samples_event_index.up.sql +2 -0
- package/native/noego_telemetry_native.node +0 -0
- package/package.json +137 -0
- package/proper.json +8 -0
|
@@ -0,0 +1,3382 @@
|
|
|
1
|
+
/* @noego/telemetry */
|
|
2
|
+
|
|
3
|
+
// src/transports/sqlite_transport.ts
|
|
4
|
+
import { randomUUID } from "crypto";
|
|
5
|
+
import { resolve as resolve2 } from "path";
|
|
6
|
+
import { fileURLToPath } from "url";
|
|
7
|
+
import { MigrationRunnerFactory } from "@noego/proper";
|
|
8
|
+
import BetterSqlite3 from "better-sqlite3";
|
|
9
|
+
import { createSqliteDb } from "sqlstack/adapters";
|
|
10
|
+
|
|
11
|
+
// src/core/helper/canonical_json.ts
|
|
12
|
+
var UTF8_ENCODER = new TextEncoder();
|
|
13
|
+
function compareCanonicalKeys(left, right) {
|
|
14
|
+
if (left === right) return 0;
|
|
15
|
+
if (/^[\x00-\x7f]*$/.test(left) && /^[\x00-\x7f]*$/.test(right)) {
|
|
16
|
+
return left < right ? -1 : 1;
|
|
17
|
+
}
|
|
18
|
+
const leftBytes = UTF8_ENCODER.encode(left);
|
|
19
|
+
const rightBytes = UTF8_ENCODER.encode(right);
|
|
20
|
+
const length = Math.min(leftBytes.length, rightBytes.length);
|
|
21
|
+
for (let index = 0; index < length; index += 1) {
|
|
22
|
+
const difference = (leftBytes[index] ?? 0) - (rightBytes[index] ?? 0);
|
|
23
|
+
if (difference !== 0) return difference;
|
|
24
|
+
}
|
|
25
|
+
return leftBytes.length - rightBytes.length;
|
|
26
|
+
}
|
|
27
|
+
function canonicalJson(value) {
|
|
28
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") {
|
|
29
|
+
return JSON.stringify(value);
|
|
30
|
+
}
|
|
31
|
+
if (typeof value === "number") {
|
|
32
|
+
if (!Number.isFinite(value)) {
|
|
33
|
+
throw new Error("Canonical telemetry JSON cannot encode a non-finite number");
|
|
34
|
+
}
|
|
35
|
+
return JSON.stringify(value);
|
|
36
|
+
}
|
|
37
|
+
if (Array.isArray(value)) {
|
|
38
|
+
const items = value.map((item) => canonicalJson(item));
|
|
39
|
+
return `[${items.join(",")}]`;
|
|
40
|
+
}
|
|
41
|
+
if (typeof value !== "object") {
|
|
42
|
+
throw new Error(`Canonical telemetry JSON cannot encode ${typeof value}`);
|
|
43
|
+
}
|
|
44
|
+
const entries = Object.entries(value).sort(([left], [right]) => compareCanonicalKeys(left, right));
|
|
45
|
+
const properties = entries.map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`);
|
|
46
|
+
return `{${properties.join(",")}}`;
|
|
47
|
+
}
|
|
48
|
+
function telemetryUtf8Length(value) {
|
|
49
|
+
return new TextEncoder().encode(value).byteLength;
|
|
50
|
+
}
|
|
51
|
+
function telemetryChecksum(value) {
|
|
52
|
+
const bytes = new TextEncoder().encode(canonicalJson(value));
|
|
53
|
+
let hash = 0xcbf29ce484222325n;
|
|
54
|
+
const prime = 0x100000001b3n;
|
|
55
|
+
for (const byte of bytes) {
|
|
56
|
+
hash ^= BigInt(byte);
|
|
57
|
+
hash = BigInt.asUintN(64, hash * prime);
|
|
58
|
+
}
|
|
59
|
+
return hash.toString(16).padStart(16, "0");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// src/storage/helper/sqlite_table_names.ts
|
|
63
|
+
var SQLITE_LOGICAL_TABLE_NAMES = [
|
|
64
|
+
"rawEvents",
|
|
65
|
+
"batchAcknowledgements",
|
|
66
|
+
"ingestWatermarks",
|
|
67
|
+
"metricDefinitions",
|
|
68
|
+
"metricSamples",
|
|
69
|
+
"metricRollups",
|
|
70
|
+
"rollupCompletions",
|
|
71
|
+
"extractionFailures",
|
|
72
|
+
"maintenanceRuns",
|
|
73
|
+
"readerLeases",
|
|
74
|
+
"committedDropCounters",
|
|
75
|
+
"migrationHistory"
|
|
76
|
+
];
|
|
77
|
+
var SQLITE_DEFAULT_TABLE_NAMES = Object.freeze({
|
|
78
|
+
rawEvents: "raw_events",
|
|
79
|
+
batchAcknowledgements: "batch_acknowledgements",
|
|
80
|
+
ingestWatermarks: "ingest_watermarks",
|
|
81
|
+
metricDefinitions: "metric_definitions",
|
|
82
|
+
metricSamples: "metric_samples",
|
|
83
|
+
metricRollups: "metric_rollups",
|
|
84
|
+
rollupCompletions: "rollup_completions",
|
|
85
|
+
extractionFailures: "extraction_failures",
|
|
86
|
+
maintenanceRuns: "maintenance_runs",
|
|
87
|
+
readerLeases: "reader_leases",
|
|
88
|
+
committedDropCounters: "committed_drop_counters",
|
|
89
|
+
migrationHistory: "migration_history"
|
|
90
|
+
});
|
|
91
|
+
var SQLITE_SCHEMA_MANIFEST_TABLE = "telemetry_schema_manifest";
|
|
92
|
+
|
|
93
|
+
// src/storage/sqlite_ownership_lease.ts
|
|
94
|
+
import {
|
|
95
|
+
closeSync,
|
|
96
|
+
existsSync,
|
|
97
|
+
mkdirSync,
|
|
98
|
+
openSync,
|
|
99
|
+
readFileSync,
|
|
100
|
+
realpathSync,
|
|
101
|
+
renameSync,
|
|
102
|
+
rmSync,
|
|
103
|
+
writeFileSync
|
|
104
|
+
} from "fs";
|
|
105
|
+
import { dirname, resolve } from "path";
|
|
106
|
+
var SqliteOwnershipLease = class {
|
|
107
|
+
constructor(options) {
|
|
108
|
+
this.options = options;
|
|
109
|
+
mkdirSync(dirname(options.databasePath), { recursive: true });
|
|
110
|
+
const canonicalDirectory = realpathSync(dirname(resolve(options.databasePath)));
|
|
111
|
+
const databaseName = resolve(options.databasePath).slice(dirname(resolve(options.databasePath)).length + 1);
|
|
112
|
+
this.canonicalDatabasePath = resolve(canonicalDirectory, databaseName);
|
|
113
|
+
this.leasePath = `${this.canonicalDatabasePath}.telemetry-owner.json`;
|
|
114
|
+
}
|
|
115
|
+
options;
|
|
116
|
+
canonicalDatabasePath;
|
|
117
|
+
leasePath;
|
|
118
|
+
held = false;
|
|
119
|
+
acquire() {
|
|
120
|
+
const firstAttempt = this.writeExclusive(this.createRecord());
|
|
121
|
+
if (firstAttempt.ok) {
|
|
122
|
+
this.held = true;
|
|
123
|
+
return firstAttempt;
|
|
124
|
+
}
|
|
125
|
+
const recovered = this.recoverStaleLease();
|
|
126
|
+
if (!recovered.ok) {
|
|
127
|
+
return recovered;
|
|
128
|
+
}
|
|
129
|
+
const secondAttempt = this.writeExclusive(this.createRecord());
|
|
130
|
+
if (secondAttempt.ok) {
|
|
131
|
+
this.held = true;
|
|
132
|
+
}
|
|
133
|
+
return secondAttempt;
|
|
134
|
+
}
|
|
135
|
+
heartbeat() {
|
|
136
|
+
if (!this.held) {
|
|
137
|
+
return { ok: false, error: "Cannot heartbeat a telemetry lease that is not held" };
|
|
138
|
+
}
|
|
139
|
+
const current = this.readRecord();
|
|
140
|
+
if (!current.ok) {
|
|
141
|
+
return current;
|
|
142
|
+
}
|
|
143
|
+
if (current.value.processToken !== this.options.processToken) {
|
|
144
|
+
this.held = false;
|
|
145
|
+
return { ok: false, error: "Telemetry ownership lease was replaced by another owner" };
|
|
146
|
+
}
|
|
147
|
+
const now = this.options.now();
|
|
148
|
+
return this.writeReplace({
|
|
149
|
+
...current.value,
|
|
150
|
+
heartbeatAtMs: now,
|
|
151
|
+
expiresAtMs: now + this.options.expiryMs
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
release() {
|
|
155
|
+
if (!this.held || !existsSync(this.leasePath)) {
|
|
156
|
+
this.held = false;
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
const current = this.readRecord();
|
|
160
|
+
if (current.ok && current.value.processToken === this.options.processToken) {
|
|
161
|
+
try {
|
|
162
|
+
rmSync(this.leasePath, { force: true });
|
|
163
|
+
} catch {
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
this.held = false;
|
|
167
|
+
}
|
|
168
|
+
isHeld() {
|
|
169
|
+
return this.held;
|
|
170
|
+
}
|
|
171
|
+
path() {
|
|
172
|
+
return this.leasePath;
|
|
173
|
+
}
|
|
174
|
+
createRecord() {
|
|
175
|
+
const now = this.options.now();
|
|
176
|
+
return {
|
|
177
|
+
formatVersion: 1,
|
|
178
|
+
targetId: this.options.targetId,
|
|
179
|
+
databasePath: this.canonicalDatabasePath,
|
|
180
|
+
mode: this.options.mode,
|
|
181
|
+
pid: process.pid,
|
|
182
|
+
processToken: this.options.processToken,
|
|
183
|
+
acquiredAtMs: now,
|
|
184
|
+
heartbeatAtMs: now,
|
|
185
|
+
expiresAtMs: now + this.options.expiryMs
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
writeExclusive(record) {
|
|
189
|
+
let descriptor = null;
|
|
190
|
+
try {
|
|
191
|
+
descriptor = openSync(this.leasePath, "wx", 384);
|
|
192
|
+
writeFileSync(descriptor, JSON.stringify(record), { encoding: "utf8" });
|
|
193
|
+
closeSync(descriptor);
|
|
194
|
+
return { ok: true, value: void 0 };
|
|
195
|
+
} catch (cause) {
|
|
196
|
+
if (descriptor !== null) {
|
|
197
|
+
try {
|
|
198
|
+
closeSync(descriptor);
|
|
199
|
+
} catch {
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
writeReplace(record) {
|
|
206
|
+
const temporaryPath = `${this.leasePath}.${this.options.processToken}.tmp`;
|
|
207
|
+
try {
|
|
208
|
+
writeFileSync(temporaryPath, JSON.stringify(record), { encoding: "utf8", mode: 384 });
|
|
209
|
+
renameSync(temporaryPath, this.leasePath);
|
|
210
|
+
return { ok: true, value: void 0 };
|
|
211
|
+
} catch (cause) {
|
|
212
|
+
try {
|
|
213
|
+
rmSync(temporaryPath, { force: true });
|
|
214
|
+
} catch {
|
|
215
|
+
}
|
|
216
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
readRecord() {
|
|
220
|
+
try {
|
|
221
|
+
const parsed = JSON.parse(readFileSync(this.leasePath, "utf8"));
|
|
222
|
+
if (!this.isLeaseRecord(parsed)) {
|
|
223
|
+
return { ok: false, error: `Invalid telemetry ownership lease at ${this.leasePath}` };
|
|
224
|
+
}
|
|
225
|
+
return { ok: true, value: parsed };
|
|
226
|
+
} catch (cause) {
|
|
227
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
recoverStaleLease() {
|
|
231
|
+
const current = this.readRecord();
|
|
232
|
+
if (!current.ok) {
|
|
233
|
+
return current;
|
|
234
|
+
}
|
|
235
|
+
const expired = current.value.expiresAtMs < this.options.now();
|
|
236
|
+
const processAlive = this.isProcessAlive(current.value.pid);
|
|
237
|
+
if (!expired || processAlive) {
|
|
238
|
+
return {
|
|
239
|
+
ok: false,
|
|
240
|
+
error: `Telemetry target ${this.options.targetId} is owned by pid ${current.value.pid} (${current.value.mode})`
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
try {
|
|
244
|
+
const stalePath = `${this.leasePath}.stale.${current.value.processToken}`;
|
|
245
|
+
renameSync(this.leasePath, stalePath);
|
|
246
|
+
rmSync(stalePath, { force: true });
|
|
247
|
+
return { ok: true, value: void 0 };
|
|
248
|
+
} catch (cause) {
|
|
249
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
isProcessAlive(pid) {
|
|
253
|
+
try {
|
|
254
|
+
process.kill(pid, 0);
|
|
255
|
+
return true;
|
|
256
|
+
} catch (cause) {
|
|
257
|
+
const code = typeof cause === "object" && cause !== null && "code" in cause ? Reflect.get(cause, "code") : null;
|
|
258
|
+
return code === "EPERM";
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
isLeaseRecord(value) {
|
|
262
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
return Reflect.get(value, "formatVersion") === 1 && typeof Reflect.get(value, "targetId") === "string" && typeof Reflect.get(value, "databasePath") === "string" && (Reflect.get(value, "mode") === "writer" || Reflect.get(value, "mode") === "offline-maintenance") && Number.isInteger(Reflect.get(value, "pid")) && typeof Reflect.get(value, "processToken") === "string" && typeof Reflect.get(value, "acquiredAtMs") === "number" && typeof Reflect.get(value, "heartbeatAtMs") === "number" && typeof Reflect.get(value, "expiresAtMs") === "number";
|
|
266
|
+
}
|
|
267
|
+
errorMessage(cause) {
|
|
268
|
+
return cause instanceof Error ? cause.message : String(cause);
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
// src/storage/telemetry_proper_sql_runner.ts
|
|
273
|
+
var TelemetryProperSqlRunner = class {
|
|
274
|
+
constructor(connection, tableMap) {
|
|
275
|
+
this.connection = connection;
|
|
276
|
+
this.tableMap = tableMap;
|
|
277
|
+
}
|
|
278
|
+
connection;
|
|
279
|
+
tableMap;
|
|
280
|
+
async query(sql, params = []) {
|
|
281
|
+
const substituted = this.substitute(sql);
|
|
282
|
+
const normalized = this.leadingKeyword(substituted);
|
|
283
|
+
if (normalized === "SELECT" || normalized === "WITH" || normalized === "PRAGMA" || normalized === "EXPLAIN") {
|
|
284
|
+
return [this.connection.prepare(substituted).all(...params), void 0];
|
|
285
|
+
}
|
|
286
|
+
if (params.length > 0) {
|
|
287
|
+
return [this.connection.prepare(substituted).run(...params), void 0];
|
|
288
|
+
}
|
|
289
|
+
this.connection.exec(substituted);
|
|
290
|
+
return [{ changes: 0 }, void 0];
|
|
291
|
+
}
|
|
292
|
+
async execute(sql, params = []) {
|
|
293
|
+
const substituted = this.substitute(sql);
|
|
294
|
+
if (params.length > 0) {
|
|
295
|
+
return [this.connection.prepare(substituted).run(...params), void 0];
|
|
296
|
+
}
|
|
297
|
+
this.connection.exec(substituted);
|
|
298
|
+
return [{ changes: 0 }, void 0];
|
|
299
|
+
}
|
|
300
|
+
async end() {
|
|
301
|
+
}
|
|
302
|
+
substitute(sql) {
|
|
303
|
+
const substituted = this.tableMap.substitute(sql);
|
|
304
|
+
if (!substituted.ok) {
|
|
305
|
+
throw new Error(substituted.error);
|
|
306
|
+
}
|
|
307
|
+
return substituted.value;
|
|
308
|
+
}
|
|
309
|
+
leadingKeyword(sql) {
|
|
310
|
+
const withoutComments = sql.replace(/--.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "").trimStart();
|
|
311
|
+
return withoutComments.split(/\s+/, 1)[0]?.toUpperCase() ?? "";
|
|
312
|
+
}
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
// src/storage/telemetry_repository.ts
|
|
316
|
+
import { createHash } from "crypto";
|
|
317
|
+
|
|
318
|
+
// src/core/metric_definition_contract.ts
|
|
319
|
+
var MetricDefinitionContract = class {
|
|
320
|
+
normalize(definition) {
|
|
321
|
+
const parsedSelector = this.parseSelector(definition.selector);
|
|
322
|
+
if (!parsedSelector.ok) {
|
|
323
|
+
return parsedSelector;
|
|
324
|
+
}
|
|
325
|
+
const bracketGroupBy = parsedSelector.value.groupBy;
|
|
326
|
+
const configuredGroupBy = definition.groupBy;
|
|
327
|
+
const bothSpecifyDifferentGrouping = bracketGroupBy.length > 0 && configuredGroupBy.length > 0 && bracketGroupBy.join(".") !== configuredGroupBy.join(".");
|
|
328
|
+
if (bothSpecifyDifferentGrouping) {
|
|
329
|
+
return { ok: false, error: `Metric ${definition.name} has conflicting selector and groupBy declarations` };
|
|
330
|
+
}
|
|
331
|
+
const groupBy = configuredGroupBy.length > 0 ? configuredGroupBy : bracketGroupBy;
|
|
332
|
+
const basicError = this.validateBasic(definition, parsedSelector.value.selector, groupBy);
|
|
333
|
+
if (basicError !== null) {
|
|
334
|
+
return { ok: false, error: basicError };
|
|
335
|
+
}
|
|
336
|
+
if (parsedSelector.value.wildcardUngrouped && definition.measurements.some((measurement) => measurement.kind !== "count")) {
|
|
337
|
+
return {
|
|
338
|
+
ok: false,
|
|
339
|
+
error: `Metric ${definition.name} uses [*] and may only declare count`
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
ok: true,
|
|
344
|
+
value: Object.freeze({
|
|
345
|
+
...definition,
|
|
346
|
+
selector: parsedSelector.value.selector,
|
|
347
|
+
groupBy: Object.freeze([...groupBy]),
|
|
348
|
+
resolutionsMs: Object.freeze([...definition.resolutionsMs].sort((left, right) => left - right)),
|
|
349
|
+
measurements: Object.freeze([...definition.measurements]),
|
|
350
|
+
targetIds: Object.freeze([...definition.targetIds]),
|
|
351
|
+
cardinality: Object.freeze({ ...definition.cardinality })
|
|
352
|
+
})
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
matches(selector, label) {
|
|
356
|
+
const selectorSegments = selector.split(".");
|
|
357
|
+
const labelSegments = label.split(".");
|
|
358
|
+
return this.matchSegments(selectorSegments, labelSegments, 0, 0);
|
|
359
|
+
}
|
|
360
|
+
parseSelector(selector) {
|
|
361
|
+
if (selector.startsWith("[")) {
|
|
362
|
+
return { ok: false, error: "A metric selector cannot omit its event label or glob" };
|
|
363
|
+
}
|
|
364
|
+
const bracketStart = selector.lastIndexOf("[");
|
|
365
|
+
if (bracketStart < 0) {
|
|
366
|
+
return { ok: true, value: { selector, groupBy: [], wildcardUngrouped: false } };
|
|
367
|
+
}
|
|
368
|
+
if (!selector.endsWith("]")) {
|
|
369
|
+
return { ok: false, error: `Invalid metric selector ${selector}` };
|
|
370
|
+
}
|
|
371
|
+
const eventSelector = selector.slice(0, bracketStart);
|
|
372
|
+
const bracket = selector.slice(bracketStart + 1, -1).trim();
|
|
373
|
+
if (!eventSelector || !bracket) {
|
|
374
|
+
return { ok: false, error: `Invalid metric selector ${selector}` };
|
|
375
|
+
}
|
|
376
|
+
return {
|
|
377
|
+
ok: true,
|
|
378
|
+
value: {
|
|
379
|
+
selector: eventSelector,
|
|
380
|
+
groupBy: bracket === "*" ? [] : [bracket],
|
|
381
|
+
wildcardUngrouped: bracket === "*"
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
validateBasic(definition, selector, groupBy) {
|
|
386
|
+
if (!/^[A-Za-z0-9_*][A-Za-z0-9_.:*\-]*$/.test(selector)) {
|
|
387
|
+
return `Metric ${definition.name} has an invalid selector`;
|
|
388
|
+
}
|
|
389
|
+
if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(definition.name)) {
|
|
390
|
+
return `Metric name ${definition.name} is invalid`;
|
|
391
|
+
}
|
|
392
|
+
if (!Number.isInteger(definition.version) || definition.version <= 0) {
|
|
393
|
+
return `Metric ${definition.name} must have a positive integer version`;
|
|
394
|
+
}
|
|
395
|
+
if (definition.measurements.length === 0) {
|
|
396
|
+
return `Metric ${definition.name} must declare at least one measurement`;
|
|
397
|
+
}
|
|
398
|
+
const invalidResolution = definition.resolutionsMs.some(
|
|
399
|
+
(resolution) => !Number.isInteger(resolution) || resolution <= 0
|
|
400
|
+
);
|
|
401
|
+
const duplicateResolution = new Set(definition.resolutionsMs).size !== definition.resolutionsMs.length;
|
|
402
|
+
if (invalidResolution || duplicateResolution || definition.resolutionsMs.length === 0) {
|
|
403
|
+
return `Metric ${definition.name} has invalid or duplicate resolutions`;
|
|
404
|
+
}
|
|
405
|
+
const missingCardinality = groupBy.find((field) => definition.cardinality[field] === void 0);
|
|
406
|
+
if (missingCardinality) {
|
|
407
|
+
return `Metric ${definition.name} is missing cardinality policy for ${missingCardinality}`;
|
|
408
|
+
}
|
|
409
|
+
const invalidCardinality = Object.values(definition.cardinality).some(
|
|
410
|
+
(rule) => !Number.isInteger(rule.maxValuesPerBucket) || rule.maxValuesPerBucket <= 0
|
|
411
|
+
);
|
|
412
|
+
if (invalidCardinality) {
|
|
413
|
+
return `Metric ${definition.name} has an invalid cardinality limit`;
|
|
414
|
+
}
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
417
|
+
matchSegments(selector, label, selectorIndex, labelIndex) {
|
|
418
|
+
if (selectorIndex === selector.length) {
|
|
419
|
+
return labelIndex === label.length;
|
|
420
|
+
}
|
|
421
|
+
const segment = selector[selectorIndex];
|
|
422
|
+
if (segment === "**") {
|
|
423
|
+
if (selectorIndex === selector.length - 1) {
|
|
424
|
+
return true;
|
|
425
|
+
}
|
|
426
|
+
for (let next = labelIndex; next <= label.length; next += 1) {
|
|
427
|
+
if (this.matchSegments(selector, label, selectorIndex + 1, next)) {
|
|
428
|
+
return true;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return false;
|
|
432
|
+
}
|
|
433
|
+
if (labelIndex >= label.length) {
|
|
434
|
+
return false;
|
|
435
|
+
}
|
|
436
|
+
const segmentMatches = segment === "*" || segment === label[labelIndex];
|
|
437
|
+
return segmentMatches && this.matchSegments(selector, label, selectorIndex + 1, labelIndex + 1);
|
|
438
|
+
}
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
// src/storage/schema/telemetry_repository_rows.ts
|
|
442
|
+
import { z } from "zod";
|
|
443
|
+
var integer = z.number().int();
|
|
444
|
+
var RawTelemetryEventRowSchema = z.object({
|
|
445
|
+
producer_id: z.string(),
|
|
446
|
+
sequence: integer.nonnegative(),
|
|
447
|
+
event_id: z.string(),
|
|
448
|
+
occurred_at_ms: integer,
|
|
449
|
+
monotonic_ns: z.string().nullable(),
|
|
450
|
+
source: z.string(),
|
|
451
|
+
process_kind: z.string(),
|
|
452
|
+
level: z.string(),
|
|
453
|
+
label: z.string(),
|
|
454
|
+
conversation_id: z.string().nullable(),
|
|
455
|
+
conversation_id_kind: z.enum(["string", "number"]).nullable(),
|
|
456
|
+
correlation_id: z.string().nullable(),
|
|
457
|
+
metadata_json: z.string(),
|
|
458
|
+
schema_version: integer.positive()
|
|
459
|
+
}).strict();
|
|
460
|
+
var BatchAcknowledgementRowSchema = z.object({
|
|
461
|
+
idempotency_key: z.string(),
|
|
462
|
+
producer_id: z.string(),
|
|
463
|
+
sequence_start: integer.nonnegative(),
|
|
464
|
+
sequence_end: integer.nonnegative(),
|
|
465
|
+
schema_version: integer.positive(),
|
|
466
|
+
checksum: z.string(),
|
|
467
|
+
accepted_events: integer.nonnegative(),
|
|
468
|
+
committed_at_ms: integer
|
|
469
|
+
}).strict();
|
|
470
|
+
var MetricSampleRowSchema = z.object({
|
|
471
|
+
metric_name: z.string(),
|
|
472
|
+
definition_version: integer.positive(),
|
|
473
|
+
occurred_at_ms: integer,
|
|
474
|
+
producer_id: z.string(),
|
|
475
|
+
sequence: integer.nonnegative(),
|
|
476
|
+
dimension_key: z.string(),
|
|
477
|
+
dimensions_json: z.string(),
|
|
478
|
+
values_json: z.string()
|
|
479
|
+
}).strict();
|
|
480
|
+
var MetricRollupRowSchema = z.object({
|
|
481
|
+
metric_name: z.string(),
|
|
482
|
+
definition_version: integer.positive(),
|
|
483
|
+
resolution_ms: integer.positive(),
|
|
484
|
+
bucket_start_ms: integer,
|
|
485
|
+
dimension_key: z.string(),
|
|
486
|
+
dimensions_json: z.string(),
|
|
487
|
+
values_json: z.string()
|
|
488
|
+
}).strict();
|
|
489
|
+
var RollupCompletionRowSchema = z.object({
|
|
490
|
+
metric_name: z.string(),
|
|
491
|
+
definition_version: integer.positive(),
|
|
492
|
+
resolution_ms: integer.positive(),
|
|
493
|
+
bucket_start_ms: integer,
|
|
494
|
+
sample_count: integer.nonnegative(),
|
|
495
|
+
sample_watermark_ms: integer,
|
|
496
|
+
completed_at_ms: integer,
|
|
497
|
+
state: z.enum(["completed", "rejected"]),
|
|
498
|
+
detail: z.string().nullable()
|
|
499
|
+
}).strict();
|
|
500
|
+
var TimelineCursorSchema = z.object({
|
|
501
|
+
version: z.literal(1),
|
|
502
|
+
order: z.enum(["asc", "desc"]),
|
|
503
|
+
occurredAtMs: integer,
|
|
504
|
+
producerId: z.string(),
|
|
505
|
+
sequence: integer.nonnegative()
|
|
506
|
+
}).strict();
|
|
507
|
+
var MetricCursorSchema = z.object({
|
|
508
|
+
version: z.literal(1),
|
|
509
|
+
order: z.enum(["asc", "desc"]),
|
|
510
|
+
bucketStartMs: integer,
|
|
511
|
+
resolutionMs: integer.positive(),
|
|
512
|
+
definitionVersion: integer.positive(),
|
|
513
|
+
dimensionKey: z.string()
|
|
514
|
+
}).strict();
|
|
515
|
+
var TelemetryJsonObjectSchema = z.record(z.string(), z.json());
|
|
516
|
+
var NumericValuesSchema = z.record(z.string(), z.number().finite());
|
|
517
|
+
var StoredMetricMeasurementSchema = z.discriminatedUnion("kind", [
|
|
518
|
+
z.object({ kind: z.literal("count") }).strict(),
|
|
519
|
+
z.object({
|
|
520
|
+
kind: z.literal("field"),
|
|
521
|
+
field: z.string().min(1),
|
|
522
|
+
aggregates: z.array(z.enum(["sum", "min", "max", "last"])).min(1)
|
|
523
|
+
}).strict()
|
|
524
|
+
]);
|
|
525
|
+
var StoredMetricDefinitionSchema = z.object({
|
|
526
|
+
name: z.string().min(1),
|
|
527
|
+
version: integer.positive(),
|
|
528
|
+
selector: z.string().min(1),
|
|
529
|
+
groupBy: z.array(z.string().min(1)),
|
|
530
|
+
measurements: z.array(StoredMetricMeasurementSchema).min(1),
|
|
531
|
+
cardinality: z.record(
|
|
532
|
+
z.string(),
|
|
533
|
+
z.object({
|
|
534
|
+
maxValuesPerBucket: integer.positive(),
|
|
535
|
+
overflow: z.enum(["other", "drop", "reject"])
|
|
536
|
+
}).strict()
|
|
537
|
+
),
|
|
538
|
+
targetIds: z.array(z.string().min(1)),
|
|
539
|
+
resolutionsMs: z.array(integer.positive()).min(1),
|
|
540
|
+
missingField: z.enum(["skip", "record-failure"])
|
|
541
|
+
}).strict();
|
|
542
|
+
var StoredMetricDefinitionRowSchema = z.object({
|
|
543
|
+
metric_name: z.string(),
|
|
544
|
+
definition_version: integer.positive(),
|
|
545
|
+
definition_json: z.string(),
|
|
546
|
+
definition_checksum: z.string(),
|
|
547
|
+
active: integer.min(0).max(1)
|
|
548
|
+
}).strict();
|
|
549
|
+
var MetricRollupValuesSchema = z.object({
|
|
550
|
+
count: integer.nonnegative(),
|
|
551
|
+
fields: z.record(
|
|
552
|
+
z.string(),
|
|
553
|
+
z.object({
|
|
554
|
+
count: integer.nonnegative(),
|
|
555
|
+
sum: z.number().finite(),
|
|
556
|
+
min: z.number().finite(),
|
|
557
|
+
max: z.number().finite(),
|
|
558
|
+
last: z.number().finite(),
|
|
559
|
+
lastOccurredAtMs: integer,
|
|
560
|
+
lastProducerId: z.string(),
|
|
561
|
+
lastSequence: integer.nonnegative()
|
|
562
|
+
}).strict()
|
|
563
|
+
)
|
|
564
|
+
}).strict();
|
|
565
|
+
|
|
566
|
+
// src/storage/telemetry_repository_row_parser.ts
|
|
567
|
+
var TelemetryRepositoryRowParser = class {
|
|
568
|
+
parseRawEnvelope(input) {
|
|
569
|
+
const parsed = RawTelemetryEventRowSchema.safeParse(input);
|
|
570
|
+
if (!parsed.success) {
|
|
571
|
+
return { ok: false, error: parsed.error.message };
|
|
572
|
+
}
|
|
573
|
+
const metadata = this.parseJsonObject(parsed.data.metadata_json);
|
|
574
|
+
if (!metadata.ok) {
|
|
575
|
+
return metadata;
|
|
576
|
+
}
|
|
577
|
+
const conversationId = this.parseConversationId(
|
|
578
|
+
parsed.data.conversation_id,
|
|
579
|
+
parsed.data.conversation_id_kind
|
|
580
|
+
);
|
|
581
|
+
if (!conversationId.ok) {
|
|
582
|
+
return conversationId;
|
|
583
|
+
}
|
|
584
|
+
return {
|
|
585
|
+
ok: true,
|
|
586
|
+
value: Object.freeze({
|
|
587
|
+
schemaVersion: parsed.data.schema_version,
|
|
588
|
+
eventId: parsed.data.event_id,
|
|
589
|
+
producerId: parsed.data.producer_id,
|
|
590
|
+
sequence: parsed.data.sequence,
|
|
591
|
+
occurredAtMs: parsed.data.occurred_at_ms,
|
|
592
|
+
monotonicNs: parsed.data.monotonic_ns,
|
|
593
|
+
source: parsed.data.source,
|
|
594
|
+
processKind: parsed.data.process_kind,
|
|
595
|
+
level: parsed.data.level,
|
|
596
|
+
label: parsed.data.label,
|
|
597
|
+
conversationId: conversationId.value,
|
|
598
|
+
correlationId: parsed.data.correlation_id,
|
|
599
|
+
metadata: metadata.value
|
|
600
|
+
})
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
parseMetricSample(input) {
|
|
604
|
+
const parsed = MetricSampleRowSchema.safeParse(input);
|
|
605
|
+
return parsed.success ? { ok: true, value: parsed.data } : { ok: false, error: parsed.error.message };
|
|
606
|
+
}
|
|
607
|
+
parseMetricRollup(input) {
|
|
608
|
+
const parsed = MetricRollupRowSchema.safeParse(input);
|
|
609
|
+
return parsed.success ? { ok: true, value: parsed.data } : { ok: false, error: parsed.error.message };
|
|
610
|
+
}
|
|
611
|
+
parseJsonObject(input) {
|
|
612
|
+
try {
|
|
613
|
+
const decoded = JSON.parse(input);
|
|
614
|
+
const parsed = TelemetryJsonObjectSchema.safeParse(decoded);
|
|
615
|
+
return parsed.success ? { ok: true, value: Object.freeze(parsed.data) } : { ok: false, error: parsed.error.message };
|
|
616
|
+
} catch (cause) {
|
|
617
|
+
return { ok: false, error: cause instanceof Error ? cause.message : String(cause) };
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
parseNumericValues(input) {
|
|
621
|
+
try {
|
|
622
|
+
const decoded = JSON.parse(input);
|
|
623
|
+
const parsed = NumericValuesSchema.safeParse(decoded);
|
|
624
|
+
return parsed.success ? { ok: true, value: Object.freeze(parsed.data) } : { ok: false, error: parsed.error.message };
|
|
625
|
+
} catch (cause) {
|
|
626
|
+
return { ok: false, error: cause instanceof Error ? cause.message : String(cause) };
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
parseRollupValues(input) {
|
|
630
|
+
try {
|
|
631
|
+
const decoded = JSON.parse(input);
|
|
632
|
+
const parsed = MetricRollupValuesSchema.safeParse(decoded);
|
|
633
|
+
return parsed.success ? { ok: true, value: parsed.data } : { ok: false, error: parsed.error.message };
|
|
634
|
+
} catch (cause) {
|
|
635
|
+
return { ok: false, error: cause instanceof Error ? cause.message : String(cause) };
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
parseTimelineCursor(input) {
|
|
639
|
+
return this.parseCursor(input, TimelineCursorSchema);
|
|
640
|
+
}
|
|
641
|
+
parseMetricCursor(input) {
|
|
642
|
+
return this.parseCursor(input, MetricCursorSchema);
|
|
643
|
+
}
|
|
644
|
+
parseCursor(input, schema) {
|
|
645
|
+
try {
|
|
646
|
+
const decodedText = Buffer.from(input, "base64url").toString("utf8");
|
|
647
|
+
const decoded = JSON.parse(decodedText);
|
|
648
|
+
const parsed = schema.safeParse(decoded);
|
|
649
|
+
return parsed.success ? { ok: true, value: parsed.data } : { ok: false, error: parsed.error.message };
|
|
650
|
+
} catch (cause) {
|
|
651
|
+
return { ok: false, error: cause instanceof Error ? cause.message : String(cause) };
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
parseConversationId(value, kind) {
|
|
655
|
+
if (value === null) {
|
|
656
|
+
return { ok: true, value: null };
|
|
657
|
+
}
|
|
658
|
+
if (kind === "string") {
|
|
659
|
+
return { ok: true, value };
|
|
660
|
+
}
|
|
661
|
+
if (kind === "number") {
|
|
662
|
+
const numeric = Number(value);
|
|
663
|
+
return Number.isFinite(numeric) ? { ok: true, value: numeric } : { ok: false, error: `Invalid numeric conversation id ${value}` };
|
|
664
|
+
}
|
|
665
|
+
return { ok: false, error: "Conversation id kind is missing" };
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
|
|
669
|
+
// src/storage/telemetry_repository.ts
|
|
670
|
+
var RETENTION_PAGE_SIZE = 500;
|
|
671
|
+
var MetricCardinalityRejectionError = class extends Error {
|
|
672
|
+
constructor(message) {
|
|
673
|
+
super(message);
|
|
674
|
+
this.name = "MetricCardinalityRejectionError";
|
|
675
|
+
}
|
|
676
|
+
};
|
|
677
|
+
var TelemetryRepository = class {
|
|
678
|
+
constructor(options) {
|
|
679
|
+
this.options = options;
|
|
680
|
+
}
|
|
681
|
+
options;
|
|
682
|
+
rowParser = new TelemetryRepositoryRowParser();
|
|
683
|
+
definitionContract = new MetricDefinitionContract();
|
|
684
|
+
definitions = /* @__PURE__ */ new Map();
|
|
685
|
+
registerDefinitions(definitions) {
|
|
686
|
+
try {
|
|
687
|
+
const normalizedDefinitions = this.normalizeDefinitions(definitions);
|
|
688
|
+
const registeredAtMs = this.options.now();
|
|
689
|
+
const byName = /* @__PURE__ */ new Map();
|
|
690
|
+
for (const definition of normalizedDefinitions) {
|
|
691
|
+
const versions = byName.get(definition.name) ?? [];
|
|
692
|
+
versions.push(definition);
|
|
693
|
+
byName.set(definition.name, versions);
|
|
694
|
+
}
|
|
695
|
+
const transaction = this.options.connection.transaction(() => {
|
|
696
|
+
const table = this.options.tableMap.quoted("metricDefinitions");
|
|
697
|
+
const deactivate = this.options.connection.prepare(
|
|
698
|
+
`UPDATE ${table} SET active = 0 WHERE target_id = ? AND metric_name = ?`
|
|
699
|
+
);
|
|
700
|
+
const existing = this.options.connection.prepare(
|
|
701
|
+
`SELECT definition_checksum FROM ${table}
|
|
702
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?`
|
|
703
|
+
);
|
|
704
|
+
const insert = this.options.connection.prepare(
|
|
705
|
+
`INSERT INTO ${table} (
|
|
706
|
+
target_id, metric_name, definition_version, definition_json,
|
|
707
|
+
definition_checksum, active, registered_at_ms
|
|
708
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
709
|
+
);
|
|
710
|
+
const activate = this.options.connection.prepare(
|
|
711
|
+
`UPDATE ${table} SET active = 1
|
|
712
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?`
|
|
713
|
+
);
|
|
714
|
+
for (const [metricName, versions] of byName) {
|
|
715
|
+
deactivate.run(this.options.targetId, metricName);
|
|
716
|
+
const latestVersion = Math.max(...versions.map((definition) => definition.version));
|
|
717
|
+
for (const definition of versions) {
|
|
718
|
+
const definitionJson = canonicalJson(definition);
|
|
719
|
+
const definitionChecksum = telemetryChecksum(definition);
|
|
720
|
+
const prior = existing.get(this.options.targetId, definition.name, definition.version);
|
|
721
|
+
if (prior !== void 0) {
|
|
722
|
+
const priorChecksum = this.readStringField(prior, "definition_checksum");
|
|
723
|
+
if (priorChecksum !== definitionChecksum) {
|
|
724
|
+
throw new Error(
|
|
725
|
+
`Metric ${definition.name} version ${definition.version} changed without a version bump`
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
} else {
|
|
729
|
+
insert.run(
|
|
730
|
+
this.options.targetId,
|
|
731
|
+
definition.name,
|
|
732
|
+
definition.version,
|
|
733
|
+
definitionJson,
|
|
734
|
+
definitionChecksum,
|
|
735
|
+
definition.version === latestVersion ? 1 : 0,
|
|
736
|
+
registeredAtMs
|
|
737
|
+
);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
activate.run(this.options.targetId, metricName, latestVersion);
|
|
741
|
+
}
|
|
742
|
+
});
|
|
743
|
+
transaction();
|
|
744
|
+
const loaded = this.loadDefinitionsReadOnly(normalizedDefinitions);
|
|
745
|
+
if (!loaded.ok) {
|
|
746
|
+
throw new Error(loaded.error);
|
|
747
|
+
}
|
|
748
|
+
return { ok: true, value: void 0 };
|
|
749
|
+
} catch (cause) {
|
|
750
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
loadDefinitionsReadOnly(expectedDefinitions) {
|
|
754
|
+
try {
|
|
755
|
+
const normalizedExpectedDefinitions = this.normalizeDefinitions(expectedDefinitions);
|
|
756
|
+
const table = this.options.tableMap.quoted("metricDefinitions");
|
|
757
|
+
const rows = this.options.connection.prepare(
|
|
758
|
+
`SELECT metric_name, definition_version, definition_json, definition_checksum, active
|
|
759
|
+
FROM ${table} WHERE target_id = ?`
|
|
760
|
+
).all(this.options.targetId);
|
|
761
|
+
const loaded = /* @__PURE__ */ new Map();
|
|
762
|
+
for (const rawRow of rows) {
|
|
763
|
+
const row = StoredMetricDefinitionRowSchema.parse(rawRow);
|
|
764
|
+
const parsedJson = JSON.parse(row.definition_json);
|
|
765
|
+
const parsedDefinition = StoredMetricDefinitionSchema.parse(parsedJson);
|
|
766
|
+
const normalized = this.definitionContract.normalize(parsedDefinition);
|
|
767
|
+
if (!normalized.ok) {
|
|
768
|
+
throw new Error(normalized.error);
|
|
769
|
+
}
|
|
770
|
+
const definition = normalized.value;
|
|
771
|
+
const identityMatches = definition.name === row.metric_name && definition.version === row.definition_version;
|
|
772
|
+
if (!identityMatches) {
|
|
773
|
+
throw new Error("Stored metric definition identity does not match its row");
|
|
774
|
+
}
|
|
775
|
+
const checksum = telemetryChecksum(definition);
|
|
776
|
+
const canonicalRowMatches = checksum === row.definition_checksum && canonicalJson(definition) === row.definition_json;
|
|
777
|
+
const legacyRowMatches = !canonicalRowMatches && createHash("sha256").update(row.definition_json, "utf8").digest("hex") === row.definition_checksum;
|
|
778
|
+
if (!canonicalRowMatches && !legacyRowMatches) {
|
|
779
|
+
if (row.active !== 1) {
|
|
780
|
+
continue;
|
|
781
|
+
}
|
|
782
|
+
throw new Error(
|
|
783
|
+
`Stored metric ${definition.name} version ${definition.version} checksum is invalid`
|
|
784
|
+
);
|
|
785
|
+
}
|
|
786
|
+
loaded.set(this.definitionKey(definition.name, definition.version), definition);
|
|
787
|
+
}
|
|
788
|
+
for (const expected of normalizedExpectedDefinitions) {
|
|
789
|
+
const stored = loaded.get(this.definitionKey(expected.name, expected.version));
|
|
790
|
+
if (stored === void 0 || telemetryChecksum(stored) !== telemetryChecksum(expected)) {
|
|
791
|
+
throw new Error(
|
|
792
|
+
`Configured metric ${expected.name} version ${expected.version} is not registered in target ${this.options.targetId}`
|
|
793
|
+
);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
this.definitions.clear();
|
|
797
|
+
for (const [identity, definition] of loaded) {
|
|
798
|
+
this.definitions.set(identity, definition);
|
|
799
|
+
}
|
|
800
|
+
return { ok: true, value: void 0 };
|
|
801
|
+
} catch (cause) {
|
|
802
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
persist(request) {
|
|
806
|
+
try {
|
|
807
|
+
const outsideHorizon = this.outsideIngestionHorizon(request.batch);
|
|
808
|
+
if (outsideHorizon !== null) {
|
|
809
|
+
return { kind: "acknowledged", acknowledgement: outsideHorizon };
|
|
810
|
+
}
|
|
811
|
+
const replay = this.findReplay(request.batch);
|
|
812
|
+
if (replay !== null) {
|
|
813
|
+
return { kind: "acknowledged", acknowledgement: replay };
|
|
814
|
+
}
|
|
815
|
+
const acknowledgement = this.persistTransaction(request);
|
|
816
|
+
return { kind: "acknowledged", acknowledgement };
|
|
817
|
+
} catch (cause) {
|
|
818
|
+
return { kind: "failed", reason: this.errorMessage(cause) };
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
timeline(query, rawRetentionCutoffMs) {
|
|
822
|
+
try {
|
|
823
|
+
const cursor = query.cursor === null ? null : this.rowParser.parseTimelineCursor(query.cursor);
|
|
824
|
+
if (cursor !== null && !cursor.ok) {
|
|
825
|
+
return { ok: false, error: { kind: "invalid-query", message: cursor.error } };
|
|
826
|
+
}
|
|
827
|
+
if (cursor !== null && cursor.value.order !== query.order) {
|
|
828
|
+
return {
|
|
829
|
+
ok: false,
|
|
830
|
+
error: { kind: "invalid-query", message: "Timeline cursor order changed" }
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
const availableFromMs = Math.max(query.fromMs, rawRetentionCutoffMs);
|
|
834
|
+
const clauses = ["target_id = ?", "occurred_at_ms >= ?", "occurred_at_ms < ?"];
|
|
835
|
+
const params = [this.options.targetId, availableFromMs, query.toMs];
|
|
836
|
+
this.addTimelineFilters(query, clauses, params);
|
|
837
|
+
if (cursor?.ok) {
|
|
838
|
+
this.addTimelineCursor(cursor.value, query.order, clauses, params);
|
|
839
|
+
}
|
|
840
|
+
const direction = query.order === "asc" ? "ASC" : "DESC";
|
|
841
|
+
const table = this.options.tableMap.quoted("rawEvents");
|
|
842
|
+
const sql = `SELECT
|
|
843
|
+
producer_id, sequence, event_id, occurred_at_ms, monotonic_ns,
|
|
844
|
+
source, process_kind, level, label, conversation_id,
|
|
845
|
+
conversation_id_kind, correlation_id, metadata_json, schema_version
|
|
846
|
+
FROM ${table}
|
|
847
|
+
WHERE ${clauses.join(" AND ")}
|
|
848
|
+
ORDER BY occurred_at_ms ${direction}, producer_id ${direction}, sequence ${direction}
|
|
849
|
+
LIMIT ?`;
|
|
850
|
+
params.push(query.limit + 1);
|
|
851
|
+
const rows = this.options.connection.prepare(sql).all(...params);
|
|
852
|
+
const hasNextPage = rows.length > query.limit;
|
|
853
|
+
const pageRows = hasNextPage ? rows.slice(0, query.limit) : rows;
|
|
854
|
+
const events = [];
|
|
855
|
+
for (const row of pageRows) {
|
|
856
|
+
const parsed = this.rowParser.parseRawEnvelope(row);
|
|
857
|
+
if (!parsed.ok) {
|
|
858
|
+
return { ok: false, error: { kind: "storage-failure", message: parsed.error } };
|
|
859
|
+
}
|
|
860
|
+
events.push(parsed.value);
|
|
861
|
+
}
|
|
862
|
+
const last = events[events.length - 1];
|
|
863
|
+
const nextCursor = hasNextPage && last ? this.encodeCursor({
|
|
864
|
+
version: 1,
|
|
865
|
+
order: query.order,
|
|
866
|
+
occurredAtMs: last.occurredAtMs,
|
|
867
|
+
producerId: last.producerId,
|
|
868
|
+
sequence: last.sequence
|
|
869
|
+
}) : null;
|
|
870
|
+
const unavailableEnd = Math.min(query.toMs, rawRetentionCutoffMs);
|
|
871
|
+
const gaps = query.fromMs < unavailableEnd ? [{ fromMs: query.fromMs, toMs: unavailableEnd, reason: "raw-retention" }] : [];
|
|
872
|
+
return {
|
|
873
|
+
ok: true,
|
|
874
|
+
value: {
|
|
875
|
+
targetId: this.options.targetId,
|
|
876
|
+
events: Object.freeze(events),
|
|
877
|
+
gaps: Object.freeze(gaps),
|
|
878
|
+
nextCursor
|
|
879
|
+
}
|
|
880
|
+
};
|
|
881
|
+
} catch (cause) {
|
|
882
|
+
return { ok: false, error: { kind: "storage-failure", message: this.errorMessage(cause) } };
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
metrics(query) {
|
|
886
|
+
try {
|
|
887
|
+
const definitions = this.metricDefinitions(query.metric);
|
|
888
|
+
if (definitions.length === 0) {
|
|
889
|
+
return {
|
|
890
|
+
ok: false,
|
|
891
|
+
error: { kind: "invalid-query", message: `Unknown metric ${query.metric}` }
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
const validationError = this.validateMetricQuery(query, definitions);
|
|
895
|
+
if (validationError !== null) {
|
|
896
|
+
return { ok: false, error: { kind: "invalid-query", message: validationError } };
|
|
897
|
+
}
|
|
898
|
+
const cursor = query.cursor === null ? null : this.rowParser.parseMetricCursor(query.cursor);
|
|
899
|
+
if (cursor !== null && !cursor.ok) {
|
|
900
|
+
return { ok: false, error: { kind: "invalid-query", message: cursor.error } };
|
|
901
|
+
}
|
|
902
|
+
if (cursor !== null && cursor.value.order !== query.order) {
|
|
903
|
+
return {
|
|
904
|
+
ok: false,
|
|
905
|
+
error: { kind: "invalid-query", message: "Metric cursor order changed" }
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
const coverage = this.planCoverage(query, definitions);
|
|
909
|
+
const rows = this.readRollupsForCoverage(query, coverage.intervals);
|
|
910
|
+
const filtered = rows.filter((row) => this.matchesWhere(row.bucket.dimensions, query.where));
|
|
911
|
+
const ordered = filtered.sort(
|
|
912
|
+
(left, right) => this.compareParsedRollups(left, right, query.order)
|
|
913
|
+
);
|
|
914
|
+
const afterCursor = cursor?.ok ? ordered.filter((row) => this.isAfterMetricCursor(row, cursor.value, query.order)) : ordered;
|
|
915
|
+
const hasNextPage = afterCursor.length > query.limit;
|
|
916
|
+
const page = hasNextPage ? afterCursor.slice(0, query.limit) : afterCursor;
|
|
917
|
+
const projected = page.map((row) => ({
|
|
918
|
+
...row,
|
|
919
|
+
bucket: {
|
|
920
|
+
...row.bucket,
|
|
921
|
+
values: this.selectMeasurements(row.bucket.values, query.measurements)
|
|
922
|
+
}
|
|
923
|
+
}));
|
|
924
|
+
const segments = this.rowsToSegments(coverage.intervals, projected);
|
|
925
|
+
const last = page[page.length - 1];
|
|
926
|
+
const nextCursor = hasNextPage && last ? this.encodeCursor({
|
|
927
|
+
version: 1,
|
|
928
|
+
order: query.order,
|
|
929
|
+
bucketStartMs: last.bucket.startMs,
|
|
930
|
+
resolutionMs: last.resolutionMs,
|
|
931
|
+
definitionVersion: last.definitionVersion,
|
|
932
|
+
dimensionKey: last.dimensionKey
|
|
933
|
+
}) : null;
|
|
934
|
+
return {
|
|
935
|
+
ok: true,
|
|
936
|
+
value: {
|
|
937
|
+
targetId: this.options.targetId,
|
|
938
|
+
metric: query.metric,
|
|
939
|
+
segments,
|
|
940
|
+
gaps: coverage.gaps,
|
|
941
|
+
nextCursor,
|
|
942
|
+
partialFailures: []
|
|
943
|
+
}
|
|
944
|
+
};
|
|
945
|
+
} catch (cause) {
|
|
946
|
+
return { ok: false, error: { kind: "storage-failure", message: this.errorMessage(cause) } };
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
materializeRollups(nowMs, maxWorkMs) {
|
|
950
|
+
const startedAt = this.options.now();
|
|
951
|
+
let affectedRows = 0;
|
|
952
|
+
let rejectedBucketCount = 0;
|
|
953
|
+
const rejectedBucketExamples = [];
|
|
954
|
+
try {
|
|
955
|
+
for (const definition of this.definitions.values()) {
|
|
956
|
+
for (const resolutionMs of definition.resolutionsMs) {
|
|
957
|
+
const dueBeforeMs = nowMs - this.options.retention.lateArrivalMs;
|
|
958
|
+
const bucketStarts = this.findDueBucketStarts(definition, resolutionMs, dueBeforeMs);
|
|
959
|
+
for (const bucketStartMs of bucketStarts) {
|
|
960
|
+
if (this.options.now() - startedAt >= maxWorkMs) {
|
|
961
|
+
return {
|
|
962
|
+
ok: true,
|
|
963
|
+
value: {
|
|
964
|
+
stage: "rollup",
|
|
965
|
+
affectedRows,
|
|
966
|
+
affectedPages: 0,
|
|
967
|
+
elapsedMs: this.options.now() - startedAt,
|
|
968
|
+
detail: this.rollupStageDetail(
|
|
969
|
+
rejectedBucketCount,
|
|
970
|
+
rejectedBucketExamples,
|
|
971
|
+
true
|
|
972
|
+
)
|
|
973
|
+
}
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
const work = this.loadBucketWork(definition, resolutionMs, bucketStartMs);
|
|
977
|
+
if (this.bucketIsCurrent(work)) {
|
|
978
|
+
continue;
|
|
979
|
+
}
|
|
980
|
+
try {
|
|
981
|
+
affectedRows += this.materializeBucket(work, nowMs);
|
|
982
|
+
} catch (cause) {
|
|
983
|
+
if (!(cause instanceof MetricCardinalityRejectionError)) {
|
|
984
|
+
throw cause;
|
|
985
|
+
}
|
|
986
|
+
this.recordRejectedBucket(work, nowMs, cause.message);
|
|
987
|
+
rejectedBucketCount += 1;
|
|
988
|
+
if (rejectedBucketExamples.length < 3) {
|
|
989
|
+
rejectedBucketExamples.push(
|
|
990
|
+
`${definition.name}:${definition.version}:${resolutionMs}:${bucketStartMs}: ${cause.message}`
|
|
991
|
+
);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
return {
|
|
998
|
+
ok: true,
|
|
999
|
+
value: {
|
|
1000
|
+
stage: "rollup",
|
|
1001
|
+
affectedRows,
|
|
1002
|
+
affectedPages: 0,
|
|
1003
|
+
elapsedMs: this.options.now() - startedAt,
|
|
1004
|
+
detail: this.rollupStageDetail(
|
|
1005
|
+
rejectedBucketCount,
|
|
1006
|
+
rejectedBucketExamples,
|
|
1007
|
+
false
|
|
1008
|
+
)
|
|
1009
|
+
}
|
|
1010
|
+
};
|
|
1011
|
+
} catch (cause) {
|
|
1012
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
beginRetentionDrain(nowMs) {
|
|
1016
|
+
const steps = [
|
|
1017
|
+
this.createRawEventRetentionStep(nowMs),
|
|
1018
|
+
this.createSampleRetentionStep(nowMs),
|
|
1019
|
+
...this.createRollupRetentionSteps(nowMs),
|
|
1020
|
+
...this.createAuxiliaryRetentionSteps(nowMs)
|
|
1021
|
+
];
|
|
1022
|
+
let index = 0;
|
|
1023
|
+
return {
|
|
1024
|
+
nextBatch: () => {
|
|
1025
|
+
try {
|
|
1026
|
+
if (steps.length === 0) {
|
|
1027
|
+
return { ok: true, value: { deletedRows: 0, hasMore: false } };
|
|
1028
|
+
}
|
|
1029
|
+
if (index >= steps.length) {
|
|
1030
|
+
index = 0;
|
|
1031
|
+
}
|
|
1032
|
+
const step = steps[index];
|
|
1033
|
+
if (step === void 0) {
|
|
1034
|
+
return { ok: true, value: { deletedRows: 0, hasMore: false } };
|
|
1035
|
+
}
|
|
1036
|
+
const outcome = step();
|
|
1037
|
+
if (outcome.done) {
|
|
1038
|
+
steps.splice(index, 1);
|
|
1039
|
+
} else {
|
|
1040
|
+
index += 1;
|
|
1041
|
+
}
|
|
1042
|
+
return {
|
|
1043
|
+
ok: true,
|
|
1044
|
+
value: { deletedRows: outcome.deleted, hasMore: steps.length > 0 }
|
|
1045
|
+
};
|
|
1046
|
+
} catch (cause) {
|
|
1047
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
};
|
|
1051
|
+
}
|
|
1052
|
+
checkpoint() {
|
|
1053
|
+
const startedAt = this.options.now();
|
|
1054
|
+
try {
|
|
1055
|
+
const rows = this.options.connection.pragma("wal_checkpoint(PASSIVE)");
|
|
1056
|
+
return {
|
|
1057
|
+
ok: true,
|
|
1058
|
+
value: {
|
|
1059
|
+
stage: "checkpoint",
|
|
1060
|
+
affectedRows: Array.isArray(rows) ? rows.length : 0,
|
|
1061
|
+
affectedPages: 0,
|
|
1062
|
+
elapsedMs: this.options.now() - startedAt,
|
|
1063
|
+
detail: null
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
} catch (cause) {
|
|
1067
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
reclaim(maxPages) {
|
|
1071
|
+
const startedAt = this.options.now();
|
|
1072
|
+
try {
|
|
1073
|
+
const before = Number(this.options.connection.pragma("freelist_count", { simple: true }));
|
|
1074
|
+
this.options.connection.pragma(`incremental_vacuum(${Math.max(0, Math.floor(maxPages))})`);
|
|
1075
|
+
const after = Number(this.options.connection.pragma("freelist_count", { simple: true }));
|
|
1076
|
+
return {
|
|
1077
|
+
ok: true,
|
|
1078
|
+
value: {
|
|
1079
|
+
stage: "reclaim",
|
|
1080
|
+
affectedRows: 0,
|
|
1081
|
+
affectedPages: Math.max(0, before - after),
|
|
1082
|
+
elapsedMs: this.options.now() - startedAt,
|
|
1083
|
+
detail: null
|
|
1084
|
+
}
|
|
1085
|
+
};
|
|
1086
|
+
} catch (cause) {
|
|
1087
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
pageStats() {
|
|
1091
|
+
try {
|
|
1092
|
+
return {
|
|
1093
|
+
ok: true,
|
|
1094
|
+
value: {
|
|
1095
|
+
pageCount: Number(this.options.connection.pragma("page_count", { simple: true })),
|
|
1096
|
+
freePages: Number(this.options.connection.pragma("freelist_count", { simple: true })),
|
|
1097
|
+
pageSize: Number(this.options.connection.pragma("page_size", { simple: true }))
|
|
1098
|
+
}
|
|
1099
|
+
};
|
|
1100
|
+
} catch (cause) {
|
|
1101
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
recordMaintenance(record) {
|
|
1105
|
+
try {
|
|
1106
|
+
const table = this.options.tableMap.quoted("maintenanceRuns");
|
|
1107
|
+
const affectedRows = record.stages.reduce((sum, stage) => sum + stage.affectedRows, 0);
|
|
1108
|
+
const affectedPages = record.stages.reduce((sum, stage) => sum + stage.affectedPages, 0);
|
|
1109
|
+
const detail = canonicalJson({
|
|
1110
|
+
stages: record.stages,
|
|
1111
|
+
failedStage: record.failedStage,
|
|
1112
|
+
error: record.error
|
|
1113
|
+
});
|
|
1114
|
+
this.options.connection.prepare(
|
|
1115
|
+
`INSERT INTO ${table} (
|
|
1116
|
+
target_id, run_id, state, stage, started_at_ms, completed_at_ms,
|
|
1117
|
+
affected_rows, affected_pages, detail
|
|
1118
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1119
|
+
ON CONFLICT(target_id, run_id, stage) DO UPDATE SET
|
|
1120
|
+
state = excluded.state,
|
|
1121
|
+
completed_at_ms = excluded.completed_at_ms,
|
|
1122
|
+
affected_rows = excluded.affected_rows,
|
|
1123
|
+
affected_pages = excluded.affected_pages,
|
|
1124
|
+
detail = excluded.detail`
|
|
1125
|
+
).run(
|
|
1126
|
+
this.options.targetId,
|
|
1127
|
+
record.runId,
|
|
1128
|
+
record.state,
|
|
1129
|
+
record.kind,
|
|
1130
|
+
record.startedAtMs,
|
|
1131
|
+
record.completedAtMs,
|
|
1132
|
+
affectedRows,
|
|
1133
|
+
affectedPages,
|
|
1134
|
+
detail
|
|
1135
|
+
);
|
|
1136
|
+
return { ok: true, value: void 0 };
|
|
1137
|
+
} catch (cause) {
|
|
1138
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
recoverInterruptedMaintenance(nowMs) {
|
|
1142
|
+
try {
|
|
1143
|
+
const table = this.options.tableMap.quoted("maintenanceRuns");
|
|
1144
|
+
const detail = canonicalJson({
|
|
1145
|
+
stages: [],
|
|
1146
|
+
failedStage: null,
|
|
1147
|
+
error: "interrupted-before-restart"
|
|
1148
|
+
});
|
|
1149
|
+
const result = this.options.connection.prepare(
|
|
1150
|
+
`UPDATE ${table}
|
|
1151
|
+
SET state = 'failed', completed_at_ms = ?, detail = ?
|
|
1152
|
+
WHERE target_id = ? AND state = 'started'`
|
|
1153
|
+
).run(nowMs, detail, this.options.targetId);
|
|
1154
|
+
return { ok: true, value: result.changes };
|
|
1155
|
+
} catch (cause) {
|
|
1156
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
committedDropCounters() {
|
|
1160
|
+
try {
|
|
1161
|
+
const table = this.options.tableMap.quoted("committedDropCounters");
|
|
1162
|
+
const rows = this.options.connection.prepare(
|
|
1163
|
+
`SELECT reason, priority, count
|
|
1164
|
+
FROM ${table}
|
|
1165
|
+
WHERE target_id = ?
|
|
1166
|
+
ORDER BY reason ASC, priority ASC`
|
|
1167
|
+
).all(this.options.targetId);
|
|
1168
|
+
const counters = rows.map((row) => this.parseCommittedDropCounter(row));
|
|
1169
|
+
return { ok: true, value: Object.freeze(counters) };
|
|
1170
|
+
} catch (cause) {
|
|
1171
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
findReplay(batch) {
|
|
1175
|
+
const table = this.options.tableMap.quoted("batchAcknowledgements");
|
|
1176
|
+
const rows = this.options.connection.prepare(
|
|
1177
|
+
`SELECT idempotency_key, producer_id, sequence_start, sequence_end,
|
|
1178
|
+
schema_version, checksum, accepted_events, committed_at_ms
|
|
1179
|
+
FROM ${table}
|
|
1180
|
+
WHERE target_id = ? AND producer_id = ?
|
|
1181
|
+
AND sequence_start <= ? AND sequence_end >= ?`
|
|
1182
|
+
).all(this.options.targetId, batch.producerId, batch.sequenceEnd, batch.sequenceStart);
|
|
1183
|
+
if (rows.length === 0) {
|
|
1184
|
+
return this.findPrunedReplayConflict(batch);
|
|
1185
|
+
}
|
|
1186
|
+
for (const row of rows) {
|
|
1187
|
+
const parsed = BatchAcknowledgementRowSchema.safeParse(row);
|
|
1188
|
+
if (!parsed.success) {
|
|
1189
|
+
throw new Error(parsed.error.message);
|
|
1190
|
+
}
|
|
1191
|
+
const identical = parsed.data.idempotency_key === batch.idempotencyKey && parsed.data.checksum === batch.checksum && parsed.data.sequence_start === batch.sequenceStart && parsed.data.sequence_end === batch.sequenceEnd;
|
|
1192
|
+
if (identical) {
|
|
1193
|
+
return {
|
|
1194
|
+
kind: "committed",
|
|
1195
|
+
idempotencyKey: batch.idempotencyKey,
|
|
1196
|
+
committedAtMs: parsed.data.committed_at_ms,
|
|
1197
|
+
acceptedEvents: parsed.data.accepted_events,
|
|
1198
|
+
duplicate: true,
|
|
1199
|
+
targetStates: [this.committedTargetState(0)]
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
return {
|
|
1204
|
+
kind: "conflict",
|
|
1205
|
+
idempotencyKey: batch.idempotencyKey,
|
|
1206
|
+
reason: "Producer sequence range overlaps a different committed batch",
|
|
1207
|
+
targetStates: [
|
|
1208
|
+
{
|
|
1209
|
+
targetId: this.options.targetId,
|
|
1210
|
+
delivery: "required",
|
|
1211
|
+
state: "failed",
|
|
1212
|
+
attempts: 1,
|
|
1213
|
+
reason: "sequence-range-conflict"
|
|
1214
|
+
}
|
|
1215
|
+
]
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
1218
|
+
outsideIngestionHorizon(batch) {
|
|
1219
|
+
const cutoffMs = this.options.now() - this.longestRetentionMs();
|
|
1220
|
+
const containsExpiredEvent = batch.envelopes.some(
|
|
1221
|
+
(envelope) => envelope.occurredAtMs < cutoffMs
|
|
1222
|
+
);
|
|
1223
|
+
if (!containsExpiredEvent) {
|
|
1224
|
+
return null;
|
|
1225
|
+
}
|
|
1226
|
+
return {
|
|
1227
|
+
kind: "rejected",
|
|
1228
|
+
idempotencyKey: batch.idempotencyKey,
|
|
1229
|
+
code: "invalid-batch",
|
|
1230
|
+
reason: "Telemetry batch is older than the configured ingestion horizon",
|
|
1231
|
+
targetStates: [
|
|
1232
|
+
{
|
|
1233
|
+
targetId: this.options.targetId,
|
|
1234
|
+
delivery: "required",
|
|
1235
|
+
state: "failed",
|
|
1236
|
+
attempts: 1,
|
|
1237
|
+
reason: "batch-outside-ingestion-horizon"
|
|
1238
|
+
}
|
|
1239
|
+
]
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
findPrunedReplayConflict(batch) {
|
|
1243
|
+
const table = this.options.tableMap.quoted("ingestWatermarks");
|
|
1244
|
+
const row = this.options.connection.prepare(
|
|
1245
|
+
`SELECT highest_sequence
|
|
1246
|
+
FROM ${table}
|
|
1247
|
+
WHERE target_id = ? AND producer_id = ?`
|
|
1248
|
+
).get(this.options.targetId, batch.producerId);
|
|
1249
|
+
if (row === void 0) {
|
|
1250
|
+
return null;
|
|
1251
|
+
}
|
|
1252
|
+
const highestSequence = this.readNumberField(row, "highest_sequence");
|
|
1253
|
+
if (batch.sequenceStart > highestSequence) {
|
|
1254
|
+
return null;
|
|
1255
|
+
}
|
|
1256
|
+
return {
|
|
1257
|
+
kind: "conflict",
|
|
1258
|
+
idempotencyKey: batch.idempotencyKey,
|
|
1259
|
+
reason: "Producer sequence range is at or below the durable ingest watermark",
|
|
1260
|
+
targetStates: [
|
|
1261
|
+
{
|
|
1262
|
+
targetId: this.options.targetId,
|
|
1263
|
+
delivery: "required",
|
|
1264
|
+
state: "failed",
|
|
1265
|
+
attempts: 1,
|
|
1266
|
+
reason: "sequence-at-or-below-watermark"
|
|
1267
|
+
}
|
|
1268
|
+
]
|
|
1269
|
+
};
|
|
1270
|
+
}
|
|
1271
|
+
persistTransaction(request) {
|
|
1272
|
+
const committedAtMs = this.options.now();
|
|
1273
|
+
const transaction = this.options.connection.transaction(() => {
|
|
1274
|
+
this.insertRawEvents(request, committedAtMs);
|
|
1275
|
+
this.insertMetricSamples(request, committedAtMs);
|
|
1276
|
+
this.insertExtractionFailures(request, committedAtMs);
|
|
1277
|
+
this.upsertCommittedDropCounters(request.batch, committedAtMs);
|
|
1278
|
+
this.insertBatchAcknowledgement(request.batch, committedAtMs);
|
|
1279
|
+
this.updateIngestWatermark(request.batch, committedAtMs);
|
|
1280
|
+
});
|
|
1281
|
+
transaction();
|
|
1282
|
+
return {
|
|
1283
|
+
kind: "committed",
|
|
1284
|
+
idempotencyKey: request.batch.idempotencyKey,
|
|
1285
|
+
committedAtMs,
|
|
1286
|
+
acceptedEvents: request.batch.envelopes.length,
|
|
1287
|
+
duplicate: false,
|
|
1288
|
+
targetStates: [this.committedTargetState(1)]
|
|
1289
|
+
};
|
|
1290
|
+
}
|
|
1291
|
+
insertRawEvents(request, insertedAtMs) {
|
|
1292
|
+
const table = this.options.tableMap.quoted("rawEvents");
|
|
1293
|
+
const insert = this.options.connection.prepare(
|
|
1294
|
+
`INSERT INTO ${table} (
|
|
1295
|
+
target_id, producer_id, sequence, event_id, batch_idempotency_key,
|
|
1296
|
+
occurred_at_ms, monotonic_ns, source, process_kind, level, label,
|
|
1297
|
+
conversation_id, conversation_id_kind, correlation_id, metadata_json,
|
|
1298
|
+
schema_version, inserted_at_ms
|
|
1299
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1300
|
+
);
|
|
1301
|
+
for (const envelope of request.batch.envelopes) {
|
|
1302
|
+
const conversationId = envelope.conversationId === null ? null : String(envelope.conversationId);
|
|
1303
|
+
const conversationKind = envelope.conversationId === null ? null : typeof envelope.conversationId;
|
|
1304
|
+
insert.run(
|
|
1305
|
+
this.options.targetId,
|
|
1306
|
+
envelope.producerId,
|
|
1307
|
+
envelope.sequence,
|
|
1308
|
+
envelope.eventId,
|
|
1309
|
+
request.batch.idempotencyKey,
|
|
1310
|
+
envelope.occurredAtMs,
|
|
1311
|
+
envelope.monotonicNs,
|
|
1312
|
+
envelope.source,
|
|
1313
|
+
envelope.processKind,
|
|
1314
|
+
envelope.level,
|
|
1315
|
+
envelope.label,
|
|
1316
|
+
conversationId,
|
|
1317
|
+
conversationKind,
|
|
1318
|
+
envelope.correlationId,
|
|
1319
|
+
canonicalJson(envelope.metadata),
|
|
1320
|
+
envelope.schemaVersion,
|
|
1321
|
+
insertedAtMs
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
insertMetricSamples(request, _insertedAtMs) {
|
|
1326
|
+
const samplesTable = this.options.tableMap.quoted("metricSamples");
|
|
1327
|
+
const completionTable = this.options.tableMap.quoted("rollupCompletions");
|
|
1328
|
+
const insert = this.options.connection.prepare(
|
|
1329
|
+
`INSERT INTO ${samplesTable} (
|
|
1330
|
+
target_id, metric_name, definition_version, occurred_at_ms,
|
|
1331
|
+
producer_id, sequence, dimension_key, dimensions_json, values_json
|
|
1332
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1333
|
+
);
|
|
1334
|
+
const invalidate = this.options.connection.prepare(
|
|
1335
|
+
`DELETE FROM ${completionTable}
|
|
1336
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1337
|
+
AND resolution_ms = ? AND bucket_start_ms = ?`
|
|
1338
|
+
);
|
|
1339
|
+
for (const sample of request.samples) {
|
|
1340
|
+
insert.run(
|
|
1341
|
+
this.options.targetId,
|
|
1342
|
+
sample.metricName,
|
|
1343
|
+
sample.definitionVersion,
|
|
1344
|
+
sample.occurredAtMs,
|
|
1345
|
+
sample.producerId,
|
|
1346
|
+
sample.sequence,
|
|
1347
|
+
sample.dimensionKey,
|
|
1348
|
+
canonicalJson(sample.dimensions),
|
|
1349
|
+
canonicalJson(sample.values)
|
|
1350
|
+
);
|
|
1351
|
+
const definition = this.definitions.get(
|
|
1352
|
+
this.definitionKey(sample.metricName, sample.definitionVersion)
|
|
1353
|
+
);
|
|
1354
|
+
if (!definition) {
|
|
1355
|
+
throw new Error(
|
|
1356
|
+
`Missing registered metric definition ${sample.metricName}:${sample.definitionVersion}`
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
for (const resolutionMs of definition.resolutionsMs) {
|
|
1360
|
+
const bucketStartMs = this.bucketStart(sample.occurredAtMs, resolutionMs);
|
|
1361
|
+
invalidate.run(
|
|
1362
|
+
this.options.targetId,
|
|
1363
|
+
sample.metricName,
|
|
1364
|
+
sample.definitionVersion,
|
|
1365
|
+
resolutionMs,
|
|
1366
|
+
bucketStartMs
|
|
1367
|
+
);
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
insertExtractionFailures(request, recordedAtMs) {
|
|
1372
|
+
const table = this.options.tableMap.quoted("extractionFailures");
|
|
1373
|
+
const insert = this.options.connection.prepare(
|
|
1374
|
+
`INSERT OR REPLACE INTO ${table} (
|
|
1375
|
+
target_id, metric_name, definition_version, producer_id,
|
|
1376
|
+
sequence, reason, recorded_at_ms
|
|
1377
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
1378
|
+
);
|
|
1379
|
+
for (const failure of request.extractionFailures) {
|
|
1380
|
+
insert.run(
|
|
1381
|
+
this.options.targetId,
|
|
1382
|
+
failure.metricName,
|
|
1383
|
+
failure.definitionVersion,
|
|
1384
|
+
failure.producerId,
|
|
1385
|
+
failure.sequence,
|
|
1386
|
+
failure.reason.slice(0, 1024),
|
|
1387
|
+
recordedAtMs
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
upsertCommittedDropCounters(batch, committedAtMs) {
|
|
1392
|
+
const table = this.options.tableMap.quoted("committedDropCounters");
|
|
1393
|
+
const upsert = this.options.connection.prepare(
|
|
1394
|
+
`INSERT INTO ${table} (
|
|
1395
|
+
target_id, reason, priority, count, first_committed_at_ms, last_committed_at_ms
|
|
1396
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
1397
|
+
ON CONFLICT(target_id, reason, priority) DO UPDATE SET
|
|
1398
|
+
count = MIN(9007199254740991, ${table}.count + excluded.count),
|
|
1399
|
+
first_committed_at_ms = MIN(
|
|
1400
|
+
${table}.first_committed_at_ms,
|
|
1401
|
+
excluded.first_committed_at_ms
|
|
1402
|
+
),
|
|
1403
|
+
last_committed_at_ms = MAX(
|
|
1404
|
+
${table}.last_committed_at_ms,
|
|
1405
|
+
excluded.last_committed_at_ms
|
|
1406
|
+
)`
|
|
1407
|
+
);
|
|
1408
|
+
for (const counter of batch.dropCounters) {
|
|
1409
|
+
if (!Number.isSafeInteger(counter.count) || counter.count <= 0) {
|
|
1410
|
+
throw new Error("Committed telemetry drop counters must be positive safe integers");
|
|
1411
|
+
}
|
|
1412
|
+
upsert.run(
|
|
1413
|
+
this.options.targetId,
|
|
1414
|
+
counter.reason,
|
|
1415
|
+
counter.priority,
|
|
1416
|
+
counter.count,
|
|
1417
|
+
committedAtMs,
|
|
1418
|
+
committedAtMs
|
|
1419
|
+
);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
insertBatchAcknowledgement(batch, committedAtMs) {
|
|
1423
|
+
const table = this.options.tableMap.quoted("batchAcknowledgements");
|
|
1424
|
+
this.options.connection.prepare(
|
|
1425
|
+
`INSERT INTO ${table} (
|
|
1426
|
+
target_id, idempotency_key, producer_id, sequence_start, sequence_end,
|
|
1427
|
+
schema_version, checksum, accepted_events, committed_at_ms
|
|
1428
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1429
|
+
).run(
|
|
1430
|
+
this.options.targetId,
|
|
1431
|
+
batch.idempotencyKey,
|
|
1432
|
+
batch.producerId,
|
|
1433
|
+
batch.sequenceStart,
|
|
1434
|
+
batch.sequenceEnd,
|
|
1435
|
+
batch.schemaVersion,
|
|
1436
|
+
batch.checksum,
|
|
1437
|
+
batch.envelopes.length,
|
|
1438
|
+
committedAtMs
|
|
1439
|
+
);
|
|
1440
|
+
}
|
|
1441
|
+
updateIngestWatermark(batch, committedAtMs) {
|
|
1442
|
+
const table = this.options.tableMap.quoted("ingestWatermarks");
|
|
1443
|
+
const lastOccurredAtMs = Math.max(...batch.envelopes.map((envelope) => envelope.occurredAtMs));
|
|
1444
|
+
const internalGapCount = batch.sequenceEnd - batch.sequenceStart + 1 - batch.envelopes.length;
|
|
1445
|
+
const initialGapCount = batch.sequenceStart + internalGapCount;
|
|
1446
|
+
this.options.connection.prepare(
|
|
1447
|
+
`INSERT INTO ${table} (
|
|
1448
|
+
target_id, producer_id, highest_sequence, last_occurred_at_ms,
|
|
1449
|
+
last_committed_at_ms, observed_gap_count
|
|
1450
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
1451
|
+
ON CONFLICT(target_id, producer_id) DO UPDATE SET
|
|
1452
|
+
observed_gap_count = ${table}.observed_gap_count
|
|
1453
|
+
+ MAX(0, ? - ${table}.highest_sequence - 1)
|
|
1454
|
+
+ ?,
|
|
1455
|
+
highest_sequence = MAX(${table}.highest_sequence, excluded.highest_sequence),
|
|
1456
|
+
last_occurred_at_ms = MAX(${table}.last_occurred_at_ms, excluded.last_occurred_at_ms),
|
|
1457
|
+
last_committed_at_ms = excluded.last_committed_at_ms`
|
|
1458
|
+
).run(
|
|
1459
|
+
this.options.targetId,
|
|
1460
|
+
batch.producerId,
|
|
1461
|
+
batch.sequenceEnd,
|
|
1462
|
+
lastOccurredAtMs,
|
|
1463
|
+
committedAtMs,
|
|
1464
|
+
initialGapCount,
|
|
1465
|
+
batch.sequenceStart,
|
|
1466
|
+
internalGapCount
|
|
1467
|
+
);
|
|
1468
|
+
}
|
|
1469
|
+
addTimelineFilters(query, clauses, params) {
|
|
1470
|
+
const filters = [
|
|
1471
|
+
[
|
|
1472
|
+
query.conversationId,
|
|
1473
|
+
"conversation_id = ?",
|
|
1474
|
+
query.conversationId === null ? null : String(query.conversationId)
|
|
1475
|
+
],
|
|
1476
|
+
[query.source, "source = ?", query.source],
|
|
1477
|
+
[query.label, "label = ?", query.label],
|
|
1478
|
+
[query.level, "level = ?", query.level],
|
|
1479
|
+
[query.producerId, "producer_id = ?", query.producerId],
|
|
1480
|
+
[query.correlationId, "correlation_id = ?", query.correlationId]
|
|
1481
|
+
];
|
|
1482
|
+
for (const [guard, clause, parameter] of filters) {
|
|
1483
|
+
if (guard !== null) {
|
|
1484
|
+
clauses.push(clause);
|
|
1485
|
+
params.push(parameter);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
addTimelineCursor(cursor, order, clauses, params) {
|
|
1490
|
+
const comparison = order === "asc" ? ">" : "<";
|
|
1491
|
+
clauses.push(`(
|
|
1492
|
+
occurred_at_ms ${comparison} ?
|
|
1493
|
+
OR (occurred_at_ms = ? AND producer_id ${comparison} ?)
|
|
1494
|
+
OR (occurred_at_ms = ? AND producer_id = ? AND sequence ${comparison} ?)
|
|
1495
|
+
)`);
|
|
1496
|
+
params.push(
|
|
1497
|
+
cursor.occurredAtMs,
|
|
1498
|
+
cursor.occurredAtMs,
|
|
1499
|
+
cursor.producerId,
|
|
1500
|
+
cursor.occurredAtMs,
|
|
1501
|
+
cursor.producerId,
|
|
1502
|
+
cursor.sequence
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1505
|
+
findDueBucketStarts(definition, resolutionMs, dueBeforeMs) {
|
|
1506
|
+
const sampleTable = this.options.tableMap.quoted("metricSamples");
|
|
1507
|
+
const completionTable = this.options.tableMap.quoted("rollupCompletions");
|
|
1508
|
+
const rows = this.options.connection.prepare(
|
|
1509
|
+
`WITH sample_buckets AS (
|
|
1510
|
+
SELECT
|
|
1511
|
+
CAST(occurred_at_ms / ? AS INTEGER) * ? AS bucket_start_ms,
|
|
1512
|
+
COUNT(*) AS sample_count,
|
|
1513
|
+
MAX(occurred_at_ms) AS sample_watermark_ms
|
|
1514
|
+
FROM ${sampleTable}
|
|
1515
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1516
|
+
AND occurred_at_ms < ?
|
|
1517
|
+
AND (CAST(occurred_at_ms / ? AS INTEGER) * ?) + ? <= ?
|
|
1518
|
+
GROUP BY bucket_start_ms
|
|
1519
|
+
)
|
|
1520
|
+
SELECT bucket.bucket_start_ms
|
|
1521
|
+
FROM sample_buckets AS bucket
|
|
1522
|
+
LEFT JOIN ${completionTable} AS completion
|
|
1523
|
+
ON completion.target_id = ?
|
|
1524
|
+
AND completion.metric_name = ?
|
|
1525
|
+
AND completion.definition_version = ?
|
|
1526
|
+
AND completion.resolution_ms = ?
|
|
1527
|
+
AND completion.bucket_start_ms = bucket.bucket_start_ms
|
|
1528
|
+
WHERE completion.bucket_start_ms IS NULL
|
|
1529
|
+
OR completion.sample_count <> bucket.sample_count
|
|
1530
|
+
OR completion.sample_watermark_ms <> bucket.sample_watermark_ms
|
|
1531
|
+
ORDER BY bucket.bucket_start_ms ASC
|
|
1532
|
+
LIMIT 256`
|
|
1533
|
+
).all(
|
|
1534
|
+
resolutionMs,
|
|
1535
|
+
resolutionMs,
|
|
1536
|
+
this.options.targetId,
|
|
1537
|
+
definition.name,
|
|
1538
|
+
definition.version,
|
|
1539
|
+
dueBeforeMs,
|
|
1540
|
+
resolutionMs,
|
|
1541
|
+
resolutionMs,
|
|
1542
|
+
resolutionMs,
|
|
1543
|
+
dueBeforeMs,
|
|
1544
|
+
this.options.targetId,
|
|
1545
|
+
definition.name,
|
|
1546
|
+
definition.version,
|
|
1547
|
+
resolutionMs
|
|
1548
|
+
);
|
|
1549
|
+
return rows.map((row) => this.readNumberField(row, "bucket_start_ms"));
|
|
1550
|
+
}
|
|
1551
|
+
loadBucketWork(definition, resolutionMs, bucketStartMs) {
|
|
1552
|
+
const table = this.options.tableMap.quoted("metricSamples");
|
|
1553
|
+
const rows = this.options.connection.prepare(
|
|
1554
|
+
`SELECT metric_name, definition_version, occurred_at_ms, producer_id,
|
|
1555
|
+
sequence, dimension_key, dimensions_json, values_json
|
|
1556
|
+
FROM ${table}
|
|
1557
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1558
|
+
AND occurred_at_ms >= ? AND occurred_at_ms < ?
|
|
1559
|
+
ORDER BY occurred_at_ms ASC, producer_id ASC, sequence ASC`
|
|
1560
|
+
).all(
|
|
1561
|
+
this.options.targetId,
|
|
1562
|
+
definition.name,
|
|
1563
|
+
definition.version,
|
|
1564
|
+
bucketStartMs,
|
|
1565
|
+
bucketStartMs + resolutionMs
|
|
1566
|
+
);
|
|
1567
|
+
const samples = rows.map((row) => {
|
|
1568
|
+
const parsed = this.rowParser.parseMetricSample(row);
|
|
1569
|
+
if (!parsed.ok) {
|
|
1570
|
+
throw new Error(parsed.error);
|
|
1571
|
+
}
|
|
1572
|
+
return parsed.value;
|
|
1573
|
+
});
|
|
1574
|
+
return { definition, resolutionMs, bucketStartMs, samples };
|
|
1575
|
+
}
|
|
1576
|
+
bucketIsCurrent(work) {
|
|
1577
|
+
const table = this.options.tableMap.quoted("rollupCompletions");
|
|
1578
|
+
const row = this.options.connection.prepare(
|
|
1579
|
+
`SELECT sample_count, sample_watermark_ms
|
|
1580
|
+
FROM ${table}
|
|
1581
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1582
|
+
AND resolution_ms = ? AND bucket_start_ms = ?`
|
|
1583
|
+
).get(
|
|
1584
|
+
this.options.targetId,
|
|
1585
|
+
work.definition.name,
|
|
1586
|
+
work.definition.version,
|
|
1587
|
+
work.resolutionMs,
|
|
1588
|
+
work.bucketStartMs
|
|
1589
|
+
);
|
|
1590
|
+
if (row === void 0) {
|
|
1591
|
+
return false;
|
|
1592
|
+
}
|
|
1593
|
+
const sampleCount = this.readNumberField(row, "sample_count");
|
|
1594
|
+
const watermark = this.readNumberField(row, "sample_watermark_ms");
|
|
1595
|
+
return sampleCount === work.samples.length && watermark === this.sampleWatermark(work);
|
|
1596
|
+
}
|
|
1597
|
+
sampleWatermark(work) {
|
|
1598
|
+
return work.samples.length === 0 ? work.bucketStartMs : Math.max(...work.samples.map((sample) => sample.occurred_at_ms));
|
|
1599
|
+
}
|
|
1600
|
+
materializeBucket(work, materializedAtMs) {
|
|
1601
|
+
const normalized = this.normalizeSamplesForCardinality(work);
|
|
1602
|
+
const groups = this.aggregateSamples(work.definition.measurements, normalized);
|
|
1603
|
+
const rollupTable = this.options.tableMap.quoted("metricRollups");
|
|
1604
|
+
const completionTable = this.options.tableMap.quoted("rollupCompletions");
|
|
1605
|
+
const transaction = this.options.connection.transaction(() => {
|
|
1606
|
+
this.options.connection.prepare(
|
|
1607
|
+
`DELETE FROM ${rollupTable}
|
|
1608
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1609
|
+
AND resolution_ms = ? AND bucket_start_ms = ?`
|
|
1610
|
+
).run(
|
|
1611
|
+
this.options.targetId,
|
|
1612
|
+
work.definition.name,
|
|
1613
|
+
work.definition.version,
|
|
1614
|
+
work.resolutionMs,
|
|
1615
|
+
work.bucketStartMs
|
|
1616
|
+
);
|
|
1617
|
+
const insert = this.options.connection.prepare(
|
|
1618
|
+
`INSERT INTO ${rollupTable} (
|
|
1619
|
+
target_id, metric_name, definition_version, resolution_ms,
|
|
1620
|
+
bucket_start_ms, dimension_key, dimensions_json, values_json,
|
|
1621
|
+
materialized_at_ms
|
|
1622
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1623
|
+
);
|
|
1624
|
+
for (const [dimensionKey, group] of groups) {
|
|
1625
|
+
insert.run(
|
|
1626
|
+
this.options.targetId,
|
|
1627
|
+
work.definition.name,
|
|
1628
|
+
work.definition.version,
|
|
1629
|
+
work.resolutionMs,
|
|
1630
|
+
work.bucketStartMs,
|
|
1631
|
+
dimensionKey,
|
|
1632
|
+
group.dimensionsJson,
|
|
1633
|
+
canonicalJson(group.values),
|
|
1634
|
+
materializedAtMs
|
|
1635
|
+
);
|
|
1636
|
+
}
|
|
1637
|
+
this.options.connection.prepare(
|
|
1638
|
+
`INSERT INTO ${completionTable} (
|
|
1639
|
+
target_id, metric_name, definition_version, resolution_ms,
|
|
1640
|
+
bucket_start_ms, sample_count, sample_watermark_ms, completed_at_ms,
|
|
1641
|
+
state, detail
|
|
1642
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'completed', NULL)
|
|
1643
|
+
ON CONFLICT(
|
|
1644
|
+
target_id, metric_name, definition_version, resolution_ms, bucket_start_ms
|
|
1645
|
+
) DO UPDATE SET
|
|
1646
|
+
sample_count = excluded.sample_count,
|
|
1647
|
+
sample_watermark_ms = excluded.sample_watermark_ms,
|
|
1648
|
+
completed_at_ms = excluded.completed_at_ms,
|
|
1649
|
+
state = 'completed',
|
|
1650
|
+
detail = NULL`
|
|
1651
|
+
).run(
|
|
1652
|
+
this.options.targetId,
|
|
1653
|
+
work.definition.name,
|
|
1654
|
+
work.definition.version,
|
|
1655
|
+
work.resolutionMs,
|
|
1656
|
+
work.bucketStartMs,
|
|
1657
|
+
work.samples.length,
|
|
1658
|
+
this.sampleWatermark(work),
|
|
1659
|
+
materializedAtMs
|
|
1660
|
+
);
|
|
1661
|
+
});
|
|
1662
|
+
transaction();
|
|
1663
|
+
return groups.size;
|
|
1664
|
+
}
|
|
1665
|
+
recordRejectedBucket(work, recordedAtMs, detail) {
|
|
1666
|
+
const rollupTable = this.options.tableMap.quoted("metricRollups");
|
|
1667
|
+
const completionTable = this.options.tableMap.quoted("rollupCompletions");
|
|
1668
|
+
const transaction = this.options.connection.transaction(() => {
|
|
1669
|
+
this.options.connection.prepare(
|
|
1670
|
+
`DELETE FROM ${rollupTable}
|
|
1671
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1672
|
+
AND resolution_ms = ? AND bucket_start_ms = ?`
|
|
1673
|
+
).run(
|
|
1674
|
+
this.options.targetId,
|
|
1675
|
+
work.definition.name,
|
|
1676
|
+
work.definition.version,
|
|
1677
|
+
work.resolutionMs,
|
|
1678
|
+
work.bucketStartMs
|
|
1679
|
+
);
|
|
1680
|
+
this.options.connection.prepare(
|
|
1681
|
+
`INSERT INTO ${completionTable} (
|
|
1682
|
+
target_id, metric_name, definition_version, resolution_ms,
|
|
1683
|
+
bucket_start_ms, sample_count, sample_watermark_ms, completed_at_ms,
|
|
1684
|
+
state, detail
|
|
1685
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'rejected', ?)
|
|
1686
|
+
ON CONFLICT(
|
|
1687
|
+
target_id, metric_name, definition_version, resolution_ms, bucket_start_ms
|
|
1688
|
+
) DO UPDATE SET
|
|
1689
|
+
sample_count = excluded.sample_count,
|
|
1690
|
+
sample_watermark_ms = excluded.sample_watermark_ms,
|
|
1691
|
+
completed_at_ms = excluded.completed_at_ms,
|
|
1692
|
+
state = 'rejected',
|
|
1693
|
+
detail = excluded.detail`
|
|
1694
|
+
).run(
|
|
1695
|
+
this.options.targetId,
|
|
1696
|
+
work.definition.name,
|
|
1697
|
+
work.definition.version,
|
|
1698
|
+
work.resolutionMs,
|
|
1699
|
+
work.bucketStartMs,
|
|
1700
|
+
work.samples.length,
|
|
1701
|
+
this.sampleWatermark(work),
|
|
1702
|
+
recordedAtMs,
|
|
1703
|
+
detail.slice(0, 1024)
|
|
1704
|
+
);
|
|
1705
|
+
});
|
|
1706
|
+
transaction();
|
|
1707
|
+
}
|
|
1708
|
+
rollupStageDetail(rejectedBucketCount, examples, workBudgetReached) {
|
|
1709
|
+
const rejected = rejectedBucketCount === 0 ? null : `cardinality-rejected-buckets=${rejectedBucketCount}; ${examples.join("; ")}`;
|
|
1710
|
+
const detail = workBudgetReached ? ["work-budget-reached", rejected].filter((value) => value !== null).join("; ") : rejected;
|
|
1711
|
+
return detail?.slice(0, 2048) ?? null;
|
|
1712
|
+
}
|
|
1713
|
+
normalizeSamplesForCardinality(work) {
|
|
1714
|
+
const admittedByField = /* @__PURE__ */ new Map();
|
|
1715
|
+
const normalized = [];
|
|
1716
|
+
for (const sample of work.samples) {
|
|
1717
|
+
const dimensions = this.rowParser.parseJsonObject(sample.dimensions_json);
|
|
1718
|
+
const values = this.rowParser.parseNumericValues(sample.values_json);
|
|
1719
|
+
if (!dimensions.ok) {
|
|
1720
|
+
throw new Error(dimensions.error);
|
|
1721
|
+
}
|
|
1722
|
+
if (!values.ok) {
|
|
1723
|
+
throw new Error(values.error);
|
|
1724
|
+
}
|
|
1725
|
+
const admittedDimensions = { ...dimensions.value };
|
|
1726
|
+
let dropSample = false;
|
|
1727
|
+
for (const field of work.definition.groupBy) {
|
|
1728
|
+
const rule = work.definition.cardinality[field];
|
|
1729
|
+
if (!rule) {
|
|
1730
|
+
throw new Error(`Missing cardinality rule for ${field}`);
|
|
1731
|
+
}
|
|
1732
|
+
const dimensionValue = admittedDimensions[field];
|
|
1733
|
+
const valueKey = canonicalJson(dimensionValue);
|
|
1734
|
+
const admittedValues = admittedByField.get(field) ?? /* @__PURE__ */ new Set();
|
|
1735
|
+
if (!admittedValues.has(valueKey) && admittedValues.size >= rule.maxValuesPerBucket) {
|
|
1736
|
+
if (rule.overflow === "other") {
|
|
1737
|
+
admittedDimensions[field] = "__telemetry_other__";
|
|
1738
|
+
} else if (rule.overflow === "reject") {
|
|
1739
|
+
throw new MetricCardinalityRejectionError(
|
|
1740
|
+
`Metric ${work.definition.name} rejected cardinality overflow for ${field}`
|
|
1741
|
+
);
|
|
1742
|
+
} else {
|
|
1743
|
+
dropSample = true;
|
|
1744
|
+
break;
|
|
1745
|
+
}
|
|
1746
|
+
} else {
|
|
1747
|
+
admittedValues.add(valueKey);
|
|
1748
|
+
admittedByField.set(field, admittedValues);
|
|
1749
|
+
}
|
|
1750
|
+
}
|
|
1751
|
+
if (dropSample) {
|
|
1752
|
+
continue;
|
|
1753
|
+
}
|
|
1754
|
+
const dimensionsJson = canonicalJson(admittedDimensions);
|
|
1755
|
+
normalized.push({
|
|
1756
|
+
dimensionKey: dimensionsJson,
|
|
1757
|
+
dimensionsJson,
|
|
1758
|
+
values: values.value,
|
|
1759
|
+
occurredAtMs: sample.occurred_at_ms,
|
|
1760
|
+
producerId: sample.producer_id,
|
|
1761
|
+
sequence: sample.sequence
|
|
1762
|
+
});
|
|
1763
|
+
}
|
|
1764
|
+
return normalized;
|
|
1765
|
+
}
|
|
1766
|
+
aggregateSamples(measurements, samples) {
|
|
1767
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1768
|
+
const fields = measurements.filter((measurement) => {
|
|
1769
|
+
return measurement.kind === "field";
|
|
1770
|
+
}).map((measurement) => measurement.field);
|
|
1771
|
+
for (const sample of samples) {
|
|
1772
|
+
const group = groups.get(sample.dimensionKey) ?? {
|
|
1773
|
+
dimensionsJson: sample.dimensionsJson,
|
|
1774
|
+
values: { count: 0, fields: {} }
|
|
1775
|
+
};
|
|
1776
|
+
group.values.count += 1;
|
|
1777
|
+
for (const field of fields) {
|
|
1778
|
+
const measured = sample.values[field];
|
|
1779
|
+
if (measured === void 0) {
|
|
1780
|
+
continue;
|
|
1781
|
+
}
|
|
1782
|
+
const aggregate = group.values.fields[field] ?? this.newFieldAggregate(measured, sample);
|
|
1783
|
+
aggregate.count += group.values.fields[field] === void 0 ? 0 : 1;
|
|
1784
|
+
aggregate.sum += group.values.fields[field] === void 0 ? 0 : measured;
|
|
1785
|
+
aggregate.min = Math.min(aggregate.min, measured);
|
|
1786
|
+
aggregate.max = Math.max(aggregate.max, measured);
|
|
1787
|
+
aggregate.last = measured;
|
|
1788
|
+
aggregate.lastOccurredAtMs = sample.occurredAtMs;
|
|
1789
|
+
aggregate.lastProducerId = sample.producerId;
|
|
1790
|
+
aggregate.lastSequence = sample.sequence;
|
|
1791
|
+
group.values.fields[field] = aggregate;
|
|
1792
|
+
}
|
|
1793
|
+
groups.set(sample.dimensionKey, group);
|
|
1794
|
+
}
|
|
1795
|
+
return groups;
|
|
1796
|
+
}
|
|
1797
|
+
newFieldAggregate(measured, sample) {
|
|
1798
|
+
return {
|
|
1799
|
+
count: 1,
|
|
1800
|
+
sum: measured,
|
|
1801
|
+
min: measured,
|
|
1802
|
+
max: measured,
|
|
1803
|
+
last: measured,
|
|
1804
|
+
lastOccurredAtMs: sample.occurredAtMs,
|
|
1805
|
+
lastProducerId: sample.producerId,
|
|
1806
|
+
lastSequence: sample.sequence
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1809
|
+
createRawEventRetentionStep(nowMs) {
|
|
1810
|
+
const cutoffMs = nowMs - this.options.retention.rawMs;
|
|
1811
|
+
const table = this.options.tableMap.quoted("rawEvents");
|
|
1812
|
+
const readFirstPage = this.options.connection.prepare(
|
|
1813
|
+
`SELECT producer_id, sequence, event_id, occurred_at_ms, monotonic_ns,
|
|
1814
|
+
source, process_kind, level, label, conversation_id,
|
|
1815
|
+
conversation_id_kind, correlation_id, metadata_json, schema_version
|
|
1816
|
+
FROM ${table}
|
|
1817
|
+
WHERE target_id = ? AND occurred_at_ms < ?
|
|
1818
|
+
ORDER BY occurred_at_ms ASC, producer_id ASC, sequence ASC
|
|
1819
|
+
LIMIT ${RETENTION_PAGE_SIZE}`
|
|
1820
|
+
);
|
|
1821
|
+
const readNextPage = this.options.connection.prepare(
|
|
1822
|
+
`SELECT producer_id, sequence, event_id, occurred_at_ms, monotonic_ns,
|
|
1823
|
+
source, process_kind, level, label, conversation_id,
|
|
1824
|
+
conversation_id_kind, correlation_id, metadata_json, schema_version
|
|
1825
|
+
FROM ${table}
|
|
1826
|
+
WHERE target_id = ? AND occurred_at_ms < ?
|
|
1827
|
+
AND (occurred_at_ms, producer_id, sequence) > (?, ?, ?)
|
|
1828
|
+
ORDER BY occurred_at_ms ASC, producer_id ASC, sequence ASC
|
|
1829
|
+
LIMIT ${RETENTION_PAGE_SIZE}`
|
|
1830
|
+
);
|
|
1831
|
+
const remove = this.options.connection.prepare(
|
|
1832
|
+
`DELETE FROM ${table} WHERE target_id = ? AND producer_id = ? AND sequence = ?`
|
|
1833
|
+
);
|
|
1834
|
+
let cursor = null;
|
|
1835
|
+
return () => {
|
|
1836
|
+
const rows = cursor === null ? readFirstPage.all(this.options.targetId, cutoffMs) : readNextPage.all(
|
|
1837
|
+
this.options.targetId,
|
|
1838
|
+
cutoffMs,
|
|
1839
|
+
cursor.occurredAtMs,
|
|
1840
|
+
cursor.producerId,
|
|
1841
|
+
cursor.sequence
|
|
1842
|
+
);
|
|
1843
|
+
if (rows.length === 0) {
|
|
1844
|
+
return { deleted: 0, done: true };
|
|
1845
|
+
}
|
|
1846
|
+
const scanned = rows.map((row) => {
|
|
1847
|
+
const parsed = this.rowParser.parseRawEnvelope(row);
|
|
1848
|
+
if (!parsed.ok) {
|
|
1849
|
+
throw new Error(parsed.error);
|
|
1850
|
+
}
|
|
1851
|
+
return parsed.value;
|
|
1852
|
+
});
|
|
1853
|
+
const eligible = scanned.filter((envelope) => this.rawEventHasRequiredRollups(envelope));
|
|
1854
|
+
const transaction = this.options.connection.transaction(() => {
|
|
1855
|
+
let deleted2 = 0;
|
|
1856
|
+
for (const envelope of eligible) {
|
|
1857
|
+
deleted2 += remove.run(
|
|
1858
|
+
this.options.targetId,
|
|
1859
|
+
envelope.producerId,
|
|
1860
|
+
envelope.sequence
|
|
1861
|
+
).changes;
|
|
1862
|
+
}
|
|
1863
|
+
return deleted2;
|
|
1864
|
+
});
|
|
1865
|
+
const deleted = transaction();
|
|
1866
|
+
const last = scanned[scanned.length - 1];
|
|
1867
|
+
if (last === void 0 || rows.length < RETENTION_PAGE_SIZE) {
|
|
1868
|
+
return { deleted, done: true };
|
|
1869
|
+
}
|
|
1870
|
+
cursor = {
|
|
1871
|
+
occurredAtMs: last.occurredAtMs,
|
|
1872
|
+
producerId: last.producerId,
|
|
1873
|
+
sequence: last.sequence
|
|
1874
|
+
};
|
|
1875
|
+
return { deleted, done: false };
|
|
1876
|
+
};
|
|
1877
|
+
}
|
|
1878
|
+
rawEventHasRequiredRollups(envelope) {
|
|
1879
|
+
const applicable = [...this.definitions.values()].filter((definition) => {
|
|
1880
|
+
return this.definitionContract.matches(definition.selector, envelope.label);
|
|
1881
|
+
});
|
|
1882
|
+
return applicable.every((definition) => {
|
|
1883
|
+
if (!this.metricSampleExists(definition, envelope)) {
|
|
1884
|
+
return true;
|
|
1885
|
+
}
|
|
1886
|
+
return definition.resolutionsMs.every((resolutionMs) => {
|
|
1887
|
+
return this.hasCompletion(
|
|
1888
|
+
definition.name,
|
|
1889
|
+
definition.version,
|
|
1890
|
+
resolutionMs,
|
|
1891
|
+
this.bucketStart(envelope.occurredAtMs, resolutionMs)
|
|
1892
|
+
);
|
|
1893
|
+
});
|
|
1894
|
+
});
|
|
1895
|
+
}
|
|
1896
|
+
metricSampleExists(definition, envelope) {
|
|
1897
|
+
const table = this.options.tableMap.quoted("metricSamples");
|
|
1898
|
+
const row = this.options.connection.prepare(
|
|
1899
|
+
`SELECT 1 AS present
|
|
1900
|
+
FROM ${table}
|
|
1901
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1902
|
+
AND producer_id = ? AND sequence = ?`
|
|
1903
|
+
).get(
|
|
1904
|
+
this.options.targetId,
|
|
1905
|
+
definition.name,
|
|
1906
|
+
definition.version,
|
|
1907
|
+
envelope.producerId,
|
|
1908
|
+
envelope.sequence
|
|
1909
|
+
);
|
|
1910
|
+
return row !== void 0;
|
|
1911
|
+
}
|
|
1912
|
+
createSampleRetentionStep(nowMs) {
|
|
1913
|
+
const cutoffMs = nowMs - this.options.retention.samplesMs;
|
|
1914
|
+
const table = this.options.tableMap.quoted("metricSamples");
|
|
1915
|
+
const readFirstPage = this.options.connection.prepare(
|
|
1916
|
+
`SELECT metric_name, definition_version, occurred_at_ms, producer_id,
|
|
1917
|
+
sequence, dimension_key, dimensions_json, values_json
|
|
1918
|
+
FROM ${table}
|
|
1919
|
+
WHERE target_id = ? AND occurred_at_ms < ?
|
|
1920
|
+
ORDER BY occurred_at_ms ASC, metric_name ASC, definition_version ASC,
|
|
1921
|
+
producer_id ASC, sequence ASC
|
|
1922
|
+
LIMIT ${RETENTION_PAGE_SIZE}`
|
|
1923
|
+
);
|
|
1924
|
+
const readNextPage = this.options.connection.prepare(
|
|
1925
|
+
`SELECT metric_name, definition_version, occurred_at_ms, producer_id,
|
|
1926
|
+
sequence, dimension_key, dimensions_json, values_json
|
|
1927
|
+
FROM ${table}
|
|
1928
|
+
WHERE target_id = ? AND occurred_at_ms < ?
|
|
1929
|
+
AND (
|
|
1930
|
+
occurred_at_ms, metric_name, definition_version, producer_id, sequence
|
|
1931
|
+
) > (?, ?, ?, ?, ?)
|
|
1932
|
+
ORDER BY occurred_at_ms ASC, metric_name ASC, definition_version ASC,
|
|
1933
|
+
producer_id ASC, sequence ASC
|
|
1934
|
+
LIMIT ${RETENTION_PAGE_SIZE}`
|
|
1935
|
+
);
|
|
1936
|
+
const remove = this.options.connection.prepare(
|
|
1937
|
+
`DELETE FROM ${table}
|
|
1938
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
1939
|
+
AND producer_id = ? AND sequence = ?`
|
|
1940
|
+
);
|
|
1941
|
+
let cursor = null;
|
|
1942
|
+
return () => {
|
|
1943
|
+
const rows = cursor === null ? readFirstPage.all(this.options.targetId, cutoffMs) : readNextPage.all(
|
|
1944
|
+
this.options.targetId,
|
|
1945
|
+
cutoffMs,
|
|
1946
|
+
cursor.occurredAtMs,
|
|
1947
|
+
cursor.metricName,
|
|
1948
|
+
cursor.definitionVersion,
|
|
1949
|
+
cursor.producerId,
|
|
1950
|
+
cursor.sequence
|
|
1951
|
+
);
|
|
1952
|
+
if (rows.length === 0) {
|
|
1953
|
+
return { deleted: 0, done: true };
|
|
1954
|
+
}
|
|
1955
|
+
const samples = rows.map((row) => {
|
|
1956
|
+
const parsed = this.rowParser.parseMetricSample(row);
|
|
1957
|
+
if (!parsed.ok) {
|
|
1958
|
+
throw new Error(parsed.error);
|
|
1959
|
+
}
|
|
1960
|
+
return parsed.value;
|
|
1961
|
+
});
|
|
1962
|
+
const transaction = this.options.connection.transaction(() => {
|
|
1963
|
+
let deleted2 = 0;
|
|
1964
|
+
for (const sample of samples) {
|
|
1965
|
+
const definition = this.definitions.get(
|
|
1966
|
+
this.definitionKey(sample.metric_name, sample.definition_version)
|
|
1967
|
+
);
|
|
1968
|
+
if (!definition) {
|
|
1969
|
+
continue;
|
|
1970
|
+
}
|
|
1971
|
+
const complete = definition.resolutionsMs.every((resolutionMs) => {
|
|
1972
|
+
return this.hasCompletion(
|
|
1973
|
+
definition.name,
|
|
1974
|
+
definition.version,
|
|
1975
|
+
resolutionMs,
|
|
1976
|
+
this.bucketStart(sample.occurred_at_ms, resolutionMs)
|
|
1977
|
+
);
|
|
1978
|
+
});
|
|
1979
|
+
if (complete) {
|
|
1980
|
+
deleted2 += remove.run(
|
|
1981
|
+
this.options.targetId,
|
|
1982
|
+
sample.metric_name,
|
|
1983
|
+
sample.definition_version,
|
|
1984
|
+
sample.producer_id,
|
|
1985
|
+
sample.sequence
|
|
1986
|
+
).changes;
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
return deleted2;
|
|
1990
|
+
});
|
|
1991
|
+
const deleted = transaction();
|
|
1992
|
+
const last = samples[samples.length - 1];
|
|
1993
|
+
if (last === void 0 || rows.length < RETENTION_PAGE_SIZE) {
|
|
1994
|
+
return { deleted, done: true };
|
|
1995
|
+
}
|
|
1996
|
+
cursor = {
|
|
1997
|
+
occurredAtMs: last.occurred_at_ms,
|
|
1998
|
+
metricName: last.metric_name,
|
|
1999
|
+
definitionVersion: last.definition_version,
|
|
2000
|
+
producerId: last.producer_id,
|
|
2001
|
+
sequence: last.sequence
|
|
2002
|
+
};
|
|
2003
|
+
return { deleted, done: false };
|
|
2004
|
+
};
|
|
2005
|
+
}
|
|
2006
|
+
createRollupRetentionSteps(nowMs) {
|
|
2007
|
+
const rollupTable = this.options.tableMap.quoted("metricRollups");
|
|
2008
|
+
const completionTable = this.options.tableMap.quoted("rollupCompletions");
|
|
2009
|
+
const steps = [];
|
|
2010
|
+
for (const tier of this.options.retention.rollups) {
|
|
2011
|
+
const cutoffMs = nowMs - tier.retainMs;
|
|
2012
|
+
steps.push(this.createExpiredRollupStep(rollupTable, tier.resolutionMs, cutoffMs, false));
|
|
2013
|
+
steps.push(this.createExpiredRollupStep(completionTable, tier.resolutionMs, cutoffMs, true));
|
|
2014
|
+
}
|
|
2015
|
+
return steps;
|
|
2016
|
+
}
|
|
2017
|
+
createExpiredRollupStep(table, resolutionMs, cutoffMs, completedOnly) {
|
|
2018
|
+
const statePredicate = completedOnly ? " AND state = 'completed'" : "";
|
|
2019
|
+
const remove = this.options.connection.prepare(
|
|
2020
|
+
`DELETE FROM ${table}
|
|
2021
|
+
WHERE rowid IN (
|
|
2022
|
+
SELECT rowid FROM ${table}
|
|
2023
|
+
WHERE target_id = ? AND resolution_ms = ? AND bucket_start_ms < ?${statePredicate}
|
|
2024
|
+
ORDER BY bucket_start_ms ASC
|
|
2025
|
+
LIMIT 500
|
|
2026
|
+
)`
|
|
2027
|
+
);
|
|
2028
|
+
return this.createPagedDeleteStep(
|
|
2029
|
+
() => remove.run(this.options.targetId, resolutionMs, cutoffMs).changes
|
|
2030
|
+
);
|
|
2031
|
+
}
|
|
2032
|
+
createPagedDeleteStep(deletePage) {
|
|
2033
|
+
return () => {
|
|
2034
|
+
const changes = deletePage();
|
|
2035
|
+
return { deleted: changes, done: changes < RETENTION_PAGE_SIZE };
|
|
2036
|
+
};
|
|
2037
|
+
}
|
|
2038
|
+
createAuxiliaryRetentionSteps(nowMs) {
|
|
2039
|
+
const acknowledgementCutoffMs = nowMs - this.acknowledgementRetentionMs();
|
|
2040
|
+
const ingestionCutoffMs = nowMs - this.longestRetentionMs();
|
|
2041
|
+
const extractionCutoffMs = nowMs - this.options.retention.samplesMs;
|
|
2042
|
+
return [
|
|
2043
|
+
this.createRowsBeforeStep("batchAcknowledgements", "committed_at_ms", acknowledgementCutoffMs),
|
|
2044
|
+
this.createRowsBeforeStep("extractionFailures", "recorded_at_ms", extractionCutoffMs),
|
|
2045
|
+
this.createFinishedMaintenanceRunStep(acknowledgementCutoffMs),
|
|
2046
|
+
...this.createReaderLeaseSteps(nowMs),
|
|
2047
|
+
this.createStaleIngestWatermarkStep(ingestionCutoffMs),
|
|
2048
|
+
this.createUnreferencedMetricDefinitionStep()
|
|
2049
|
+
];
|
|
2050
|
+
}
|
|
2051
|
+
createRowsBeforeStep(logicalTable, timestampColumn, cutoffMs) {
|
|
2052
|
+
const table = this.options.tableMap.quoted(logicalTable);
|
|
2053
|
+
const remove = this.options.connection.prepare(
|
|
2054
|
+
`DELETE FROM ${table}
|
|
2055
|
+
WHERE rowid IN (
|
|
2056
|
+
SELECT rowid FROM ${table}
|
|
2057
|
+
WHERE target_id = ? AND ${timestampColumn} < ?
|
|
2058
|
+
ORDER BY ${timestampColumn} ASC
|
|
2059
|
+
LIMIT 500
|
|
2060
|
+
)`
|
|
2061
|
+
);
|
|
2062
|
+
return this.createPagedDeleteStep(
|
|
2063
|
+
() => remove.run(this.options.targetId, cutoffMs).changes
|
|
2064
|
+
);
|
|
2065
|
+
}
|
|
2066
|
+
createFinishedMaintenanceRunStep(cutoffMs) {
|
|
2067
|
+
const table = this.options.tableMap.quoted("maintenanceRuns");
|
|
2068
|
+
const remove = this.options.connection.prepare(
|
|
2069
|
+
`DELETE FROM ${table}
|
|
2070
|
+
WHERE rowid IN (
|
|
2071
|
+
SELECT rowid FROM ${table}
|
|
2072
|
+
WHERE target_id = ? AND state IN ('completed', 'failed')
|
|
2073
|
+
AND COALESCE(completed_at_ms, started_at_ms) < ?
|
|
2074
|
+
ORDER BY COALESCE(completed_at_ms, started_at_ms) ASC
|
|
2075
|
+
LIMIT 500
|
|
2076
|
+
)`
|
|
2077
|
+
);
|
|
2078
|
+
return this.createPagedDeleteStep(
|
|
2079
|
+
() => remove.run(this.options.targetId, cutoffMs).changes
|
|
2080
|
+
);
|
|
2081
|
+
}
|
|
2082
|
+
createReaderLeaseSteps(nowMs) {
|
|
2083
|
+
const table = this.options.tableMap.quoted("readerLeases");
|
|
2084
|
+
const expire = this.options.connection.prepare(
|
|
2085
|
+
`UPDATE ${table}
|
|
2086
|
+
SET state = 'expired'
|
|
2087
|
+
WHERE rowid IN (
|
|
2088
|
+
SELECT rowid FROM ${table}
|
|
2089
|
+
WHERE target_id = ? AND state = 'active' AND expires_at_ms <= ?
|
|
2090
|
+
ORDER BY expires_at_ms ASC
|
|
2091
|
+
LIMIT 500
|
|
2092
|
+
)`
|
|
2093
|
+
);
|
|
2094
|
+
const remove = this.options.connection.prepare(
|
|
2095
|
+
`DELETE FROM ${table}
|
|
2096
|
+
WHERE rowid IN (
|
|
2097
|
+
SELECT rowid FROM ${table}
|
|
2098
|
+
WHERE target_id = ? AND state IN ('released', 'expired')
|
|
2099
|
+
AND expires_at_ms <= ?
|
|
2100
|
+
ORDER BY expires_at_ms ASC
|
|
2101
|
+
LIMIT 500
|
|
2102
|
+
)`
|
|
2103
|
+
);
|
|
2104
|
+
return [
|
|
2105
|
+
this.createPagedDeleteStep(() => expire.run(this.options.targetId, nowMs).changes),
|
|
2106
|
+
this.createPagedDeleteStep(() => remove.run(this.options.targetId, nowMs).changes)
|
|
2107
|
+
];
|
|
2108
|
+
}
|
|
2109
|
+
createStaleIngestWatermarkStep(cutoffMs) {
|
|
2110
|
+
const watermarks = this.options.tableMap.quoted("ingestWatermarks");
|
|
2111
|
+
const acknowledgements = this.options.tableMap.quoted("batchAcknowledgements");
|
|
2112
|
+
const rawEvents = this.options.tableMap.quoted("rawEvents");
|
|
2113
|
+
const samples = this.options.tableMap.quoted("metricSamples");
|
|
2114
|
+
const failures = this.options.tableMap.quoted("extractionFailures");
|
|
2115
|
+
const remove = this.options.connection.prepare(
|
|
2116
|
+
`DELETE FROM ${watermarks}
|
|
2117
|
+
WHERE rowid IN (
|
|
2118
|
+
SELECT watermark.rowid
|
|
2119
|
+
FROM ${watermarks} AS watermark
|
|
2120
|
+
WHERE watermark.target_id = ?
|
|
2121
|
+
AND watermark.last_occurred_at_ms < ?
|
|
2122
|
+
AND watermark.last_committed_at_ms < ?
|
|
2123
|
+
AND NOT EXISTS (
|
|
2124
|
+
SELECT 1 FROM ${acknowledgements} AS acknowledgement
|
|
2125
|
+
WHERE acknowledgement.target_id = watermark.target_id
|
|
2126
|
+
AND acknowledgement.producer_id = watermark.producer_id
|
|
2127
|
+
)
|
|
2128
|
+
AND NOT EXISTS (
|
|
2129
|
+
SELECT 1 FROM ${rawEvents} AS raw_event
|
|
2130
|
+
WHERE raw_event.target_id = watermark.target_id
|
|
2131
|
+
AND raw_event.producer_id = watermark.producer_id
|
|
2132
|
+
)
|
|
2133
|
+
AND NOT EXISTS (
|
|
2134
|
+
SELECT 1 FROM ${samples} AS sample
|
|
2135
|
+
WHERE sample.target_id = watermark.target_id
|
|
2136
|
+
AND sample.producer_id = watermark.producer_id
|
|
2137
|
+
)
|
|
2138
|
+
AND NOT EXISTS (
|
|
2139
|
+
SELECT 1 FROM ${failures} AS failure
|
|
2140
|
+
WHERE failure.target_id = watermark.target_id
|
|
2141
|
+
AND failure.producer_id = watermark.producer_id
|
|
2142
|
+
)
|
|
2143
|
+
ORDER BY watermark.last_committed_at_ms ASC
|
|
2144
|
+
LIMIT 500
|
|
2145
|
+
)`
|
|
2146
|
+
);
|
|
2147
|
+
return this.createPagedDeleteStep(
|
|
2148
|
+
() => remove.run(this.options.targetId, cutoffMs, cutoffMs).changes
|
|
2149
|
+
);
|
|
2150
|
+
}
|
|
2151
|
+
createUnreferencedMetricDefinitionStep() {
|
|
2152
|
+
const definitions = this.options.tableMap.quoted("metricDefinitions");
|
|
2153
|
+
const samples = this.options.tableMap.quoted("metricSamples");
|
|
2154
|
+
const rollups = this.options.tableMap.quoted("metricRollups");
|
|
2155
|
+
const completions = this.options.tableMap.quoted("rollupCompletions");
|
|
2156
|
+
const failures = this.options.tableMap.quoted("extractionFailures");
|
|
2157
|
+
const read = this.options.connection.prepare(
|
|
2158
|
+
`SELECT definition.metric_name, definition.definition_version
|
|
2159
|
+
FROM ${definitions} AS definition
|
|
2160
|
+
WHERE definition.target_id = ? AND definition.active = 0
|
|
2161
|
+
AND NOT EXISTS (
|
|
2162
|
+
SELECT 1 FROM ${samples} AS sample
|
|
2163
|
+
WHERE sample.target_id = definition.target_id
|
|
2164
|
+
AND sample.metric_name = definition.metric_name
|
|
2165
|
+
AND sample.definition_version = definition.definition_version
|
|
2166
|
+
)
|
|
2167
|
+
AND NOT EXISTS (
|
|
2168
|
+
SELECT 1 FROM ${rollups} AS rollup
|
|
2169
|
+
WHERE rollup.target_id = definition.target_id
|
|
2170
|
+
AND rollup.metric_name = definition.metric_name
|
|
2171
|
+
AND rollup.definition_version = definition.definition_version
|
|
2172
|
+
)
|
|
2173
|
+
AND NOT EXISTS (
|
|
2174
|
+
SELECT 1 FROM ${completions} AS completion
|
|
2175
|
+
WHERE completion.target_id = definition.target_id
|
|
2176
|
+
AND completion.metric_name = definition.metric_name
|
|
2177
|
+
AND completion.definition_version = definition.definition_version
|
|
2178
|
+
)
|
|
2179
|
+
AND NOT EXISTS (
|
|
2180
|
+
SELECT 1 FROM ${failures} AS failure
|
|
2181
|
+
WHERE failure.target_id = definition.target_id
|
|
2182
|
+
AND failure.metric_name = definition.metric_name
|
|
2183
|
+
AND failure.definition_version = definition.definition_version
|
|
2184
|
+
)
|
|
2185
|
+
ORDER BY definition.registered_at_ms ASC
|
|
2186
|
+
LIMIT 500`
|
|
2187
|
+
);
|
|
2188
|
+
const remove = this.options.connection.prepare(
|
|
2189
|
+
`DELETE FROM ${definitions}
|
|
2190
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ? AND active = 0`
|
|
2191
|
+
);
|
|
2192
|
+
return () => {
|
|
2193
|
+
const rows = read.all(this.options.targetId).map((row) => ({
|
|
2194
|
+
metricName: this.readStringField(row, "metric_name"),
|
|
2195
|
+
definitionVersion: this.readNumberField(row, "definition_version")
|
|
2196
|
+
}));
|
|
2197
|
+
if (rows.length === 0) {
|
|
2198
|
+
return { deleted: 0, done: true };
|
|
2199
|
+
}
|
|
2200
|
+
const transaction = this.options.connection.transaction(() => {
|
|
2201
|
+
let deleted2 = 0;
|
|
2202
|
+
for (const row of rows) {
|
|
2203
|
+
deleted2 += remove.run(
|
|
2204
|
+
this.options.targetId,
|
|
2205
|
+
row.metricName,
|
|
2206
|
+
row.definitionVersion
|
|
2207
|
+
).changes;
|
|
2208
|
+
}
|
|
2209
|
+
return deleted2;
|
|
2210
|
+
});
|
|
2211
|
+
const deleted = transaction();
|
|
2212
|
+
for (const row of rows) {
|
|
2213
|
+
this.definitions.delete(this.definitionKey(row.metricName, row.definitionVersion));
|
|
2214
|
+
}
|
|
2215
|
+
return { deleted, done: deleted === 0 || rows.length < RETENTION_PAGE_SIZE };
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
acknowledgementRetentionMs() {
|
|
2219
|
+
return this.options.retention.rawMs + this.options.retention.lateArrivalMs;
|
|
2220
|
+
}
|
|
2221
|
+
longestRetentionMs() {
|
|
2222
|
+
return Math.max(
|
|
2223
|
+
this.options.retention.rawMs,
|
|
2224
|
+
this.options.retention.samplesMs,
|
|
2225
|
+
...this.options.retention.rollups.map((tier) => tier.retainMs)
|
|
2226
|
+
);
|
|
2227
|
+
}
|
|
2228
|
+
hasCompletion(metricName, definitionVersion, resolutionMs, bucketStartMs) {
|
|
2229
|
+
const table = this.options.tableMap.quoted("rollupCompletions");
|
|
2230
|
+
const row = this.options.connection.prepare(
|
|
2231
|
+
`SELECT 1 AS present FROM ${table}
|
|
2232
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
2233
|
+
AND resolution_ms = ? AND bucket_start_ms = ?
|
|
2234
|
+
AND state = 'completed'`
|
|
2235
|
+
).get(this.options.targetId, metricName, definitionVersion, resolutionMs, bucketStartMs);
|
|
2236
|
+
return row !== void 0;
|
|
2237
|
+
}
|
|
2238
|
+
metricDefinitions(metricName) {
|
|
2239
|
+
return [...this.definitions.values()].filter((definition) => definition.name === metricName).sort((left, right) => left.version - right.version);
|
|
2240
|
+
}
|
|
2241
|
+
validateMetricQuery(query, definitions) {
|
|
2242
|
+
const dimensionSets = definitions.map((definition) => new Set(definition.groupBy));
|
|
2243
|
+
const invalidDimension = Object.keys(query.where).find((field) => {
|
|
2244
|
+
return dimensionSets.some((dimensions) => !dimensions.has(field));
|
|
2245
|
+
});
|
|
2246
|
+
if (invalidDimension) {
|
|
2247
|
+
return `Metric ${query.metric} does not declare dimension ${invalidDimension}`;
|
|
2248
|
+
}
|
|
2249
|
+
if (query.resolution !== "auto") {
|
|
2250
|
+
const explicitResolution = query.resolution;
|
|
2251
|
+
const unavailable = definitions.every(
|
|
2252
|
+
(definition) => !definition.resolutionsMs.includes(explicitResolution)
|
|
2253
|
+
);
|
|
2254
|
+
if (unavailable) {
|
|
2255
|
+
return `Resolution ${query.resolution} is not configured for metric ${query.metric}`;
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
const invalidMeasurement = query.measurements.find((measurement) => {
|
|
2259
|
+
return definitions.some(
|
|
2260
|
+
(definition) => !this.definitionHasMeasurement(definition, measurement)
|
|
2261
|
+
);
|
|
2262
|
+
});
|
|
2263
|
+
return invalidMeasurement ? `Metric ${query.metric} does not declare measurement ${invalidMeasurement}` : null;
|
|
2264
|
+
}
|
|
2265
|
+
definitionHasMeasurement(definition, requested) {
|
|
2266
|
+
if (requested === "count") {
|
|
2267
|
+
return definition.measurements.some((measurement) => measurement.kind === "count");
|
|
2268
|
+
}
|
|
2269
|
+
const [field, aggregate] = requested.split(".");
|
|
2270
|
+
if (!field || !aggregate) {
|
|
2271
|
+
return false;
|
|
2272
|
+
}
|
|
2273
|
+
return definition.measurements.some((measurement) => {
|
|
2274
|
+
return measurement.kind === "field" && measurement.field === field && (aggregate === "avg" && measurement.aggregates.includes("sum") || this.isMetricAggregate(aggregate) && measurement.aggregates.includes(aggregate));
|
|
2275
|
+
});
|
|
2276
|
+
}
|
|
2277
|
+
planCoverage(query, definitions) {
|
|
2278
|
+
const table = this.options.tableMap.quoted("rollupCompletions");
|
|
2279
|
+
const rows = this.options.connection.prepare(
|
|
2280
|
+
`SELECT definition_version, resolution_ms, bucket_start_ms
|
|
2281
|
+
FROM ${table}
|
|
2282
|
+
WHERE target_id = ? AND metric_name = ?
|
|
2283
|
+
AND state = 'completed'
|
|
2284
|
+
AND bucket_start_ms < ?
|
|
2285
|
+
AND bucket_start_ms + resolution_ms > ?
|
|
2286
|
+
ORDER BY bucket_start_ms ASC, resolution_ms ASC, definition_version DESC`
|
|
2287
|
+
).all(this.options.targetId, query.metric, query.toMs, query.fromMs);
|
|
2288
|
+
const definitionVersions = new Set(definitions.map((definition) => definition.version));
|
|
2289
|
+
const available = rows.map((row) => ({
|
|
2290
|
+
fromMs: this.readNumberField(row, "bucket_start_ms"),
|
|
2291
|
+
toMs: this.readNumberField(row, "bucket_start_ms") + this.readNumberField(row, "resolution_ms"),
|
|
2292
|
+
definitionVersion: this.readNumberField(row, "definition_version"),
|
|
2293
|
+
resolutionMs: this.readNumberField(row, "resolution_ms")
|
|
2294
|
+
})).filter((row) => {
|
|
2295
|
+
return definitionVersions.has(row.definitionVersion) && (query.resolution === "auto" || row.resolutionMs === query.resolution);
|
|
2296
|
+
});
|
|
2297
|
+
const intervals = [];
|
|
2298
|
+
for (const definition of definitions) {
|
|
2299
|
+
const versionCandidates = available.filter(
|
|
2300
|
+
(candidate) => candidate.definitionVersion === definition.version
|
|
2301
|
+
);
|
|
2302
|
+
intervals.push(...this.planDefinitionCoverage(query, versionCandidates));
|
|
2303
|
+
}
|
|
2304
|
+
const orderedIntervals = intervals.sort((left, right) => {
|
|
2305
|
+
return left.fromMs - right.fromMs || left.definitionVersion - right.definitionVersion || left.resolutionMs - right.resolutionMs;
|
|
2306
|
+
});
|
|
2307
|
+
return {
|
|
2308
|
+
intervals: orderedIntervals,
|
|
2309
|
+
gaps: this.coverageGaps(query, orderedIntervals, available)
|
|
2310
|
+
};
|
|
2311
|
+
}
|
|
2312
|
+
planDefinitionCoverage(query, candidates) {
|
|
2313
|
+
const contained = candidates.filter((candidate) => {
|
|
2314
|
+
return candidate.fromMs >= query.fromMs && candidate.toMs <= query.toMs;
|
|
2315
|
+
});
|
|
2316
|
+
const intervals = [];
|
|
2317
|
+
let cursor = query.fromMs;
|
|
2318
|
+
while (cursor < query.toMs) {
|
|
2319
|
+
const starting = contained.filter((candidate) => candidate.fromMs === cursor).sort((left, right) => left.resolutionMs - right.resolutionMs);
|
|
2320
|
+
const chosen = starting[0];
|
|
2321
|
+
if (chosen) {
|
|
2322
|
+
this.appendCoverageInterval(intervals, chosen);
|
|
2323
|
+
cursor = chosen.toMs;
|
|
2324
|
+
continue;
|
|
2325
|
+
}
|
|
2326
|
+
const nextStart = contained.map((candidate) => candidate.fromMs).filter((startMs) => startMs > cursor).sort((left, right) => left - right)[0];
|
|
2327
|
+
cursor = nextStart ?? query.toMs;
|
|
2328
|
+
}
|
|
2329
|
+
return intervals;
|
|
2330
|
+
}
|
|
2331
|
+
appendCoverageInterval(intervals, candidate) {
|
|
2332
|
+
const previous = intervals[intervals.length - 1];
|
|
2333
|
+
const extendsPrevious = previous !== void 0 && previous.toMs === candidate.fromMs && previous.resolutionMs === candidate.resolutionMs && previous.definitionVersion === candidate.definitionVersion;
|
|
2334
|
+
if (extendsPrevious) {
|
|
2335
|
+
intervals[intervals.length - 1] = { ...previous, toMs: candidate.toMs };
|
|
2336
|
+
return;
|
|
2337
|
+
}
|
|
2338
|
+
intervals.push(candidate);
|
|
2339
|
+
}
|
|
2340
|
+
coverageGaps(query, intervals, candidates) {
|
|
2341
|
+
const coverage = [...intervals].sort(
|
|
2342
|
+
(left, right) => left.fromMs - right.fromMs || right.toMs - left.toMs
|
|
2343
|
+
);
|
|
2344
|
+
const gaps = [];
|
|
2345
|
+
let cursor = query.fromMs;
|
|
2346
|
+
for (const interval of coverage) {
|
|
2347
|
+
if (interval.fromMs > cursor) {
|
|
2348
|
+
this.appendSplitCoverageGaps(gaps, cursor, interval.fromMs, query, candidates);
|
|
2349
|
+
}
|
|
2350
|
+
cursor = Math.max(cursor, interval.toMs);
|
|
2351
|
+
}
|
|
2352
|
+
if (cursor < query.toMs) {
|
|
2353
|
+
this.appendSplitCoverageGaps(gaps, cursor, query.toMs, query, candidates);
|
|
2354
|
+
}
|
|
2355
|
+
return Object.freeze(gaps);
|
|
2356
|
+
}
|
|
2357
|
+
appendSplitCoverageGaps(gaps, fromMs, toMs, query, candidates) {
|
|
2358
|
+
const partialCandidates = candidates.filter((candidate) => {
|
|
2359
|
+
const outsideQuery = candidate.fromMs < query.fromMs || candidate.toMs > query.toMs;
|
|
2360
|
+
return outsideQuery && candidate.fromMs < toMs && candidate.toMs > fromMs;
|
|
2361
|
+
});
|
|
2362
|
+
const boundaries = /* @__PURE__ */ new Set([fromMs, toMs]);
|
|
2363
|
+
for (const candidate of partialCandidates) {
|
|
2364
|
+
boundaries.add(Math.max(fromMs, candidate.fromMs));
|
|
2365
|
+
boundaries.add(Math.min(toMs, candidate.toMs));
|
|
2366
|
+
}
|
|
2367
|
+
const ordered = [...boundaries].sort((left, right) => left - right);
|
|
2368
|
+
for (let index = 0; index < ordered.length - 1; index += 1) {
|
|
2369
|
+
const gapStart = ordered[index];
|
|
2370
|
+
const gapEnd = ordered[index + 1];
|
|
2371
|
+
if (gapStart === void 0 || gapEnd === void 0 || gapStart >= gapEnd) {
|
|
2372
|
+
continue;
|
|
2373
|
+
}
|
|
2374
|
+
const isPartialBucket = partialCandidates.some((candidate) => {
|
|
2375
|
+
return candidate.fromMs < gapEnd && candidate.toMs > gapStart;
|
|
2376
|
+
});
|
|
2377
|
+
this.appendMetricGap(gaps, {
|
|
2378
|
+
fromMs: gapStart,
|
|
2379
|
+
toMs: gapEnd,
|
|
2380
|
+
reason: isPartialBucket ? "partial-bucket" : "not-materialized"
|
|
2381
|
+
});
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
appendMetricGap(gaps, gap) {
|
|
2385
|
+
const previous = gaps[gaps.length - 1];
|
|
2386
|
+
if (previous && previous.toMs === gap.fromMs && previous.reason === gap.reason) {
|
|
2387
|
+
gaps[gaps.length - 1] = { ...previous, toMs: gap.toMs };
|
|
2388
|
+
return;
|
|
2389
|
+
}
|
|
2390
|
+
gaps.push(gap);
|
|
2391
|
+
}
|
|
2392
|
+
readRollupsForCoverage(query, intervals) {
|
|
2393
|
+
const table = this.options.tableMap.quoted("metricRollups");
|
|
2394
|
+
const parsedRows = [];
|
|
2395
|
+
for (const interval of intervals) {
|
|
2396
|
+
const rows = this.options.connection.prepare(
|
|
2397
|
+
`SELECT metric_name, definition_version, resolution_ms, bucket_start_ms,
|
|
2398
|
+
dimension_key, dimensions_json, values_json
|
|
2399
|
+
FROM ${table}
|
|
2400
|
+
WHERE target_id = ? AND metric_name = ? AND definition_version = ?
|
|
2401
|
+
AND resolution_ms = ? AND bucket_start_ms >= ?
|
|
2402
|
+
AND bucket_start_ms + resolution_ms <= ?`
|
|
2403
|
+
).all(
|
|
2404
|
+
this.options.targetId,
|
|
2405
|
+
query.metric,
|
|
2406
|
+
interval.definitionVersion,
|
|
2407
|
+
interval.resolutionMs,
|
|
2408
|
+
interval.fromMs,
|
|
2409
|
+
interval.toMs
|
|
2410
|
+
);
|
|
2411
|
+
for (const row of rows) {
|
|
2412
|
+
const parsed = this.rowParser.parseMetricRollup(row);
|
|
2413
|
+
if (!parsed.ok) {
|
|
2414
|
+
throw new Error(parsed.error);
|
|
2415
|
+
}
|
|
2416
|
+
const dimensions = this.rowParser.parseJsonObject(parsed.value.dimensions_json);
|
|
2417
|
+
const values = this.rowParser.parseRollupValues(parsed.value.values_json);
|
|
2418
|
+
if (!dimensions.ok) {
|
|
2419
|
+
throw new Error(dimensions.error);
|
|
2420
|
+
}
|
|
2421
|
+
if (!values.ok) {
|
|
2422
|
+
throw new Error(values.error);
|
|
2423
|
+
}
|
|
2424
|
+
parsedRows.push({
|
|
2425
|
+
bucket: {
|
|
2426
|
+
startMs: parsed.value.bucket_start_ms,
|
|
2427
|
+
endMs: parsed.value.bucket_start_ms + parsed.value.resolution_ms,
|
|
2428
|
+
dimensions: dimensions.value,
|
|
2429
|
+
values: this.rollupValuesToJson(values.value)
|
|
2430
|
+
},
|
|
2431
|
+
resolutionMs: parsed.value.resolution_ms,
|
|
2432
|
+
definitionVersion: parsed.value.definition_version,
|
|
2433
|
+
dimensionKey: parsed.value.dimension_key
|
|
2434
|
+
});
|
|
2435
|
+
}
|
|
2436
|
+
}
|
|
2437
|
+
return parsedRows;
|
|
2438
|
+
}
|
|
2439
|
+
rollupValuesToJson(values) {
|
|
2440
|
+
const fields = {};
|
|
2441
|
+
for (const [field, aggregate] of Object.entries(values.fields)) {
|
|
2442
|
+
fields[field] = {
|
|
2443
|
+
count: aggregate.count,
|
|
2444
|
+
sum: aggregate.sum,
|
|
2445
|
+
min: aggregate.min,
|
|
2446
|
+
max: aggregate.max,
|
|
2447
|
+
last: aggregate.last,
|
|
2448
|
+
avg: aggregate.count === 0 ? 0 : aggregate.sum / aggregate.count
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
return Object.freeze({ count: values.count, ...fields });
|
|
2452
|
+
}
|
|
2453
|
+
selectMeasurements(values, measurements) {
|
|
2454
|
+
const selected = {};
|
|
2455
|
+
for (const measurement of measurements) {
|
|
2456
|
+
if (measurement === "count") {
|
|
2457
|
+
selected.count = values.count ?? 0;
|
|
2458
|
+
continue;
|
|
2459
|
+
}
|
|
2460
|
+
const [field, aggregate] = measurement.split(".");
|
|
2461
|
+
const fieldValues = field ? values[field] : null;
|
|
2462
|
+
if (!field || !aggregate) {
|
|
2463
|
+
continue;
|
|
2464
|
+
}
|
|
2465
|
+
if (!this.isJsonObject(fieldValues)) {
|
|
2466
|
+
continue;
|
|
2467
|
+
}
|
|
2468
|
+
const existing = selected[field];
|
|
2469
|
+
const selectedField = this.isJsonObject(existing) ? { ...existing } : {};
|
|
2470
|
+
const aggregateValue = fieldValues[aggregate];
|
|
2471
|
+
if (aggregateValue !== void 0) {
|
|
2472
|
+
selectedField[aggregate] = aggregateValue;
|
|
2473
|
+
}
|
|
2474
|
+
selected[field] = selectedField;
|
|
2475
|
+
}
|
|
2476
|
+
return Object.freeze(selected);
|
|
2477
|
+
}
|
|
2478
|
+
matchesWhere(dimensions, where) {
|
|
2479
|
+
return Object.entries(where).every(([field, expected]) => {
|
|
2480
|
+
const actual = dimensions[field];
|
|
2481
|
+
const candidates = Array.isArray(expected) ? expected : [expected];
|
|
2482
|
+
return candidates.some((candidate) => canonicalJson(candidate) === canonicalJson(actual));
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
compareParsedRollups(left, right, order) {
|
|
2486
|
+
const direction = order === "asc" ? 1 : -1;
|
|
2487
|
+
return direction * this.compareRollupIdentity(
|
|
2488
|
+
left.bucket.startMs,
|
|
2489
|
+
left.resolutionMs,
|
|
2490
|
+
left.definitionVersion,
|
|
2491
|
+
left.dimensionKey,
|
|
2492
|
+
right.bucket.startMs,
|
|
2493
|
+
right.resolutionMs,
|
|
2494
|
+
right.definitionVersion,
|
|
2495
|
+
right.dimensionKey
|
|
2496
|
+
);
|
|
2497
|
+
}
|
|
2498
|
+
isAfterMetricCursor(row, cursor, order) {
|
|
2499
|
+
const comparison = this.compareRollupIdentity(
|
|
2500
|
+
row.bucket.startMs,
|
|
2501
|
+
row.resolutionMs,
|
|
2502
|
+
row.definitionVersion,
|
|
2503
|
+
row.dimensionKey,
|
|
2504
|
+
cursor.bucketStartMs,
|
|
2505
|
+
cursor.resolutionMs,
|
|
2506
|
+
cursor.definitionVersion,
|
|
2507
|
+
cursor.dimensionKey
|
|
2508
|
+
);
|
|
2509
|
+
return order === "asc" ? comparison > 0 : comparison < 0;
|
|
2510
|
+
}
|
|
2511
|
+
compareRollupIdentity(leftStartMs, leftResolutionMs, leftDefinitionVersion, leftDimensionKey, rightStartMs, rightResolutionMs, rightDefinitionVersion, rightDimensionKey) {
|
|
2512
|
+
return leftStartMs - rightStartMs || leftResolutionMs - rightResolutionMs || leftDefinitionVersion - rightDefinitionVersion || leftDimensionKey.localeCompare(rightDimensionKey);
|
|
2513
|
+
}
|
|
2514
|
+
rowsToSegments(intervals, rows) {
|
|
2515
|
+
return Object.freeze(
|
|
2516
|
+
intervals.map((interval) => ({
|
|
2517
|
+
fromMs: interval.fromMs,
|
|
2518
|
+
toMs: interval.toMs,
|
|
2519
|
+
resolutionMs: interval.resolutionMs,
|
|
2520
|
+
isAggregated: true,
|
|
2521
|
+
definitionVersion: interval.definitionVersion,
|
|
2522
|
+
buckets: Object.freeze(
|
|
2523
|
+
rows.filter(
|
|
2524
|
+
(row) => row.resolutionMs === interval.resolutionMs && row.definitionVersion === interval.definitionVersion && row.bucket.startMs >= interval.fromMs && row.bucket.endMs <= interval.toMs
|
|
2525
|
+
).map((row) => row.bucket)
|
|
2526
|
+
)
|
|
2527
|
+
}))
|
|
2528
|
+
);
|
|
2529
|
+
}
|
|
2530
|
+
committedTargetState(attempts) {
|
|
2531
|
+
return {
|
|
2532
|
+
targetId: this.options.targetId,
|
|
2533
|
+
delivery: "required",
|
|
2534
|
+
state: "committed",
|
|
2535
|
+
attempts,
|
|
2536
|
+
reason: null
|
|
2537
|
+
};
|
|
2538
|
+
}
|
|
2539
|
+
definitionKey(name, version) {
|
|
2540
|
+
return `${name}:${version}`;
|
|
2541
|
+
}
|
|
2542
|
+
normalizeDefinitions(definitions) {
|
|
2543
|
+
return definitions.map((definition) => {
|
|
2544
|
+
const normalized = this.definitionContract.normalize(definition);
|
|
2545
|
+
if (!normalized.ok) {
|
|
2546
|
+
throw new Error(normalized.error);
|
|
2547
|
+
}
|
|
2548
|
+
return normalized.value;
|
|
2549
|
+
});
|
|
2550
|
+
}
|
|
2551
|
+
bucketStart(occurredAtMs, resolutionMs) {
|
|
2552
|
+
return Math.floor(occurredAtMs / resolutionMs) * resolutionMs;
|
|
2553
|
+
}
|
|
2554
|
+
encodeCursor(cursor) {
|
|
2555
|
+
return Buffer.from(canonicalJson(cursor), "utf8").toString("base64url");
|
|
2556
|
+
}
|
|
2557
|
+
readStringField(row, field) {
|
|
2558
|
+
if (typeof row !== "object" || row === null || Array.isArray(row)) {
|
|
2559
|
+
throw new Error(`Expected a SQLite row while reading ${field}`);
|
|
2560
|
+
}
|
|
2561
|
+
const value = Reflect.get(row, field);
|
|
2562
|
+
if (typeof value !== "string") {
|
|
2563
|
+
throw new Error(`Expected string field ${field}`);
|
|
2564
|
+
}
|
|
2565
|
+
return value;
|
|
2566
|
+
}
|
|
2567
|
+
readNumberField(row, field) {
|
|
2568
|
+
if (typeof row !== "object" || row === null || Array.isArray(row)) {
|
|
2569
|
+
throw new Error(`Expected a SQLite row while reading ${field}`);
|
|
2570
|
+
}
|
|
2571
|
+
const value = Reflect.get(row, field);
|
|
2572
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
2573
|
+
throw new Error(`Expected numeric field ${field}`);
|
|
2574
|
+
}
|
|
2575
|
+
return value;
|
|
2576
|
+
}
|
|
2577
|
+
parseCommittedDropCounter(row) {
|
|
2578
|
+
const reason = this.readStringField(row, "reason");
|
|
2579
|
+
const priority = this.readStringField(row, "priority");
|
|
2580
|
+
const count = this.readNumberField(row, "count");
|
|
2581
|
+
if (!this.isDropReason(reason) || !this.isDropPriority(priority)) {
|
|
2582
|
+
throw new Error("Stored committed telemetry drop counter identity is invalid");
|
|
2583
|
+
}
|
|
2584
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
2585
|
+
throw new Error("Stored committed telemetry drop counter value is invalid");
|
|
2586
|
+
}
|
|
2587
|
+
return Object.freeze({ reason, priority, count });
|
|
2588
|
+
}
|
|
2589
|
+
errorMessage(cause) {
|
|
2590
|
+
return cause instanceof Error ? cause.message : String(cause);
|
|
2591
|
+
}
|
|
2592
|
+
isJsonObject(value) {
|
|
2593
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2594
|
+
}
|
|
2595
|
+
isMetricAggregate(value) {
|
|
2596
|
+
return value === "sum" || value === "min" || value === "max" || value === "last";
|
|
2597
|
+
}
|
|
2598
|
+
isDropReason(value) {
|
|
2599
|
+
return value === "capacity" || value === "sampled" || value === "stopped" || value === "invalid" || value === "transport";
|
|
2600
|
+
}
|
|
2601
|
+
isDropPriority(value) {
|
|
2602
|
+
return value === "low" || value === "normal" || value === "high";
|
|
2603
|
+
}
|
|
2604
|
+
};
|
|
2605
|
+
|
|
2606
|
+
// src/storage/telemetry_table_map.ts
|
|
2607
|
+
var TelemetryTableMap = class _TelemetryTableMap {
|
|
2608
|
+
constructor(mapping, checksum) {
|
|
2609
|
+
this.mapping = mapping;
|
|
2610
|
+
this.checksum = checksum;
|
|
2611
|
+
this.tokenMap = Object.freeze({
|
|
2612
|
+
...mapping,
|
|
2613
|
+
rawEventsOccurredIndex: this.indexName(mapping.rawEvents, "occurred"),
|
|
2614
|
+
rawEventsConversationIndex: this.indexName(mapping.rawEvents, "conversation"),
|
|
2615
|
+
rawEventsLabelIndex: this.indexName(mapping.rawEvents, "label"),
|
|
2616
|
+
metricSamplesTimeIndex: this.indexName(mapping.metricSamples, "time"),
|
|
2617
|
+
metricSamplesEventIndex: this.indexName(mapping.metricSamples, "event"),
|
|
2618
|
+
metricRollupsTimeIndex: this.indexName(mapping.metricRollups, "time")
|
|
2619
|
+
});
|
|
2620
|
+
}
|
|
2621
|
+
mapping;
|
|
2622
|
+
checksum;
|
|
2623
|
+
tokenMap;
|
|
2624
|
+
static create(configuration) {
|
|
2625
|
+
const prefixError = _TelemetryTableMap.validateOptionalPrefix(configuration.tablePrefix);
|
|
2626
|
+
if (prefixError !== null) {
|
|
2627
|
+
return { ok: false, error: prefixError };
|
|
2628
|
+
}
|
|
2629
|
+
const entries = SQLITE_LOGICAL_TABLE_NAMES.map((logicalName) => {
|
|
2630
|
+
const override = configuration.tables[logicalName];
|
|
2631
|
+
const defaultName = SQLITE_DEFAULT_TABLE_NAMES[logicalName];
|
|
2632
|
+
const physicalName = override ?? (configuration.tablePrefix === null ? defaultName : `${configuration.tablePrefix}${defaultName}`);
|
|
2633
|
+
return [logicalName, physicalName];
|
|
2634
|
+
});
|
|
2635
|
+
for (const [logicalName, physicalName] of entries) {
|
|
2636
|
+
const identifierError = _TelemetryTableMap.validateIdentifier(physicalName);
|
|
2637
|
+
if (identifierError !== null) {
|
|
2638
|
+
return { ok: false, error: `${logicalName}: ${identifierError}` };
|
|
2639
|
+
}
|
|
2640
|
+
if (physicalName.toLowerCase() === SQLITE_SCHEMA_MANIFEST_TABLE) {
|
|
2641
|
+
return { ok: false, error: `${logicalName} cannot use the fixed schema manifest name` };
|
|
2642
|
+
}
|
|
2643
|
+
}
|
|
2644
|
+
const foldedNames = entries.map(([, physicalName]) => physicalName.toLowerCase());
|
|
2645
|
+
if (new Set(foldedNames).size !== foldedNames.length) {
|
|
2646
|
+
return { ok: false, error: "Resolved telemetry table names must be unique after case folding" };
|
|
2647
|
+
}
|
|
2648
|
+
const mapping = Object.freeze(Object.fromEntries(entries));
|
|
2649
|
+
const checksum = telemetryChecksum(mapping);
|
|
2650
|
+
return { ok: true, value: new _TelemetryTableMap(mapping, checksum) };
|
|
2651
|
+
}
|
|
2652
|
+
physical(logicalName) {
|
|
2653
|
+
return this.mapping[logicalName];
|
|
2654
|
+
}
|
|
2655
|
+
quoted(logicalName) {
|
|
2656
|
+
return this.quote(this.mapping[logicalName]);
|
|
2657
|
+
}
|
|
2658
|
+
manifestMapping() {
|
|
2659
|
+
return this.mapping;
|
|
2660
|
+
}
|
|
2661
|
+
substitute(sql) {
|
|
2662
|
+
const unknownTokens = [];
|
|
2663
|
+
const substituted = sql.replace(/\{\{([A-Za-z][A-Za-z0-9]*)\}\}/g, (_match, token) => {
|
|
2664
|
+
const identifier = this.tokenMap[token];
|
|
2665
|
+
if (identifier === void 0) {
|
|
2666
|
+
unknownTokens.push(token);
|
|
2667
|
+
return `{{${token}}}`;
|
|
2668
|
+
}
|
|
2669
|
+
return this.quote(identifier);
|
|
2670
|
+
});
|
|
2671
|
+
if (unknownTokens.length > 0 || substituted.includes("{{")) {
|
|
2672
|
+
return {
|
|
2673
|
+
ok: false,
|
|
2674
|
+
error: `Unknown telemetry migration identifier token(s): ${unknownTokens.join(", ") || "malformed token"}`
|
|
2675
|
+
};
|
|
2676
|
+
}
|
|
2677
|
+
return { ok: true, value: substituted };
|
|
2678
|
+
}
|
|
2679
|
+
quote(identifier) {
|
|
2680
|
+
return `"${identifier.replaceAll('"', '""')}"`;
|
|
2681
|
+
}
|
|
2682
|
+
indexName(tableName, purpose) {
|
|
2683
|
+
const raw = `${tableName}_${purpose}_idx`;
|
|
2684
|
+
if (raw.length <= 63) {
|
|
2685
|
+
return raw;
|
|
2686
|
+
}
|
|
2687
|
+
const suffix = telemetryChecksum(raw).slice(0, 8);
|
|
2688
|
+
return `${raw.slice(0, 54)}_${suffix}`;
|
|
2689
|
+
}
|
|
2690
|
+
static validateOptionalPrefix(prefix) {
|
|
2691
|
+
if (prefix === null) {
|
|
2692
|
+
return null;
|
|
2693
|
+
}
|
|
2694
|
+
return this.validateIdentifier(prefix);
|
|
2695
|
+
}
|
|
2696
|
+
static validateIdentifier(identifier) {
|
|
2697
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
|
|
2698
|
+
return `Invalid SQLite identifier ${identifier}`;
|
|
2699
|
+
}
|
|
2700
|
+
if (identifier.length > 63) {
|
|
2701
|
+
return `SQLite identifier ${identifier} exceeds 63 characters`;
|
|
2702
|
+
}
|
|
2703
|
+
if (identifier.toLowerCase().startsWith("sqlite_")) {
|
|
2704
|
+
return `SQLite identifier ${identifier} uses the reserved sqlite_ prefix`;
|
|
2705
|
+
}
|
|
2706
|
+
return null;
|
|
2707
|
+
}
|
|
2708
|
+
};
|
|
2709
|
+
|
|
2710
|
+
// src/transports/schema/telemetry_schema_manifest.ts
|
|
2711
|
+
import { z as z2 } from "zod";
|
|
2712
|
+
var TelemetrySchemaManifestRowSchema = z2.object({
|
|
2713
|
+
target_id: z2.string(),
|
|
2714
|
+
format_version: z2.number().int().positive(),
|
|
2715
|
+
mapping_json: z2.string(),
|
|
2716
|
+
mapping_checksum: z2.string(),
|
|
2717
|
+
state: z2.enum(["ready", "renaming"]),
|
|
2718
|
+
pending_mapping_json: z2.string().nullable(),
|
|
2719
|
+
updated_at_ms: z2.number().int()
|
|
2720
|
+
}).strict();
|
|
2721
|
+
|
|
2722
|
+
// src/transports/sqlite_transport.ts
|
|
2723
|
+
var DEFAULT_MIGRATIONS_DIRECTORY = typeof __dirname === "string" ? resolve2(__dirname, "../../migrations") : fileURLToPath(new URL("../../migrations/", import.meta.url));
|
|
2724
|
+
var SQLiteTelemetryTransport = class {
|
|
2725
|
+
constructor(options) {
|
|
2726
|
+
this.options = options;
|
|
2727
|
+
this.targetId = options.configuration.targetId;
|
|
2728
|
+
this.validateConfiguration();
|
|
2729
|
+
}
|
|
2730
|
+
options;
|
|
2731
|
+
targetId;
|
|
2732
|
+
capabilities = Object.freeze({
|
|
2733
|
+
rawPersistence: true,
|
|
2734
|
+
metricSamples: true,
|
|
2735
|
+
rollups: true,
|
|
2736
|
+
queries: true,
|
|
2737
|
+
retention: true,
|
|
2738
|
+
maintenance: true
|
|
2739
|
+
});
|
|
2740
|
+
state = "created";
|
|
2741
|
+
detail = null;
|
|
2742
|
+
tableMap = null;
|
|
2743
|
+
connection = null;
|
|
2744
|
+
repository = null;
|
|
2745
|
+
definitions = Object.freeze([]);
|
|
2746
|
+
lease = null;
|
|
2747
|
+
heartbeatHandle = null;
|
|
2748
|
+
activeReaders = 0;
|
|
2749
|
+
activeWrites = 0;
|
|
2750
|
+
accepting = false;
|
|
2751
|
+
static defaults(configuration) {
|
|
2752
|
+
return {
|
|
2753
|
+
configuration,
|
|
2754
|
+
migrationsDirectory: DEFAULT_MIGRATIONS_DIRECTORY,
|
|
2755
|
+
processToken: randomUUID(),
|
|
2756
|
+
now: () => Date.now(),
|
|
2757
|
+
createInterval: (work, intervalMs) => setInterval(work, intervalMs),
|
|
2758
|
+
clearInterval: (handle) => clearInterval(handle),
|
|
2759
|
+
delay: (durationMs) => new Promise((resolve3) => setTimeout(resolve3, durationMs))
|
|
2760
|
+
};
|
|
2761
|
+
}
|
|
2762
|
+
async start(definitions) {
|
|
2763
|
+
if (this.state !== "created" && this.state !== "stopped") {
|
|
2764
|
+
return { ok: false, error: `Cannot start SQLite telemetry target from state ${this.state}` };
|
|
2765
|
+
}
|
|
2766
|
+
this.state = "starting";
|
|
2767
|
+
this.detail = null;
|
|
2768
|
+
const tableMap = TelemetryTableMap.create(this.options.configuration.schema);
|
|
2769
|
+
if (!tableMap.ok) {
|
|
2770
|
+
return this.failStart(tableMap.error);
|
|
2771
|
+
}
|
|
2772
|
+
this.tableMap = tableMap.value;
|
|
2773
|
+
this.lease = new SqliteOwnershipLease({
|
|
2774
|
+
targetId: this.targetId,
|
|
2775
|
+
databasePath: this.options.configuration.path,
|
|
2776
|
+
mode: "writer",
|
|
2777
|
+
expiryMs: this.options.configuration.leaseExpiryMs,
|
|
2778
|
+
processToken: this.options.processToken,
|
|
2779
|
+
now: this.options.now
|
|
2780
|
+
});
|
|
2781
|
+
const acquired = this.lease.acquire();
|
|
2782
|
+
if (!acquired.ok) {
|
|
2783
|
+
return this.failStart(acquired.error);
|
|
2784
|
+
}
|
|
2785
|
+
try {
|
|
2786
|
+
const connection = new BetterSqlite3(this.options.configuration.path);
|
|
2787
|
+
this.connection = connection;
|
|
2788
|
+
this.configureConnection(connection);
|
|
2789
|
+
const manifestNeedsUpgrade = this.initializeAndVerifyManifest(connection, tableMap.value);
|
|
2790
|
+
await this.runMigrations(connection, tableMap.value);
|
|
2791
|
+
this.verifySchema(connection, tableMap.value);
|
|
2792
|
+
if (manifestNeedsUpgrade) {
|
|
2793
|
+
this.upgradeManifestMapping(connection, tableMap.value);
|
|
2794
|
+
}
|
|
2795
|
+
const stackDatabase = await createSqliteDb(connection);
|
|
2796
|
+
this.repository = new TelemetryRepository({
|
|
2797
|
+
targetId: this.targetId,
|
|
2798
|
+
connection,
|
|
2799
|
+
stackDatabase,
|
|
2800
|
+
tableMap: tableMap.value,
|
|
2801
|
+
retention: this.options.configuration.retention,
|
|
2802
|
+
now: this.options.now
|
|
2803
|
+
});
|
|
2804
|
+
const recovered = this.repository.recoverInterruptedMaintenance(this.options.now());
|
|
2805
|
+
if (!recovered.ok) {
|
|
2806
|
+
throw new Error(recovered.error);
|
|
2807
|
+
}
|
|
2808
|
+
const registered = this.repository.registerDefinitions(definitions);
|
|
2809
|
+
if (!registered.ok) {
|
|
2810
|
+
throw new Error(registered.error);
|
|
2811
|
+
}
|
|
2812
|
+
this.definitions = Object.freeze([...definitions]);
|
|
2813
|
+
this.startHeartbeat();
|
|
2814
|
+
this.accepting = true;
|
|
2815
|
+
this.state = "ready";
|
|
2816
|
+
return { ok: true, value: void 0 };
|
|
2817
|
+
} catch (cause) {
|
|
2818
|
+
this.closeOwnedResources();
|
|
2819
|
+
return this.failStart(this.errorMessage(cause));
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
async persist(request) {
|
|
2823
|
+
if (!this.accepting || this.state !== "ready" || this.repository === null) {
|
|
2824
|
+
return this.closedAcknowledgement(request.batch.idempotencyKey);
|
|
2825
|
+
}
|
|
2826
|
+
this.activeWrites += 1;
|
|
2827
|
+
try {
|
|
2828
|
+
const persisted = this.repository.persist(request);
|
|
2829
|
+
if (persisted.kind === "acknowledged") {
|
|
2830
|
+
return persisted.acknowledgement;
|
|
2831
|
+
}
|
|
2832
|
+
this.detail = persisted.reason;
|
|
2833
|
+
return {
|
|
2834
|
+
kind: "retryable",
|
|
2835
|
+
idempotencyKey: request.batch.idempotencyKey,
|
|
2836
|
+
reason: persisted.reason,
|
|
2837
|
+
retryAfterMs: 100,
|
|
2838
|
+
targetStates: [{
|
|
2839
|
+
targetId: this.targetId,
|
|
2840
|
+
delivery: this.options.configuration.delivery,
|
|
2841
|
+
state: "failed",
|
|
2842
|
+
attempts: 1,
|
|
2843
|
+
reason: persisted.reason
|
|
2844
|
+
}]
|
|
2845
|
+
};
|
|
2846
|
+
} finally {
|
|
2847
|
+
this.activeWrites -= 1;
|
|
2848
|
+
}
|
|
2849
|
+
}
|
|
2850
|
+
async timeline(query) {
|
|
2851
|
+
const queryError = this.validateTimelineQuery(query);
|
|
2852
|
+
if (queryError !== null) {
|
|
2853
|
+
return { ok: false, error: { kind: "invalid-query", message: queryError } };
|
|
2854
|
+
}
|
|
2855
|
+
const result = this.withReaderLease(() => {
|
|
2856
|
+
const repository = this.requireRepository();
|
|
2857
|
+
const rawCutoffMs = this.options.now() - this.options.configuration.retention.rawMs;
|
|
2858
|
+
return repository.timeline(query, rawCutoffMs);
|
|
2859
|
+
});
|
|
2860
|
+
return this.enforceResponseBytes(result);
|
|
2861
|
+
}
|
|
2862
|
+
async metrics(query) {
|
|
2863
|
+
const queryError = this.validateMetricQuery(query);
|
|
2864
|
+
if (queryError !== null) {
|
|
2865
|
+
return { ok: false, error: { kind: "invalid-query", message: queryError } };
|
|
2866
|
+
}
|
|
2867
|
+
const result = this.withReaderLease(() => this.requireRepository().metrics(query));
|
|
2868
|
+
if (result.ok) {
|
|
2869
|
+
const bucketCount = result.value.segments.reduce((count, segment) => count + segment.buckets.length, 0);
|
|
2870
|
+
if (bucketCount > this.options.configuration.query.maxBuckets) {
|
|
2871
|
+
return {
|
|
2872
|
+
ok: false,
|
|
2873
|
+
error: { kind: "invalid-query", message: "Metric result exceeds the configured bucket maximum" }
|
|
2874
|
+
};
|
|
2875
|
+
}
|
|
2876
|
+
}
|
|
2877
|
+
return this.enforceResponseBytes(result);
|
|
2878
|
+
}
|
|
2879
|
+
async materializeRollups(nowMs, maxWorkMs) {
|
|
2880
|
+
return this.requireRepositoryResult((repository) => repository.materializeRollups(nowMs, maxWorkMs));
|
|
2881
|
+
}
|
|
2882
|
+
async enforceRetention(nowMs, shouldContinue = () => true) {
|
|
2883
|
+
const startedAt = this.options.now();
|
|
2884
|
+
let drain;
|
|
2885
|
+
try {
|
|
2886
|
+
drain = this.requireRepository().beginRetentionDrain(nowMs);
|
|
2887
|
+
} catch (cause) {
|
|
2888
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
2889
|
+
}
|
|
2890
|
+
let affectedRows = 0;
|
|
2891
|
+
let deletionRounds = 0;
|
|
2892
|
+
let interrupted = false;
|
|
2893
|
+
for (; ; ) {
|
|
2894
|
+
if (!shouldContinue()) {
|
|
2895
|
+
interrupted = true;
|
|
2896
|
+
break;
|
|
2897
|
+
}
|
|
2898
|
+
const batch = drain.nextBatch();
|
|
2899
|
+
if (!batch.ok) {
|
|
2900
|
+
return batch;
|
|
2901
|
+
}
|
|
2902
|
+
affectedRows += batch.value.deletedRows;
|
|
2903
|
+
deletionRounds += 1;
|
|
2904
|
+
if (!batch.value.hasMore) {
|
|
2905
|
+
break;
|
|
2906
|
+
}
|
|
2907
|
+
await this.options.delay(0);
|
|
2908
|
+
}
|
|
2909
|
+
return {
|
|
2910
|
+
ok: true,
|
|
2911
|
+
value: {
|
|
2912
|
+
stage: "retention",
|
|
2913
|
+
affectedRows,
|
|
2914
|
+
affectedPages: 0,
|
|
2915
|
+
elapsedMs: this.options.now() - startedAt,
|
|
2916
|
+
detail: interrupted ? "interrupted" : null,
|
|
2917
|
+
deletionRounds,
|
|
2918
|
+
completed: !interrupted,
|
|
2919
|
+
interrupted
|
|
2920
|
+
}
|
|
2921
|
+
};
|
|
2922
|
+
}
|
|
2923
|
+
async checkpoint() {
|
|
2924
|
+
return this.requireRepositoryResult((repository) => repository.checkpoint());
|
|
2925
|
+
}
|
|
2926
|
+
async reclaim(maxPages) {
|
|
2927
|
+
return this.requireRepositoryResult((repository) => repository.reclaim(maxPages));
|
|
2928
|
+
}
|
|
2929
|
+
async compact(deadlineMs) {
|
|
2930
|
+
const startedAt = this.options.now();
|
|
2931
|
+
this.accepting = false;
|
|
2932
|
+
while ((this.activeReaders > 0 || this.activeWrites > 0) && this.options.now() < deadlineMs) {
|
|
2933
|
+
await this.options.delay(1);
|
|
2934
|
+
}
|
|
2935
|
+
if (this.activeReaders > 0 || this.activeWrites > 0) {
|
|
2936
|
+
this.accepting = true;
|
|
2937
|
+
return { ok: false, error: "Timed out waiting for telemetry readers/writers to quiesce" };
|
|
2938
|
+
}
|
|
2939
|
+
try {
|
|
2940
|
+
const tableMap = this.requireTableMap();
|
|
2941
|
+
this.closeDatabaseConnectionForCompaction();
|
|
2942
|
+
this.compactClosedDatabase(tableMap);
|
|
2943
|
+
await this.reopenDatabaseAfterCompaction(tableMap);
|
|
2944
|
+
this.accepting = true;
|
|
2945
|
+
return {
|
|
2946
|
+
ok: true,
|
|
2947
|
+
value: {
|
|
2948
|
+
stage: "compaction",
|
|
2949
|
+
affectedRows: 0,
|
|
2950
|
+
affectedPages: 0,
|
|
2951
|
+
elapsedMs: this.options.now() - startedAt,
|
|
2952
|
+
detail: null
|
|
2953
|
+
}
|
|
2954
|
+
};
|
|
2955
|
+
} catch (cause) {
|
|
2956
|
+
const compactionError = this.errorMessage(cause);
|
|
2957
|
+
try {
|
|
2958
|
+
await this.reopenDatabaseAfterCompaction(this.requireTableMap());
|
|
2959
|
+
this.accepting = true;
|
|
2960
|
+
} catch (reopenCause) {
|
|
2961
|
+
this.state = "failed";
|
|
2962
|
+
this.detail = this.errorMessage(reopenCause);
|
|
2963
|
+
}
|
|
2964
|
+
return { ok: false, error: compactionError };
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
async recordMaintenance(record) {
|
|
2968
|
+
try {
|
|
2969
|
+
return this.requireRepository().recordMaintenance(record);
|
|
2970
|
+
} catch (cause) {
|
|
2971
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
status() {
|
|
2975
|
+
const committedDropCounters = this.repository?.committedDropCounters();
|
|
2976
|
+
return Object.freeze({
|
|
2977
|
+
targetId: this.targetId,
|
|
2978
|
+
state: this.state,
|
|
2979
|
+
detail: this.detail,
|
|
2980
|
+
writerLeaseHeld: this.lease?.isHeld() ?? false,
|
|
2981
|
+
committedDropCounters: committedDropCounters?.ok === true ? committedDropCounters.value : Object.freeze([])
|
|
2982
|
+
});
|
|
2983
|
+
}
|
|
2984
|
+
maintenanceConfiguration() {
|
|
2985
|
+
return this.options.configuration.maintenance;
|
|
2986
|
+
}
|
|
2987
|
+
async stop(deadlineMs) {
|
|
2988
|
+
this.accepting = false;
|
|
2989
|
+
while ((this.activeReaders > 0 || this.activeWrites > 0) && this.options.now() < deadlineMs) {
|
|
2990
|
+
await this.options.delay(1);
|
|
2991
|
+
}
|
|
2992
|
+
this.closeOwnedResources();
|
|
2993
|
+
this.state = "stopped";
|
|
2994
|
+
this.detail = null;
|
|
2995
|
+
}
|
|
2996
|
+
configureConnection(connection) {
|
|
2997
|
+
const userTableCount = this.readCount(connection.prepare(
|
|
2998
|
+
`SELECT COUNT(*) AS count FROM sqlite_master
|
|
2999
|
+
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`
|
|
3000
|
+
).get());
|
|
3001
|
+
if (userTableCount === 0) {
|
|
3002
|
+
connection.pragma("auto_vacuum = INCREMENTAL");
|
|
3003
|
+
connection.exec("VACUUM");
|
|
3004
|
+
}
|
|
3005
|
+
const autoVacuum = Number(connection.pragma("auto_vacuum", { simple: true }));
|
|
3006
|
+
if (autoVacuum !== 2) {
|
|
3007
|
+
throw new Error("Telemetry SQLite target must use auto_vacuum=INCREMENTAL");
|
|
3008
|
+
}
|
|
3009
|
+
connection.pragma("journal_mode = WAL");
|
|
3010
|
+
connection.pragma("synchronous = NORMAL");
|
|
3011
|
+
connection.pragma("foreign_keys = ON");
|
|
3012
|
+
connection.pragma("busy_timeout = 5000");
|
|
3013
|
+
}
|
|
3014
|
+
closeDatabaseConnectionForCompaction() {
|
|
3015
|
+
const connection = this.requireConnection();
|
|
3016
|
+
connection.pragma("wal_checkpoint(TRUNCATE)");
|
|
3017
|
+
this.repository = null;
|
|
3018
|
+
this.connection = null;
|
|
3019
|
+
connection.close();
|
|
3020
|
+
}
|
|
3021
|
+
compactClosedDatabase(tableMap) {
|
|
3022
|
+
const connection = new BetterSqlite3(this.options.configuration.path);
|
|
3023
|
+
try {
|
|
3024
|
+
this.configureConnection(connection);
|
|
3025
|
+
connection.exec("VACUUM");
|
|
3026
|
+
this.verifySchema(connection, tableMap);
|
|
3027
|
+
} finally {
|
|
3028
|
+
connection.close();
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
3031
|
+
async reopenDatabaseAfterCompaction(tableMap) {
|
|
3032
|
+
if (this.connection !== null && this.repository !== null) {
|
|
3033
|
+
return;
|
|
3034
|
+
}
|
|
3035
|
+
const connection = new BetterSqlite3(this.options.configuration.path);
|
|
3036
|
+
try {
|
|
3037
|
+
this.configureConnection(connection);
|
|
3038
|
+
this.initializeAndVerifyManifest(connection, tableMap);
|
|
3039
|
+
this.verifySchema(connection, tableMap);
|
|
3040
|
+
const stackDatabase = await createSqliteDb(connection);
|
|
3041
|
+
const repository = new TelemetryRepository({
|
|
3042
|
+
targetId: this.targetId,
|
|
3043
|
+
connection,
|
|
3044
|
+
stackDatabase,
|
|
3045
|
+
tableMap,
|
|
3046
|
+
retention: this.options.configuration.retention,
|
|
3047
|
+
now: this.options.now
|
|
3048
|
+
});
|
|
3049
|
+
const loaded = repository.loadDefinitionsReadOnly(this.definitions);
|
|
3050
|
+
if (!loaded.ok) {
|
|
3051
|
+
throw new Error(loaded.error);
|
|
3052
|
+
}
|
|
3053
|
+
this.connection = connection;
|
|
3054
|
+
this.repository = repository;
|
|
3055
|
+
} catch (cause) {
|
|
3056
|
+
connection.close();
|
|
3057
|
+
throw cause;
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
initializeAndVerifyManifest(connection, tableMap) {
|
|
3061
|
+
connection.exec(
|
|
3062
|
+
`CREATE TABLE IF NOT EXISTS "${SQLITE_SCHEMA_MANIFEST_TABLE}" (
|
|
3063
|
+
target_id TEXT PRIMARY KEY,
|
|
3064
|
+
format_version INTEGER NOT NULL,
|
|
3065
|
+
mapping_json TEXT NOT NULL,
|
|
3066
|
+
mapping_checksum TEXT NOT NULL,
|
|
3067
|
+
state TEXT NOT NULL CHECK (state IN ('ready', 'renaming')),
|
|
3068
|
+
pending_mapping_json TEXT,
|
|
3069
|
+
updated_at_ms INTEGER NOT NULL
|
|
3070
|
+
)`
|
|
3071
|
+
);
|
|
3072
|
+
const rows = connection.prepare(
|
|
3073
|
+
`SELECT target_id, format_version, mapping_json, mapping_checksum,
|
|
3074
|
+
state, pending_mapping_json, updated_at_ms
|
|
3075
|
+
FROM "${SQLITE_SCHEMA_MANIFEST_TABLE}"`
|
|
3076
|
+
).all();
|
|
3077
|
+
if (rows.length === 0) {
|
|
3078
|
+
connection.prepare(
|
|
3079
|
+
`INSERT INTO "${SQLITE_SCHEMA_MANIFEST_TABLE}" (
|
|
3080
|
+
target_id, format_version, mapping_json, mapping_checksum,
|
|
3081
|
+
state, pending_mapping_json, updated_at_ms
|
|
3082
|
+
) VALUES (?, ?, ?, ?, 'ready', NULL, ?)`
|
|
3083
|
+
).run(
|
|
3084
|
+
this.targetId,
|
|
3085
|
+
2,
|
|
3086
|
+
JSON.stringify(tableMap.manifestMapping()),
|
|
3087
|
+
tableMap.checksum,
|
|
3088
|
+
this.options.now()
|
|
3089
|
+
);
|
|
3090
|
+
return false;
|
|
3091
|
+
}
|
|
3092
|
+
if (rows.length !== 1) {
|
|
3093
|
+
throw new Error("Telemetry schema manifest must contain exactly one target mapping");
|
|
3094
|
+
}
|
|
3095
|
+
const parsed = TelemetrySchemaManifestRowSchema.safeParse(rows[0]);
|
|
3096
|
+
if (!parsed.success) {
|
|
3097
|
+
throw new Error(parsed.error.message);
|
|
3098
|
+
}
|
|
3099
|
+
if (parsed.data.target_id !== this.targetId) {
|
|
3100
|
+
throw new Error(`Telemetry database belongs to target ${parsed.data.target_id}, not ${this.targetId}`);
|
|
3101
|
+
}
|
|
3102
|
+
if (parsed.data.state !== "ready") {
|
|
3103
|
+
throw new Error("Telemetry schema rename recovery is required before startup");
|
|
3104
|
+
}
|
|
3105
|
+
if (parsed.data.mapping_checksum === tableMap.checksum) {
|
|
3106
|
+
return false;
|
|
3107
|
+
}
|
|
3108
|
+
if (this.isLegacyManifestMapping(parsed.data, tableMap)) {
|
|
3109
|
+
return true;
|
|
3110
|
+
}
|
|
3111
|
+
throw new Error("Telemetry table mapping differs from the schema manifest; run offline schema rename");
|
|
3112
|
+
}
|
|
3113
|
+
isLegacyManifestMapping(manifest, tableMap) {
|
|
3114
|
+
if (manifest.format_version !== 1) {
|
|
3115
|
+
return false;
|
|
3116
|
+
}
|
|
3117
|
+
const parsed = JSON.parse(manifest.mapping_json);
|
|
3118
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
3119
|
+
return false;
|
|
3120
|
+
}
|
|
3121
|
+
if (telemetryChecksum(parsed) !== manifest.mapping_checksum) {
|
|
3122
|
+
return false;
|
|
3123
|
+
}
|
|
3124
|
+
const expected = tableMap.manifestMapping();
|
|
3125
|
+
const legacyEntries = Object.entries(expected).filter(
|
|
3126
|
+
([logicalName]) => logicalName !== "committedDropCounters"
|
|
3127
|
+
);
|
|
3128
|
+
const actualEntries = Object.entries(parsed);
|
|
3129
|
+
return actualEntries.length === legacyEntries.length && legacyEntries.every(([logicalName, physicalName]) => {
|
|
3130
|
+
return Reflect.get(parsed, logicalName) === physicalName;
|
|
3131
|
+
});
|
|
3132
|
+
}
|
|
3133
|
+
upgradeManifestMapping(connection, tableMap) {
|
|
3134
|
+
connection.prepare(
|
|
3135
|
+
`UPDATE "${SQLITE_SCHEMA_MANIFEST_TABLE}"
|
|
3136
|
+
SET format_version = 2, mapping_json = ?, mapping_checksum = ?, updated_at_ms = ?
|
|
3137
|
+
WHERE target_id = ?`
|
|
3138
|
+
).run(
|
|
3139
|
+
JSON.stringify(tableMap.manifestMapping()),
|
|
3140
|
+
tableMap.checksum,
|
|
3141
|
+
this.options.now(),
|
|
3142
|
+
this.targetId
|
|
3143
|
+
);
|
|
3144
|
+
}
|
|
3145
|
+
async runMigrations(connection, tableMap) {
|
|
3146
|
+
const sqlRunner = new TelemetryProperSqlRunner(connection, tableMap);
|
|
3147
|
+
const migrationRunner = await new MigrationRunnerFactory().create({
|
|
3148
|
+
migration_folder: this.options.migrationsDirectory,
|
|
3149
|
+
migration_table: tableMap.quoted("migrationHistory"),
|
|
3150
|
+
database: "sqlite",
|
|
3151
|
+
sqlite: { database: this.options.configuration.path }
|
|
3152
|
+
}, sqlRunner);
|
|
3153
|
+
const pending = await migrationRunner.getPendingMigrations();
|
|
3154
|
+
await migrationRunner.migrate(pending, true);
|
|
3155
|
+
}
|
|
3156
|
+
verifySchema(connection, tableMap) {
|
|
3157
|
+
const rows = connection.prepare(
|
|
3158
|
+
`SELECT name FROM sqlite_master WHERE type = 'table'`
|
|
3159
|
+
).all();
|
|
3160
|
+
const names = new Set(rows.map((row) => this.readName(row)));
|
|
3161
|
+
if (!names.has(SQLITE_SCHEMA_MANIFEST_TABLE)) {
|
|
3162
|
+
throw new Error("Telemetry schema manifest is missing");
|
|
3163
|
+
}
|
|
3164
|
+
const missing = SQLITE_LOGICAL_TABLE_NAMES.map((logicalName) => tableMap.physical(logicalName)).filter((physicalName) => !names.has(physicalName));
|
|
3165
|
+
if (missing.length > 0) {
|
|
3166
|
+
throw new Error(`Telemetry schema is missing table(s): ${missing.join(", ")}`);
|
|
3167
|
+
}
|
|
3168
|
+
}
|
|
3169
|
+
startHeartbeat() {
|
|
3170
|
+
const intervalMs = Math.max(1e3, Math.floor(this.options.configuration.leaseExpiryMs / 3));
|
|
3171
|
+
this.heartbeatHandle = this.options.createInterval(() => {
|
|
3172
|
+
const heartbeat = this.lease?.heartbeat();
|
|
3173
|
+
if (heartbeat && !heartbeat.ok) {
|
|
3174
|
+
this.accepting = false;
|
|
3175
|
+
this.state = "failed";
|
|
3176
|
+
this.detail = heartbeat.error;
|
|
3177
|
+
}
|
|
3178
|
+
}, intervalMs);
|
|
3179
|
+
}
|
|
3180
|
+
withReaderLease(work) {
|
|
3181
|
+
if (!this.accepting || this.state !== "ready") {
|
|
3182
|
+
return { ok: false, error: { kind: "storage-failure", message: "Telemetry target is not accepting readers" } };
|
|
3183
|
+
}
|
|
3184
|
+
this.activeReaders += 1;
|
|
3185
|
+
const leaseId = randomUUID();
|
|
3186
|
+
const table = this.requireTableMap().quoted("readerLeases");
|
|
3187
|
+
const connection = this.requireConnection();
|
|
3188
|
+
try {
|
|
3189
|
+
const acquiredAtMs = this.options.now();
|
|
3190
|
+
connection.prepare(
|
|
3191
|
+
`INSERT INTO ${table} (
|
|
3192
|
+
target_id, lease_id, owner_id, acquired_at_ms, expires_at_ms, state
|
|
3193
|
+
) VALUES (?, ?, ?, ?, ?, 'active')`
|
|
3194
|
+
).run(
|
|
3195
|
+
this.targetId,
|
|
3196
|
+
leaseId,
|
|
3197
|
+
this.options.processToken,
|
|
3198
|
+
acquiredAtMs,
|
|
3199
|
+
acquiredAtMs + this.options.configuration.leaseExpiryMs
|
|
3200
|
+
);
|
|
3201
|
+
return work();
|
|
3202
|
+
} finally {
|
|
3203
|
+
connection.prepare(
|
|
3204
|
+
`UPDATE ${table} SET state = 'released' WHERE target_id = ? AND lease_id = ?`
|
|
3205
|
+
).run(this.targetId, leaseId);
|
|
3206
|
+
this.activeReaders -= 1;
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
requireRepositoryResult(work) {
|
|
3210
|
+
try {
|
|
3211
|
+
return work(this.requireRepository());
|
|
3212
|
+
} catch (cause) {
|
|
3213
|
+
return { ok: false, error: this.errorMessage(cause) };
|
|
3214
|
+
}
|
|
3215
|
+
}
|
|
3216
|
+
validateTimelineQuery(query) {
|
|
3217
|
+
const commonError = this.validateQueryRange(query.targetId, query.fromMs, query.toMs, query.limit);
|
|
3218
|
+
return commonError;
|
|
3219
|
+
}
|
|
3220
|
+
validateMetricQuery(query) {
|
|
3221
|
+
const rangeError = this.validateQueryRange(query.targetId, query.fromMs, query.toMs, query.limit);
|
|
3222
|
+
if (rangeError !== null) {
|
|
3223
|
+
return rangeError;
|
|
3224
|
+
}
|
|
3225
|
+
if (query.limit > this.options.configuration.query.maxBuckets) {
|
|
3226
|
+
return "Metric query limit exceeds the configured bucket maximum";
|
|
3227
|
+
}
|
|
3228
|
+
return null;
|
|
3229
|
+
}
|
|
3230
|
+
validateQueryRange(targetId, fromMs, toMs, limit) {
|
|
3231
|
+
if (targetId !== null && targetId !== this.targetId) {
|
|
3232
|
+
return `Query target ${targetId} does not match ${this.targetId}`;
|
|
3233
|
+
}
|
|
3234
|
+
if (!Number.isSafeInteger(fromMs) || !Number.isSafeInteger(toMs) || fromMs >= toMs) {
|
|
3235
|
+
return "Query time range must use safe integer milliseconds and be non-empty";
|
|
3236
|
+
}
|
|
3237
|
+
if (toMs - fromMs > this.options.configuration.query.maxRangeMs) {
|
|
3238
|
+
return "Query time range exceeds the configured maximum";
|
|
3239
|
+
}
|
|
3240
|
+
if (!Number.isInteger(limit) || limit <= 0 || limit > this.options.configuration.query.maxRows) {
|
|
3241
|
+
return "Query row limit is outside the configured bounds";
|
|
3242
|
+
}
|
|
3243
|
+
return null;
|
|
3244
|
+
}
|
|
3245
|
+
validateConfiguration() {
|
|
3246
|
+
const configuration = this.options.configuration;
|
|
3247
|
+
if (!configuration.targetId.trim() || !configuration.path.trim()) {
|
|
3248
|
+
throw new Error("SQLite telemetry target id and path are required");
|
|
3249
|
+
}
|
|
3250
|
+
if (!Number.isInteger(configuration.leaseExpiryMs) || configuration.leaseExpiryMs < 1e3) {
|
|
3251
|
+
throw new Error("SQLite telemetry ownership lease expiry must be at least one second");
|
|
3252
|
+
}
|
|
3253
|
+
const queryLimits = Object.values(configuration.query);
|
|
3254
|
+
if (queryLimits.some((limit) => !Number.isSafeInteger(limit) || limit <= 0)) {
|
|
3255
|
+
throw new Error("SQLite telemetry query limits must be positive safe integers");
|
|
3256
|
+
}
|
|
3257
|
+
const retention = configuration.retention;
|
|
3258
|
+
const invalidRetention = retention.rawMs <= 0 || retention.lateArrivalMs < 0 || retention.samplesMs < retention.rawMs + retention.lateArrivalMs;
|
|
3259
|
+
if (invalidRetention) {
|
|
3260
|
+
throw new Error("SQLite telemetry retention values are invalid");
|
|
3261
|
+
}
|
|
3262
|
+
this.validateRollupTiers();
|
|
3263
|
+
}
|
|
3264
|
+
validateRollupTiers() {
|
|
3265
|
+
const retention = this.options.configuration.retention;
|
|
3266
|
+
const tiers = retention.rollups;
|
|
3267
|
+
if (tiers.length === 0) {
|
|
3268
|
+
throw new Error("SQLite telemetry requires at least one rollup retention tier");
|
|
3269
|
+
}
|
|
3270
|
+
for (let index = 0; index < tiers.length; index += 1) {
|
|
3271
|
+
const tier = tiers[index];
|
|
3272
|
+
const previous = tiers[index - 1];
|
|
3273
|
+
if (!tier || tier.resolutionMs <= 0 || tier.retainMs < retention.rawMs) {
|
|
3274
|
+
throw new Error("SQLite telemetry rollup tiers are invalid");
|
|
3275
|
+
}
|
|
3276
|
+
if (previous && (tier.resolutionMs <= previous.resolutionMs || tier.retainMs < previous.retainMs)) {
|
|
3277
|
+
throw new Error("SQLite telemetry rollup tiers must increase by resolution and retention");
|
|
3278
|
+
}
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
enforceResponseBytes(result) {
|
|
3282
|
+
if (!result.ok) {
|
|
3283
|
+
return result;
|
|
3284
|
+
}
|
|
3285
|
+
try {
|
|
3286
|
+
const bytes = telemetryUtf8Length(canonicalJson(result.value));
|
|
3287
|
+
if (bytes > this.options.configuration.query.maxResponseBytes) {
|
|
3288
|
+
return {
|
|
3289
|
+
ok: false,
|
|
3290
|
+
error: { kind: "invalid-query", message: "Query result exceeds the configured response byte maximum" }
|
|
3291
|
+
};
|
|
3292
|
+
}
|
|
3293
|
+
return result;
|
|
3294
|
+
} catch (cause) {
|
|
3295
|
+
return {
|
|
3296
|
+
ok: false,
|
|
3297
|
+
error: { kind: "storage-failure", message: `Unable to size telemetry query result: ${this.errorMessage(cause)}` }
|
|
3298
|
+
};
|
|
3299
|
+
}
|
|
3300
|
+
}
|
|
3301
|
+
failStart(error) {
|
|
3302
|
+
this.state = "failed";
|
|
3303
|
+
this.detail = error;
|
|
3304
|
+
return { ok: false, error };
|
|
3305
|
+
}
|
|
3306
|
+
closedAcknowledgement(idempotencyKey) {
|
|
3307
|
+
return {
|
|
3308
|
+
kind: "retryable",
|
|
3309
|
+
idempotencyKey,
|
|
3310
|
+
reason: "SQLite telemetry target is not ready",
|
|
3311
|
+
retryAfterMs: 100,
|
|
3312
|
+
targetStates: [{
|
|
3313
|
+
targetId: this.targetId,
|
|
3314
|
+
delivery: this.options.configuration.delivery,
|
|
3315
|
+
state: "failed",
|
|
3316
|
+
attempts: 0,
|
|
3317
|
+
reason: "target-not-ready"
|
|
3318
|
+
}]
|
|
3319
|
+
};
|
|
3320
|
+
}
|
|
3321
|
+
closeOwnedResources() {
|
|
3322
|
+
if (this.heartbeatHandle !== null) {
|
|
3323
|
+
this.options.clearInterval(this.heartbeatHandle);
|
|
3324
|
+
this.heartbeatHandle = null;
|
|
3325
|
+
}
|
|
3326
|
+
if (this.connection?.open) {
|
|
3327
|
+
try {
|
|
3328
|
+
this.connection.close();
|
|
3329
|
+
} catch {
|
|
3330
|
+
}
|
|
3331
|
+
}
|
|
3332
|
+
this.connection = null;
|
|
3333
|
+
this.repository = null;
|
|
3334
|
+
this.lease?.release();
|
|
3335
|
+
this.lease = null;
|
|
3336
|
+
}
|
|
3337
|
+
requireRepository() {
|
|
3338
|
+
if (this.repository === null) {
|
|
3339
|
+
throw new Error("SQLite telemetry repository is not initialized");
|
|
3340
|
+
}
|
|
3341
|
+
return this.repository;
|
|
3342
|
+
}
|
|
3343
|
+
requireConnection() {
|
|
3344
|
+
if (this.connection === null) {
|
|
3345
|
+
throw new Error("SQLite telemetry connection is not initialized");
|
|
3346
|
+
}
|
|
3347
|
+
return this.connection;
|
|
3348
|
+
}
|
|
3349
|
+
requireTableMap() {
|
|
3350
|
+
if (this.tableMap === null) {
|
|
3351
|
+
throw new Error("SQLite telemetry table map is not initialized");
|
|
3352
|
+
}
|
|
3353
|
+
return this.tableMap;
|
|
3354
|
+
}
|
|
3355
|
+
readCount(row) {
|
|
3356
|
+
if (typeof row !== "object" || row === null || Array.isArray(row)) {
|
|
3357
|
+
throw new Error("SQLite count query returned no row");
|
|
3358
|
+
}
|
|
3359
|
+
const count = Reflect.get(row, "count");
|
|
3360
|
+
if (typeof count !== "number") {
|
|
3361
|
+
throw new Error("SQLite count query returned an invalid count");
|
|
3362
|
+
}
|
|
3363
|
+
return count;
|
|
3364
|
+
}
|
|
3365
|
+
readName(row) {
|
|
3366
|
+
if (typeof row !== "object" || row === null || Array.isArray(row)) {
|
|
3367
|
+
throw new Error("SQLite schema query returned an invalid row");
|
|
3368
|
+
}
|
|
3369
|
+
const name = Reflect.get(row, "name");
|
|
3370
|
+
if (typeof name !== "string") {
|
|
3371
|
+
throw new Error("SQLite schema query returned an invalid table name");
|
|
3372
|
+
}
|
|
3373
|
+
return name;
|
|
3374
|
+
}
|
|
3375
|
+
errorMessage(cause) {
|
|
3376
|
+
return cause instanceof Error ? cause.message : String(cause);
|
|
3377
|
+
}
|
|
3378
|
+
};
|
|
3379
|
+
export {
|
|
3380
|
+
SQLiteTelemetryTransport
|
|
3381
|
+
};
|
|
3382
|
+
//# sourceMappingURL=index.js.map
|