@dudousxd/nestjs-telescope-mikro-orm 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +334 -0
- package/README.md +28 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/mikro-orm-storage.provider.d.ts +49 -18
- package/dist/mikro-orm-storage.provider.d.ts.map +1 -1
- package/dist/mikro-orm-storage.provider.js +519 -28
- package/dist/mikro-orm-storage.provider.js.map +1 -1
- package/dist/schema.d.ts +18 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +38 -0
- package/dist/schema.js.map +1 -0
- package/dist/telescope-entry.entity.d.ts.map +1 -1
- package/dist/telescope-entry.entity.js +6 -1
- package/dist/telescope-entry.entity.js.map +1 -1
- package/dist/telescope-lease.entity.d.ts +11 -0
- package/dist/telescope-lease.entity.d.ts.map +1 -0
- package/dist/telescope-lease.entity.js +39 -0
- package/dist/telescope-lease.entity.js.map +1 -0
- package/dist/telescope-mikro-orm.logger.d.ts +26 -4
- package/dist/telescope-mikro-orm.logger.d.ts.map +1 -1
- package/dist/telescope-mikro-orm.logger.js +28 -16
- package/dist/telescope-mikro-orm.logger.js.map +1 -1
- package/dist/telescope-schema-meta.entity.d.ts +11 -0
- package/dist/telescope-schema-meta.entity.d.ts.map +1 -0
- package/dist/telescope-schema-meta.entity.js +33 -0
- package/dist/telescope-schema-meta.entity.js.map +1 -0
- package/package.json +32 -26
|
@@ -1,8 +1,33 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { HISTOGRAM_BUCKET_COUNT, decodeCursor, encodeCursor, isBatchOrigin, mergeHistograms, normalizeHistogram, } from '@dudousxd/nestjs-telescope';
|
|
2
3
|
import { MikroORM, raw } from '@mikro-orm/core';
|
|
3
4
|
import { TelescopeEntry } from './telescope-entry.entity.js';
|
|
5
|
+
import { TelescopeLease } from './telescope-lease.entity.js';
|
|
4
6
|
import { TelescopeRollup } from './telescope-rollup.entity.js';
|
|
7
|
+
import { TelescopeSchemaMeta } from './telescope-schema-meta.entity.js';
|
|
5
8
|
const DEFAULT_LIMIT = 50;
|
|
9
|
+
/**
|
|
10
|
+
* Hand-bump escape hatch for DDL changes that are NOT visible in entity metadata
|
|
11
|
+
* (e.g. a collation tweak, an index expression, anything `schema.update` would
|
|
12
|
+
* emit but the fingerprint inputs below can't see). Incrementing this forces
|
|
13
|
+
* every pod to re-run the additive `schema.update` once, then re-cache.
|
|
14
|
+
*/
|
|
15
|
+
const SCHEMA_REVISION = 1;
|
|
16
|
+
/** Stable PK of the single fingerprint marker row. */
|
|
17
|
+
const SCHEMA_META_ID = 'mikro-orm';
|
|
18
|
+
/**
|
|
19
|
+
* Cross-driver DDL for the fingerprint marker table. `if not exists` keeps it
|
|
20
|
+
* idempotent and introspection-free (no `information_schema` round-trip), and
|
|
21
|
+
* the unquoted lowercase identifiers + portable `varchar`/`bigint` types parse
|
|
22
|
+
* on MySQL, PostgreSQL, and SQLite alike. Mirrors `TelescopeSchemaMeta`.
|
|
23
|
+
*/
|
|
24
|
+
const CREATE_SCHEMA_META_TABLE_SQL = 'create table if not exists telescope_schema_meta (' +
|
|
25
|
+
'id varchar(32) not null, ' +
|
|
26
|
+
'fingerprint varchar(64) not null, ' +
|
|
27
|
+
'applied_at bigint not null, ' +
|
|
28
|
+
'primary key (id))';
|
|
29
|
+
/** A shared advisory-lock name/key both lock dialects derive from. */
|
|
30
|
+
const SCHEMA_LOCK_NAME = 'telescope_schema';
|
|
6
31
|
/**
|
|
7
32
|
* Every persisted column EXCEPT `content`, used as the MikroORM `fields`
|
|
8
33
|
* projection for content-less aggregate scans (omitContent). Selecting these
|
|
@@ -57,6 +82,68 @@ function rowToEntry(row, omitContent = false) {
|
|
|
57
82
|
function entryToRowData(entry) {
|
|
58
83
|
return { ...entry, tagsText: padTags(entry.tags) };
|
|
59
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* `createdAt < before` plus the type selector (single type, or all-except a set,
|
|
87
|
+
* or — when neither is given — no type filter). MikroORM renders the type clause
|
|
88
|
+
* to `type = ?` / `type not in (?, …)` on every driver. Shared by the unbounded
|
|
89
|
+
* and the batched prune so the two can never select different rows.
|
|
90
|
+
*/
|
|
91
|
+
function scopedWhere(scope) {
|
|
92
|
+
const where = { createdAt: { $lt: scope.before } };
|
|
93
|
+
if (scope.type !== undefined) {
|
|
94
|
+
where.type = scope.type;
|
|
95
|
+
}
|
|
96
|
+
else if (scope.excludeTypes !== undefined && scope.excludeTypes.length > 0) {
|
|
97
|
+
where.type = { $nin: scope.excludeTypes };
|
|
98
|
+
}
|
|
99
|
+
return where;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* The oldest `limit` rows of `where`, deleted: SELECT their ids, then DELETE BY PRIMARY KEY.
|
|
103
|
+
*
|
|
104
|
+
* This shape is chosen for PORTABILITY, and the obvious alternative is not portable: `DELETE ...
|
|
105
|
+
* ORDER BY created_at LIMIT n` is MySQL/MariaDB-only. PostgreSQL has no ORDER BY/LIMIT on DELETE at
|
|
106
|
+
* all (it needs exactly this subquery on the primary key), SQLite only has it when compiled with
|
|
107
|
+
* SQLITE_ENABLE_UPDATE_DELETE_LIMIT, and SQL Server spells it `DELETE TOP (n)` with no ordering.
|
|
108
|
+
* Expressed at the ORM level as find-then-delete it renders correctly on every driver MikroORM
|
|
109
|
+
* supports, takes no row locks to read, and the DELETE touches exactly those primary keys.
|
|
110
|
+
*
|
|
111
|
+
* With a `type = ?` filter the read is a range of the `(type, created_at, id)` index, already in
|
|
112
|
+
* the ORDER BY's order: it starts at that type's oldest row, stops after `limit` matches, and costs
|
|
113
|
+
* one index seek when nothing of that type has expired. Without it the planner sorts or walks
|
|
114
|
+
* another index past the rows of every other type, on PostgreSQL every row of the table.
|
|
115
|
+
*
|
|
116
|
+
* `hasMore` comes from what was SELECTED, not from what was deleted: another pruner racing us can
|
|
117
|
+
* delete some of these rows first, which lowers `deleted` while saying nothing about whether work
|
|
118
|
+
* remains.
|
|
119
|
+
*/
|
|
120
|
+
async function pruneOldestBatch(em, where, limit) {
|
|
121
|
+
const doomed = await em.find(TelescopeEntry, where, {
|
|
122
|
+
orderBy: { createdAt: 'asc', id: 'asc' },
|
|
123
|
+
limit,
|
|
124
|
+
fields: ['id'],
|
|
125
|
+
});
|
|
126
|
+
if (doomed.length === 0)
|
|
127
|
+
return { deleted: 0, hasMore: false };
|
|
128
|
+
const deleted = await em.nativeDelete(TelescopeEntry, {
|
|
129
|
+
id: { $in: doomed.map((row) => row.id) },
|
|
130
|
+
});
|
|
131
|
+
return { deleted, hasMore: doomed.length === limit };
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Every `type` present, one index seek each (`type > ?`, first in order): a skip scan by hand, since
|
|
135
|
+
* a plain `DISTINCT` reads the whole index on engines without one (PostgreSQL before 18).
|
|
136
|
+
*/
|
|
137
|
+
async function distinctTypes(em) {
|
|
138
|
+
const types = [];
|
|
139
|
+
for (;;) {
|
|
140
|
+
const previous = types[types.length - 1];
|
|
141
|
+
const [next] = await em.find(TelescopeEntry, previous === undefined ? {} : { type: { $gt: previous } }, { orderBy: { type: 'asc' }, fields: ['type'], limit: 1 });
|
|
142
|
+
if (!next)
|
|
143
|
+
return types;
|
|
144
|
+
types.push(next.type);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
60
147
|
function resolveLimit(limit) {
|
|
61
148
|
return typeof limit === 'number' && Number.isInteger(limit) && limit > 0 ? limit : DEFAULT_LIMIT;
|
|
62
149
|
}
|
|
@@ -99,6 +186,185 @@ function pickHostConnection(hostConfig) {
|
|
|
99
186
|
connection.timezone = hostConfig.timezone;
|
|
100
187
|
return connection;
|
|
101
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Table names the gate owns and fingerprints — `telescope_entries`,
|
|
191
|
+
* `telescope_rollups`, and `telescope_leases`. Derived from metadata (never
|
|
192
|
+
* hardcoded) so a future tableName rename flows through automatically.
|
|
193
|
+
* Deliberately EXCLUDES the marker table, whose own shape must never invalidate
|
|
194
|
+
* the gate.
|
|
195
|
+
*
|
|
196
|
+
* `telescope_leases` is INSIDE the fingerprint on purpose: that is what makes an
|
|
197
|
+
* existing deployment notice the new table and create it on the next boot,
|
|
198
|
+
* through the same additive `schema.update` path as any other schema change. If
|
|
199
|
+
* it were excluded, the fingerprint would still match and the lease table would
|
|
200
|
+
* never be created.
|
|
201
|
+
*/
|
|
202
|
+
function collectOwnedTableNames(orm) {
|
|
203
|
+
const metadata = orm.getMetadata();
|
|
204
|
+
return [
|
|
205
|
+
metadata.get(TelescopeEntry).tableName,
|
|
206
|
+
metadata.get(TelescopeRollup).tableName,
|
|
207
|
+
metadata.get(TelescopeLease).tableName,
|
|
208
|
+
];
|
|
209
|
+
}
|
|
210
|
+
function compareStrings(left, right) {
|
|
211
|
+
if (left < right)
|
|
212
|
+
return -1;
|
|
213
|
+
if (left > right)
|
|
214
|
+
return 1;
|
|
215
|
+
return 0;
|
|
216
|
+
}
|
|
217
|
+
/** Maps an entity's properties to per-column fingerprints, expanding any
|
|
218
|
+
* multi-column property by its `fieldNames`. Columns are sorted by name. */
|
|
219
|
+
function columnsOf(meta) {
|
|
220
|
+
const columns = [];
|
|
221
|
+
for (const prop of meta.props) {
|
|
222
|
+
prop.fieldNames.forEach((fieldName, index) => {
|
|
223
|
+
columns.push({
|
|
224
|
+
autoincrement: prop.autoincrement === true,
|
|
225
|
+
default: prop.default ?? null,
|
|
226
|
+
name: fieldName,
|
|
227
|
+
nullable: prop.nullable === true,
|
|
228
|
+
primary: prop.primary === true,
|
|
229
|
+
type: prop.columnTypes[index] ?? prop.columnTypes[0] ?? prop.type,
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
return columns.sort((left, right) => compareStrings(left.name, right.name));
|
|
234
|
+
}
|
|
235
|
+
function indexNameFromFlag(flag) {
|
|
236
|
+
return typeof flag === 'string' ? flag : '';
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Index fingerprints from BOTH property-level flags (`{ index: true }` /
|
|
240
|
+
* `{ unique: true }`, which is how the telescope entities declare every index)
|
|
241
|
+
* AND any entity-level `meta.indexes`. Without the property-level pass, flipping
|
|
242
|
+
* an existing column to indexed (no new column) would slip past the gate. Sorted
|
|
243
|
+
* by name then properties for determinism.
|
|
244
|
+
*/
|
|
245
|
+
function indexesOf(meta) {
|
|
246
|
+
const indexes = [];
|
|
247
|
+
for (const prop of meta.props) {
|
|
248
|
+
if (prop.index) {
|
|
249
|
+
indexes.push({
|
|
250
|
+
name: indexNameFromFlag(prop.index),
|
|
251
|
+
properties: [...prop.fieldNames].sort(),
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
if (prop.unique) {
|
|
255
|
+
indexes.push({
|
|
256
|
+
name: indexNameFromFlag(prop.unique),
|
|
257
|
+
properties: [...prop.fieldNames].sort(),
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
for (const index of meta.indexes) {
|
|
262
|
+
const declared = index.properties;
|
|
263
|
+
const properties = Array.isArray(declared)
|
|
264
|
+
? [...declared]
|
|
265
|
+
: declared !== undefined
|
|
266
|
+
? [declared]
|
|
267
|
+
: [];
|
|
268
|
+
indexes.push({ name: index.name ?? '', properties: properties.map(String).sort() });
|
|
269
|
+
}
|
|
270
|
+
return indexes.sort((left, right) => compareStrings(left.name, right.name) ||
|
|
271
|
+
compareStrings(left.properties.join(','), right.properties.join(',')));
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Pure, in-memory sha256 of a CANONICAL serialization of the owned tables' shape
|
|
275
|
+
* (sorted tables, sorted columns, sorted indexes; object keys authored
|
|
276
|
+
* alphabetically so `JSON.stringify` is order-stable). The platform name is
|
|
277
|
+
* folded in so a SQLite CI marker can never satisfy a MySQL boot, the configured
|
|
278
|
+
* collation so a collation change re-heals, and SCHEMA_REVISION as the manual
|
|
279
|
+
* escape hatch. Never reads enumeration order — everything is sorted.
|
|
280
|
+
*/
|
|
281
|
+
function computeExpectedFingerprint(orm, ownedTableNames) {
|
|
282
|
+
const owned = new Set(ownedTableNames);
|
|
283
|
+
const tables = [];
|
|
284
|
+
for (const meta of orm.getMetadata()) {
|
|
285
|
+
if (!owned.has(meta.tableName))
|
|
286
|
+
continue;
|
|
287
|
+
tables.push({ columns: columnsOf(meta), indexes: indexesOf(meta), tableName: meta.tableName });
|
|
288
|
+
}
|
|
289
|
+
tables.sort((left, right) => compareStrings(left.tableName, right.tableName));
|
|
290
|
+
const collate = orm.config.get('collate');
|
|
291
|
+
const payload = {
|
|
292
|
+
collate: collate ?? null,
|
|
293
|
+
platform: orm.em.getPlatform().constructor.name,
|
|
294
|
+
revision: SCHEMA_REVISION,
|
|
295
|
+
tables,
|
|
296
|
+
};
|
|
297
|
+
return createHash('sha256').update(JSON.stringify(payload)).digest('hex');
|
|
298
|
+
}
|
|
299
|
+
/** Reads the single stored fingerprint, or null when the marker row is absent. */
|
|
300
|
+
async function readStoredFingerprint(orm) {
|
|
301
|
+
const em = orm.em.fork();
|
|
302
|
+
const row = await em.findOne(TelescopeSchemaMeta, { id: SCHEMA_META_ID });
|
|
303
|
+
return row ? row.fingerprint : null;
|
|
304
|
+
}
|
|
305
|
+
/** Driver-agnostic upsert of the single marker row with the reconciled fingerprint. */
|
|
306
|
+
async function writeStoredFingerprint(orm, fingerprint) {
|
|
307
|
+
const em = orm.em.fork();
|
|
308
|
+
await em.upsert(TelescopeSchemaMeta, {
|
|
309
|
+
id: SCHEMA_META_ID,
|
|
310
|
+
fingerprint,
|
|
311
|
+
appliedAt: Date.now(),
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Best-effort advisory lock so concurrent pods serialize the one-time heal
|
|
316
|
+
* instead of all introspecting at once. MySQL/MariaDB use `GET_LOCK`,
|
|
317
|
+
* PostgreSQL `pg_advisory_lock(hashtext(...))`; SQLite and unknown drivers get a
|
|
318
|
+
* no-op (single-writer or unsupported). Acquisition failures (permissions,
|
|
319
|
+
* timeout) degrade to a no-op so the heal still proceeds — the re-SELECT after
|
|
320
|
+
* locking is what actually prevents duplicate work, the lock just reduces it.
|
|
321
|
+
*/
|
|
322
|
+
async function acquireSchemaLock(orm) {
|
|
323
|
+
const platform = orm.em.getPlatform().constructor.name.toLowerCase();
|
|
324
|
+
const connection = orm.em.getConnection();
|
|
325
|
+
const noop = {
|
|
326
|
+
release: async () => {
|
|
327
|
+
/* nothing acquired */
|
|
328
|
+
},
|
|
329
|
+
};
|
|
330
|
+
try {
|
|
331
|
+
if (platform.includes('mysql') || platform.includes('maria')) {
|
|
332
|
+
await connection.execute(`select get_lock('${SCHEMA_LOCK_NAME}', 10) as locked`);
|
|
333
|
+
return {
|
|
334
|
+
release: async () => {
|
|
335
|
+
try {
|
|
336
|
+
await connection.execute(`select release_lock('${SCHEMA_LOCK_NAME}')`);
|
|
337
|
+
}
|
|
338
|
+
catch {
|
|
339
|
+
/* lock auto-releases on connection close */
|
|
340
|
+
}
|
|
341
|
+
},
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
if (platform.includes('postgre')) {
|
|
345
|
+
await connection.execute(`select pg_advisory_lock(hashtext('${SCHEMA_LOCK_NAME}'))`);
|
|
346
|
+
return {
|
|
347
|
+
release: async () => {
|
|
348
|
+
try {
|
|
349
|
+
await connection.execute(`select pg_advisory_unlock(hashtext('${SCHEMA_LOCK_NAME}'))`);
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
/* lock auto-releases on connection close */
|
|
353
|
+
}
|
|
354
|
+
},
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
/* advisory lock unavailable → proceed without it */
|
|
360
|
+
}
|
|
361
|
+
return noop;
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Declared against {@link BoundedPruneCapable} and {@link LeaseCapableStorage}
|
|
365
|
+
* (both extend `StorageProvider`) so dropping either capability is a compile
|
|
366
|
+
* error rather than a silent fall back to unbounded, unlocked pruning.
|
|
367
|
+
*/
|
|
102
368
|
export class MikroOrmStorageProvider {
|
|
103
369
|
source;
|
|
104
370
|
ensureSchema;
|
|
@@ -120,7 +386,7 @@ export class MikroOrmStorageProvider {
|
|
|
120
386
|
const connection = pickHostConnection(this.source.config.getAll());
|
|
121
387
|
return {
|
|
122
388
|
...connection,
|
|
123
|
-
entities: [TelescopeEntry, TelescopeRollup],
|
|
389
|
+
entities: [TelescopeEntry, TelescopeRollup, TelescopeSchemaMeta, TelescopeLease],
|
|
124
390
|
contextName: TELESCOPE_CONTEXT_NAME,
|
|
125
391
|
allowGlobalContext: true,
|
|
126
392
|
};
|
|
@@ -128,7 +394,7 @@ export class MikroOrmStorageProvider {
|
|
|
128
394
|
const { ensureSchema: _ensureSchema, entities: _entities, ...rest } = this.source;
|
|
129
395
|
return {
|
|
130
396
|
...rest,
|
|
131
|
-
entities: [TelescopeEntry, TelescopeRollup],
|
|
397
|
+
entities: [TelescopeEntry, TelescopeRollup, TelescopeSchemaMeta, TelescopeLease],
|
|
132
398
|
contextName: TELESCOPE_CONTEXT_NAME,
|
|
133
399
|
allowGlobalContext: true,
|
|
134
400
|
};
|
|
@@ -138,11 +404,52 @@ export class MikroOrmStorageProvider {
|
|
|
138
404
|
return;
|
|
139
405
|
this.orm = await MikroORM.init(this.buildOrmOptions());
|
|
140
406
|
if (this.ensureSchema) {
|
|
141
|
-
await this.orm
|
|
407
|
+
await this.reconcileSchema(this.orm);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Fingerprint-gated schema reconciliation. Replaces a bare
|
|
412
|
+
* `ensureDatabase(); schema.update()` so steady-state boots skip the
|
|
413
|
+
* whole-database introspection (see the gate notes above the helpers).
|
|
414
|
+
*
|
|
415
|
+
* FRESH-DB ORDERING: `ensureDatabase()` runs FIRST, before the marker table is
|
|
416
|
+
* touched. The provider owns a DEDICATED connection, so on a brand-new DB the
|
|
417
|
+
* database itself may not exist yet — a `create table telescope_schema_meta`
|
|
418
|
+
* issued before `ensureDatabase()` would hit "unknown database" on MySQL/PG.
|
|
419
|
+
* `ensureDatabase()` creates it when missing and is a single cheap existence
|
|
420
|
+
* check when it already exists (the common case: the host app is already
|
|
421
|
+
* connected to that database), so paying for it every boot is negligible next
|
|
422
|
+
* to the multi-second `schema.update` introspection the gate eliminates.
|
|
423
|
+
*/
|
|
424
|
+
async reconcileSchema(orm) {
|
|
425
|
+
// (0) Create the database if the owned connection points at one that does
|
|
426
|
+
// not exist yet. MUST precede any statement on the connection.
|
|
427
|
+
await orm.schema.ensureDatabase();
|
|
428
|
+
const ownedTableNames = collectOwnedTableNames(orm);
|
|
429
|
+
// (1) Idempotent, introspection-free marker table (empty-DB safe).
|
|
430
|
+
await orm.em.getConnection().execute(CREATE_SCHEMA_META_TABLE_SQL);
|
|
431
|
+
// (2) One PK read + (3) in-memory expected fingerprint.
|
|
432
|
+
const stored = await readStoredFingerprint(orm);
|
|
433
|
+
const expected = computeExpectedFingerprint(orm, ownedTableNames);
|
|
434
|
+
// (4) Steady state: identical fingerprint ⇒ skip the whole-DB schema.update.
|
|
435
|
+
if (stored === expected)
|
|
436
|
+
return;
|
|
437
|
+
// (5) Absent/mismatch ⇒ heal once, under a best-effort advisory lock so
|
|
438
|
+
// concurrent pods don't all introspect.
|
|
439
|
+
const lock = await acquireSchemaLock(orm);
|
|
440
|
+
try {
|
|
441
|
+
// Another pod may have healed between our SELECT and the lock; re-check.
|
|
442
|
+
const reStored = await readStoredFingerprint(orm);
|
|
443
|
+
if (reStored === expected)
|
|
444
|
+
return;
|
|
142
445
|
// `{ safe: true }` disables BOTH table and column dropping, so the update
|
|
143
|
-
// is additive-only: it creates
|
|
144
|
-
//
|
|
145
|
-
await
|
|
446
|
+
// is additive-only: it creates tables and adds missing columns/indexes but
|
|
447
|
+
// never drops anything that already exists.
|
|
448
|
+
await orm.schema.update({ safe: true });
|
|
449
|
+
await writeStoredFingerprint(orm, expected);
|
|
450
|
+
}
|
|
451
|
+
finally {
|
|
452
|
+
await lock.release();
|
|
146
453
|
}
|
|
147
454
|
}
|
|
148
455
|
async close() {
|
|
@@ -157,14 +464,27 @@ export class MikroOrmStorageProvider {
|
|
|
157
464
|
}
|
|
158
465
|
return this.orm.em.fork();
|
|
159
466
|
}
|
|
467
|
+
/**
|
|
468
|
+
* Plain multi-row INSERTs, not `em.create()` + `flush()`: entries are write-once, so managed
|
|
469
|
+
* entities and change sets bought nothing and cost several times the INSERT itself on a busy
|
|
470
|
+
* host. Chunked by the ORM's `batchSize` so a large flush stays under every driver's
|
|
471
|
+
* bound-parameter limit; a flush spanning several chunks runs in one transaction so it lands
|
|
472
|
+
* whole or not at all, and the recorder's retry never re-inserts rows a failed attempt wrote.
|
|
473
|
+
*/
|
|
160
474
|
async store(entries) {
|
|
161
475
|
if (entries.length === 0)
|
|
162
476
|
return;
|
|
163
477
|
const em = this.forkEm();
|
|
164
|
-
|
|
165
|
-
|
|
478
|
+
const chunk = em.config.get('batchSize');
|
|
479
|
+
if (entries.length <= chunk) {
|
|
480
|
+
await em.insertMany(TelescopeEntry, entries.map(entryToRowData));
|
|
481
|
+
return;
|
|
166
482
|
}
|
|
167
|
-
await em.
|
|
483
|
+
await em.transactional(async (tx) => {
|
|
484
|
+
for (let offset = 0; offset < entries.length; offset += chunk) {
|
|
485
|
+
await tx.insertMany(TelescopeEntry, entries.slice(offset, offset + chunk).map(entryToRowData));
|
|
486
|
+
}
|
|
487
|
+
});
|
|
168
488
|
}
|
|
169
489
|
async update(id, patch) {
|
|
170
490
|
const em = this.forkEm();
|
|
@@ -198,15 +518,18 @@ export class MikroOrmStorageProvider {
|
|
|
198
518
|
}
|
|
199
519
|
where.id = { $in: query.ids };
|
|
200
520
|
}
|
|
201
|
-
|
|
521
|
+
// Presence-based, not truthy: an explicit empty-string filter must still
|
|
522
|
+
// apply (matching the in-memory/sqlite/redis providers, which all use
|
|
523
|
+
// `!== undefined`). A truthy check would silently drop a `type: ''` filter.
|
|
524
|
+
if (query.type !== undefined)
|
|
202
525
|
where.type = query.type;
|
|
203
|
-
if (query.familyHash)
|
|
526
|
+
if (query.familyHash !== undefined)
|
|
204
527
|
where.familyHash = query.familyHash;
|
|
205
|
-
if (query.batchId)
|
|
528
|
+
if (query.batchId !== undefined)
|
|
206
529
|
where.batchId = query.batchId;
|
|
207
530
|
if (query.traceId !== undefined)
|
|
208
531
|
where.traceId = query.traceId;
|
|
209
|
-
if (query.tag)
|
|
532
|
+
if (query.tag !== undefined)
|
|
210
533
|
where.tagsText = { $like: `% ${query.tag} %` };
|
|
211
534
|
if (query.before || query.after) {
|
|
212
535
|
where.createdAt = {
|
|
@@ -251,18 +574,28 @@ export class MikroOrmStorageProvider {
|
|
|
251
574
|
const rows = await em.find(TelescopeEntry, { batchId }, { orderBy: { sequence: 'asc' } });
|
|
252
575
|
return rows.map((row) => rowToEntry(row));
|
|
253
576
|
}
|
|
254
|
-
async tags(prefix) {
|
|
577
|
+
async tags(prefix, query) {
|
|
255
578
|
const em = this.forkEm();
|
|
256
579
|
const rows = await em.find(TelescopeEntry, {}, { orderBy: { createdAt: 'desc', id: 'desc' }, limit: 10000, fields: ['tags'] });
|
|
580
|
+
const search = query?.search?.trim().toLowerCase();
|
|
257
581
|
const counts = new Map();
|
|
258
582
|
for (const row of rows) {
|
|
259
583
|
for (const tag of row.tags) {
|
|
260
584
|
if (prefix && !tag.startsWith(prefix))
|
|
261
585
|
continue;
|
|
586
|
+
if (search && !tag.toLowerCase().includes(search))
|
|
587
|
+
continue;
|
|
262
588
|
counts.set(tag, (counts.get(tag) ?? 0) + 1);
|
|
263
589
|
}
|
|
264
590
|
}
|
|
265
|
-
|
|
591
|
+
// Count descending, ties by tag: the total order `offset` pages over.
|
|
592
|
+
const ordered = [...counts.entries()]
|
|
593
|
+
.map(([tag, count]) => ({ tag, count }))
|
|
594
|
+
.sort((a, b) => b.count - a.count || (a.tag < b.tag ? -1 : a.tag > b.tag ? 1 : 0));
|
|
595
|
+
if (query?.limit === undefined)
|
|
596
|
+
return ordered;
|
|
597
|
+
const offset = query.offset ?? 0;
|
|
598
|
+
return ordered.slice(offset, offset + query.limit);
|
|
266
599
|
}
|
|
267
600
|
async prune(olderThan, keepLast) {
|
|
268
601
|
const em = this.forkEm();
|
|
@@ -275,6 +608,78 @@ export class MikroOrmStorageProvider {
|
|
|
275
608
|
return 0;
|
|
276
609
|
return em.nativeDelete(TelescopeEntry, { id: { $in: toDelete } });
|
|
277
610
|
}
|
|
611
|
+
async pruneScopedBatch(input) {
|
|
612
|
+
const em = this.forkEm();
|
|
613
|
+
const limit = input.limit;
|
|
614
|
+
if (input.type !== undefined || !input.excludeTypes?.length) {
|
|
615
|
+
return pruneOldestBatch(em, scopedWhere(input), limit);
|
|
616
|
+
}
|
|
617
|
+
// All types but some, one type at a time. As one `type NOT IN (…)` query the planner walks the
|
|
618
|
+
// `created_at` index from the oldest row and steps over every row of the excluded types, which
|
|
619
|
+
// are excluded because they are kept longer: on a table where they hold most of the rows, each
|
|
620
|
+
// batch read the whole retained backlog to find a few hundred rows (10 s for 1,000 rows next to
|
|
621
|
+
// 2.8M kept `durable` entries). Per type it reads only that type's expired rows.
|
|
622
|
+
const excluded = new Set(input.excludeTypes);
|
|
623
|
+
let deleted = 0;
|
|
624
|
+
for (const type of await distinctTypes(em)) {
|
|
625
|
+
if (excluded.has(type))
|
|
626
|
+
continue;
|
|
627
|
+
const batch = await pruneOldestBatch(em, scopedWhere({ before: input.before, type }), limit - deleted);
|
|
628
|
+
deleted += batch.deleted;
|
|
629
|
+
if (batch.hasMore)
|
|
630
|
+
return { deleted, hasMore: true };
|
|
631
|
+
}
|
|
632
|
+
return { deleted, hasMore: false };
|
|
633
|
+
}
|
|
634
|
+
async tryAcquireLease(key, owner, ttlMs, nowMs) {
|
|
635
|
+
const em = this.forkEm();
|
|
636
|
+
const expiresAt = nowMs + ttlMs;
|
|
637
|
+
// Atomicity comes from the CONDITIONAL UPDATE, not from the transaction
|
|
638
|
+
// isolation level: `update ... where key = ? and (expires_at <= ? or owner = ?)`
|
|
639
|
+
// is a single statement that takes a row lock, so exactly one of two racing
|
|
640
|
+
// pods can match. Whoever matches gets the lease; the other's update affects
|
|
641
|
+
// zero rows.
|
|
642
|
+
return em.transactional(async (tx) => {
|
|
643
|
+
const updated = await tx.nativeUpdate(TelescopeLease, { key, $or: [{ expiresAt: { $lte: nowMs } }, { owner }] }, { owner, expiresAt });
|
|
644
|
+
if (updated > 0)
|
|
645
|
+
return true;
|
|
646
|
+
// No row matched: either somebody else holds a live lease, or there is no
|
|
647
|
+
// row at all. Distinguish by trying the insert — the primary key makes a
|
|
648
|
+
// concurrent double-insert impossible, and a duplicate-key failure simply
|
|
649
|
+
// means the other pod won the race.
|
|
650
|
+
const existing = await tx.findOne(TelescopeLease, { key });
|
|
651
|
+
if (existing !== null)
|
|
652
|
+
return false;
|
|
653
|
+
try {
|
|
654
|
+
await tx.insert(TelescopeLease, { key, owner, expiresAt });
|
|
655
|
+
return true;
|
|
656
|
+
}
|
|
657
|
+
catch {
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
async releaseLease(key, owner) {
|
|
663
|
+
const em = this.forkEm();
|
|
664
|
+
// Owner-scoped: a holder whose lease already expired and was re-granted must
|
|
665
|
+
// not delete the new holder's row.
|
|
666
|
+
await em.nativeDelete(TelescopeLease, { key, owner });
|
|
667
|
+
}
|
|
668
|
+
async pruneScoped(input) {
|
|
669
|
+
const em = this.forkEm();
|
|
670
|
+
const where = scopedWhere(input);
|
|
671
|
+
if (input.keepLast === undefined) {
|
|
672
|
+
return em.nativeDelete(TelescopeEntry, where);
|
|
673
|
+
}
|
|
674
|
+
const older = await em.find(TelescopeEntry, where, {
|
|
675
|
+
orderBy: { createdAt: 'desc', id: 'desc' },
|
|
676
|
+
fields: ['id'],
|
|
677
|
+
});
|
|
678
|
+
const toDelete = older.slice(input.keepLast).map((row) => row.id);
|
|
679
|
+
if (toDelete.length === 0)
|
|
680
|
+
return 0;
|
|
681
|
+
return em.nativeDelete(TelescopeEntry, { id: { $in: toDelete } });
|
|
682
|
+
}
|
|
278
683
|
async clear() {
|
|
279
684
|
const em = this.forkEm();
|
|
280
685
|
await em.nativeDelete(TelescopeEntry, {});
|
|
@@ -282,25 +687,44 @@ export class MikroOrmStorageProvider {
|
|
|
282
687
|
}
|
|
283
688
|
// ── RollupStore SPI ────────────────────────────────────────────────────────
|
|
284
689
|
/**
|
|
285
|
-
* Additively merge deltas into `telescope_rollups
|
|
286
|
-
*
|
|
287
|
-
* semantics: accumulate count/sum and keep a running max per
|
|
690
|
+
* Additively merge deltas into `telescope_rollups`: count/sum accumulate, `max`
|
|
691
|
+
* keeps the running maximum and the histogram adds element-wise, per
|
|
288
692
|
* (metric, bucketStart).
|
|
289
693
|
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
293
|
-
*
|
|
294
|
-
*
|
|
295
|
-
*
|
|
296
|
-
* on the touched PKs — safe to call repeatedly and concurrently: concurrent
|
|
297
|
-
* writers serialize on the same (metric, bucketStart) rows, so no increment is
|
|
298
|
-
* lost. New buckets are inserted; existing ones accumulate.
|
|
694
|
+
* On MySQL/MariaDB, PostgreSQL and SQLite this is one native upsert per chunk
|
|
695
|
+
* of buckets, with every merge done by the database against the row as it is
|
|
696
|
+
* at write time, so concurrent writers (other pods) never lose each other's
|
|
697
|
+
* increments. Buckets are written in key order, so two writers touching the
|
|
698
|
+
* same buckets lock them in the same order. Other drivers fall back to a
|
|
699
|
+
* transactional read-modify-write.
|
|
299
700
|
*/
|
|
300
701
|
async recordRollups(deltas) {
|
|
301
702
|
if (deltas.length === 0)
|
|
302
703
|
return;
|
|
303
704
|
const em = this.forkEm();
|
|
705
|
+
const platform = em.getPlatform();
|
|
706
|
+
const dialect = rollupUpsertDialect(platform.constructor.name);
|
|
707
|
+
if (dialect === null) {
|
|
708
|
+
await this.recordRollupsReadModifyWrite(em, deltas);
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
const tableName = em.getMetadata().get(TelescopeRollup).tableName;
|
|
712
|
+
const quote = (identifier) => platform.quoteIdentifier(identifier);
|
|
713
|
+
const buckets = mergeDeltasByBucket(deltas);
|
|
714
|
+
const connection = em.getConnection();
|
|
715
|
+
for (let offset = 0; offset < buckets.length; offset += ROLLUP_UPSERT_CHUNK) {
|
|
716
|
+
const chunk = buckets.slice(offset, offset + ROLLUP_UPSERT_CHUNK);
|
|
717
|
+
await connection.execute(rollupUpsertSql(dialect, quote, tableName, chunk.length), chunk.flatMap((delta) => [
|
|
718
|
+
delta.metric,
|
|
719
|
+
delta.bucketStart,
|
|
720
|
+
delta.count,
|
|
721
|
+
delta.sum,
|
|
722
|
+
delta.max,
|
|
723
|
+
JSON.stringify(normalizeHistogram(delta.histogram)),
|
|
724
|
+
]), 'run');
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
async recordRollupsReadModifyWrite(em, deltas) {
|
|
304
728
|
await em.transactional(async (tx) => {
|
|
305
729
|
for (const delta of deltas) {
|
|
306
730
|
const existing = await tx.findOne(TelescopeRollup, {
|
|
@@ -345,4 +769,71 @@ export class MikroOrmStorageProvider {
|
|
|
345
769
|
}));
|
|
346
770
|
}
|
|
347
771
|
}
|
|
772
|
+
/** Buckets per rollup upsert statement (6 bound values each). */
|
|
773
|
+
const ROLLUP_UPSERT_CHUNK = 100;
|
|
774
|
+
/** The upsert flavour for a MikroORM platform class name, or null when it has none here. */
|
|
775
|
+
function rollupUpsertDialect(platformName) {
|
|
776
|
+
const name = platformName.toLowerCase();
|
|
777
|
+
if (name.includes('mysql') || name.includes('maria'))
|
|
778
|
+
return 'mysql';
|
|
779
|
+
if (name.includes('postgre'))
|
|
780
|
+
return 'postgresql';
|
|
781
|
+
if (name.includes('sqlite') || name.includes('libsql'))
|
|
782
|
+
return 'sqlite';
|
|
783
|
+
return null;
|
|
784
|
+
}
|
|
785
|
+
/** One delta per (metric, bucketStart), sorted by that key: an upsert may touch a row once. */
|
|
786
|
+
function mergeDeltasByBucket(deltas) {
|
|
787
|
+
const byKey = new Map();
|
|
788
|
+
for (const delta of deltas) {
|
|
789
|
+
const key = `${delta.metric}\u0000${delta.bucketStart}`;
|
|
790
|
+
const existing = byKey.get(key);
|
|
791
|
+
if (existing === undefined) {
|
|
792
|
+
byKey.set(key, { ...delta, histogram: normalizeHistogram(delta.histogram) });
|
|
793
|
+
}
|
|
794
|
+
else {
|
|
795
|
+
existing.count += delta.count;
|
|
796
|
+
existing.sum += delta.sum;
|
|
797
|
+
existing.max = Math.max(existing.max, delta.max);
|
|
798
|
+
existing.histogram = mergeHistograms(existing.histogram, delta.histogram);
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
return [...byKey.values()].sort((a, b) => (a.metric < b.metric ? -1 : a.metric > b.metric ? 1 : 0) || a.bucketStart - b.bucketStart);
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* `INSERT ... ON CONFLICT/ON DUPLICATE KEY UPDATE` for `rows` rollup buckets.
|
|
805
|
+
* The histogram is a fixed-length JSON array, rebuilt cell by cell as the sum
|
|
806
|
+
* of the stored cell (0 when the column is null or short) and the new one.
|
|
807
|
+
*/
|
|
808
|
+
function rollupUpsertSql(dialect, quote, tableName, rows) {
|
|
809
|
+
const table = quote(tableName);
|
|
810
|
+
const columns = ['metric', 'bucket_start', 'count', 'sum', 'max', 'histogram'];
|
|
811
|
+
const values = Array.from({ length: rows }, () => `(${columns.map(() => '?').join(', ')})`);
|
|
812
|
+
const insert = `insert into ${table} (${columns.map(quote).join(', ')}) values ${values.join(', ')}`;
|
|
813
|
+
const [count, sum, max, histogram] = ['count', 'sum', 'max', 'histogram'].map(quote);
|
|
814
|
+
const cells = Array.from({ length: HISTOGRAM_BUCKET_COUNT }, (_, index) => index);
|
|
815
|
+
if (dialect === 'mysql') {
|
|
816
|
+
const cell = (source, index) => `cast(coalesce(json_extract(${source}, '$[${index}]'), 0) as signed)`;
|
|
817
|
+
const merged = cells
|
|
818
|
+
.map((index) => `${cell(histogram, index)} + ${cell(`values(${histogram})`, index)}`)
|
|
819
|
+
.join(', ');
|
|
820
|
+
return `${insert} on duplicate key update ${count} = ${count} + values(${count}), ${sum} = ${sum} + values(${sum}), ${max} = greatest(${max}, values(${max})), ${histogram} = json_array(${merged})`;
|
|
821
|
+
}
|
|
822
|
+
const current = (column) => `${table}.${column}`;
|
|
823
|
+
const incoming = (column) => `excluded.${column}`;
|
|
824
|
+
const conflict = `on conflict (${quote('metric')}, ${quote('bucket_start')}) do update set`;
|
|
825
|
+
const totals = `${count} = ${current(count)} + ${incoming(count)}, ${sum} = ${current(sum)} + ${incoming(sum)}`;
|
|
826
|
+
if (dialect === 'postgresql') {
|
|
827
|
+
const cell = (source, index) => `coalesce((${source} ->> ${index})::bigint, 0)`;
|
|
828
|
+
const merged = cells
|
|
829
|
+
.map((index) => `${cell(current(histogram), index)} + ${cell(incoming(histogram), index)}`)
|
|
830
|
+
.join(', ');
|
|
831
|
+
return `${insert} ${conflict} ${totals}, ${max} = greatest(${current(max)}, ${incoming(max)}), ${histogram} = jsonb_build_array(${merged})`;
|
|
832
|
+
}
|
|
833
|
+
const cell = (source, index) => `coalesce(json_extract(${source}, '$[${index}]'), 0)`;
|
|
834
|
+
const merged = cells
|
|
835
|
+
.map((index) => `${cell(current(histogram), index)} + ${cell(incoming(histogram), index)}`)
|
|
836
|
+
.join(', ');
|
|
837
|
+
return `${insert} ${conflict} ${totals}, ${max} = max(${current(max)}, ${incoming(max)}), ${histogram} = json_array(${merged})`;
|
|
838
|
+
}
|
|
348
839
|
//# sourceMappingURL=mikro-orm-storage.provider.js.map
|