@lensjs/core 2.2.2 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/abstracts/store.cjs +3 -0
- package/dist/abstracts/store.d.cts +3 -1
- package/dist/abstracts/store.d.ts +3 -1
- package/dist/abstracts/store.js +3 -0
- package/dist/core/lens.cjs +151 -24
- package/dist/core/lens.d.cts +3 -2
- package/dist/core/lens.d.ts +3 -2
- package/dist/core/lens.js +151 -24
- package/dist/{index-CMvlRWcQ.d.cts → index-CZsa0Zcm.d.ts} +3 -1
- package/dist/{index-CMvlRWcQ.d.ts → index-QmOJr0K-.d.cts} +3 -1
- package/dist/index.cjs +199 -29
- package/dist/index.d.cts +5 -2
- package/dist/index.d.ts +5 -2
- package/dist/index.js +198 -29
- package/dist/mixins/queued_store.cjs +114 -0
- package/dist/mixins/queued_store.d.cts +38 -0
- package/dist/mixins/queued_store.d.ts +38 -0
- package/dist/mixins/queued_store.js +79 -0
- package/dist/stores/better_sqlite.cjs +36 -0
- package/dist/stores/better_sqlite.d.cts +3 -0
- package/dist/stores/better_sqlite.d.ts +3 -0
- package/dist/stores/better_sqlite.js +36 -0
- package/dist/stores/index.cjs +129 -2
- package/dist/stores/index.d.cts +2 -0
- package/dist/stores/index.d.ts +2 -0
- package/dist/stores/index.js +127 -1
- package/dist/stores/queued_sqlite.cjs +313 -0
- package/dist/stores/queued_sqlite.d.cts +43 -0
- package/dist/stores/queued_sqlite.d.ts +43 -0
- package/dist/stores/queued_sqlite.js +280 -0
- package/dist/types/index.d.cts +16 -2
- package/dist/types/index.d.ts +16 -2
- package/dist/ui/assets/{CacheActionBadge-CL10Xlw7.js → CacheActionBadge-BB4uokI1.js} +1 -1
- package/dist/ui/assets/CacheEntriesTable-B8cUXhos.js +1 -0
- package/dist/ui/assets/CacheEntryContainer-WkdnGvnu.js +2 -0
- package/dist/ui/assets/CacheEntryDetails-BeZnoIpm.js +1 -0
- package/dist/ui/assets/CacheEntryDetailsContainer-DI0mEvpu.js +2 -0
- package/dist/ui/assets/ExceptionContainer-YNcR0F5U.js +2 -0
- package/dist/ui/assets/{ExceptionDetails-Bzj8OZ70.js → ExceptionDetails-BKHzv6hf.js} +1 -1
- package/dist/ui/assets/ExceptionDetailsContainer-CJHILjb3.js +2 -0
- package/dist/ui/assets/{ExceptionTable-BkFSFGbn.js → ExceptionTable-DzBmQLLa.js} +1 -1
- package/dist/ui/assets/JsonViewer-D-KPN089.js +1 -0
- package/dist/ui/assets/{LoadMore-Du7yL-Hr.js → LoadMore-CLPR6Zd4.js} +1 -1
- package/dist/ui/assets/QueriesContainer-B_PmBkHR.js +2 -0
- package/dist/ui/assets/{QueryDetailsContainer-BtbvRyBf.js → QueryDetailsContainer-Cqj3E6Dr.js} +16 -26
- package/dist/ui/assets/{QueryTable-tJVEncUM.js → QueryTable-DmWdZSnJ.js} +1 -1
- package/dist/ui/assets/{RequestDetails-C6tqSqg9.js → RequestDetails-CF338Kcv.js} +1 -1
- package/dist/ui/assets/{RequestDetailsContainer---KvdZKp.js → RequestDetailsContainer-aW4GLool.js} +2 -2
- package/dist/ui/assets/{RequestsContainer-Bogurt1b.js → RequestsContainer-DdLSvAbl.js} +2 -2
- package/dist/ui/assets/{RequetsTable-BMrYHd0d.js → RequetsTable-Bdp_PhGU.js} +1 -1
- package/dist/ui/assets/{StatusCode-DpZO0dUJ.js → StatusCode-C605nHvd.js} +1 -1
- package/dist/ui/assets/TabbedDataViewer-ofhEq_Wj.js +2 -0
- package/dist/ui/assets/{Table-BtkmKVTF.js → Table-kak5sL5X.js} +1 -1
- package/dist/ui/assets/{columns-D_NhXbk6.js → columns-BEyDhUNq.js} +1 -1
- package/dist/ui/assets/{columns-DlGaMv5C.js → columns-Bu5psHyp.js} +1 -1
- package/dist/ui/assets/{columns-DF7BR0z_.js → columns-BvIUTkjN.js} +1 -1
- package/dist/ui/assets/copy-DzXuP4eO.js +11 -0
- package/dist/ui/assets/index-CsnKQ5Mh.css +1 -0
- package/dist/ui/assets/{index-EQXljT95.js → index-TW_-MgRG.js} +25 -25
- package/dist/ui/assets/{useCacheEntries-jC9XYsV_.js → useCacheEntries-Pvte_aNc.js} +1 -1
- package/dist/ui/assets/{useExceptions-CwwK33mG.js → useExceptions-P3cnURvN.js} +1 -1
- package/dist/ui/assets/{useLensApi-CPvDlyGv.js → useLensApi-BFdsfrzR.js} +1 -1
- package/dist/ui/assets/{useLoadMore-C2bqGSaL.js → useLoadMore-JCWak1Dg.js} +1 -1
- package/dist/ui/assets/{useQueries-CzbIajH6.js → useQueries-CNquFtm0.js} +1 -1
- package/dist/ui/index.html +2 -2
- package/dist/utils/compose.cjs +32 -0
- package/dist/utils/compose.d.cts +11 -0
- package/dist/utils/compose.d.ts +11 -0
- package/dist/utils/compose.js +7 -0
- package/dist/utils/index.cjs +10 -1
- package/dist/utils/index.d.cts +3 -1
- package/dist/utils/index.d.ts +3 -1
- package/dist/utils/index.js +9 -1
- package/dist/watchers/index.cjs +45 -5
- package/dist/watchers/index.js +45 -5
- package/dist/watchers/request_watcher.cjs +45 -5
- package/dist/watchers/request_watcher.d.cts +12 -1
- package/dist/watchers/request_watcher.d.ts +12 -1
- package/dist/watchers/request_watcher.js +45 -5
- package/package.json +2 -1
- package/dist/ui/assets/CacheEntriesTable-DuLoeu0e.js +0 -1
- package/dist/ui/assets/CacheEntryContainer-DqHm-jQl.js +0 -2
- package/dist/ui/assets/CacheEntryDetails-k-74LsSb.js +0 -1
- package/dist/ui/assets/CacheEntryDetailsContainer-DTI7gtUq.js +0 -2
- package/dist/ui/assets/ExceptionContainer-0dCs6QMQ.js +0 -2
- package/dist/ui/assets/ExceptionDetailsContainer-Bim0gTpE.js +0 -2
- package/dist/ui/assets/QueriesContainer-5xlqsYl0.js +0 -2
- package/dist/ui/assets/TabbedDataViewer-C60W9bqz.js +0 -1
- package/dist/ui/assets/index-CMJVCuvo.css +0 -1
package/dist/abstracts/store.cjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { WatcherTypeEnum, PaginationParams, Paginator, LensEntry } from '../types/index.cjs';
|
|
1
|
+
import { QueuedStoreConfig, WatcherTypeEnum, PaginationParams, Paginator, LensEntry } from '../types/index.cjs';
|
|
2
2
|
import 'sql-formatter';
|
|
3
3
|
|
|
4
4
|
type MinimalPaginatePromise = Promise<Paginator<Omit<LensEntry, "data">[]>>;
|
|
5
5
|
declare abstract class Store {
|
|
6
|
+
protected storeConfig?: QueuedStoreConfig;
|
|
7
|
+
constructor(...args: any[]);
|
|
6
8
|
abstract initialize(): Promise<void>;
|
|
7
9
|
abstract save(entry: {
|
|
8
10
|
id?: string;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { WatcherTypeEnum, PaginationParams, Paginator, LensEntry } from '../types/index.js';
|
|
1
|
+
import { QueuedStoreConfig, WatcherTypeEnum, PaginationParams, Paginator, LensEntry } from '../types/index.js';
|
|
2
2
|
import 'sql-formatter';
|
|
3
3
|
|
|
4
4
|
type MinimalPaginatePromise = Promise<Paginator<Omit<LensEntry, "data">[]>>;
|
|
5
5
|
declare abstract class Store {
|
|
6
|
+
protected storeConfig?: QueuedStoreConfig;
|
|
7
|
+
constructor(...args: any[]);
|
|
6
8
|
abstract initialize(): Promise<void>;
|
|
7
9
|
abstract save(entry: {
|
|
8
10
|
id?: string;
|
package/dist/abstracts/store.js
CHANGED
package/dist/core/lens.cjs
CHANGED
|
@@ -204,6 +204,9 @@ var path2 = __toESM(require("path"), 1);
|
|
|
204
204
|
|
|
205
205
|
// src/abstracts/store.ts
|
|
206
206
|
var Store = class {
|
|
207
|
+
storeConfig;
|
|
208
|
+
constructor(...args) {
|
|
209
|
+
}
|
|
207
210
|
getAllExceptions(_paginationParams) {
|
|
208
211
|
return this.defaultMinimalPaginate();
|
|
209
212
|
}
|
|
@@ -234,6 +237,8 @@ var import_crypto = require("crypto");
|
|
|
234
237
|
var import_libsql = __toESM(require("libsql"), 1);
|
|
235
238
|
var import_date = require("@lensjs/date");
|
|
236
239
|
var TABLE_NAME = "lens_entries";
|
|
240
|
+
var BYTES_IN_GB = 1024 * 1024 * 1024;
|
|
241
|
+
var PRUNE_BATCH_SIZE = 1e3;
|
|
237
242
|
var BetterSqliteStore = class extends Store {
|
|
238
243
|
connection;
|
|
239
244
|
async initialize() {
|
|
@@ -255,6 +260,7 @@ var BetterSqliteStore = class extends Store {
|
|
|
255
260
|
lens_entry_id: entry.requestId || null,
|
|
256
261
|
minimalData: this.stringifyData(entry.minimal_data ?? {})
|
|
257
262
|
});
|
|
263
|
+
this.maybePruneDatabase();
|
|
258
264
|
}
|
|
259
265
|
async getAllQueries(pagination) {
|
|
260
266
|
return await this.paginate("query" /* QUERY */, pagination);
|
|
@@ -330,6 +336,36 @@ var BetterSqliteStore = class extends Store {
|
|
|
330
336
|
this.connection.exec(createIndex);
|
|
331
337
|
this.connection.exec(lensEntryIdIndex);
|
|
332
338
|
}
|
|
339
|
+
maybePruneDatabase() {
|
|
340
|
+
const maxGb = this.storeConfig?.dbMaxSizeGb;
|
|
341
|
+
const pruneGb = this.storeConfig?.dbPruneSizeGb;
|
|
342
|
+
if (!maxGb || !pruneGb) return;
|
|
343
|
+
const maxBytes = maxGb * BYTES_IN_GB;
|
|
344
|
+
const pruneBytes = pruneGb * BYTES_IN_GB;
|
|
345
|
+
if (maxBytes <= 0 || pruneBytes <= 0) return;
|
|
346
|
+
const targetBytes = Math.max(0, maxBytes - pruneBytes);
|
|
347
|
+
let usedBytes = this.getDatabaseUsedBytes();
|
|
348
|
+
if (usedBytes < maxBytes) return;
|
|
349
|
+
while (usedBytes > targetBytes) {
|
|
350
|
+
const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
|
|
351
|
+
if (deletedRows === 0) break;
|
|
352
|
+
usedBytes = this.getDatabaseUsedBytes();
|
|
353
|
+
}
|
|
354
|
+
this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
355
|
+
}
|
|
356
|
+
getDatabaseUsedBytes() {
|
|
357
|
+
const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
|
|
358
|
+
const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
|
|
359
|
+
const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
|
|
360
|
+
const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
|
|
361
|
+
return usedPages * pageSizeResult.page_size;
|
|
362
|
+
}
|
|
363
|
+
deleteOldestEntries(batchSize) {
|
|
364
|
+
const result = this.connection.prepare(
|
|
365
|
+
`DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
|
|
366
|
+
).run(batchSize);
|
|
367
|
+
return Number(result.changes ?? 0);
|
|
368
|
+
}
|
|
333
369
|
mapRow(row, includeFullData = true) {
|
|
334
370
|
let data = includeFullData ? JSON.parse(row.data) : {};
|
|
335
371
|
if (!includeFullData) {
|
|
@@ -355,6 +391,95 @@ var BetterSqliteStore = class extends Store {
|
|
|
355
391
|
}
|
|
356
392
|
};
|
|
357
393
|
|
|
394
|
+
// src/mixins/queued_store.ts
|
|
395
|
+
var import_denque = __toESM(require("denque"), 1);
|
|
396
|
+
function QueuedStore(Base) {
|
|
397
|
+
return class Queued extends Base {
|
|
398
|
+
queue;
|
|
399
|
+
processingInterval = null;
|
|
400
|
+
BATCH_SIZE;
|
|
401
|
+
PROCESS_INTERVAL_MS;
|
|
402
|
+
WARN_THRESHOLD;
|
|
403
|
+
PREALLOCATE;
|
|
404
|
+
constructor(...args) {
|
|
405
|
+
super(...args);
|
|
406
|
+
const config = args[0] || {};
|
|
407
|
+
this.storeConfig = config;
|
|
408
|
+
this.BATCH_SIZE = config.batchSize ?? 100;
|
|
409
|
+
this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
|
|
410
|
+
this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
|
|
411
|
+
this.PREALLOCATE = config.preallocate ?? true;
|
|
412
|
+
this.queue = this.PREALLOCATE ? new import_denque.default([], { capacity: this.WARN_THRESHOLD * 2 }) : new import_denque.default();
|
|
413
|
+
}
|
|
414
|
+
async initialize() {
|
|
415
|
+
if (super["initialize"]) {
|
|
416
|
+
await super["initialize"].call(this);
|
|
417
|
+
}
|
|
418
|
+
this.startProcessingQueue();
|
|
419
|
+
process.on("SIGINT", () => this.shutdown());
|
|
420
|
+
process.on("SIGTERM", () => this.shutdown());
|
|
421
|
+
}
|
|
422
|
+
async truncate() {
|
|
423
|
+
this.queue.clear();
|
|
424
|
+
if (super["truncate"]) {
|
|
425
|
+
await super["truncate"].call(this);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
async save(entry) {
|
|
429
|
+
this.queue.push(entry);
|
|
430
|
+
if (this.queue.length > this.WARN_THRESHOLD) {
|
|
431
|
+
console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
startProcessingQueue() {
|
|
435
|
+
if (this.processingInterval) clearInterval(this.processingInterval);
|
|
436
|
+
this.processingInterval = setInterval(
|
|
437
|
+
() => this.processQueue(),
|
|
438
|
+
this.PROCESS_INTERVAL_MS
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
async processQueue() {
|
|
442
|
+
if (this.queue.isEmpty()) return;
|
|
443
|
+
const batchSize = Math.min(
|
|
444
|
+
this.BATCH_SIZE,
|
|
445
|
+
Math.max(10, Math.floor(this.queue.length / 10))
|
|
446
|
+
);
|
|
447
|
+
const entriesToProcess = this.queue.remove(0, batchSize);
|
|
448
|
+
if (!entriesToProcess.length) return;
|
|
449
|
+
for (const entry of entriesToProcess) {
|
|
450
|
+
super["save"]?.call(this, entry).catch((error) => {
|
|
451
|
+
console.error("Error saving queued entry:", error);
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
async stopProcessingQueue() {
|
|
456
|
+
if (this.processingInterval) {
|
|
457
|
+
clearInterval(this.processingInterval);
|
|
458
|
+
this.processingInterval = null;
|
|
459
|
+
}
|
|
460
|
+
if (!this.queue.isEmpty()) {
|
|
461
|
+
await this.processQueue();
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
async shutdown() {
|
|
465
|
+
await this.stopProcessingQueue();
|
|
466
|
+
process.exit(0);
|
|
467
|
+
}
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// src/utils/compose.ts
|
|
472
|
+
function compose(superclass, ...mixins) {
|
|
473
|
+
return mixins.reduce((c, mixin) => mixin(c), superclass);
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// src/stores/queued_sqlite.ts
|
|
477
|
+
var QueuedSqliteStore = class extends compose(
|
|
478
|
+
BetterSqliteStore,
|
|
479
|
+
QueuedStore
|
|
480
|
+
) {
|
|
481
|
+
};
|
|
482
|
+
|
|
358
483
|
// src/utils/index.ts
|
|
359
484
|
var import_luxon = require("luxon");
|
|
360
485
|
var import_sql_formatter = require("sql-formatter");
|
|
@@ -362,7 +487,7 @@ var import_crypto2 = require("crypto");
|
|
|
362
487
|
var import_url = require("url");
|
|
363
488
|
var path = __toESM(require("path"), 1);
|
|
364
489
|
function getMeta(metaUrl) {
|
|
365
|
-
const isESM = typeof __dirname === "undefined";
|
|
490
|
+
const isESM = typeof __dirname === "undefined" || typeof __filename === "undefined";
|
|
366
491
|
if (isESM) {
|
|
367
492
|
if (!metaUrl) {
|
|
368
493
|
throw new Error("In ESM, you must pass import.meta.url to getMeta()");
|
|
@@ -381,6 +506,7 @@ var Lens = class {
|
|
|
381
506
|
static watchers = /* @__PURE__ */ new Map();
|
|
382
507
|
static store;
|
|
383
508
|
static adapter;
|
|
509
|
+
static config;
|
|
384
510
|
static watch(watcher) {
|
|
385
511
|
this.watchers.set(watcher.name, watcher);
|
|
386
512
|
return this;
|
|
@@ -390,22 +516,23 @@ var Lens = class {
|
|
|
390
516
|
return this;
|
|
391
517
|
}
|
|
392
518
|
static async start(config = {
|
|
393
|
-
|
|
519
|
+
path: "lens",
|
|
394
520
|
appName: "Lens",
|
|
395
521
|
enabled: true
|
|
396
522
|
}) {
|
|
397
523
|
if (!config.enabled) {
|
|
398
524
|
return;
|
|
399
525
|
}
|
|
400
|
-
|
|
526
|
+
this.config = config;
|
|
527
|
+
await this.bindContainerDeps();
|
|
401
528
|
let adapter = this.getAdapter();
|
|
402
529
|
adapter.setWatchers(Array.from(this.watchers.values())).setup();
|
|
403
530
|
const { apiRoutes } = this.getRoutes({
|
|
404
|
-
|
|
531
|
+
path: config.path
|
|
405
532
|
});
|
|
406
533
|
adapter.registerRoutes(apiRoutes);
|
|
407
534
|
const uiPath = this.getUiPath();
|
|
408
|
-
adapter.serveUI(uiPath, config.
|
|
535
|
+
adapter.serveUI(uiPath, config.path, getUiConfig());
|
|
409
536
|
}
|
|
410
537
|
static setStore(store) {
|
|
411
538
|
this.store = store;
|
|
@@ -428,7 +555,7 @@ var Lens = class {
|
|
|
428
555
|
const { __dirname: __dirname2 } = getMeta(import_meta.url);
|
|
429
556
|
return path2.resolve(this.normalizeDirName(__dirname2), "ui");
|
|
430
557
|
}
|
|
431
|
-
static getRoutes({
|
|
558
|
+
static getRoutes({ path: path3 }) {
|
|
432
559
|
const apiRoutes = [
|
|
433
560
|
{
|
|
434
561
|
method: "GET",
|
|
@@ -437,71 +564,71 @@ var Lens = class {
|
|
|
437
564
|
},
|
|
438
565
|
{
|
|
439
566
|
method: "GET",
|
|
440
|
-
path: `/${
|
|
567
|
+
path: `/${path3}/api/requests`,
|
|
441
568
|
handler: async (data) => await ApiController.getRequests(data)
|
|
442
569
|
},
|
|
443
570
|
{
|
|
444
571
|
method: "GET",
|
|
445
|
-
path: `/${
|
|
572
|
+
path: `/${path3}/api/requests/:id`,
|
|
446
573
|
handler: async (data) => await ApiController.getRequest(data)
|
|
447
574
|
},
|
|
448
575
|
{
|
|
449
576
|
method: "GET",
|
|
450
|
-
path: `/${
|
|
577
|
+
path: `/${path3}/api/queries`,
|
|
451
578
|
handler: async (data) => await ApiController.getQueries(data)
|
|
452
579
|
},
|
|
453
580
|
{
|
|
454
581
|
method: "GET",
|
|
455
|
-
path: `/${
|
|
582
|
+
path: `/${path3}/api/queries/:id`,
|
|
456
583
|
handler: async (data) => await ApiController.getQuery(data)
|
|
457
584
|
},
|
|
458
585
|
{
|
|
459
586
|
method: "GET",
|
|
460
|
-
path: `/${
|
|
587
|
+
path: `/${path3}/api/cache`,
|
|
461
588
|
handler: async (data) => await ApiController.getCacheEntries(data)
|
|
462
589
|
},
|
|
463
590
|
{
|
|
464
591
|
method: "GET",
|
|
465
|
-
path: `/${
|
|
592
|
+
path: `/${path3}/api/cache/:id`,
|
|
466
593
|
handler: async (data) => await ApiController.getCacheEntry(data)
|
|
467
594
|
},
|
|
468
595
|
{
|
|
469
596
|
method: "GET",
|
|
470
|
-
path: `/${
|
|
597
|
+
path: `/${path3}/api/exceptions`,
|
|
471
598
|
handler: async (data) => await ApiController.getExceptions(data)
|
|
472
599
|
},
|
|
473
600
|
{
|
|
474
601
|
method: "GET",
|
|
475
|
-
path: `/${
|
|
602
|
+
path: `/${path3}/api/exceptions/:id`,
|
|
476
603
|
handler: async (data) => await ApiController.getException(data)
|
|
477
604
|
},
|
|
478
605
|
{
|
|
479
606
|
method: "DELETE",
|
|
480
|
-
path: `/${
|
|
607
|
+
path: `/${path3}/api/truncate`,
|
|
481
608
|
handler: async () => await ApiController.truncate()
|
|
482
609
|
}
|
|
483
610
|
];
|
|
484
611
|
return { apiRoutes };
|
|
485
612
|
}
|
|
486
|
-
static async bindContainerDeps(
|
|
613
|
+
static async bindContainerDeps() {
|
|
487
614
|
const dbStore = await this.getStore();
|
|
488
615
|
Container.singleton("store", () => dbStore);
|
|
489
616
|
Container.singleton("uiConfig", () => {
|
|
490
617
|
return {
|
|
491
|
-
appName: config.appName,
|
|
492
|
-
path: `/${config.
|
|
618
|
+
appName: this.config.appName,
|
|
619
|
+
path: `/${this.config.path}`,
|
|
493
620
|
api: {
|
|
494
|
-
requests: `/${config.
|
|
495
|
-
queries: `/${config.
|
|
496
|
-
cache: `/${config.
|
|
497
|
-
exceptions: `/${config.
|
|
498
|
-
truncate: `/${config.
|
|
621
|
+
requests: `/${this.config.path}/api/requests`,
|
|
622
|
+
queries: `/${this.config.path}/api/queries`,
|
|
623
|
+
cache: `/${this.config.path}/api/cache`,
|
|
624
|
+
exceptions: `/${this.config.path}/api/exceptions`,
|
|
625
|
+
truncate: `/${this.config.path}/api/truncate`
|
|
499
626
|
}
|
|
500
627
|
};
|
|
501
628
|
});
|
|
502
629
|
}
|
|
503
630
|
static async getDefaultStore() {
|
|
504
|
-
const store = new
|
|
631
|
+
const store = new QueuedSqliteStore(this.config.storeQueueConfig);
|
|
505
632
|
await store.initialize();
|
|
506
633
|
return store;
|
|
507
634
|
}
|
package/dist/core/lens.d.cts
CHANGED
|
@@ -8,6 +8,7 @@ declare class Lens {
|
|
|
8
8
|
private static watchers;
|
|
9
9
|
private static store;
|
|
10
10
|
private static adapter;
|
|
11
|
+
private static config;
|
|
11
12
|
static watch(watcher: Watcher): typeof Lens;
|
|
12
13
|
static setWatchers(watchers: Watcher[]): typeof Lens;
|
|
13
14
|
static start(config?: LensConfig): Promise<void>;
|
|
@@ -16,8 +17,8 @@ declare class Lens {
|
|
|
16
17
|
static setAdapter(adapter: Adapter): typeof Lens;
|
|
17
18
|
static getAdapter(): Adapter;
|
|
18
19
|
static getUiPath(): string;
|
|
19
|
-
static getRoutes({
|
|
20
|
-
|
|
20
|
+
static getRoutes({ path }: {
|
|
21
|
+
path: string;
|
|
21
22
|
}): {
|
|
22
23
|
apiRoutes: ({
|
|
23
24
|
method: "GET";
|
package/dist/core/lens.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ declare class Lens {
|
|
|
8
8
|
private static watchers;
|
|
9
9
|
private static store;
|
|
10
10
|
private static adapter;
|
|
11
|
+
private static config;
|
|
11
12
|
static watch(watcher: Watcher): typeof Lens;
|
|
12
13
|
static setWatchers(watchers: Watcher[]): typeof Lens;
|
|
13
14
|
static start(config?: LensConfig): Promise<void>;
|
|
@@ -16,8 +17,8 @@ declare class Lens {
|
|
|
16
17
|
static setAdapter(adapter: Adapter): typeof Lens;
|
|
17
18
|
static getAdapter(): Adapter;
|
|
18
19
|
static getUiPath(): string;
|
|
19
|
-
static getRoutes({
|
|
20
|
-
|
|
20
|
+
static getRoutes({ path }: {
|
|
21
|
+
path: string;
|
|
21
22
|
}): {
|
|
22
23
|
apiRoutes: ({
|
|
23
24
|
method: "GET";
|
package/dist/core/lens.js
CHANGED
|
@@ -168,6 +168,9 @@ import * as path2 from "path";
|
|
|
168
168
|
|
|
169
169
|
// src/abstracts/store.ts
|
|
170
170
|
var Store = class {
|
|
171
|
+
storeConfig;
|
|
172
|
+
constructor(...args) {
|
|
173
|
+
}
|
|
171
174
|
getAllExceptions(_paginationParams) {
|
|
172
175
|
return this.defaultMinimalPaginate();
|
|
173
176
|
}
|
|
@@ -198,6 +201,8 @@ import { randomUUID } from "crypto";
|
|
|
198
201
|
import Database from "libsql";
|
|
199
202
|
import { nowISO } from "@lensjs/date";
|
|
200
203
|
var TABLE_NAME = "lens_entries";
|
|
204
|
+
var BYTES_IN_GB = 1024 * 1024 * 1024;
|
|
205
|
+
var PRUNE_BATCH_SIZE = 1e3;
|
|
201
206
|
var BetterSqliteStore = class extends Store {
|
|
202
207
|
connection;
|
|
203
208
|
async initialize() {
|
|
@@ -219,6 +224,7 @@ var BetterSqliteStore = class extends Store {
|
|
|
219
224
|
lens_entry_id: entry.requestId || null,
|
|
220
225
|
minimalData: this.stringifyData(entry.minimal_data ?? {})
|
|
221
226
|
});
|
|
227
|
+
this.maybePruneDatabase();
|
|
222
228
|
}
|
|
223
229
|
async getAllQueries(pagination) {
|
|
224
230
|
return await this.paginate("query" /* QUERY */, pagination);
|
|
@@ -294,6 +300,36 @@ var BetterSqliteStore = class extends Store {
|
|
|
294
300
|
this.connection.exec(createIndex);
|
|
295
301
|
this.connection.exec(lensEntryIdIndex);
|
|
296
302
|
}
|
|
303
|
+
maybePruneDatabase() {
|
|
304
|
+
const maxGb = this.storeConfig?.dbMaxSizeGb;
|
|
305
|
+
const pruneGb = this.storeConfig?.dbPruneSizeGb;
|
|
306
|
+
if (!maxGb || !pruneGb) return;
|
|
307
|
+
const maxBytes = maxGb * BYTES_IN_GB;
|
|
308
|
+
const pruneBytes = pruneGb * BYTES_IN_GB;
|
|
309
|
+
if (maxBytes <= 0 || pruneBytes <= 0) return;
|
|
310
|
+
const targetBytes = Math.max(0, maxBytes - pruneBytes);
|
|
311
|
+
let usedBytes = this.getDatabaseUsedBytes();
|
|
312
|
+
if (usedBytes < maxBytes) return;
|
|
313
|
+
while (usedBytes > targetBytes) {
|
|
314
|
+
const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
|
|
315
|
+
if (deletedRows === 0) break;
|
|
316
|
+
usedBytes = this.getDatabaseUsedBytes();
|
|
317
|
+
}
|
|
318
|
+
this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
319
|
+
}
|
|
320
|
+
getDatabaseUsedBytes() {
|
|
321
|
+
const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
|
|
322
|
+
const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
|
|
323
|
+
const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
|
|
324
|
+
const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
|
|
325
|
+
return usedPages * pageSizeResult.page_size;
|
|
326
|
+
}
|
|
327
|
+
deleteOldestEntries(batchSize) {
|
|
328
|
+
const result = this.connection.prepare(
|
|
329
|
+
`DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
|
|
330
|
+
).run(batchSize);
|
|
331
|
+
return Number(result.changes ?? 0);
|
|
332
|
+
}
|
|
297
333
|
mapRow(row, includeFullData = true) {
|
|
298
334
|
let data = includeFullData ? JSON.parse(row.data) : {};
|
|
299
335
|
if (!includeFullData) {
|
|
@@ -319,6 +355,95 @@ var BetterSqliteStore = class extends Store {
|
|
|
319
355
|
}
|
|
320
356
|
};
|
|
321
357
|
|
|
358
|
+
// src/mixins/queued_store.ts
|
|
359
|
+
import Denque from "denque";
|
|
360
|
+
function QueuedStore(Base) {
|
|
361
|
+
return class Queued extends Base {
|
|
362
|
+
queue;
|
|
363
|
+
processingInterval = null;
|
|
364
|
+
BATCH_SIZE;
|
|
365
|
+
PROCESS_INTERVAL_MS;
|
|
366
|
+
WARN_THRESHOLD;
|
|
367
|
+
PREALLOCATE;
|
|
368
|
+
constructor(...args) {
|
|
369
|
+
super(...args);
|
|
370
|
+
const config = args[0] || {};
|
|
371
|
+
this.storeConfig = config;
|
|
372
|
+
this.BATCH_SIZE = config.batchSize ?? 100;
|
|
373
|
+
this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
|
|
374
|
+
this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
|
|
375
|
+
this.PREALLOCATE = config.preallocate ?? true;
|
|
376
|
+
this.queue = this.PREALLOCATE ? new Denque([], { capacity: this.WARN_THRESHOLD * 2 }) : new Denque();
|
|
377
|
+
}
|
|
378
|
+
async initialize() {
|
|
379
|
+
if (super["initialize"]) {
|
|
380
|
+
await super["initialize"].call(this);
|
|
381
|
+
}
|
|
382
|
+
this.startProcessingQueue();
|
|
383
|
+
process.on("SIGINT", () => this.shutdown());
|
|
384
|
+
process.on("SIGTERM", () => this.shutdown());
|
|
385
|
+
}
|
|
386
|
+
async truncate() {
|
|
387
|
+
this.queue.clear();
|
|
388
|
+
if (super["truncate"]) {
|
|
389
|
+
await super["truncate"].call(this);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
async save(entry) {
|
|
393
|
+
this.queue.push(entry);
|
|
394
|
+
if (this.queue.length > this.WARN_THRESHOLD) {
|
|
395
|
+
console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
startProcessingQueue() {
|
|
399
|
+
if (this.processingInterval) clearInterval(this.processingInterval);
|
|
400
|
+
this.processingInterval = setInterval(
|
|
401
|
+
() => this.processQueue(),
|
|
402
|
+
this.PROCESS_INTERVAL_MS
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
async processQueue() {
|
|
406
|
+
if (this.queue.isEmpty()) return;
|
|
407
|
+
const batchSize = Math.min(
|
|
408
|
+
this.BATCH_SIZE,
|
|
409
|
+
Math.max(10, Math.floor(this.queue.length / 10))
|
|
410
|
+
);
|
|
411
|
+
const entriesToProcess = this.queue.remove(0, batchSize);
|
|
412
|
+
if (!entriesToProcess.length) return;
|
|
413
|
+
for (const entry of entriesToProcess) {
|
|
414
|
+
super["save"]?.call(this, entry).catch((error) => {
|
|
415
|
+
console.error("Error saving queued entry:", error);
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
async stopProcessingQueue() {
|
|
420
|
+
if (this.processingInterval) {
|
|
421
|
+
clearInterval(this.processingInterval);
|
|
422
|
+
this.processingInterval = null;
|
|
423
|
+
}
|
|
424
|
+
if (!this.queue.isEmpty()) {
|
|
425
|
+
await this.processQueue();
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
async shutdown() {
|
|
429
|
+
await this.stopProcessingQueue();
|
|
430
|
+
process.exit(0);
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// src/utils/compose.ts
|
|
436
|
+
function compose(superclass, ...mixins) {
|
|
437
|
+
return mixins.reduce((c, mixin) => mixin(c), superclass);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// src/stores/queued_sqlite.ts
|
|
441
|
+
var QueuedSqliteStore = class extends compose(
|
|
442
|
+
BetterSqliteStore,
|
|
443
|
+
QueuedStore
|
|
444
|
+
) {
|
|
445
|
+
};
|
|
446
|
+
|
|
322
447
|
// src/utils/index.ts
|
|
323
448
|
import { DateTime } from "luxon";
|
|
324
449
|
import { format } from "sql-formatter";
|
|
@@ -326,7 +451,7 @@ import { randomUUID as randomUUID2 } from "crypto";
|
|
|
326
451
|
import { fileURLToPath } from "url";
|
|
327
452
|
import * as path from "path";
|
|
328
453
|
function getMeta(metaUrl) {
|
|
329
|
-
const isESM = typeof __dirname === "undefined";
|
|
454
|
+
const isESM = typeof __dirname === "undefined" || typeof __filename === "undefined";
|
|
330
455
|
if (isESM) {
|
|
331
456
|
if (!metaUrl) {
|
|
332
457
|
throw new Error("In ESM, you must pass import.meta.url to getMeta()");
|
|
@@ -344,6 +469,7 @@ var Lens = class {
|
|
|
344
469
|
static watchers = /* @__PURE__ */ new Map();
|
|
345
470
|
static store;
|
|
346
471
|
static adapter;
|
|
472
|
+
static config;
|
|
347
473
|
static watch(watcher) {
|
|
348
474
|
this.watchers.set(watcher.name, watcher);
|
|
349
475
|
return this;
|
|
@@ -353,22 +479,23 @@ var Lens = class {
|
|
|
353
479
|
return this;
|
|
354
480
|
}
|
|
355
481
|
static async start(config = {
|
|
356
|
-
|
|
482
|
+
path: "lens",
|
|
357
483
|
appName: "Lens",
|
|
358
484
|
enabled: true
|
|
359
485
|
}) {
|
|
360
486
|
if (!config.enabled) {
|
|
361
487
|
return;
|
|
362
488
|
}
|
|
363
|
-
|
|
489
|
+
this.config = config;
|
|
490
|
+
await this.bindContainerDeps();
|
|
364
491
|
let adapter = this.getAdapter();
|
|
365
492
|
adapter.setWatchers(Array.from(this.watchers.values())).setup();
|
|
366
493
|
const { apiRoutes } = this.getRoutes({
|
|
367
|
-
|
|
494
|
+
path: config.path
|
|
368
495
|
});
|
|
369
496
|
adapter.registerRoutes(apiRoutes);
|
|
370
497
|
const uiPath = this.getUiPath();
|
|
371
|
-
adapter.serveUI(uiPath, config.
|
|
498
|
+
adapter.serveUI(uiPath, config.path, getUiConfig());
|
|
372
499
|
}
|
|
373
500
|
static setStore(store) {
|
|
374
501
|
this.store = store;
|
|
@@ -391,7 +518,7 @@ var Lens = class {
|
|
|
391
518
|
const { __dirname: __dirname2 } = getMeta(import.meta.url);
|
|
392
519
|
return path2.resolve(this.normalizeDirName(__dirname2), "ui");
|
|
393
520
|
}
|
|
394
|
-
static getRoutes({
|
|
521
|
+
static getRoutes({ path: path3 }) {
|
|
395
522
|
const apiRoutes = [
|
|
396
523
|
{
|
|
397
524
|
method: "GET",
|
|
@@ -400,71 +527,71 @@ var Lens = class {
|
|
|
400
527
|
},
|
|
401
528
|
{
|
|
402
529
|
method: "GET",
|
|
403
|
-
path: `/${
|
|
530
|
+
path: `/${path3}/api/requests`,
|
|
404
531
|
handler: async (data) => await ApiController.getRequests(data)
|
|
405
532
|
},
|
|
406
533
|
{
|
|
407
534
|
method: "GET",
|
|
408
|
-
path: `/${
|
|
535
|
+
path: `/${path3}/api/requests/:id`,
|
|
409
536
|
handler: async (data) => await ApiController.getRequest(data)
|
|
410
537
|
},
|
|
411
538
|
{
|
|
412
539
|
method: "GET",
|
|
413
|
-
path: `/${
|
|
540
|
+
path: `/${path3}/api/queries`,
|
|
414
541
|
handler: async (data) => await ApiController.getQueries(data)
|
|
415
542
|
},
|
|
416
543
|
{
|
|
417
544
|
method: "GET",
|
|
418
|
-
path: `/${
|
|
545
|
+
path: `/${path3}/api/queries/:id`,
|
|
419
546
|
handler: async (data) => await ApiController.getQuery(data)
|
|
420
547
|
},
|
|
421
548
|
{
|
|
422
549
|
method: "GET",
|
|
423
|
-
path: `/${
|
|
550
|
+
path: `/${path3}/api/cache`,
|
|
424
551
|
handler: async (data) => await ApiController.getCacheEntries(data)
|
|
425
552
|
},
|
|
426
553
|
{
|
|
427
554
|
method: "GET",
|
|
428
|
-
path: `/${
|
|
555
|
+
path: `/${path3}/api/cache/:id`,
|
|
429
556
|
handler: async (data) => await ApiController.getCacheEntry(data)
|
|
430
557
|
},
|
|
431
558
|
{
|
|
432
559
|
method: "GET",
|
|
433
|
-
path: `/${
|
|
560
|
+
path: `/${path3}/api/exceptions`,
|
|
434
561
|
handler: async (data) => await ApiController.getExceptions(data)
|
|
435
562
|
},
|
|
436
563
|
{
|
|
437
564
|
method: "GET",
|
|
438
|
-
path: `/${
|
|
565
|
+
path: `/${path3}/api/exceptions/:id`,
|
|
439
566
|
handler: async (data) => await ApiController.getException(data)
|
|
440
567
|
},
|
|
441
568
|
{
|
|
442
569
|
method: "DELETE",
|
|
443
|
-
path: `/${
|
|
570
|
+
path: `/${path3}/api/truncate`,
|
|
444
571
|
handler: async () => await ApiController.truncate()
|
|
445
572
|
}
|
|
446
573
|
];
|
|
447
574
|
return { apiRoutes };
|
|
448
575
|
}
|
|
449
|
-
static async bindContainerDeps(
|
|
576
|
+
static async bindContainerDeps() {
|
|
450
577
|
const dbStore = await this.getStore();
|
|
451
578
|
Container.singleton("store", () => dbStore);
|
|
452
579
|
Container.singleton("uiConfig", () => {
|
|
453
580
|
return {
|
|
454
|
-
appName: config.appName,
|
|
455
|
-
path: `/${config.
|
|
581
|
+
appName: this.config.appName,
|
|
582
|
+
path: `/${this.config.path}`,
|
|
456
583
|
api: {
|
|
457
|
-
requests: `/${config.
|
|
458
|
-
queries: `/${config.
|
|
459
|
-
cache: `/${config.
|
|
460
|
-
exceptions: `/${config.
|
|
461
|
-
truncate: `/${config.
|
|
584
|
+
requests: `/${this.config.path}/api/requests`,
|
|
585
|
+
queries: `/${this.config.path}/api/queries`,
|
|
586
|
+
cache: `/${this.config.path}/api/cache`,
|
|
587
|
+
exceptions: `/${this.config.path}/api/exceptions`,
|
|
588
|
+
truncate: `/${this.config.path}/api/truncate`
|
|
462
589
|
}
|
|
463
590
|
};
|
|
464
591
|
});
|
|
465
592
|
}
|
|
466
593
|
static async getDefaultStore() {
|
|
467
|
-
const store = new
|
|
594
|
+
const store = new QueuedSqliteStore(this.config.storeQueueConfig);
|
|
468
595
|
await store.initialize();
|
|
469
596
|
return store;
|
|
470
597
|
}
|