@lensjs/core 2.2.2 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/abstracts/store.cjs +3 -0
- package/dist/abstracts/store.d.cts +3 -1
- package/dist/abstracts/store.d.ts +3 -1
- package/dist/abstracts/store.js +3 -0
- package/dist/core/lens.cjs +151 -24
- package/dist/core/lens.d.cts +3 -2
- package/dist/core/lens.d.ts +3 -2
- package/dist/core/lens.js +151 -24
- package/dist/{index-CMvlRWcQ.d.cts → index-CZsa0Zcm.d.ts} +3 -1
- package/dist/{index-CMvlRWcQ.d.ts → index-QmOJr0K-.d.cts} +3 -1
- package/dist/index.cjs +199 -29
- package/dist/index.d.cts +5 -2
- package/dist/index.d.ts +5 -2
- package/dist/index.js +198 -29
- package/dist/mixins/queued_store.cjs +114 -0
- package/dist/mixins/queued_store.d.cts +38 -0
- package/dist/mixins/queued_store.d.ts +38 -0
- package/dist/mixins/queued_store.js +79 -0
- package/dist/stores/better_sqlite.cjs +36 -0
- package/dist/stores/better_sqlite.d.cts +3 -0
- package/dist/stores/better_sqlite.d.ts +3 -0
- package/dist/stores/better_sqlite.js +36 -0
- package/dist/stores/index.cjs +129 -2
- package/dist/stores/index.d.cts +2 -0
- package/dist/stores/index.d.ts +2 -0
- package/dist/stores/index.js +127 -1
- package/dist/stores/queued_sqlite.cjs +313 -0
- package/dist/stores/queued_sqlite.d.cts +43 -0
- package/dist/stores/queued_sqlite.d.ts +43 -0
- package/dist/stores/queued_sqlite.js +280 -0
- package/dist/types/index.d.cts +16 -2
- package/dist/types/index.d.ts +16 -2
- package/dist/ui/assets/{CacheActionBadge-CL10Xlw7.js → CacheActionBadge-BB4uokI1.js} +1 -1
- package/dist/ui/assets/CacheEntriesTable-B8cUXhos.js +1 -0
- package/dist/ui/assets/CacheEntryContainer-WkdnGvnu.js +2 -0
- package/dist/ui/assets/CacheEntryDetails-BeZnoIpm.js +1 -0
- package/dist/ui/assets/CacheEntryDetailsContainer-DI0mEvpu.js +2 -0
- package/dist/ui/assets/ExceptionContainer-YNcR0F5U.js +2 -0
- package/dist/ui/assets/{ExceptionDetails-Bzj8OZ70.js → ExceptionDetails-BKHzv6hf.js} +1 -1
- package/dist/ui/assets/ExceptionDetailsContainer-CJHILjb3.js +2 -0
- package/dist/ui/assets/{ExceptionTable-BkFSFGbn.js → ExceptionTable-DzBmQLLa.js} +1 -1
- package/dist/ui/assets/JsonViewer-D-KPN089.js +1 -0
- package/dist/ui/assets/{LoadMore-Du7yL-Hr.js → LoadMore-CLPR6Zd4.js} +1 -1
- package/dist/ui/assets/QueriesContainer-B_PmBkHR.js +2 -0
- package/dist/ui/assets/{QueryDetailsContainer-BtbvRyBf.js → QueryDetailsContainer-Cqj3E6Dr.js} +16 -26
- package/dist/ui/assets/{QueryTable-tJVEncUM.js → QueryTable-DmWdZSnJ.js} +1 -1
- package/dist/ui/assets/{RequestDetails-C6tqSqg9.js → RequestDetails-CF338Kcv.js} +1 -1
- package/dist/ui/assets/{RequestDetailsContainer---KvdZKp.js → RequestDetailsContainer-aW4GLool.js} +2 -2
- package/dist/ui/assets/{RequestsContainer-Bogurt1b.js → RequestsContainer-DdLSvAbl.js} +2 -2
- package/dist/ui/assets/{RequetsTable-BMrYHd0d.js → RequetsTable-Bdp_PhGU.js} +1 -1
- package/dist/ui/assets/{StatusCode-DpZO0dUJ.js → StatusCode-C605nHvd.js} +1 -1
- package/dist/ui/assets/TabbedDataViewer-ofhEq_Wj.js +2 -0
- package/dist/ui/assets/{Table-BtkmKVTF.js → Table-kak5sL5X.js} +1 -1
- package/dist/ui/assets/{columns-D_NhXbk6.js → columns-BEyDhUNq.js} +1 -1
- package/dist/ui/assets/{columns-DlGaMv5C.js → columns-Bu5psHyp.js} +1 -1
- package/dist/ui/assets/{columns-DF7BR0z_.js → columns-BvIUTkjN.js} +1 -1
- package/dist/ui/assets/copy-DzXuP4eO.js +11 -0
- package/dist/ui/assets/index-CsnKQ5Mh.css +1 -0
- package/dist/ui/assets/{index-EQXljT95.js → index-TW_-MgRG.js} +25 -25
- package/dist/ui/assets/{useCacheEntries-jC9XYsV_.js → useCacheEntries-Pvte_aNc.js} +1 -1
- package/dist/ui/assets/{useExceptions-CwwK33mG.js → useExceptions-P3cnURvN.js} +1 -1
- package/dist/ui/assets/{useLensApi-CPvDlyGv.js → useLensApi-BFdsfrzR.js} +1 -1
- package/dist/ui/assets/{useLoadMore-C2bqGSaL.js → useLoadMore-JCWak1Dg.js} +1 -1
- package/dist/ui/assets/{useQueries-CzbIajH6.js → useQueries-CNquFtm0.js} +1 -1
- package/dist/ui/index.html +2 -2
- package/dist/utils/compose.cjs +32 -0
- package/dist/utils/compose.d.cts +11 -0
- package/dist/utils/compose.d.ts +11 -0
- package/dist/utils/compose.js +7 -0
- package/dist/utils/index.cjs +10 -1
- package/dist/utils/index.d.cts +3 -1
- package/dist/utils/index.d.ts +3 -1
- package/dist/utils/index.js +9 -1
- package/dist/watchers/index.cjs +45 -5
- package/dist/watchers/index.js +45 -5
- package/dist/watchers/request_watcher.cjs +45 -5
- package/dist/watchers/request_watcher.d.cts +12 -1
- package/dist/watchers/request_watcher.d.ts +12 -1
- package/dist/watchers/request_watcher.js +45 -5
- package/package.json +2 -1
- package/dist/ui/assets/CacheEntriesTable-DuLoeu0e.js +0 -1
- package/dist/ui/assets/CacheEntryContainer-DqHm-jQl.js +0 -2
- package/dist/ui/assets/CacheEntryDetails-k-74LsSb.js +0 -1
- package/dist/ui/assets/CacheEntryDetailsContainer-DTI7gtUq.js +0 -2
- package/dist/ui/assets/ExceptionContainer-0dCs6QMQ.js +0 -2
- package/dist/ui/assets/ExceptionDetailsContainer-Bim0gTpE.js +0 -2
- package/dist/ui/assets/QueriesContainer-5xlqsYl0.js +0 -2
- package/dist/ui/assets/TabbedDataViewer-C60W9bqz.js +0 -1
- package/dist/ui/assets/index-CMJVCuvo.css +0 -1
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/stores/queued_sqlite.ts
|
|
31
|
+
var queued_sqlite_exports = {};
|
|
32
|
+
__export(queued_sqlite_exports, {
|
|
33
|
+
default: () => QueuedSqliteStore
|
|
34
|
+
});
|
|
35
|
+
module.exports = __toCommonJS(queued_sqlite_exports);
|
|
36
|
+
|
|
37
|
+
// src/mixins/queued_store.ts
|
|
38
|
+
var import_denque = __toESM(require("denque"), 1);
|
|
39
|
+
function QueuedStore(Base) {
|
|
40
|
+
return class Queued extends Base {
|
|
41
|
+
queue;
|
|
42
|
+
processingInterval = null;
|
|
43
|
+
BATCH_SIZE;
|
|
44
|
+
PROCESS_INTERVAL_MS;
|
|
45
|
+
WARN_THRESHOLD;
|
|
46
|
+
PREALLOCATE;
|
|
47
|
+
constructor(...args) {
|
|
48
|
+
super(...args);
|
|
49
|
+
const config = args[0] || {};
|
|
50
|
+
this.storeConfig = config;
|
|
51
|
+
this.BATCH_SIZE = config.batchSize ?? 100;
|
|
52
|
+
this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
|
|
53
|
+
this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
|
|
54
|
+
this.PREALLOCATE = config.preallocate ?? true;
|
|
55
|
+
this.queue = this.PREALLOCATE ? new import_denque.default([], { capacity: this.WARN_THRESHOLD * 2 }) : new import_denque.default();
|
|
56
|
+
}
|
|
57
|
+
async initialize() {
|
|
58
|
+
if (super["initialize"]) {
|
|
59
|
+
await super["initialize"].call(this);
|
|
60
|
+
}
|
|
61
|
+
this.startProcessingQueue();
|
|
62
|
+
process.on("SIGINT", () => this.shutdown());
|
|
63
|
+
process.on("SIGTERM", () => this.shutdown());
|
|
64
|
+
}
|
|
65
|
+
async truncate() {
|
|
66
|
+
this.queue.clear();
|
|
67
|
+
if (super["truncate"]) {
|
|
68
|
+
await super["truncate"].call(this);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
async save(entry) {
|
|
72
|
+
this.queue.push(entry);
|
|
73
|
+
if (this.queue.length > this.WARN_THRESHOLD) {
|
|
74
|
+
console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
startProcessingQueue() {
|
|
78
|
+
if (this.processingInterval) clearInterval(this.processingInterval);
|
|
79
|
+
this.processingInterval = setInterval(
|
|
80
|
+
() => this.processQueue(),
|
|
81
|
+
this.PROCESS_INTERVAL_MS
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
async processQueue() {
|
|
85
|
+
if (this.queue.isEmpty()) return;
|
|
86
|
+
const batchSize = Math.min(
|
|
87
|
+
this.BATCH_SIZE,
|
|
88
|
+
Math.max(10, Math.floor(this.queue.length / 10))
|
|
89
|
+
);
|
|
90
|
+
const entriesToProcess = this.queue.remove(0, batchSize);
|
|
91
|
+
if (!entriesToProcess.length) return;
|
|
92
|
+
for (const entry of entriesToProcess) {
|
|
93
|
+
super["save"]?.call(this, entry).catch((error) => {
|
|
94
|
+
console.error("Error saving queued entry:", error);
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async stopProcessingQueue() {
|
|
99
|
+
if (this.processingInterval) {
|
|
100
|
+
clearInterval(this.processingInterval);
|
|
101
|
+
this.processingInterval = null;
|
|
102
|
+
}
|
|
103
|
+
if (!this.queue.isEmpty()) {
|
|
104
|
+
await this.processQueue();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async shutdown() {
|
|
108
|
+
await this.stopProcessingQueue();
|
|
109
|
+
process.exit(0);
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// src/utils/compose.ts
|
|
115
|
+
function compose(superclass, ...mixins) {
|
|
116
|
+
return mixins.reduce((c, mixin) => mixin(c), superclass);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// src/abstracts/store.ts
|
|
120
|
+
var Store = class {
|
|
121
|
+
storeConfig;
|
|
122
|
+
constructor(...args) {
|
|
123
|
+
}
|
|
124
|
+
getAllExceptions(_paginationParams) {
|
|
125
|
+
return this.defaultMinimalPaginate();
|
|
126
|
+
}
|
|
127
|
+
stringifyData(data) {
|
|
128
|
+
if (typeof data === "string") {
|
|
129
|
+
return data;
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
return JSON.stringify(data);
|
|
133
|
+
} catch (e) {
|
|
134
|
+
console.error(`Failed to stringify lens data: ${e}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
defaultMinimalPaginate() {
|
|
138
|
+
return Promise.resolve({
|
|
139
|
+
data: [],
|
|
140
|
+
meta: {
|
|
141
|
+
currentPage: 0,
|
|
142
|
+
lastPage: 0,
|
|
143
|
+
total: 0
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
// src/stores/better_sqlite.ts
|
|
150
|
+
var import_crypto = require("crypto");
|
|
151
|
+
var import_libsql = __toESM(require("libsql"), 1);
|
|
152
|
+
var import_date = require("@lensjs/date");
|
|
153
|
+
var TABLE_NAME = "lens_entries";
|
|
154
|
+
var BYTES_IN_GB = 1024 * 1024 * 1024;
|
|
155
|
+
var PRUNE_BATCH_SIZE = 1e3;
|
|
156
|
+
var BetterSqliteStore = class extends Store {
|
|
157
|
+
connection;
|
|
158
|
+
async initialize() {
|
|
159
|
+
this.connection = new import_libsql.default("lens.db");
|
|
160
|
+
this.setupSchema();
|
|
161
|
+
console.log("Connected to Lens (SQLite) database.");
|
|
162
|
+
}
|
|
163
|
+
async truncate() {
|
|
164
|
+
this.connection.prepare(`DELETE FROM ${TABLE_NAME};`).run();
|
|
165
|
+
}
|
|
166
|
+
async save(entry) {
|
|
167
|
+
this.connection.prepare(
|
|
168
|
+
`INSERT INTO ${TABLE_NAME} (id, data, type, created_at, lens_entry_id, minimal_data) values($id, $data, $type, $created_at, $lens_entry_id, $minimalData)`
|
|
169
|
+
).run({
|
|
170
|
+
id: entry.id ?? (0, import_crypto.randomUUID)(),
|
|
171
|
+
data: this.stringifyData(entry.data),
|
|
172
|
+
type: entry.type,
|
|
173
|
+
created_at: entry.timestamp ?? (0, import_date.nowISO)(),
|
|
174
|
+
lens_entry_id: entry.requestId || null,
|
|
175
|
+
minimalData: this.stringifyData(entry.minimal_data ?? {})
|
|
176
|
+
});
|
|
177
|
+
this.maybePruneDatabase();
|
|
178
|
+
}
|
|
179
|
+
async getAllQueries(pagination) {
|
|
180
|
+
return await this.paginate("query" /* QUERY */, pagination);
|
|
181
|
+
}
|
|
182
|
+
async getAllRequests(pagination) {
|
|
183
|
+
return await this.paginate("request" /* REQUEST */, pagination, false);
|
|
184
|
+
}
|
|
185
|
+
async getAllCacheEntries(pagination) {
|
|
186
|
+
return await this.paginate("cache" /* CACHE */, pagination);
|
|
187
|
+
}
|
|
188
|
+
async getAllExceptions(pagination) {
|
|
189
|
+
return await this.paginate("exception" /* EXCEPTION */, pagination, false);
|
|
190
|
+
}
|
|
191
|
+
async allByRequestId(requestId, type, includeFullData = true) {
|
|
192
|
+
const rows = this.connection.prepare(
|
|
193
|
+
`${this.getSelectedColumns(includeFullData)} FROM ${TABLE_NAME} WHERE type = $type AND lens_entry_id = $requestId ORDER BY created_at DESC`
|
|
194
|
+
).all({ type, requestId });
|
|
195
|
+
return this.mapRows(rows, includeFullData);
|
|
196
|
+
}
|
|
197
|
+
async paginate(type, { page, perPage }, includeFullData = true) {
|
|
198
|
+
const offset = (page - 1) * perPage;
|
|
199
|
+
const query = `${this.getSelectedColumns(
|
|
200
|
+
includeFullData
|
|
201
|
+
)} FROM ${TABLE_NAME} WHERE type = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`;
|
|
202
|
+
const count = await this.count(type);
|
|
203
|
+
const rows = this.connection.prepare(query).all(type, perPage, offset);
|
|
204
|
+
const mappedRows = this.mapRows(rows, includeFullData);
|
|
205
|
+
return {
|
|
206
|
+
meta: {
|
|
207
|
+
total: count,
|
|
208
|
+
lastPage: Math.ceil(count / perPage),
|
|
209
|
+
currentPage: page
|
|
210
|
+
},
|
|
211
|
+
data: mappedRows
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
async count(type) {
|
|
215
|
+
const result = this.connection.prepare(`SELECT count(*) as count FROM ${TABLE_NAME} WHERE type = ?`).get(type);
|
|
216
|
+
return Number(result.count);
|
|
217
|
+
}
|
|
218
|
+
async find(type, id) {
|
|
219
|
+
const row = this.connection.prepare(
|
|
220
|
+
`${this.getSelectedColumns()} FROM ${TABLE_NAME} WHERE id = ? AND type = ? LIMIT 1`
|
|
221
|
+
).get(id, type);
|
|
222
|
+
if (!row) {
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
return this.mapRow(row, true);
|
|
226
|
+
}
|
|
227
|
+
setupSchema() {
|
|
228
|
+
this.connection.exec("PRAGMA journal_mode = WAL;");
|
|
229
|
+
this.connection.exec("PRAGMA synchronous = NORMAL;");
|
|
230
|
+
const createTable = `
|
|
231
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (
|
|
232
|
+
id TEXT PRIMARY KEY,
|
|
233
|
+
minimal_data TEXT,
|
|
234
|
+
data TEXT NOT NULL,
|
|
235
|
+
type TEXT NOT NULL,
|
|
236
|
+
created_at TEXT NOT NULL,
|
|
237
|
+
updated_at TEXT,
|
|
238
|
+
lens_entry_id TEXT NULL
|
|
239
|
+
);
|
|
240
|
+
`;
|
|
241
|
+
const createIndex = `
|
|
242
|
+
CREATE INDEX IF NOT EXISTS lens_entries_id_type_index
|
|
243
|
+
ON ${TABLE_NAME} (id, type);
|
|
244
|
+
`;
|
|
245
|
+
const lensEntryIdIndex = `
|
|
246
|
+
CREATE INDEX IF NOT EXISTS lens_entry_id_index
|
|
247
|
+
ON ${TABLE_NAME} (lens_entry_id);
|
|
248
|
+
`;
|
|
249
|
+
this.connection.exec(createTable);
|
|
250
|
+
this.connection.exec(createIndex);
|
|
251
|
+
this.connection.exec(lensEntryIdIndex);
|
|
252
|
+
}
|
|
253
|
+
maybePruneDatabase() {
|
|
254
|
+
const maxGb = this.storeConfig?.dbMaxSizeGb;
|
|
255
|
+
const pruneGb = this.storeConfig?.dbPruneSizeGb;
|
|
256
|
+
if (!maxGb || !pruneGb) return;
|
|
257
|
+
const maxBytes = maxGb * BYTES_IN_GB;
|
|
258
|
+
const pruneBytes = pruneGb * BYTES_IN_GB;
|
|
259
|
+
if (maxBytes <= 0 || pruneBytes <= 0) return;
|
|
260
|
+
const targetBytes = Math.max(0, maxBytes - pruneBytes);
|
|
261
|
+
let usedBytes = this.getDatabaseUsedBytes();
|
|
262
|
+
if (usedBytes < maxBytes) return;
|
|
263
|
+
while (usedBytes > targetBytes) {
|
|
264
|
+
const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
|
|
265
|
+
if (deletedRows === 0) break;
|
|
266
|
+
usedBytes = this.getDatabaseUsedBytes();
|
|
267
|
+
}
|
|
268
|
+
this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
269
|
+
}
|
|
270
|
+
getDatabaseUsedBytes() {
|
|
271
|
+
const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
|
|
272
|
+
const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
|
|
273
|
+
const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
|
|
274
|
+
const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
|
|
275
|
+
return usedPages * pageSizeResult.page_size;
|
|
276
|
+
}
|
|
277
|
+
deleteOldestEntries(batchSize) {
|
|
278
|
+
const result = this.connection.prepare(
|
|
279
|
+
`DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
|
|
280
|
+
).run(batchSize);
|
|
281
|
+
return Number(result.changes ?? 0);
|
|
282
|
+
}
|
|
283
|
+
mapRow(row, includeFullData = true) {
|
|
284
|
+
let data = includeFullData ? JSON.parse(row.data) : {};
|
|
285
|
+
if (!includeFullData) {
|
|
286
|
+
data = JSON.parse(row.minimal_data);
|
|
287
|
+
}
|
|
288
|
+
return {
|
|
289
|
+
id: row.id,
|
|
290
|
+
type: row.type,
|
|
291
|
+
created_at: row.created_at,
|
|
292
|
+
lens_entry_id: row.lens_entry_id,
|
|
293
|
+
data
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
mapRows(rows, includeFullData = true) {
|
|
297
|
+
let mappedRows = [];
|
|
298
|
+
for (const row of rows) {
|
|
299
|
+
mappedRows.push(this.mapRow(row, includeFullData));
|
|
300
|
+
}
|
|
301
|
+
return mappedRows;
|
|
302
|
+
}
|
|
303
|
+
getSelectedColumns(includeFullData = true) {
|
|
304
|
+
return `SELECT id, minimal_data, type, created_at, lens_entry_id ${includeFullData ? ",data" : ""}`;
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
// src/stores/queued_sqlite.ts
|
|
309
|
+
var QueuedSqliteStore = class extends compose(
|
|
310
|
+
BetterSqliteStore,
|
|
311
|
+
QueuedStore
|
|
312
|
+
) {
|
|
313
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import BetterSqliteStore from './better_sqlite.cjs';
|
|
2
|
+
import * as denque from 'denque';
|
|
3
|
+
import { WatcherTypeEnum } from '../types/index.cjs';
|
|
4
|
+
import '../abstracts/store.cjs';
|
|
5
|
+
import 'sql-formatter';
|
|
6
|
+
import 'libsql';
|
|
7
|
+
|
|
8
|
+
declare const QueuedSqliteStore_base: {
|
|
9
|
+
new (...args: any[]): {
|
|
10
|
+
[x: string]: any;
|
|
11
|
+
queue: denque<{
|
|
12
|
+
id?: string;
|
|
13
|
+
data: Record<string, any>;
|
|
14
|
+
minimal_data?: Record<string, any>;
|
|
15
|
+
type: WatcherTypeEnum;
|
|
16
|
+
timestamp?: string;
|
|
17
|
+
requestId?: string;
|
|
18
|
+
}>;
|
|
19
|
+
processingInterval: NodeJS.Timeout | null;
|
|
20
|
+
readonly BATCH_SIZE: number;
|
|
21
|
+
readonly PROCESS_INTERVAL_MS: number;
|
|
22
|
+
readonly WARN_THRESHOLD: number;
|
|
23
|
+
readonly PREALLOCATE: boolean;
|
|
24
|
+
initialize(): Promise<void>;
|
|
25
|
+
truncate(): Promise<void>;
|
|
26
|
+
save(entry: {
|
|
27
|
+
id?: string;
|
|
28
|
+
data: Record<string, any>;
|
|
29
|
+
minimal_data?: Record<string, any>;
|
|
30
|
+
type: WatcherTypeEnum;
|
|
31
|
+
timestamp?: string;
|
|
32
|
+
requestId?: string;
|
|
33
|
+
}): Promise<void>;
|
|
34
|
+
startProcessingQueue(): void;
|
|
35
|
+
processQueue(): Promise<void>;
|
|
36
|
+
stopProcessingQueue(): Promise<void>;
|
|
37
|
+
shutdown(): Promise<void>;
|
|
38
|
+
};
|
|
39
|
+
} & typeof BetterSqliteStore;
|
|
40
|
+
declare class QueuedSqliteStore extends QueuedSqliteStore_base {
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export { QueuedSqliteStore as default };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import BetterSqliteStore from './better_sqlite.js';
|
|
2
|
+
import * as denque from 'denque';
|
|
3
|
+
import { WatcherTypeEnum } from '../types/index.js';
|
|
4
|
+
import '../abstracts/store.js';
|
|
5
|
+
import 'sql-formatter';
|
|
6
|
+
import 'libsql';
|
|
7
|
+
|
|
8
|
+
declare const QueuedSqliteStore_base: {
|
|
9
|
+
new (...args: any[]): {
|
|
10
|
+
[x: string]: any;
|
|
11
|
+
queue: denque<{
|
|
12
|
+
id?: string;
|
|
13
|
+
data: Record<string, any>;
|
|
14
|
+
minimal_data?: Record<string, any>;
|
|
15
|
+
type: WatcherTypeEnum;
|
|
16
|
+
timestamp?: string;
|
|
17
|
+
requestId?: string;
|
|
18
|
+
}>;
|
|
19
|
+
processingInterval: NodeJS.Timeout | null;
|
|
20
|
+
readonly BATCH_SIZE: number;
|
|
21
|
+
readonly PROCESS_INTERVAL_MS: number;
|
|
22
|
+
readonly WARN_THRESHOLD: number;
|
|
23
|
+
readonly PREALLOCATE: boolean;
|
|
24
|
+
initialize(): Promise<void>;
|
|
25
|
+
truncate(): Promise<void>;
|
|
26
|
+
save(entry: {
|
|
27
|
+
id?: string;
|
|
28
|
+
data: Record<string, any>;
|
|
29
|
+
minimal_data?: Record<string, any>;
|
|
30
|
+
type: WatcherTypeEnum;
|
|
31
|
+
timestamp?: string;
|
|
32
|
+
requestId?: string;
|
|
33
|
+
}): Promise<void>;
|
|
34
|
+
startProcessingQueue(): void;
|
|
35
|
+
processQueue(): Promise<void>;
|
|
36
|
+
stopProcessingQueue(): Promise<void>;
|
|
37
|
+
shutdown(): Promise<void>;
|
|
38
|
+
};
|
|
39
|
+
} & typeof BetterSqliteStore;
|
|
40
|
+
declare class QueuedSqliteStore extends QueuedSqliteStore_base {
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export { QueuedSqliteStore as default };
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
// src/mixins/queued_store.ts
|
|
2
|
+
import Denque from "denque";
|
|
3
|
+
function QueuedStore(Base) {
|
|
4
|
+
return class Queued extends Base {
|
|
5
|
+
queue;
|
|
6
|
+
processingInterval = null;
|
|
7
|
+
BATCH_SIZE;
|
|
8
|
+
PROCESS_INTERVAL_MS;
|
|
9
|
+
WARN_THRESHOLD;
|
|
10
|
+
PREALLOCATE;
|
|
11
|
+
constructor(...args) {
|
|
12
|
+
super(...args);
|
|
13
|
+
const config = args[0] || {};
|
|
14
|
+
this.storeConfig = config;
|
|
15
|
+
this.BATCH_SIZE = config.batchSize ?? 100;
|
|
16
|
+
this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
|
|
17
|
+
this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
|
|
18
|
+
this.PREALLOCATE = config.preallocate ?? true;
|
|
19
|
+
this.queue = this.PREALLOCATE ? new Denque([], { capacity: this.WARN_THRESHOLD * 2 }) : new Denque();
|
|
20
|
+
}
|
|
21
|
+
async initialize() {
|
|
22
|
+
if (super["initialize"]) {
|
|
23
|
+
await super["initialize"].call(this);
|
|
24
|
+
}
|
|
25
|
+
this.startProcessingQueue();
|
|
26
|
+
process.on("SIGINT", () => this.shutdown());
|
|
27
|
+
process.on("SIGTERM", () => this.shutdown());
|
|
28
|
+
}
|
|
29
|
+
async truncate() {
|
|
30
|
+
this.queue.clear();
|
|
31
|
+
if (super["truncate"]) {
|
|
32
|
+
await super["truncate"].call(this);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async save(entry) {
|
|
36
|
+
this.queue.push(entry);
|
|
37
|
+
if (this.queue.length > this.WARN_THRESHOLD) {
|
|
38
|
+
console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
startProcessingQueue() {
|
|
42
|
+
if (this.processingInterval) clearInterval(this.processingInterval);
|
|
43
|
+
this.processingInterval = setInterval(
|
|
44
|
+
() => this.processQueue(),
|
|
45
|
+
this.PROCESS_INTERVAL_MS
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
async processQueue() {
|
|
49
|
+
if (this.queue.isEmpty()) return;
|
|
50
|
+
const batchSize = Math.min(
|
|
51
|
+
this.BATCH_SIZE,
|
|
52
|
+
Math.max(10, Math.floor(this.queue.length / 10))
|
|
53
|
+
);
|
|
54
|
+
const entriesToProcess = this.queue.remove(0, batchSize);
|
|
55
|
+
if (!entriesToProcess.length) return;
|
|
56
|
+
for (const entry of entriesToProcess) {
|
|
57
|
+
super["save"]?.call(this, entry).catch((error) => {
|
|
58
|
+
console.error("Error saving queued entry:", error);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async stopProcessingQueue() {
|
|
63
|
+
if (this.processingInterval) {
|
|
64
|
+
clearInterval(this.processingInterval);
|
|
65
|
+
this.processingInterval = null;
|
|
66
|
+
}
|
|
67
|
+
if (!this.queue.isEmpty()) {
|
|
68
|
+
await this.processQueue();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
async shutdown() {
|
|
72
|
+
await this.stopProcessingQueue();
|
|
73
|
+
process.exit(0);
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// src/utils/compose.ts
|
|
79
|
+
function compose(superclass, ...mixins) {
|
|
80
|
+
return mixins.reduce((c, mixin) => mixin(c), superclass);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// src/abstracts/store.ts
|
|
84
|
+
var Store = class {
|
|
85
|
+
storeConfig;
|
|
86
|
+
constructor(...args) {
|
|
87
|
+
}
|
|
88
|
+
getAllExceptions(_paginationParams) {
|
|
89
|
+
return this.defaultMinimalPaginate();
|
|
90
|
+
}
|
|
91
|
+
stringifyData(data) {
|
|
92
|
+
if (typeof data === "string") {
|
|
93
|
+
return data;
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
return JSON.stringify(data);
|
|
97
|
+
} catch (e) {
|
|
98
|
+
console.error(`Failed to stringify lens data: ${e}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
defaultMinimalPaginate() {
|
|
102
|
+
return Promise.resolve({
|
|
103
|
+
data: [],
|
|
104
|
+
meta: {
|
|
105
|
+
currentPage: 0,
|
|
106
|
+
lastPage: 0,
|
|
107
|
+
total: 0
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// src/stores/better_sqlite.ts
|
|
114
|
+
import { randomUUID } from "crypto";
|
|
115
|
+
import Database from "libsql";
|
|
116
|
+
import { nowISO } from "@lensjs/date";
|
|
117
|
+
var TABLE_NAME = "lens_entries";
|
|
118
|
+
var BYTES_IN_GB = 1024 * 1024 * 1024;
|
|
119
|
+
var PRUNE_BATCH_SIZE = 1e3;
|
|
120
|
+
var BetterSqliteStore = class extends Store {
|
|
121
|
+
connection;
|
|
122
|
+
async initialize() {
|
|
123
|
+
this.connection = new Database("lens.db");
|
|
124
|
+
this.setupSchema();
|
|
125
|
+
console.log("Connected to Lens (SQLite) database.");
|
|
126
|
+
}
|
|
127
|
+
async truncate() {
|
|
128
|
+
this.connection.prepare(`DELETE FROM ${TABLE_NAME};`).run();
|
|
129
|
+
}
|
|
130
|
+
async save(entry) {
|
|
131
|
+
this.connection.prepare(
|
|
132
|
+
`INSERT INTO ${TABLE_NAME} (id, data, type, created_at, lens_entry_id, minimal_data) values($id, $data, $type, $created_at, $lens_entry_id, $minimalData)`
|
|
133
|
+
).run({
|
|
134
|
+
id: entry.id ?? randomUUID(),
|
|
135
|
+
data: this.stringifyData(entry.data),
|
|
136
|
+
type: entry.type,
|
|
137
|
+
created_at: entry.timestamp ?? nowISO(),
|
|
138
|
+
lens_entry_id: entry.requestId || null,
|
|
139
|
+
minimalData: this.stringifyData(entry.minimal_data ?? {})
|
|
140
|
+
});
|
|
141
|
+
this.maybePruneDatabase();
|
|
142
|
+
}
|
|
143
|
+
async getAllQueries(pagination) {
|
|
144
|
+
return await this.paginate("query" /* QUERY */, pagination);
|
|
145
|
+
}
|
|
146
|
+
async getAllRequests(pagination) {
|
|
147
|
+
return await this.paginate("request" /* REQUEST */, pagination, false);
|
|
148
|
+
}
|
|
149
|
+
async getAllCacheEntries(pagination) {
|
|
150
|
+
return await this.paginate("cache" /* CACHE */, pagination);
|
|
151
|
+
}
|
|
152
|
+
async getAllExceptions(pagination) {
|
|
153
|
+
return await this.paginate("exception" /* EXCEPTION */, pagination, false);
|
|
154
|
+
}
|
|
155
|
+
async allByRequestId(requestId, type, includeFullData = true) {
|
|
156
|
+
const rows = this.connection.prepare(
|
|
157
|
+
`${this.getSelectedColumns(includeFullData)} FROM ${TABLE_NAME} WHERE type = $type AND lens_entry_id = $requestId ORDER BY created_at DESC`
|
|
158
|
+
).all({ type, requestId });
|
|
159
|
+
return this.mapRows(rows, includeFullData);
|
|
160
|
+
}
|
|
161
|
+
async paginate(type, { page, perPage }, includeFullData = true) {
|
|
162
|
+
const offset = (page - 1) * perPage;
|
|
163
|
+
const query = `${this.getSelectedColumns(
|
|
164
|
+
includeFullData
|
|
165
|
+
)} FROM ${TABLE_NAME} WHERE type = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`;
|
|
166
|
+
const count = await this.count(type);
|
|
167
|
+
const rows = this.connection.prepare(query).all(type, perPage, offset);
|
|
168
|
+
const mappedRows = this.mapRows(rows, includeFullData);
|
|
169
|
+
return {
|
|
170
|
+
meta: {
|
|
171
|
+
total: count,
|
|
172
|
+
lastPage: Math.ceil(count / perPage),
|
|
173
|
+
currentPage: page
|
|
174
|
+
},
|
|
175
|
+
data: mappedRows
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
async count(type) {
|
|
179
|
+
const result = this.connection.prepare(`SELECT count(*) as count FROM ${TABLE_NAME} WHERE type = ?`).get(type);
|
|
180
|
+
return Number(result.count);
|
|
181
|
+
}
|
|
182
|
+
async find(type, id) {
|
|
183
|
+
const row = this.connection.prepare(
|
|
184
|
+
`${this.getSelectedColumns()} FROM ${TABLE_NAME} WHERE id = ? AND type = ? LIMIT 1`
|
|
185
|
+
).get(id, type);
|
|
186
|
+
if (!row) {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
return this.mapRow(row, true);
|
|
190
|
+
}
|
|
191
|
+
setupSchema() {
|
|
192
|
+
this.connection.exec("PRAGMA journal_mode = WAL;");
|
|
193
|
+
this.connection.exec("PRAGMA synchronous = NORMAL;");
|
|
194
|
+
const createTable = `
|
|
195
|
+
CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (
|
|
196
|
+
id TEXT PRIMARY KEY,
|
|
197
|
+
minimal_data TEXT,
|
|
198
|
+
data TEXT NOT NULL,
|
|
199
|
+
type TEXT NOT NULL,
|
|
200
|
+
created_at TEXT NOT NULL,
|
|
201
|
+
updated_at TEXT,
|
|
202
|
+
lens_entry_id TEXT NULL
|
|
203
|
+
);
|
|
204
|
+
`;
|
|
205
|
+
const createIndex = `
|
|
206
|
+
CREATE INDEX IF NOT EXISTS lens_entries_id_type_index
|
|
207
|
+
ON ${TABLE_NAME} (id, type);
|
|
208
|
+
`;
|
|
209
|
+
const lensEntryIdIndex = `
|
|
210
|
+
CREATE INDEX IF NOT EXISTS lens_entry_id_index
|
|
211
|
+
ON ${TABLE_NAME} (lens_entry_id);
|
|
212
|
+
`;
|
|
213
|
+
this.connection.exec(createTable);
|
|
214
|
+
this.connection.exec(createIndex);
|
|
215
|
+
this.connection.exec(lensEntryIdIndex);
|
|
216
|
+
}
|
|
217
|
+
maybePruneDatabase() {
|
|
218
|
+
const maxGb = this.storeConfig?.dbMaxSizeGb;
|
|
219
|
+
const pruneGb = this.storeConfig?.dbPruneSizeGb;
|
|
220
|
+
if (!maxGb || !pruneGb) return;
|
|
221
|
+
const maxBytes = maxGb * BYTES_IN_GB;
|
|
222
|
+
const pruneBytes = pruneGb * BYTES_IN_GB;
|
|
223
|
+
if (maxBytes <= 0 || pruneBytes <= 0) return;
|
|
224
|
+
const targetBytes = Math.max(0, maxBytes - pruneBytes);
|
|
225
|
+
let usedBytes = this.getDatabaseUsedBytes();
|
|
226
|
+
if (usedBytes < maxBytes) return;
|
|
227
|
+
while (usedBytes > targetBytes) {
|
|
228
|
+
const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
|
|
229
|
+
if (deletedRows === 0) break;
|
|
230
|
+
usedBytes = this.getDatabaseUsedBytes();
|
|
231
|
+
}
|
|
232
|
+
this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
233
|
+
}
|
|
234
|
+
getDatabaseUsedBytes() {
|
|
235
|
+
const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
|
|
236
|
+
const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
|
|
237
|
+
const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
|
|
238
|
+
const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
|
|
239
|
+
return usedPages * pageSizeResult.page_size;
|
|
240
|
+
}
|
|
241
|
+
deleteOldestEntries(batchSize) {
|
|
242
|
+
const result = this.connection.prepare(
|
|
243
|
+
`DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
|
|
244
|
+
).run(batchSize);
|
|
245
|
+
return Number(result.changes ?? 0);
|
|
246
|
+
}
|
|
247
|
+
mapRow(row, includeFullData = true) {
|
|
248
|
+
let data = includeFullData ? JSON.parse(row.data) : {};
|
|
249
|
+
if (!includeFullData) {
|
|
250
|
+
data = JSON.parse(row.minimal_data);
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
id: row.id,
|
|
254
|
+
type: row.type,
|
|
255
|
+
created_at: row.created_at,
|
|
256
|
+
lens_entry_id: row.lens_entry_id,
|
|
257
|
+
data
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
mapRows(rows, includeFullData = true) {
|
|
261
|
+
let mappedRows = [];
|
|
262
|
+
for (const row of rows) {
|
|
263
|
+
mappedRows.push(this.mapRow(row, includeFullData));
|
|
264
|
+
}
|
|
265
|
+
return mappedRows;
|
|
266
|
+
}
|
|
267
|
+
getSelectedColumns(includeFullData = true) {
|
|
268
|
+
return `SELECT id, minimal_data, type, created_at, lens_entry_id ${includeFullData ? ",data" : ""}`;
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
// src/stores/queued_sqlite.ts
|
|
273
|
+
var QueuedSqliteStore = class extends compose(
|
|
274
|
+
BetterSqliteStore,
|
|
275
|
+
QueuedStore
|
|
276
|
+
) {
|
|
277
|
+
};
|
|
278
|
+
export {
|
|
279
|
+
QueuedSqliteStore as default
|
|
280
|
+
};
|