@http-client-toolkit/store-sqlite 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +93 -0
- package/lib/index.cjs +1311 -0
- package/lib/index.cjs.map +1 -0
- package/lib/index.d.cts +537 -0
- package/lib/index.d.ts +537 -0
- package/lib/index.js +1299 -0
- package/lib/index.js.map +1 -0
- package/package.json +79 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,1299 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import { sql, eq, count, and, gt, lt, gte } from 'drizzle-orm';
|
|
3
|
+
import { drizzle } from 'drizzle-orm/better-sqlite3';
|
|
4
|
+
import { sqliteTable, integer, blob, text } from 'drizzle-orm/sqlite-core';
|
|
5
|
+
import { randomUUID } from 'crypto';
|
|
6
|
+
import { DEFAULT_RATE_LIMIT, AdaptiveCapacityCalculator } from '@http-client-toolkit/core';
|
|
7
|
+
|
|
8
|
+
var __async = (__this, __arguments, generator) => {
|
|
9
|
+
return new Promise((resolve, reject) => {
|
|
10
|
+
var fulfilled = (value) => {
|
|
11
|
+
try {
|
|
12
|
+
step(generator.next(value));
|
|
13
|
+
} catch (e) {
|
|
14
|
+
reject(e);
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
var rejected = (value) => {
|
|
18
|
+
try {
|
|
19
|
+
step(generator.throw(value));
|
|
20
|
+
} catch (e) {
|
|
21
|
+
reject(e);
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
|
|
25
|
+
step((generator = generator.apply(__this, __arguments)).next());
|
|
26
|
+
});
|
|
27
|
+
};
|
|
28
|
+
var cacheTable = sqliteTable("cache", {
|
|
29
|
+
hash: text("hash").primaryKey(),
|
|
30
|
+
value: blob("value", { mode: "json" }).notNull(),
|
|
31
|
+
expiresAt: integer("expires_at").notNull(),
|
|
32
|
+
createdAt: integer("created_at").notNull()
|
|
33
|
+
});
|
|
34
|
+
var dedupeTable = sqliteTable("dedupe_jobs", {
|
|
35
|
+
hash: text("hash").primaryKey(),
|
|
36
|
+
jobId: text("job_id").notNull(),
|
|
37
|
+
status: text("status").notNull(),
|
|
38
|
+
// 'pending', 'completed', 'failed'
|
|
39
|
+
result: blob("result", { mode: "json" }),
|
|
40
|
+
error: text("error"),
|
|
41
|
+
createdAt: integer("created_at").notNull(),
|
|
42
|
+
updatedAt: integer("updated_at").notNull()
|
|
43
|
+
});
|
|
44
|
+
var rateLimitTable = sqliteTable("rate_limits", {
|
|
45
|
+
resource: text("resource").notNull(),
|
|
46
|
+
timestamp: integer("timestamp").notNull(),
|
|
47
|
+
id: integer("id").primaryKey({ autoIncrement: true })
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
// src/sqlite-cache-store.ts
|
|
51
|
+
var SQLiteCacheStore = class {
|
|
52
|
+
constructor({
|
|
53
|
+
/** File path or existing `better-sqlite3` connection. Defaults to `':memory:'`. */
|
|
54
|
+
database = ":memory:",
|
|
55
|
+
/** Cleanup interval in milliseconds. Defaults to 1 minute. */
|
|
56
|
+
cleanupIntervalMs = 6e4,
|
|
57
|
+
/** Maximum allowed size (in bytes) for a single cache entry. Defaults to 5 MiB. */
|
|
58
|
+
maxEntrySizeBytes = 5 * 1024 * 1024
|
|
59
|
+
} = {}) {
|
|
60
|
+
/** Indicates whether this store is responsible for managing (and therefore closing) the SQLite connection */
|
|
61
|
+
this.isConnectionManaged = false;
|
|
62
|
+
this.isDestroyed = false;
|
|
63
|
+
let sqliteInstance;
|
|
64
|
+
let isConnectionManaged = false;
|
|
65
|
+
if (typeof database === "string") {
|
|
66
|
+
sqliteInstance = new Database(database);
|
|
67
|
+
isConnectionManaged = true;
|
|
68
|
+
} else {
|
|
69
|
+
sqliteInstance = database;
|
|
70
|
+
}
|
|
71
|
+
this.sqlite = sqliteInstance;
|
|
72
|
+
this.isConnectionManaged = isConnectionManaged;
|
|
73
|
+
this.db = drizzle(sqliteInstance);
|
|
74
|
+
this.cleanupIntervalMs = cleanupIntervalMs;
|
|
75
|
+
this.maxEntrySizeBytes = maxEntrySizeBytes;
|
|
76
|
+
this.initializeDatabase();
|
|
77
|
+
this.startCleanupInterval();
|
|
78
|
+
}
|
|
79
|
+
get(hash) {
|
|
80
|
+
return __async(this, null, function* () {
|
|
81
|
+
if (this.isDestroyed) {
|
|
82
|
+
throw new Error("Cache store has been destroyed");
|
|
83
|
+
}
|
|
84
|
+
const result = yield this.db.select().from(cacheTable).where(eq(cacheTable.hash, hash)).limit(1);
|
|
85
|
+
if (result.length === 0) {
|
|
86
|
+
return void 0;
|
|
87
|
+
}
|
|
88
|
+
const item = result[0];
|
|
89
|
+
if (!item) {
|
|
90
|
+
return void 0;
|
|
91
|
+
}
|
|
92
|
+
const now = Date.now();
|
|
93
|
+
if (item.expiresAt > 0 && now >= item.expiresAt) {
|
|
94
|
+
yield this.db.delete(cacheTable).where(eq(cacheTable.hash, hash));
|
|
95
|
+
return void 0;
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
if (item.value === "__UNDEFINED__") {
|
|
99
|
+
return void 0;
|
|
100
|
+
}
|
|
101
|
+
return JSON.parse(item.value);
|
|
102
|
+
} catch (e) {
|
|
103
|
+
yield this.db.delete(cacheTable).where(eq(cacheTable.hash, hash));
|
|
104
|
+
return void 0;
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
set(hash, value, ttlSeconds) {
|
|
109
|
+
return __async(this, null, function* () {
|
|
110
|
+
if (this.isDestroyed) {
|
|
111
|
+
throw new Error("Cache store has been destroyed");
|
|
112
|
+
}
|
|
113
|
+
const now = Date.now();
|
|
114
|
+
const expiresAt = ttlSeconds < 0 ? now : ttlSeconds === 0 ? 0 : now + ttlSeconds * 1e3;
|
|
115
|
+
let serializedValue;
|
|
116
|
+
try {
|
|
117
|
+
if (value === void 0) {
|
|
118
|
+
serializedValue = "__UNDEFINED__";
|
|
119
|
+
} else {
|
|
120
|
+
serializedValue = JSON.stringify(value);
|
|
121
|
+
}
|
|
122
|
+
} catch (error) {
|
|
123
|
+
throw new Error(
|
|
124
|
+
`Failed to serialize value: ${error instanceof Error ? error.message : String(error)}`
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
if (Buffer.byteLength(serializedValue, "utf8") > this.maxEntrySizeBytes) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
yield this.db.insert(cacheTable).values({
|
|
131
|
+
hash,
|
|
132
|
+
value: serializedValue,
|
|
133
|
+
expiresAt,
|
|
134
|
+
createdAt: now
|
|
135
|
+
}).onConflictDoUpdate({
|
|
136
|
+
target: cacheTable.hash,
|
|
137
|
+
set: {
|
|
138
|
+
value: serializedValue,
|
|
139
|
+
expiresAt,
|
|
140
|
+
createdAt: now
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
delete(hash) {
|
|
146
|
+
return __async(this, null, function* () {
|
|
147
|
+
if (this.isDestroyed) {
|
|
148
|
+
throw new Error("Cache store has been destroyed");
|
|
149
|
+
}
|
|
150
|
+
yield this.db.delete(cacheTable).where(eq(cacheTable.hash, hash));
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
clear() {
|
|
154
|
+
return __async(this, null, function* () {
|
|
155
|
+
if (this.isDestroyed) {
|
|
156
|
+
throw new Error("Cache store has been destroyed");
|
|
157
|
+
}
|
|
158
|
+
yield this.db.delete(cacheTable);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Get cache statistics
|
|
163
|
+
*/
|
|
164
|
+
getStats() {
|
|
165
|
+
return __async(this, null, function* () {
|
|
166
|
+
var _a, _b, _c, _d;
|
|
167
|
+
const now = Date.now();
|
|
168
|
+
const totalResult = yield this.db.select({ count: count() }).from(cacheTable);
|
|
169
|
+
const expiredResult = yield this.db.select({ count: count() }).from(cacheTable).where(and(gt(cacheTable.expiresAt, 0), lt(cacheTable.expiresAt, now)));
|
|
170
|
+
const pageCount = Number(
|
|
171
|
+
this.sqlite.pragma("page_count", { simple: true })
|
|
172
|
+
);
|
|
173
|
+
const pageSize = Number(this.sqlite.pragma("page_size", { simple: true }));
|
|
174
|
+
const safePageCount = Number.isFinite(pageCount) ? pageCount : 0;
|
|
175
|
+
const safePageSize = Number.isFinite(pageSize) ? pageSize : 0;
|
|
176
|
+
const databaseSizeKB = Math.round(safePageCount * safePageSize / 1024);
|
|
177
|
+
return {
|
|
178
|
+
databaseSizeKB,
|
|
179
|
+
expiredItems: (_b = (_a = expiredResult[0]) == null ? void 0 : _a.count) != null ? _b : 0,
|
|
180
|
+
totalItems: (_d = (_c = totalResult[0]) == null ? void 0 : _c.count) != null ? _d : 0
|
|
181
|
+
};
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Manually trigger cleanup of expired items
|
|
186
|
+
*/
|
|
187
|
+
cleanup() {
|
|
188
|
+
return __async(this, null, function* () {
|
|
189
|
+
const now = Date.now();
|
|
190
|
+
yield this.db.delete(cacheTable).where(and(gt(cacheTable.expiresAt, 0), lt(cacheTable.expiresAt, now)));
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Close the database connection
|
|
195
|
+
*/
|
|
196
|
+
close() {
|
|
197
|
+
return __async(this, null, function* () {
|
|
198
|
+
if (this.cleanupInterval) {
|
|
199
|
+
clearInterval(this.cleanupInterval);
|
|
200
|
+
this.cleanupInterval = void 0;
|
|
201
|
+
}
|
|
202
|
+
this.isDestroyed = true;
|
|
203
|
+
if (this.isConnectionManaged && typeof this.sqlite.close === "function") {
|
|
204
|
+
this.sqlite.close();
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Alias for close() to match test expectations
|
|
210
|
+
*/
|
|
211
|
+
destroy() {
|
|
212
|
+
this.close();
|
|
213
|
+
}
|
|
214
|
+
initializeDatabase() {
|
|
215
|
+
this.db.run(sql`
|
|
216
|
+
CREATE TABLE IF NOT EXISTS cache (
|
|
217
|
+
hash TEXT PRIMARY KEY,
|
|
218
|
+
value BLOB NOT NULL,
|
|
219
|
+
expires_at INTEGER NOT NULL,
|
|
220
|
+
created_at INTEGER NOT NULL
|
|
221
|
+
)
|
|
222
|
+
`);
|
|
223
|
+
this.db.run(sql`
|
|
224
|
+
CREATE INDEX IF NOT EXISTS idx_cache_expires_at ON cache(expires_at)
|
|
225
|
+
`);
|
|
226
|
+
}
|
|
227
|
+
startCleanupInterval() {
|
|
228
|
+
if (this.cleanupIntervalMs <= 0) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
this.cleanupInterval = setInterval(() => __async(this, null, function* () {
|
|
232
|
+
yield this.cleanup();
|
|
233
|
+
}), this.cleanupIntervalMs);
|
|
234
|
+
if (typeof this.cleanupInterval.unref === "function") {
|
|
235
|
+
this.cleanupInterval.unref();
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
cleanupExpiredItems() {
|
|
239
|
+
return __async(this, null, function* () {
|
|
240
|
+
const now = Date.now();
|
|
241
|
+
yield this.db.delete(cacheTable).where(lt(cacheTable.expiresAt, now));
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
var SQLiteDedupeStore = class {
|
|
246
|
+
constructor({
|
|
247
|
+
/** File path or existing `better-sqlite3` Database instance. Defaults to `':memory:'`. */
|
|
248
|
+
database = ":memory:",
|
|
249
|
+
/** Job timeout in milliseconds. Preferred over timeoutMs. */
|
|
250
|
+
jobTimeoutMs,
|
|
251
|
+
/** Legacy alias for jobTimeoutMs. */
|
|
252
|
+
timeoutMs,
|
|
253
|
+
/** Cleanup interval in milliseconds. Defaults to 1 minute. */
|
|
254
|
+
cleanupIntervalMs = 6e4,
|
|
255
|
+
/** Poll interval for checking pending jobs in milliseconds. Defaults to 100ms. */
|
|
256
|
+
pollIntervalMs = 100
|
|
257
|
+
} = {}) {
|
|
258
|
+
/** Indicates whether this store manages (and should close) the SQLite connection */
|
|
259
|
+
this.isConnectionManaged = false;
|
|
260
|
+
this.jobPromises = /* @__PURE__ */ new Map();
|
|
261
|
+
this.jobSettlers = /* @__PURE__ */ new Map();
|
|
262
|
+
this.isDestroyed = false;
|
|
263
|
+
var _a;
|
|
264
|
+
let sqliteInstance;
|
|
265
|
+
let isConnectionManaged = false;
|
|
266
|
+
if (typeof database === "string") {
|
|
267
|
+
sqliteInstance = new Database(database);
|
|
268
|
+
isConnectionManaged = true;
|
|
269
|
+
} else {
|
|
270
|
+
sqliteInstance = database;
|
|
271
|
+
}
|
|
272
|
+
this.sqlite = sqliteInstance;
|
|
273
|
+
this.isConnectionManaged = isConnectionManaged;
|
|
274
|
+
this.db = drizzle(sqliteInstance);
|
|
275
|
+
this.jobTimeoutMs = (_a = timeoutMs != null ? timeoutMs : jobTimeoutMs) != null ? _a : 3e5;
|
|
276
|
+
this.cleanupIntervalMs = cleanupIntervalMs;
|
|
277
|
+
this.pollIntervalMs = pollIntervalMs;
|
|
278
|
+
this.initializeDatabase();
|
|
279
|
+
this.startCleanupInterval();
|
|
280
|
+
}
|
|
281
|
+
startCleanupInterval() {
|
|
282
|
+
if (this.cleanupIntervalMs > 0) {
|
|
283
|
+
this.cleanupInterval = setInterval(() => {
|
|
284
|
+
this.cleanupExpiredJobs().catch(() => {
|
|
285
|
+
});
|
|
286
|
+
}, this.cleanupIntervalMs);
|
|
287
|
+
if (typeof this.cleanupInterval.unref === "function") {
|
|
288
|
+
this.cleanupInterval.unref();
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
cleanupExpiredJobs() {
|
|
293
|
+
return __async(this, null, function* () {
|
|
294
|
+
const noTimeoutConfigured = this.jobTimeoutMs <= 0;
|
|
295
|
+
if (noTimeoutConfigured) {
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
const now = Date.now();
|
|
299
|
+
const expiredThreshold = now - this.jobTimeoutMs;
|
|
300
|
+
yield this.db.delete(dedupeTable).where(
|
|
301
|
+
and(
|
|
302
|
+
eq(dedupeTable.status, "pending"),
|
|
303
|
+
lt(dedupeTable.createdAt, expiredThreshold)
|
|
304
|
+
)
|
|
305
|
+
);
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
waitFor(hash) {
|
|
309
|
+
return __async(this, null, function* () {
|
|
310
|
+
if (this.isDestroyed) {
|
|
311
|
+
throw new Error("Dedupe store has been destroyed");
|
|
312
|
+
}
|
|
313
|
+
const existingPromise = this.jobPromises.get(hash);
|
|
314
|
+
if (existingPromise) {
|
|
315
|
+
return existingPromise;
|
|
316
|
+
}
|
|
317
|
+
let result;
|
|
318
|
+
try {
|
|
319
|
+
result = yield this.db.select().from(dedupeTable).where(eq(dedupeTable.hash, hash)).limit(1);
|
|
320
|
+
} catch (e) {
|
|
321
|
+
return void 0;
|
|
322
|
+
}
|
|
323
|
+
if (result.length === 0) {
|
|
324
|
+
return void 0;
|
|
325
|
+
}
|
|
326
|
+
const job = result[0];
|
|
327
|
+
if (!job) {
|
|
328
|
+
return void 0;
|
|
329
|
+
}
|
|
330
|
+
if (job.status === "completed") {
|
|
331
|
+
return this.deserializeResult(job.result);
|
|
332
|
+
}
|
|
333
|
+
if (job.status === "failed") {
|
|
334
|
+
return void 0;
|
|
335
|
+
}
|
|
336
|
+
const promise = new Promise((resolve) => {
|
|
337
|
+
let settled = false;
|
|
338
|
+
let timeoutHandle;
|
|
339
|
+
const settle = (value) => {
|
|
340
|
+
if (settled) {
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
settled = true;
|
|
344
|
+
if (timeoutHandle) {
|
|
345
|
+
clearTimeout(timeoutHandle);
|
|
346
|
+
}
|
|
347
|
+
clearInterval(pollHandle);
|
|
348
|
+
this.jobSettlers.delete(hash);
|
|
349
|
+
this.jobPromises.delete(hash);
|
|
350
|
+
resolve(value);
|
|
351
|
+
};
|
|
352
|
+
this.jobSettlers.set(hash, settle);
|
|
353
|
+
const poll = () => __async(this, null, function* () {
|
|
354
|
+
if (this.isDestroyed) {
|
|
355
|
+
settle(void 0);
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
try {
|
|
359
|
+
const latest = yield this.db.select().from(dedupeTable).where(eq(dedupeTable.hash, hash)).limit(1);
|
|
360
|
+
const latestJob = latest[0];
|
|
361
|
+
if (!latestJob) {
|
|
362
|
+
settle(void 0);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
const isExpired = this.jobTimeoutMs > 0 && Date.now() - latestJob.createdAt >= this.jobTimeoutMs;
|
|
366
|
+
if (isExpired) {
|
|
367
|
+
yield this.db.update(dedupeTable).set({
|
|
368
|
+
status: "failed",
|
|
369
|
+
error: "Job timed out",
|
|
370
|
+
updatedAt: Date.now()
|
|
371
|
+
}).where(eq(dedupeTable.hash, hash));
|
|
372
|
+
settle(void 0);
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
if (latestJob.status === "completed") {
|
|
376
|
+
settle(this.deserializeResult(latestJob.result));
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
if (latestJob.status === "failed") {
|
|
380
|
+
settle(void 0);
|
|
381
|
+
}
|
|
382
|
+
} catch (e) {
|
|
383
|
+
settle(void 0);
|
|
384
|
+
}
|
|
385
|
+
});
|
|
386
|
+
const pollHandle = setInterval(() => {
|
|
387
|
+
void poll();
|
|
388
|
+
}, this.pollIntervalMs);
|
|
389
|
+
if (typeof pollHandle.unref === "function") {
|
|
390
|
+
pollHandle.unref();
|
|
391
|
+
}
|
|
392
|
+
void poll();
|
|
393
|
+
if (this.jobTimeoutMs > 0) {
|
|
394
|
+
timeoutHandle = setTimeout(() => {
|
|
395
|
+
if (this.isDestroyed) {
|
|
396
|
+
settle(void 0);
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
void (() => __async(this, null, function* () {
|
|
400
|
+
try {
|
|
401
|
+
yield this.db.update(dedupeTable).set({
|
|
402
|
+
status: "failed",
|
|
403
|
+
error: "Job timed out",
|
|
404
|
+
updatedAt: Date.now()
|
|
405
|
+
}).where(eq(dedupeTable.hash, hash));
|
|
406
|
+
} catch (e) {
|
|
407
|
+
} finally {
|
|
408
|
+
settle(void 0);
|
|
409
|
+
}
|
|
410
|
+
}))();
|
|
411
|
+
}, this.jobTimeoutMs);
|
|
412
|
+
if (typeof timeoutHandle.unref === "function") {
|
|
413
|
+
timeoutHandle.unref();
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
});
|
|
417
|
+
this.jobPromises.set(hash, promise);
|
|
418
|
+
return promise;
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
register(hash) {
|
|
422
|
+
return __async(this, null, function* () {
|
|
423
|
+
const registration = yield this.registerOrJoin(hash);
|
|
424
|
+
return registration.jobId;
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
registerOrJoin(hash) {
|
|
428
|
+
return __async(this, null, function* () {
|
|
429
|
+
if (this.isDestroyed) {
|
|
430
|
+
throw new Error("Dedupe store has been destroyed");
|
|
431
|
+
}
|
|
432
|
+
const now = Date.now();
|
|
433
|
+
const candidateJobId = randomUUID();
|
|
434
|
+
const registerTransaction = this.sqlite.transaction(
|
|
435
|
+
(requestHash, createdAt, jobId) => {
|
|
436
|
+
const existing = this.sqlite.prepare(
|
|
437
|
+
"SELECT job_id as jobId, status FROM dedupe_jobs WHERE hash = ? LIMIT 1"
|
|
438
|
+
).get(requestHash);
|
|
439
|
+
if (existing && existing.status === "pending") {
|
|
440
|
+
return {
|
|
441
|
+
jobId: existing.jobId,
|
|
442
|
+
isOwner: false
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
this.sqlite.prepare(
|
|
446
|
+
`
|
|
447
|
+
INSERT INTO dedupe_jobs (hash, job_id, status, result, error, created_at, updated_at)
|
|
448
|
+
VALUES (?, ?, 'pending', NULL, NULL, ?, ?)
|
|
449
|
+
ON CONFLICT(hash) DO UPDATE SET
|
|
450
|
+
job_id = excluded.job_id,
|
|
451
|
+
status = excluded.status,
|
|
452
|
+
result = NULL,
|
|
453
|
+
error = NULL,
|
|
454
|
+
created_at = excluded.created_at,
|
|
455
|
+
updated_at = excluded.updated_at
|
|
456
|
+
`
|
|
457
|
+
).run(requestHash, jobId, createdAt, createdAt);
|
|
458
|
+
return {
|
|
459
|
+
jobId,
|
|
460
|
+
isOwner: true
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
);
|
|
464
|
+
return registerTransaction(hash, now, candidateJobId);
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
complete(hash, value) {
|
|
468
|
+
return __async(this, null, function* () {
|
|
469
|
+
var _a;
|
|
470
|
+
if (this.isDestroyed) {
|
|
471
|
+
throw new Error("Dedupe store has been destroyed");
|
|
472
|
+
}
|
|
473
|
+
let serializedResult;
|
|
474
|
+
if (value === void 0) {
|
|
475
|
+
serializedResult = "__UNDEFINED__";
|
|
476
|
+
} else if (value === null) {
|
|
477
|
+
serializedResult = "__NULL__";
|
|
478
|
+
} else {
|
|
479
|
+
try {
|
|
480
|
+
serializedResult = JSON.stringify(value);
|
|
481
|
+
} catch (error) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
`Failed to serialize result: ${error instanceof Error ? error.message : String(error)}`
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
const now = Date.now();
|
|
488
|
+
const existingJob = yield this.db.select().from(dedupeTable).where(eq(dedupeTable.hash, hash)).limit(1);
|
|
489
|
+
if (existingJob.length > 0 && ((_a = existingJob[0]) == null ? void 0 : _a.status) === "completed") {
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
yield this.db.update(dedupeTable).set({
|
|
493
|
+
status: "completed",
|
|
494
|
+
result: serializedResult,
|
|
495
|
+
updatedAt: now
|
|
496
|
+
}).where(eq(dedupeTable.hash, hash));
|
|
497
|
+
const settle = this.jobSettlers.get(hash);
|
|
498
|
+
if (settle) {
|
|
499
|
+
settle(value);
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
fail(hash, error) {
|
|
504
|
+
return __async(this, null, function* () {
|
|
505
|
+
if (this.isDestroyed) {
|
|
506
|
+
throw new Error("Dedupe store has been destroyed");
|
|
507
|
+
}
|
|
508
|
+
const now = Date.now();
|
|
509
|
+
yield this.db.update(dedupeTable).set({
|
|
510
|
+
status: "failed",
|
|
511
|
+
error: error.message,
|
|
512
|
+
updatedAt: now
|
|
513
|
+
}).where(eq(dedupeTable.hash, hash));
|
|
514
|
+
const settle = this.jobSettlers.get(hash);
|
|
515
|
+
if (settle) {
|
|
516
|
+
settle(void 0);
|
|
517
|
+
}
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
isInProgress(hash) {
|
|
521
|
+
return __async(this, null, function* () {
|
|
522
|
+
if (this.isDestroyed) {
|
|
523
|
+
throw new Error("Dedupe store has been destroyed");
|
|
524
|
+
}
|
|
525
|
+
const result = yield this.db.select().from(dedupeTable).where(eq(dedupeTable.hash, hash)).limit(1);
|
|
526
|
+
if (result.length === 0) {
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
const job = result[0];
|
|
530
|
+
if (!job) {
|
|
531
|
+
return false;
|
|
532
|
+
}
|
|
533
|
+
const jobExpired = this.jobTimeoutMs > 0 && Date.now() - job.createdAt >= this.jobTimeoutMs;
|
|
534
|
+
if (jobExpired) {
|
|
535
|
+
yield this.db.delete(dedupeTable).where(eq(dedupeTable.hash, hash));
|
|
536
|
+
return false;
|
|
537
|
+
}
|
|
538
|
+
return job.status === "pending";
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
getResult(hash) {
|
|
542
|
+
return __async(this, null, function* () {
|
|
543
|
+
const result = yield this.db.select().from(dedupeTable).where(eq(dedupeTable.hash, hash)).limit(1);
|
|
544
|
+
if (result.length === 0) {
|
|
545
|
+
return void 0;
|
|
546
|
+
}
|
|
547
|
+
const job = result[0];
|
|
548
|
+
if (!job) {
|
|
549
|
+
return void 0;
|
|
550
|
+
}
|
|
551
|
+
const now = Date.now();
|
|
552
|
+
const isExpired = now - job.createdAt > this.jobTimeoutMs;
|
|
553
|
+
if (isExpired) {
|
|
554
|
+
yield this.db.delete(dedupeTable).where(eq(dedupeTable.hash, hash));
|
|
555
|
+
return void 0;
|
|
556
|
+
}
|
|
557
|
+
if (job.status === "completed") {
|
|
558
|
+
try {
|
|
559
|
+
if (job.result === "__UNDEFINED__") {
|
|
560
|
+
return void 0;
|
|
561
|
+
} else if (job.result === "__NULL__") {
|
|
562
|
+
return null;
|
|
563
|
+
} else if (job.result) {
|
|
564
|
+
return JSON.parse(job.result);
|
|
565
|
+
}
|
|
566
|
+
return void 0;
|
|
567
|
+
} catch (e) {
|
|
568
|
+
return void 0;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
return void 0;
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Get statistics about dedupe jobs
|
|
576
|
+
*/
|
|
577
|
+
getStats() {
|
|
578
|
+
return __async(this, null, function* () {
|
|
579
|
+
var _a, _b, _c, _d, _e;
|
|
580
|
+
const now = Date.now();
|
|
581
|
+
const expiredTime = now - this.jobTimeoutMs;
|
|
582
|
+
const totalResult = yield this.db.select({ count: count() }).from(dedupeTable);
|
|
583
|
+
const pendingResult = yield this.db.select({ count: count() }).from(dedupeTable).where(eq(dedupeTable.status, "pending"));
|
|
584
|
+
const completedResult = yield this.db.select({ count: count() }).from(dedupeTable).where(eq(dedupeTable.status, "completed"));
|
|
585
|
+
const failedResult = yield this.db.select({ count: count() }).from(dedupeTable).where(eq(dedupeTable.status, "failed"));
|
|
586
|
+
const expiredResult = yield this.db.select({ count: count() }).from(dedupeTable).where(lt(dedupeTable.createdAt, expiredTime));
|
|
587
|
+
return {
|
|
588
|
+
totalJobs: ((_a = totalResult[0]) == null ? void 0 : _a.count) || 0,
|
|
589
|
+
pendingJobs: ((_b = pendingResult[0]) == null ? void 0 : _b.count) || 0,
|
|
590
|
+
completedJobs: ((_c = completedResult[0]) == null ? void 0 : _c.count) || 0,
|
|
591
|
+
failedJobs: ((_d = failedResult[0]) == null ? void 0 : _d.count) || 0,
|
|
592
|
+
expiredJobs: ((_e = expiredResult[0]) == null ? void 0 : _e.count) || 0
|
|
593
|
+
};
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Clean up expired jobs
|
|
598
|
+
*/
|
|
599
|
+
cleanup() {
|
|
600
|
+
return __async(this, null, function* () {
|
|
601
|
+
const now = Date.now();
|
|
602
|
+
const expiredTime = now - this.jobTimeoutMs;
|
|
603
|
+
yield this.db.delete(dedupeTable).where(lt(dedupeTable.createdAt, expiredTime));
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* Clear all jobs
|
|
608
|
+
*/
|
|
609
|
+
clear() {
|
|
610
|
+
return __async(this, null, function* () {
|
|
611
|
+
yield this.db.delete(dedupeTable);
|
|
612
|
+
for (const settle of this.jobSettlers.values()) {
|
|
613
|
+
settle(void 0);
|
|
614
|
+
}
|
|
615
|
+
this.jobPromises.clear();
|
|
616
|
+
this.jobSettlers.clear();
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Close the database connection
|
|
621
|
+
*/
|
|
622
|
+
close() {
|
|
623
|
+
return __async(this, null, function* () {
|
|
624
|
+
if (this.cleanupInterval) {
|
|
625
|
+
clearInterval(this.cleanupInterval);
|
|
626
|
+
this.cleanupInterval = void 0;
|
|
627
|
+
}
|
|
628
|
+
this.isDestroyed = true;
|
|
629
|
+
for (const settle of this.jobSettlers.values()) {
|
|
630
|
+
settle(void 0);
|
|
631
|
+
}
|
|
632
|
+
this.jobPromises.clear();
|
|
633
|
+
this.jobSettlers.clear();
|
|
634
|
+
if (this.isConnectionManaged && typeof this.sqlite.close === "function") {
|
|
635
|
+
this.sqlite.close();
|
|
636
|
+
}
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Alias for close() to match test expectations
|
|
641
|
+
*/
|
|
642
|
+
destroy() {
|
|
643
|
+
this.close();
|
|
644
|
+
}
|
|
645
|
+
deserializeResult(serializedResult) {
|
|
646
|
+
try {
|
|
647
|
+
if (serializedResult === "__UNDEFINED__") {
|
|
648
|
+
return void 0;
|
|
649
|
+
}
|
|
650
|
+
if (serializedResult === "__NULL__") {
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
if (serializedResult) {
|
|
654
|
+
return JSON.parse(serializedResult);
|
|
655
|
+
}
|
|
656
|
+
return void 0;
|
|
657
|
+
} catch (e) {
|
|
658
|
+
return void 0;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
initializeDatabase() {
|
|
662
|
+
this.db.run(sql`
|
|
663
|
+
CREATE TABLE IF NOT EXISTS dedupe_jobs (
|
|
664
|
+
hash TEXT PRIMARY KEY,
|
|
665
|
+
job_id TEXT NOT NULL,
|
|
666
|
+
status TEXT NOT NULL,
|
|
667
|
+
result BLOB,
|
|
668
|
+
error TEXT,
|
|
669
|
+
created_at INTEGER NOT NULL,
|
|
670
|
+
updated_at INTEGER NOT NULL
|
|
671
|
+
)
|
|
672
|
+
`);
|
|
673
|
+
this.db.run(sql`
|
|
674
|
+
CREATE INDEX IF NOT EXISTS idx_dedupe_status ON dedupe_jobs(status)
|
|
675
|
+
`);
|
|
676
|
+
}
|
|
677
|
+
};
|
|
678
|
+
var SQLiteRateLimitStore = class {
|
|
679
|
+
constructor({
|
|
680
|
+
/** File path or existing `better-sqlite3` Database instance. Defaults to `':memory:'`. */
|
|
681
|
+
database = ":memory:",
|
|
682
|
+
/** Global/default rate-limit config applied when a resource-specific override is not provided. */
|
|
683
|
+
defaultConfig = DEFAULT_RATE_LIMIT,
|
|
684
|
+
/** Optional per-resource overrides. */
|
|
685
|
+
resourceConfigs = /* @__PURE__ */ new Map()
|
|
686
|
+
} = {}) {
|
|
687
|
+
/** Indicates whether this store manages (and should close) the SQLite connection */
|
|
688
|
+
this.isConnectionManaged = false;
|
|
689
|
+
this.resourceConfigs = /* @__PURE__ */ new Map();
|
|
690
|
+
this.isDestroyed = false;
|
|
691
|
+
let sqliteInstance;
|
|
692
|
+
let isConnectionManaged = false;
|
|
693
|
+
if (typeof database === "string") {
|
|
694
|
+
sqliteInstance = new Database(database);
|
|
695
|
+
isConnectionManaged = true;
|
|
696
|
+
} else {
|
|
697
|
+
sqliteInstance = database;
|
|
698
|
+
}
|
|
699
|
+
this.sqlite = sqliteInstance;
|
|
700
|
+
this.isConnectionManaged = isConnectionManaged;
|
|
701
|
+
this.db = drizzle(sqliteInstance);
|
|
702
|
+
this.defaultConfig = defaultConfig;
|
|
703
|
+
this.resourceConfigs = resourceConfigs;
|
|
704
|
+
this.initializeDatabase();
|
|
705
|
+
}
|
|
706
|
+
canProceed(resource) {
|
|
707
|
+
return __async(this, null, function* () {
|
|
708
|
+
var _a;
|
|
709
|
+
if (this.isDestroyed) {
|
|
710
|
+
throw new Error("Rate limit store has been destroyed");
|
|
711
|
+
}
|
|
712
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
713
|
+
const now = Date.now();
|
|
714
|
+
const windowStart = now - config.windowMs;
|
|
715
|
+
yield this.cleanupExpiredRequests(resource, windowStart);
|
|
716
|
+
const result = yield this.db.select({ count: count() }).from(rateLimitTable).where(
|
|
717
|
+
and(
|
|
718
|
+
eq(rateLimitTable.resource, resource),
|
|
719
|
+
gte(rateLimitTable.timestamp, windowStart)
|
|
720
|
+
)
|
|
721
|
+
);
|
|
722
|
+
const currentCount = ((_a = result[0]) == null ? void 0 : _a.count) || 0;
|
|
723
|
+
return currentCount < config.limit;
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
record(resource) {
|
|
727
|
+
return __async(this, null, function* () {
|
|
728
|
+
if (this.isDestroyed) {
|
|
729
|
+
throw new Error("Rate limit store has been destroyed");
|
|
730
|
+
}
|
|
731
|
+
const now = Date.now();
|
|
732
|
+
yield this.db.insert(rateLimitTable).values({
|
|
733
|
+
resource,
|
|
734
|
+
timestamp: now
|
|
735
|
+
});
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
getStatus(resource) {
|
|
739
|
+
return __async(this, null, function* () {
|
|
740
|
+
var _a;
|
|
741
|
+
if (this.isDestroyed) {
|
|
742
|
+
throw new Error("Rate limit store has been destroyed");
|
|
743
|
+
}
|
|
744
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
745
|
+
const now = Date.now();
|
|
746
|
+
const windowStart = now - config.windowMs;
|
|
747
|
+
yield this.cleanupExpiredRequests(resource, windowStart);
|
|
748
|
+
const result = yield this.db.select({ count: count() }).from(rateLimitTable).where(
|
|
749
|
+
and(
|
|
750
|
+
eq(rateLimitTable.resource, resource),
|
|
751
|
+
gte(rateLimitTable.timestamp, windowStart)
|
|
752
|
+
)
|
|
753
|
+
);
|
|
754
|
+
const currentRequests = ((_a = result[0]) == null ? void 0 : _a.count) || 0;
|
|
755
|
+
const remaining = Math.max(0, config.limit - currentRequests);
|
|
756
|
+
const resetTime = new Date(now + config.windowMs);
|
|
757
|
+
return {
|
|
758
|
+
remaining,
|
|
759
|
+
resetTime,
|
|
760
|
+
limit: config.limit
|
|
761
|
+
};
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
reset(resource) {
|
|
765
|
+
return __async(this, null, function* () {
|
|
766
|
+
if (this.isDestroyed) {
|
|
767
|
+
throw new Error("Rate limit store has been destroyed");
|
|
768
|
+
}
|
|
769
|
+
yield this.db.delete(rateLimitTable).where(eq(rateLimitTable.resource, resource));
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
getWaitTime(resource) {
|
|
773
|
+
return __async(this, null, function* () {
|
|
774
|
+
var _a, _b;
|
|
775
|
+
if (this.isDestroyed) {
|
|
776
|
+
throw new Error("Rate limit store has been destroyed");
|
|
777
|
+
}
|
|
778
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
779
|
+
if (config.limit === 0) {
|
|
780
|
+
return config.windowMs;
|
|
781
|
+
}
|
|
782
|
+
const now = Date.now();
|
|
783
|
+
const windowStart = now - config.windowMs;
|
|
784
|
+
yield this.cleanupExpiredRequests(resource, windowStart);
|
|
785
|
+
const countResult = yield this.db.select({ count: count() }).from(rateLimitTable).where(
|
|
786
|
+
and(
|
|
787
|
+
eq(rateLimitTable.resource, resource),
|
|
788
|
+
gte(rateLimitTable.timestamp, windowStart)
|
|
789
|
+
)
|
|
790
|
+
);
|
|
791
|
+
const currentRequests = ((_a = countResult[0]) == null ? void 0 : _a.count) || 0;
|
|
792
|
+
if (currentRequests < config.limit) {
|
|
793
|
+
return 0;
|
|
794
|
+
}
|
|
795
|
+
const oldestResult = yield this.db.select({ timestamp: rateLimitTable.timestamp }).from(rateLimitTable).where(
|
|
796
|
+
and(
|
|
797
|
+
eq(rateLimitTable.resource, resource),
|
|
798
|
+
gte(rateLimitTable.timestamp, windowStart)
|
|
799
|
+
)
|
|
800
|
+
).orderBy(rateLimitTable.timestamp).limit(1);
|
|
801
|
+
if (oldestResult.length === 0) {
|
|
802
|
+
return 0;
|
|
803
|
+
}
|
|
804
|
+
const oldestTimestamp = (_b = oldestResult[0]) == null ? void 0 : _b.timestamp;
|
|
805
|
+
if (oldestTimestamp === void 0) {
|
|
806
|
+
return 0;
|
|
807
|
+
}
|
|
808
|
+
const timeUntilOldestExpires = oldestTimestamp + config.windowMs - now;
|
|
809
|
+
return Math.max(0, timeUntilOldestExpires);
|
|
810
|
+
});
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Set rate limit configuration for a specific resource
|
|
814
|
+
*/
|
|
815
|
+
setResourceConfig(resource, config) {
|
|
816
|
+
this.resourceConfigs.set(resource, config);
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Get rate limit configuration for a resource
|
|
820
|
+
*/
|
|
821
|
+
getResourceConfig(resource) {
|
|
822
|
+
return this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Get statistics for all resources
|
|
826
|
+
*/
|
|
827
|
+
getStats() {
|
|
828
|
+
return __async(this, null, function* () {
|
|
829
|
+
var _a;
|
|
830
|
+
if (this.isDestroyed) {
|
|
831
|
+
throw new Error("Rate limit store has been destroyed");
|
|
832
|
+
}
|
|
833
|
+
const totalResult = yield this.db.select({ count: count() }).from(rateLimitTable);
|
|
834
|
+
const resourcesResult = yield this.db.select({ resource: rateLimitTable.resource }).from(rateLimitTable).groupBy(rateLimitTable.resource);
|
|
835
|
+
const uniqueResources = resourcesResult.length;
|
|
836
|
+
const rateLimitedResources = [];
|
|
837
|
+
for (const { resource } of resourcesResult) {
|
|
838
|
+
const canProceed = yield this.canProceed(resource);
|
|
839
|
+
if (!canProceed) {
|
|
840
|
+
rateLimitedResources.push(resource);
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
return {
|
|
844
|
+
totalRequests: ((_a = totalResult[0]) == null ? void 0 : _a.count) || 0,
|
|
845
|
+
uniqueResources,
|
|
846
|
+
rateLimitedResources
|
|
847
|
+
};
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* Clean up all rate limit data
|
|
852
|
+
*/
|
|
853
|
+
clear() {
|
|
854
|
+
return __async(this, null, function* () {
|
|
855
|
+
if (this.isDestroyed) {
|
|
856
|
+
throw new Error("Rate limit store has been destroyed");
|
|
857
|
+
}
|
|
858
|
+
yield this.db.delete(rateLimitTable);
|
|
859
|
+
});
|
|
860
|
+
}
|
|
861
|
+
/**
|
|
862
|
+
* Clean up expired requests for all resources
|
|
863
|
+
*/
|
|
864
|
+
cleanup() {
|
|
865
|
+
return __async(this, null, function* () {
|
|
866
|
+
const now = Date.now();
|
|
867
|
+
const resources = yield this.db.select({ resource: rateLimitTable.resource }).from(rateLimitTable).groupBy(rateLimitTable.resource);
|
|
868
|
+
for (const { resource } of resources) {
|
|
869
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
870
|
+
const windowStart = now - config.windowMs;
|
|
871
|
+
yield this.cleanupExpiredRequests(resource, windowStart);
|
|
872
|
+
}
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
/**
|
|
876
|
+
* Close the database connection
|
|
877
|
+
*/
|
|
878
|
+
close() {
|
|
879
|
+
return __async(this, null, function* () {
|
|
880
|
+
this.isDestroyed = true;
|
|
881
|
+
if (this.isConnectionManaged && typeof this.sqlite.close === "function") {
|
|
882
|
+
this.sqlite.close();
|
|
883
|
+
}
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
/**
|
|
887
|
+
* Alias for close() to match test expectations
|
|
888
|
+
*/
|
|
889
|
+
destroy() {
|
|
890
|
+
this.close();
|
|
891
|
+
}
|
|
892
|
+
cleanupExpiredRequests(resource, windowStart) {
|
|
893
|
+
return __async(this, null, function* () {
|
|
894
|
+
yield this.db.delete(rateLimitTable).where(
|
|
895
|
+
and(
|
|
896
|
+
eq(rateLimitTable.resource, resource),
|
|
897
|
+
lt(rateLimitTable.timestamp, windowStart)
|
|
898
|
+
)
|
|
899
|
+
);
|
|
900
|
+
});
|
|
901
|
+
}
|
|
902
|
+
initializeDatabase() {
|
|
903
|
+
this.db.run(sql`
|
|
904
|
+
CREATE TABLE IF NOT EXISTS rate_limits (
|
|
905
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
906
|
+
resource TEXT NOT NULL,
|
|
907
|
+
timestamp INTEGER NOT NULL
|
|
908
|
+
)
|
|
909
|
+
`);
|
|
910
|
+
this.db.run(sql`
|
|
911
|
+
CREATE INDEX IF NOT EXISTS idx_rate_limit_resource ON rate_limits(resource)
|
|
912
|
+
`);
|
|
913
|
+
this.db.run(sql`
|
|
914
|
+
CREATE INDEX IF NOT EXISTS idx_rate_limit_timestamp ON rate_limits(timestamp)
|
|
915
|
+
`);
|
|
916
|
+
}
|
|
917
|
+
};
|
|
918
|
+
var DEFAULT_RATE_LIMIT2 = {
|
|
919
|
+
limit: 200,
|
|
920
|
+
windowMs: 36e5
|
|
921
|
+
// 1 hour
|
|
922
|
+
};
|
|
923
|
+
var SqliteAdaptiveRateLimitStore = class {
|
|
924
|
+
constructor({
|
|
925
|
+
database = ":memory:",
|
|
926
|
+
defaultConfig = DEFAULT_RATE_LIMIT2,
|
|
927
|
+
resourceConfigs = /* @__PURE__ */ new Map(),
|
|
928
|
+
adaptiveConfig = {}
|
|
929
|
+
} = {}) {
|
|
930
|
+
/** Indicates whether this store manages (and should close) the SQLite connection */
|
|
931
|
+
this.isConnectionManaged = false;
|
|
932
|
+
this.resourceConfigs = /* @__PURE__ */ new Map();
|
|
933
|
+
this.isDestroyed = false;
|
|
934
|
+
this.activityMetrics = /* @__PURE__ */ new Map();
|
|
935
|
+
this.lastCapacityUpdate = /* @__PURE__ */ new Map();
|
|
936
|
+
this.cachedCapacity = /* @__PURE__ */ new Map();
|
|
937
|
+
let sqliteInstance;
|
|
938
|
+
let isConnectionManaged = false;
|
|
939
|
+
if (typeof database === "string") {
|
|
940
|
+
sqliteInstance = new Database(database);
|
|
941
|
+
isConnectionManaged = true;
|
|
942
|
+
} else {
|
|
943
|
+
sqliteInstance = database;
|
|
944
|
+
}
|
|
945
|
+
this.sqlite = sqliteInstance;
|
|
946
|
+
this.isConnectionManaged = isConnectionManaged;
|
|
947
|
+
this.db = drizzle(sqliteInstance);
|
|
948
|
+
this.defaultConfig = defaultConfig;
|
|
949
|
+
this.resourceConfigs = resourceConfigs;
|
|
950
|
+
this.capacityCalculator = new AdaptiveCapacityCalculator(adaptiveConfig);
|
|
951
|
+
this.initializeDatabase();
|
|
952
|
+
}
|
|
953
|
+
canProceed(resource, priority = "background") {
|
|
954
|
+
return __async(this, null, function* () {
|
|
955
|
+
if (this.isDestroyed) {
|
|
956
|
+
throw new Error("Rate limit store has been destroyed");
|
|
957
|
+
}
|
|
958
|
+
yield this.ensureActivityMetrics(resource);
|
|
959
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
960
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
961
|
+
if (priority === "background" && capacity.backgroundPaused) {
|
|
962
|
+
return false;
|
|
963
|
+
}
|
|
964
|
+
const currentUserRequests = yield this.getCurrentUsage(resource, "user");
|
|
965
|
+
const currentBackgroundRequests = yield this.getCurrentUsage(
|
|
966
|
+
resource,
|
|
967
|
+
"background"
|
|
968
|
+
);
|
|
969
|
+
if (priority === "user") {
|
|
970
|
+
return currentUserRequests < capacity.userReserved;
|
|
971
|
+
} else {
|
|
972
|
+
return currentBackgroundRequests < capacity.backgroundMax;
|
|
973
|
+
}
|
|
974
|
+
});
|
|
975
|
+
}
|
|
976
|
+
record(resource, priority = "background") {
|
|
977
|
+
return __async(this, null, function* () {
|
|
978
|
+
if (this.isDestroyed) {
|
|
979
|
+
throw new Error("Rate limit store has been destroyed");
|
|
980
|
+
}
|
|
981
|
+
const now = Date.now();
|
|
982
|
+
this.db.run(sql`
|
|
983
|
+
INSERT INTO rate_limits (resource, timestamp, priority)
|
|
984
|
+
VALUES (${resource}, ${now}, ${priority})
|
|
985
|
+
`);
|
|
986
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
987
|
+
if (priority === "user") {
|
|
988
|
+
metrics.recentUserRequests.push(now);
|
|
989
|
+
this.cleanupOldRequests(metrics.recentUserRequests);
|
|
990
|
+
} else {
|
|
991
|
+
metrics.recentBackgroundRequests.push(now);
|
|
992
|
+
this.cleanupOldRequests(metrics.recentBackgroundRequests);
|
|
993
|
+
}
|
|
994
|
+
metrics.userActivityTrend = this.capacityCalculator.calculateActivityTrend(
|
|
995
|
+
metrics.recentUserRequests
|
|
996
|
+
);
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
getStatus(resource) {
|
|
1000
|
+
return __async(this, null, function* () {
|
|
1001
|
+
if (this.isDestroyed) {
|
|
1002
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1003
|
+
}
|
|
1004
|
+
yield this.ensureActivityMetrics(resource);
|
|
1005
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1006
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
1007
|
+
const currentUserUsage = yield this.getCurrentUsage(resource, "user");
|
|
1008
|
+
const currentBackgroundUsage = yield this.getCurrentUsage(
|
|
1009
|
+
resource,
|
|
1010
|
+
"background"
|
|
1011
|
+
);
|
|
1012
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
1013
|
+
return {
|
|
1014
|
+
remaining: capacity.userReserved - currentUserUsage + (capacity.backgroundMax - currentBackgroundUsage),
|
|
1015
|
+
resetTime: new Date(Date.now() + config.windowMs),
|
|
1016
|
+
limit: this.getResourceLimit(resource),
|
|
1017
|
+
adaptive: {
|
|
1018
|
+
userReserved: capacity.userReserved,
|
|
1019
|
+
backgroundMax: capacity.backgroundMax,
|
|
1020
|
+
backgroundPaused: capacity.backgroundPaused,
|
|
1021
|
+
recentUserActivity: this.capacityCalculator.getRecentActivity(
|
|
1022
|
+
metrics.recentUserRequests
|
|
1023
|
+
),
|
|
1024
|
+
reason: capacity.reason
|
|
1025
|
+
}
|
|
1026
|
+
};
|
|
1027
|
+
});
|
|
1028
|
+
}
|
|
1029
|
+
reset(resource) {
|
|
1030
|
+
return __async(this, null, function* () {
|
|
1031
|
+
if (this.isDestroyed) {
|
|
1032
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1033
|
+
}
|
|
1034
|
+
yield this.db.delete(rateLimitTable).where(eq(rateLimitTable.resource, resource));
|
|
1035
|
+
this.activityMetrics.delete(resource);
|
|
1036
|
+
this.cachedCapacity.delete(resource);
|
|
1037
|
+
this.lastCapacityUpdate.delete(resource);
|
|
1038
|
+
});
|
|
1039
|
+
}
|
|
1040
|
+
getWaitTime(resource, priority = "background") {
|
|
1041
|
+
return __async(this, null, function* () {
|
|
1042
|
+
if (this.isDestroyed) {
|
|
1043
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1044
|
+
}
|
|
1045
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
1046
|
+
if (config.limit === 0) {
|
|
1047
|
+
return config.windowMs;
|
|
1048
|
+
}
|
|
1049
|
+
const canProceed = yield this.canProceed(resource, priority);
|
|
1050
|
+
if (canProceed) {
|
|
1051
|
+
return 0;
|
|
1052
|
+
}
|
|
1053
|
+
yield this.ensureActivityMetrics(resource);
|
|
1054
|
+
const metrics = this.getOrCreateActivityMetrics(resource);
|
|
1055
|
+
const capacity = this.calculateCurrentCapacity(resource, metrics);
|
|
1056
|
+
if (priority === "background" && capacity.backgroundPaused) {
|
|
1057
|
+
return this.capacityCalculator.config.recalculationIntervalMs;
|
|
1058
|
+
}
|
|
1059
|
+
const now = Date.now();
|
|
1060
|
+
const windowStart = now - config.windowMs;
|
|
1061
|
+
const oldestResult = this.sqlite.prepare(
|
|
1062
|
+
`
|
|
1063
|
+
SELECT timestamp
|
|
1064
|
+
FROM rate_limits
|
|
1065
|
+
WHERE resource = ? AND COALESCE(priority, 'background') = ? AND timestamp >= ?
|
|
1066
|
+
ORDER BY timestamp
|
|
1067
|
+
LIMIT 1
|
|
1068
|
+
`
|
|
1069
|
+
).get(resource, priority, windowStart);
|
|
1070
|
+
if (!oldestResult) {
|
|
1071
|
+
return 0;
|
|
1072
|
+
}
|
|
1073
|
+
const oldestTimestamp = oldestResult.timestamp;
|
|
1074
|
+
if (!oldestTimestamp) {
|
|
1075
|
+
return 0;
|
|
1076
|
+
}
|
|
1077
|
+
const timeUntilOldestExpires = oldestTimestamp + config.windowMs - now;
|
|
1078
|
+
return Math.max(0, timeUntilOldestExpires);
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
/**
|
|
1082
|
+
* Set rate limit configuration for a specific resource
|
|
1083
|
+
*/
|
|
1084
|
+
setResourceConfig(resource, config) {
|
|
1085
|
+
this.resourceConfigs.set(resource, config);
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* Get rate limit configuration for a resource
|
|
1089
|
+
*/
|
|
1090
|
+
getResourceConfig(resource) {
|
|
1091
|
+
return this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* Get statistics for all resources
|
|
1095
|
+
*/
|
|
1096
|
+
getStats() {
|
|
1097
|
+
return __async(this, null, function* () {
|
|
1098
|
+
var _a;
|
|
1099
|
+
if (this.isDestroyed) {
|
|
1100
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1101
|
+
}
|
|
1102
|
+
const totalResult = yield this.db.select({ count: count() }).from(rateLimitTable);
|
|
1103
|
+
const resourcesResult = yield this.db.select({ resource: rateLimitTable.resource }).from(rateLimitTable).groupBy(rateLimitTable.resource);
|
|
1104
|
+
const uniqueResources = resourcesResult.length;
|
|
1105
|
+
const rateLimitedResources = [];
|
|
1106
|
+
for (const { resource } of resourcesResult) {
|
|
1107
|
+
const canProceed = yield this.canProceed(resource);
|
|
1108
|
+
if (!canProceed) {
|
|
1109
|
+
rateLimitedResources.push(resource);
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
return {
|
|
1113
|
+
totalRequests: ((_a = totalResult[0]) == null ? void 0 : _a.count) || 0,
|
|
1114
|
+
uniqueResources,
|
|
1115
|
+
rateLimitedResources
|
|
1116
|
+
};
|
|
1117
|
+
});
|
|
1118
|
+
}
|
|
1119
|
+
/**
|
|
1120
|
+
* Clean up all rate limit data
|
|
1121
|
+
*/
|
|
1122
|
+
clear() {
|
|
1123
|
+
return __async(this, null, function* () {
|
|
1124
|
+
if (this.isDestroyed) {
|
|
1125
|
+
throw new Error("Rate limit store has been destroyed");
|
|
1126
|
+
}
|
|
1127
|
+
yield this.db.delete(rateLimitTable);
|
|
1128
|
+
this.activityMetrics.clear();
|
|
1129
|
+
this.cachedCapacity.clear();
|
|
1130
|
+
this.lastCapacityUpdate.clear();
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Clean up expired requests for all resources
|
|
1135
|
+
*/
|
|
1136
|
+
cleanup() {
|
|
1137
|
+
return __async(this, null, function* () {
|
|
1138
|
+
const now = Date.now();
|
|
1139
|
+
const resources = yield this.db.select({ resource: rateLimitTable.resource }).from(rateLimitTable).groupBy(rateLimitTable.resource);
|
|
1140
|
+
for (const { resource } of resources) {
|
|
1141
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
1142
|
+
const windowStart = now - config.windowMs;
|
|
1143
|
+
yield this.cleanupExpiredRequests(resource, windowStart);
|
|
1144
|
+
}
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* Close the database connection
|
|
1149
|
+
*/
|
|
1150
|
+
close() {
|
|
1151
|
+
return __async(this, null, function* () {
|
|
1152
|
+
this.isDestroyed = true;
|
|
1153
|
+
if (this.isConnectionManaged && typeof this.sqlite.close === "function") {
|
|
1154
|
+
this.sqlite.close();
|
|
1155
|
+
}
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* Alias for close() to match test expectations
|
|
1160
|
+
*/
|
|
1161
|
+
destroy() {
|
|
1162
|
+
this.close();
|
|
1163
|
+
}
|
|
1164
|
+
// Private helper methods for adaptive functionality
|
|
1165
|
+
calculateCurrentCapacity(resource, metrics) {
|
|
1166
|
+
const lastUpdate = this.lastCapacityUpdate.get(resource) || 0;
|
|
1167
|
+
const recalcInterval = this.capacityCalculator.config.recalculationIntervalMs;
|
|
1168
|
+
if (Date.now() - lastUpdate < recalcInterval) {
|
|
1169
|
+
return this.cachedCapacity.get(resource) || this.getDefaultCapacity(resource);
|
|
1170
|
+
}
|
|
1171
|
+
const totalLimit = this.getResourceLimit(resource);
|
|
1172
|
+
const capacity = this.capacityCalculator.calculateDynamicCapacity(
|
|
1173
|
+
resource,
|
|
1174
|
+
totalLimit,
|
|
1175
|
+
metrics
|
|
1176
|
+
);
|
|
1177
|
+
this.cachedCapacity.set(resource, capacity);
|
|
1178
|
+
this.lastCapacityUpdate.set(resource, Date.now());
|
|
1179
|
+
return capacity;
|
|
1180
|
+
}
|
|
1181
|
+
getOrCreateActivityMetrics(resource) {
|
|
1182
|
+
if (!this.activityMetrics.has(resource)) {
|
|
1183
|
+
this.activityMetrics.set(resource, {
|
|
1184
|
+
recentUserRequests: [],
|
|
1185
|
+
recentBackgroundRequests: [],
|
|
1186
|
+
userActivityTrend: "none"
|
|
1187
|
+
});
|
|
1188
|
+
}
|
|
1189
|
+
return this.activityMetrics.get(resource);
|
|
1190
|
+
}
|
|
1191
|
+
ensureActivityMetrics(resource) {
|
|
1192
|
+
return __async(this, null, function* () {
|
|
1193
|
+
if (this.activityMetrics.has(resource)) {
|
|
1194
|
+
return;
|
|
1195
|
+
}
|
|
1196
|
+
const now = Date.now();
|
|
1197
|
+
const windowStart = now - this.capacityCalculator.config.monitoringWindowMs;
|
|
1198
|
+
const recentRequests = this.sqlite.prepare(
|
|
1199
|
+
`
|
|
1200
|
+
SELECT timestamp, COALESCE(priority, 'background') as priority
|
|
1201
|
+
FROM rate_limits
|
|
1202
|
+
WHERE resource = ? AND timestamp >= ?
|
|
1203
|
+
ORDER BY timestamp
|
|
1204
|
+
`
|
|
1205
|
+
).all(resource, windowStart);
|
|
1206
|
+
const metrics = {
|
|
1207
|
+
recentUserRequests: [],
|
|
1208
|
+
recentBackgroundRequests: [],
|
|
1209
|
+
userActivityTrend: "none"
|
|
1210
|
+
};
|
|
1211
|
+
for (const request of recentRequests) {
|
|
1212
|
+
if (request.priority === "user") {
|
|
1213
|
+
metrics.recentUserRequests.push(request.timestamp);
|
|
1214
|
+
} else {
|
|
1215
|
+
metrics.recentBackgroundRequests.push(request.timestamp);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
metrics.userActivityTrend = this.capacityCalculator.calculateActivityTrend(
|
|
1219
|
+
metrics.recentUserRequests
|
|
1220
|
+
);
|
|
1221
|
+
this.activityMetrics.set(resource, metrics);
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
getCurrentUsage(resource, priority) {
|
|
1225
|
+
return __async(this, null, function* () {
|
|
1226
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
1227
|
+
const now = Date.now();
|
|
1228
|
+
const windowStart = now - config.windowMs;
|
|
1229
|
+
yield this.cleanupExpiredRequests(resource, windowStart);
|
|
1230
|
+
const result = this.sqlite.prepare(
|
|
1231
|
+
`
|
|
1232
|
+
SELECT COUNT(*) as count
|
|
1233
|
+
FROM rate_limits
|
|
1234
|
+
WHERE resource = ? AND priority = ? AND timestamp >= ?
|
|
1235
|
+
`
|
|
1236
|
+
).get(resource, priority, windowStart);
|
|
1237
|
+
return result.count || 0;
|
|
1238
|
+
});
|
|
1239
|
+
}
|
|
1240
|
+
cleanupOldRequests(requests) {
|
|
1241
|
+
const cutoff = Date.now() - this.capacityCalculator.config.monitoringWindowMs;
|
|
1242
|
+
while (requests.length > 0 && requests[0] < cutoff) {
|
|
1243
|
+
requests.shift();
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
getResourceLimit(resource) {
|
|
1247
|
+
const config = this.resourceConfigs.get(resource) || this.defaultConfig;
|
|
1248
|
+
return config.limit;
|
|
1249
|
+
}
|
|
1250
|
+
getDefaultCapacity(resource) {
|
|
1251
|
+
const limit = this.getResourceLimit(resource);
|
|
1252
|
+
return {
|
|
1253
|
+
userReserved: Math.floor(limit * 0.3),
|
|
1254
|
+
backgroundMax: Math.floor(limit * 0.7),
|
|
1255
|
+
backgroundPaused: false,
|
|
1256
|
+
reason: "Default capacity allocation"
|
|
1257
|
+
};
|
|
1258
|
+
}
|
|
1259
|
+
cleanupExpiredRequests(resource, windowStart) {
|
|
1260
|
+
return __async(this, null, function* () {
|
|
1261
|
+
yield this.db.delete(rateLimitTable).where(
|
|
1262
|
+
and(
|
|
1263
|
+
eq(rateLimitTable.resource, resource),
|
|
1264
|
+
lt(rateLimitTable.timestamp, windowStart)
|
|
1265
|
+
)
|
|
1266
|
+
);
|
|
1267
|
+
});
|
|
1268
|
+
}
|
|
1269
|
+
initializeDatabase() {
|
|
1270
|
+
this.db.run(sql`
|
|
1271
|
+
CREATE TABLE IF NOT EXISTS rate_limits (
|
|
1272
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1273
|
+
resource TEXT NOT NULL,
|
|
1274
|
+
timestamp INTEGER NOT NULL,
|
|
1275
|
+
priority TEXT NOT NULL DEFAULT 'background'
|
|
1276
|
+
)
|
|
1277
|
+
`);
|
|
1278
|
+
try {
|
|
1279
|
+
this.db.run(sql`
|
|
1280
|
+
ALTER TABLE rate_limits ADD COLUMN priority TEXT DEFAULT 'background'
|
|
1281
|
+
`);
|
|
1282
|
+
} catch (e) {
|
|
1283
|
+
}
|
|
1284
|
+
this.db.run(sql`
|
|
1285
|
+
CREATE INDEX IF NOT EXISTS idx_rate_limit_resource ON rate_limits(resource)
|
|
1286
|
+
`);
|
|
1287
|
+
this.db.run(sql`
|
|
1288
|
+
CREATE INDEX IF NOT EXISTS idx_rate_limit_timestamp ON rate_limits(timestamp)
|
|
1289
|
+
`);
|
|
1290
|
+
this.db.run(sql`
|
|
1291
|
+
CREATE INDEX IF NOT EXISTS idx_rate_limit_resource_priority_timestamp
|
|
1292
|
+
ON rate_limits(resource, priority, timestamp)
|
|
1293
|
+
`);
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
|
|
1297
|
+
export { SQLiteCacheStore, SQLiteDedupeStore, SQLiteRateLimitStore, SqliteAdaptiveRateLimitStore, cacheTable, dedupeTable, rateLimitTable };
|
|
1298
|
+
//# sourceMappingURL=index.js.map
|
|
1299
|
+
//# sourceMappingURL=index.js.map
|