@lensjs/core 2.2.2 → 2.3.1

package/dist/stores/better_sqlite.cjs

@@ -36,6 +36,9 @@ module.exports = __toCommonJS(better_sqlite_exports);
 
 // src/abstracts/store.ts
 var Store = class {
+  storeConfig;
+  constructor(...args) {
+  }
   getAllExceptions(_paginationParams) {
     return this.defaultMinimalPaginate();
   }
@@ -66,6 +69,8 @@ var import_crypto = require("crypto");
 var import_libsql = __toESM(require("libsql"), 1);
 var import_date = require("@lensjs/date");
 var TABLE_NAME = "lens_entries";
+var BYTES_IN_GB = 1024 * 1024 * 1024;
+var PRUNE_BATCH_SIZE = 1e3;
 var BetterSqliteStore = class extends Store {
   connection;
   async initialize() {
@@ -87,6 +92,7 @@ var BetterSqliteStore = class extends Store {
       lens_entry_id: entry.requestId || null,
       minimalData: this.stringifyData(entry.minimal_data ?? {})
     });
+    this.maybePruneDatabase();
   }
   async getAllQueries(pagination) {
     return await this.paginate("query" /* QUERY */, pagination);
@@ -162,6 +168,36 @@ var BetterSqliteStore = class extends Store {
     this.connection.exec(createIndex);
     this.connection.exec(lensEntryIdIndex);
   }
+  maybePruneDatabase() {
+    const maxGb = this.storeConfig?.dbMaxSizeGb;
+    const pruneGb = this.storeConfig?.dbPruneSizeGb;
+    if (!maxGb || !pruneGb) return;
+    const maxBytes = maxGb * BYTES_IN_GB;
+    const pruneBytes = pruneGb * BYTES_IN_GB;
+    if (maxBytes <= 0 || pruneBytes <= 0) return;
+    const targetBytes = Math.max(0, maxBytes - pruneBytes);
+    let usedBytes = this.getDatabaseUsedBytes();
+    if (usedBytes < maxBytes) return;
+    while (usedBytes > targetBytes) {
+      const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
+      if (deletedRows === 0) break;
+      usedBytes = this.getDatabaseUsedBytes();
+    }
+    this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+  }
+  getDatabaseUsedBytes() {
+    const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
+    const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
+    const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
+    const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
+    return usedPages * pageSizeResult.page_size;
+  }
+  deleteOldestEntries(batchSize) {
+    const result = this.connection.prepare(
+      `DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
+    ).run(batchSize);
+    return Number(result.changes ?? 0);
+  }
   mapRow(row, includeFullData = true) {
     let data = includeFullData ? JSON.parse(row.data) : {};
     if (!includeFullData) {

package/dist/stores/better_sqlite.d.cts

@@ -59,6 +59,9 @@ declare class BetterSqliteStore extends Store {
     count(type: WatcherTypeEnum): Promise<number>;
     find(type: WatcherTypeEnum, id: string): Promise<LensEntry | null>;
     private setupSchema;
+    private maybePruneDatabase;
+    private getDatabaseUsedBytes;
+    private deleteOldestEntries;
     protected mapRow(row: any, includeFullData?: boolean): LensEntry;
     protected mapRows(rows: any[], includeFullData?: boolean): LensEntry[];
     protected getSelectedColumns(includeFullData?: boolean): string;

package/dist/stores/better_sqlite.d.ts

@@ -59,6 +59,9 @@ declare class BetterSqliteStore extends Store {
     count(type: WatcherTypeEnum): Promise<number>;
     find(type: WatcherTypeEnum, id: string): Promise<LensEntry | null>;
     private setupSchema;
+    private maybePruneDatabase;
+    private getDatabaseUsedBytes;
+    private deleteOldestEntries;
     protected mapRow(row: any, includeFullData?: boolean): LensEntry;
     protected mapRows(rows: any[], includeFullData?: boolean): LensEntry[];
     protected getSelectedColumns(includeFullData?: boolean): string;

package/dist/stores/better_sqlite.js

@@ -1,5 +1,8 @@
 // src/abstracts/store.ts
 var Store = class {
+  storeConfig;
+  constructor(...args) {
+  }
   getAllExceptions(_paginationParams) {
     return this.defaultMinimalPaginate();
   }
@@ -30,6 +33,8 @@ import { randomUUID } from "crypto";
 import Database from "libsql";
 import { nowISO } from "@lensjs/date";
 var TABLE_NAME = "lens_entries";
+var BYTES_IN_GB = 1024 * 1024 * 1024;
+var PRUNE_BATCH_SIZE = 1e3;
 var BetterSqliteStore = class extends Store {
   connection;
   async initialize() {
@@ -51,6 +56,7 @@ var BetterSqliteStore = class extends Store {
       lens_entry_id: entry.requestId || null,
       minimalData: this.stringifyData(entry.minimal_data ?? {})
     });
+    this.maybePruneDatabase();
   }
   async getAllQueries(pagination) {
     return await this.paginate("query" /* QUERY */, pagination);
@@ -126,6 +132,36 @@ var BetterSqliteStore = class extends Store {
     this.connection.exec(createIndex);
     this.connection.exec(lensEntryIdIndex);
   }
+  maybePruneDatabase() {
+    const maxGb = this.storeConfig?.dbMaxSizeGb;
+    const pruneGb = this.storeConfig?.dbPruneSizeGb;
+    if (!maxGb || !pruneGb) return;
+    const maxBytes = maxGb * BYTES_IN_GB;
+    const pruneBytes = pruneGb * BYTES_IN_GB;
+    if (maxBytes <= 0 || pruneBytes <= 0) return;
+    const targetBytes = Math.max(0, maxBytes - pruneBytes);
+    let usedBytes = this.getDatabaseUsedBytes();
+    if (usedBytes < maxBytes) return;
+    while (usedBytes > targetBytes) {
+      const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
+      if (deletedRows === 0) break;
+      usedBytes = this.getDatabaseUsedBytes();
+    }
+    this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+  }
+  getDatabaseUsedBytes() {
+    const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
+    const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
+    const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
+    const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
+    return usedPages * pageSizeResult.page_size;
+  }
+  deleteOldestEntries(batchSize) {
+    const result = this.connection.prepare(
+      `DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
+    ).run(batchSize);
+    return Number(result.changes ?? 0);
+  }
   mapRow(row, includeFullData = true) {
     let data = includeFullData ? JSON.parse(row.data) : {};
     if (!includeFullData) {

package/dist/stores/index.cjs

@@ -30,12 +30,16 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/stores/index.ts
 var stores_exports = {};
 __export(stores_exports, {
-  BetterSqliteStore: () => BetterSqliteStore
+  BetterSqliteStore: () => BetterSqliteStore,
+  QueuedSqliteStore: () => QueuedSqliteStore
 });
 module.exports = __toCommonJS(stores_exports);
 
 // src/abstracts/store.ts
 var Store = class {
+  storeConfig;
+  constructor(...args) {
+  }
   getAllExceptions(_paginationParams) {
     return this.defaultMinimalPaginate();
   }
@@ -66,6 +70,8 @@ var import_crypto = require("crypto");
 var import_libsql = __toESM(require("libsql"), 1);
 var import_date = require("@lensjs/date");
 var TABLE_NAME = "lens_entries";
+var BYTES_IN_GB = 1024 * 1024 * 1024;
+var PRUNE_BATCH_SIZE = 1e3;
 var BetterSqliteStore = class extends Store {
   connection;
   async initialize() {
@@ -87,6 +93,7 @@ var BetterSqliteStore = class extends Store {
       lens_entry_id: entry.requestId || null,
       minimalData: this.stringifyData(entry.minimal_data ?? {})
     });
+    this.maybePruneDatabase();
   }
   async getAllQueries(pagination) {
     return await this.paginate("query" /* QUERY */, pagination);
@@ -162,6 +169,36 @@ var BetterSqliteStore = class extends Store {
     this.connection.exec(createIndex);
     this.connection.exec(lensEntryIdIndex);
   }
+  maybePruneDatabase() {
+    const maxGb = this.storeConfig?.dbMaxSizeGb;
+    const pruneGb = this.storeConfig?.dbPruneSizeGb;
+    if (!maxGb || !pruneGb) return;
+    const maxBytes = maxGb * BYTES_IN_GB;
+    const pruneBytes = pruneGb * BYTES_IN_GB;
+    if (maxBytes <= 0 || pruneBytes <= 0) return;
+    const targetBytes = Math.max(0, maxBytes - pruneBytes);
+    let usedBytes = this.getDatabaseUsedBytes();
+    if (usedBytes < maxBytes) return;
+    while (usedBytes > targetBytes) {
+      const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
+      if (deletedRows === 0) break;
+      usedBytes = this.getDatabaseUsedBytes();
+    }
+    this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+  }
+  getDatabaseUsedBytes() {
+    const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
+    const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
+    const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
+    const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
+    return usedPages * pageSizeResult.page_size;
+  }
+  deleteOldestEntries(batchSize) {
+    const result = this.connection.prepare(
+      `DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
+    ).run(batchSize);
+    return Number(result.changes ?? 0);
+  }
   mapRow(row, includeFullData = true) {
     let data = includeFullData ? JSON.parse(row.data) : {};
     if (!includeFullData) {
@@ -186,7 +223,97 @@ var BetterSqliteStore = class extends Store {
     return `SELECT id, minimal_data, type, created_at, lens_entry_id ${includeFullData ? ",data" : ""}`;
   }
 };
+
+// src/mixins/queued_store.ts
+var import_denque = __toESM(require("denque"), 1);
+function QueuedStore(Base) {
+  return class Queued extends Base {
+    queue;
+    processingInterval = null;
+    BATCH_SIZE;
+    PROCESS_INTERVAL_MS;
+    WARN_THRESHOLD;
+    PREALLOCATE;
+    constructor(...args) {
+      super(...args);
+      const config = args[0] || {};
+      this.storeConfig = config;
+      this.BATCH_SIZE = config.batchSize ?? 100;
+      this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
+      this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
+      this.PREALLOCATE = config.preallocate ?? true;
+      this.queue = this.PREALLOCATE ? new import_denque.default([], { capacity: this.WARN_THRESHOLD * 2 }) : new import_denque.default();
+    }
+    async initialize() {
+      if (super["initialize"]) {
+        await super["initialize"].call(this);
+      }
+      this.startProcessingQueue();
+      process.on("SIGINT", () => this.shutdown());
+      process.on("SIGTERM", () => this.shutdown());
+    }
+    async truncate() {
+      this.queue.clear();
+      if (super["truncate"]) {
+        await super["truncate"].call(this);
+      }
+    }
+    async save(entry) {
+      this.queue.push(entry);
+      if (this.queue.length > this.WARN_THRESHOLD) {
+        console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
+      }
+    }
+    startProcessingQueue() {
+      if (this.processingInterval) clearInterval(this.processingInterval);
+      this.processingInterval = setInterval(
+        () => this.processQueue(),
+        this.PROCESS_INTERVAL_MS
+      );
+    }
+    async processQueue() {
+      if (this.queue.isEmpty()) return;
+      const batchSize = Math.min(
+        this.BATCH_SIZE,
+        Math.max(10, Math.floor(this.queue.length / 10))
+      );
+      const entriesToProcess = this.queue.remove(0, batchSize);
+      if (!entriesToProcess.length) return;
+      for (const entry of entriesToProcess) {
+        super["save"]?.call(this, entry).catch((error) => {
+          console.error("Error saving queued entry:", error);
+        });
+      }
+    }
+    async stopProcessingQueue() {
+      if (this.processingInterval) {
+        clearInterval(this.processingInterval);
+        this.processingInterval = null;
+      }
+      if (!this.queue.isEmpty()) {
+        await this.processQueue();
+      }
+    }
+    async shutdown() {
+      await this.stopProcessingQueue();
+      process.exit(0);
+    }
+  };
+}
+
+// src/utils/compose.ts
+function compose(superclass, ...mixins) {
+  return mixins.reduce((c, mixin) => mixin(c), superclass);
+}
+
+// src/stores/queued_sqlite.ts
+var QueuedSqliteStore = class extends compose(
+  BetterSqliteStore,
+  QueuedStore
+) {
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  BetterSqliteStore
+  BetterSqliteStore,
+  QueuedSqliteStore
 });

package/dist/stores/index.d.cts

@@ -1,5 +1,7 @@
 export { default as BetterSqliteStore } from './better_sqlite.cjs';
+export { default as QueuedSqliteStore } from './queued_sqlite.cjs';
 import '../abstracts/store.cjs';
 import '../types/index.cjs';
 import 'sql-formatter';
 import 'libsql';
+import 'denque';

package/dist/stores/index.d.ts

@@ -1,5 +1,7 @@
 export { default as BetterSqliteStore } from './better_sqlite.js';
+export { default as QueuedSqliteStore } from './queued_sqlite.js';
 import '../abstracts/store.js';
 import '../types/index.js';
 import 'sql-formatter';
 import 'libsql';
+import 'denque';

package/dist/stores/index.js

@@ -1,5 +1,8 @@
 // src/abstracts/store.ts
 var Store = class {
+  storeConfig;
+  constructor(...args) {
+  }
   getAllExceptions(_paginationParams) {
     return this.defaultMinimalPaginate();
   }
@@ -30,6 +33,8 @@ import { randomUUID } from "crypto";
 import Database from "libsql";
 import { nowISO } from "@lensjs/date";
 var TABLE_NAME = "lens_entries";
+var BYTES_IN_GB = 1024 * 1024 * 1024;
+var PRUNE_BATCH_SIZE = 1e3;
 var BetterSqliteStore = class extends Store {
   connection;
   async initialize() {
@@ -51,6 +56,7 @@ var BetterSqliteStore = class extends Store {
       lens_entry_id: entry.requestId || null,
       minimalData: this.stringifyData(entry.minimal_data ?? {})
     });
+    this.maybePruneDatabase();
   }
   async getAllQueries(pagination) {
     return await this.paginate("query" /* QUERY */, pagination);
@@ -126,6 +132,36 @@ var BetterSqliteStore = class extends Store {
     this.connection.exec(createIndex);
     this.connection.exec(lensEntryIdIndex);
   }
+  maybePruneDatabase() {
+    const maxGb = this.storeConfig?.dbMaxSizeGb;
+    const pruneGb = this.storeConfig?.dbPruneSizeGb;
+    if (!maxGb || !pruneGb) return;
+    const maxBytes = maxGb * BYTES_IN_GB;
+    const pruneBytes = pruneGb * BYTES_IN_GB;
+    if (maxBytes <= 0 || pruneBytes <= 0) return;
+    const targetBytes = Math.max(0, maxBytes - pruneBytes);
+    let usedBytes = this.getDatabaseUsedBytes();
+    if (usedBytes < maxBytes) return;
+    while (usedBytes > targetBytes) {
+      const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
+      if (deletedRows === 0) break;
+      usedBytes = this.getDatabaseUsedBytes();
+    }
+    this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+  }
+  getDatabaseUsedBytes() {
+    const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
+    const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
+    const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
+    const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
+    return usedPages * pageSizeResult.page_size;
+  }
+  deleteOldestEntries(batchSize) {
+    const result = this.connection.prepare(
+      `DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
+    ).run(batchSize);
+    return Number(result.changes ?? 0);
+  }
   mapRow(row, includeFullData = true) {
     let data = includeFullData ? JSON.parse(row.data) : {};
     if (!includeFullData) {
@@ -150,6 +186,96 @@ var BetterSqliteStore = class extends Store {
     return `SELECT id, minimal_data, type, created_at, lens_entry_id ${includeFullData ? ",data" : ""}`;
   }
 };
+
+// src/mixins/queued_store.ts
+import Denque from "denque";
+function QueuedStore(Base) {
+  return class Queued extends Base {
+    queue;
+    processingInterval = null;
+    BATCH_SIZE;
+    PROCESS_INTERVAL_MS;
+    WARN_THRESHOLD;
+    PREALLOCATE;
+    constructor(...args) {
+      super(...args);
+      const config = args[0] || {};
+      this.storeConfig = config;
+      this.BATCH_SIZE = config.batchSize ?? 100;
+      this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
+      this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
+      this.PREALLOCATE = config.preallocate ?? true;
+      this.queue = this.PREALLOCATE ? new Denque([], { capacity: this.WARN_THRESHOLD * 2 }) : new Denque();
+    }
+    async initialize() {
+      if (super["initialize"]) {
+        await super["initialize"].call(this);
+      }
+      this.startProcessingQueue();
+      process.on("SIGINT", () => this.shutdown());
+      process.on("SIGTERM", () => this.shutdown());
+    }
+    async truncate() {
+      this.queue.clear();
+      if (super["truncate"]) {
+        await super["truncate"].call(this);
+      }
+    }
+    async save(entry) {
+      this.queue.push(entry);
+      if (this.queue.length > this.WARN_THRESHOLD) {
+        console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
+      }
+    }
+    startProcessingQueue() {
+      if (this.processingInterval) clearInterval(this.processingInterval);
+      this.processingInterval = setInterval(
+        () => this.processQueue(),
+        this.PROCESS_INTERVAL_MS
+      );
+    }
+    async processQueue() {
+      if (this.queue.isEmpty()) return;
+      const batchSize = Math.min(
+        this.BATCH_SIZE,
+        Math.max(10, Math.floor(this.queue.length / 10))
+      );
+      const entriesToProcess = this.queue.remove(0, batchSize);
+      if (!entriesToProcess.length) return;
+      for (const entry of entriesToProcess) {
+        super["save"]?.call(this, entry).catch((error) => {
+          console.error("Error saving queued entry:", error);
+        });
+      }
+    }
+    async stopProcessingQueue() {
+      if (this.processingInterval) {
+        clearInterval(this.processingInterval);
+        this.processingInterval = null;
+      }
+      if (!this.queue.isEmpty()) {
+        await this.processQueue();
+      }
+    }
+    async shutdown() {
+      await this.stopProcessingQueue();
+      process.exit(0);
+    }
+  };
+}
+
+// src/utils/compose.ts
+function compose(superclass, ...mixins) {
+  return mixins.reduce((c, mixin) => mixin(c), superclass);
+}
+
+// src/stores/queued_sqlite.ts
+var QueuedSqliteStore = class extends compose(
+  BetterSqliteStore,
+  QueuedStore
+) {
+};
 export {
-  BetterSqliteStore
+  BetterSqliteStore,
+  QueuedSqliteStore
 };