@lensjs/core 2.2.2 → 2.3.1

package/dist/index.js

@@ -183,6 +183,9 @@ import * as path2 from "path";
 
 // src/abstracts/store.ts
 var Store = class {
+  storeConfig;
+  constructor(...args) {
+  }
   getAllExceptions(_paginationParams) {
     return this.defaultMinimalPaginate();
   }
@@ -213,6 +216,8 @@ import { randomUUID } from "crypto";
 import Database from "libsql";
 import { nowISO } from "@lensjs/date";
 var TABLE_NAME = "lens_entries";
+var BYTES_IN_GB = 1024 * 1024 * 1024;
+var PRUNE_BATCH_SIZE = 1e3;
 var BetterSqliteStore = class extends Store {
   connection;
   async initialize() {
@@ -234,6 +239,7 @@ var BetterSqliteStore = class extends Store {
       lens_entry_id: entry.requestId || null,
       minimalData: this.stringifyData(entry.minimal_data ?? {})
     });
+    this.maybePruneDatabase();
   }
   async getAllQueries(pagination) {
     return await this.paginate("query" /* QUERY */, pagination);
@@ -309,6 +315,36 @@ var BetterSqliteStore = class extends Store {
     this.connection.exec(createIndex);
     this.connection.exec(lensEntryIdIndex);
   }
+  maybePruneDatabase() {
+    const maxGb = this.storeConfig?.dbMaxSizeGb;
+    const pruneGb = this.storeConfig?.dbPruneSizeGb;
+    if (!maxGb || !pruneGb) return;
+    const maxBytes = maxGb * BYTES_IN_GB;
+    const pruneBytes = pruneGb * BYTES_IN_GB;
+    if (maxBytes <= 0 || pruneBytes <= 0) return;
+    const targetBytes = Math.max(0, maxBytes - pruneBytes);
+    let usedBytes = this.getDatabaseUsedBytes();
+    if (usedBytes < maxBytes) return;
+    while (usedBytes > targetBytes) {
+      const deletedRows = this.deleteOldestEntries(PRUNE_BATCH_SIZE);
+      if (deletedRows === 0) break;
+      usedBytes = this.getDatabaseUsedBytes();
+    }
+    this.connection.exec("PRAGMA wal_checkpoint(TRUNCATE);");
+  }
+  getDatabaseUsedBytes() {
+    const pageSizeResult = this.connection.prepare("PRAGMA page_size;").get();
+    const pageCountResult = this.connection.prepare("PRAGMA page_count;").get();
+    const freelistCountResult = this.connection.prepare("PRAGMA freelist_count;").get();
+    const usedPages = pageCountResult.page_count - freelistCountResult.freelist_count;
+    return usedPages * pageSizeResult.page_size;
+  }
+  deleteOldestEntries(batchSize) {
+    const result = this.connection.prepare(
+      `DELETE FROM ${TABLE_NAME} WHERE id IN (SELECT id FROM ${TABLE_NAME} ORDER BY created_at ASC LIMIT ?)`
+    ).run(batchSize);
+    return Number(result.changes ?? 0);
+  }
   mapRow(row, includeFullData = true) {
     let data = includeFullData ? JSON.parse(row.data) : {};
     if (!includeFullData) {
@@ -334,9 +370,99 @@ var BetterSqliteStore = class extends Store {
   }
 };
 
+// src/mixins/queued_store.ts
+import Denque from "denque";
+function QueuedStore(Base) {
+  return class Queued extends Base {
+    queue;
+    processingInterval = null;
+    BATCH_SIZE;
+    PROCESS_INTERVAL_MS;
+    WARN_THRESHOLD;
+    PREALLOCATE;
+    constructor(...args) {
+      super(...args);
+      const config = args[0] || {};
+      this.storeConfig = config;
+      this.BATCH_SIZE = config.batchSize ?? 100;
+      this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
+      this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
+      this.PREALLOCATE = config.preallocate ?? true;
+      this.queue = this.PREALLOCATE ? new Denque([], { capacity: this.WARN_THRESHOLD * 2 }) : new Denque();
+    }
+    async initialize() {
+      if (super["initialize"]) {
+        await super["initialize"].call(this);
+      }
+      this.startProcessingQueue();
+      process.on("SIGINT", () => this.shutdown());
+      process.on("SIGTERM", () => this.shutdown());
+    }
+    async truncate() {
+      this.queue.clear();
+      if (super["truncate"]) {
+        await super["truncate"].call(this);
+      }
+    }
+    async save(entry) {
+      this.queue.push(entry);
+      if (this.queue.length > this.WARN_THRESHOLD) {
+        console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
+      }
+    }
+    startProcessingQueue() {
+      if (this.processingInterval) clearInterval(this.processingInterval);
+      this.processingInterval = setInterval(
+        () => this.processQueue(),
+        this.PROCESS_INTERVAL_MS
+      );
+    }
+    async processQueue() {
+      if (this.queue.isEmpty()) return;
+      const batchSize = Math.min(
+        this.BATCH_SIZE,
+        Math.max(10, Math.floor(this.queue.length / 10))
+      );
+      const entriesToProcess = this.queue.remove(0, batchSize);
+      if (!entriesToProcess.length) return;
+      for (const entry of entriesToProcess) {
+        super["save"]?.call(this, entry).catch((error) => {
+          console.error("Error saving queued entry:", error);
+        });
+      }
+    }
+    async stopProcessingQueue() {
+      if (this.processingInterval) {
+        clearInterval(this.processingInterval);
+        this.processingInterval = null;
+      }
+      if (!this.queue.isEmpty()) {
+        await this.processQueue();
+      }
+    }
+    async shutdown() {
+      await this.stopProcessingQueue();
+      process.exit(0);
+    }
+  };
+}
+
+// src/utils/compose.ts
+function compose(superclass, ...mixins) {
+  return mixins.reduce((c, mixin) => mixin(c), superclass);
+}
+
+// src/stores/queued_sqlite.ts
+var QueuedSqliteStore = class extends compose(
+  BetterSqliteStore,
+  QueuedStore
+) {
+};
+
 // src/utils/index.ts
 var utils_exports = {};
 __export(utils_exports, {
+  compose: () => compose,
   formatSqlQuery: () => formatSqlQuery,
   generateRandomUuid: () => generateRandomUuid,
   getMeta: () => getMeta,
@@ -411,7 +537,7 @@ var formatSqlQuery = (query, language) => {
   });
 };
 function getMeta(metaUrl) {
-  const isESM = typeof __dirname === "undefined";
+  const isESM = typeof __dirname === "undefined" || typeof __filename === "undefined";
   if (isESM) {
     if (!metaUrl) {
       throw new Error("In ESM, you must pass import.meta.url to getMeta()");
@@ -476,6 +602,7 @@ var Lens = class {
   static watchers = /* @__PURE__ */ new Map();
   static store;
   static adapter;
+  static config;
   static watch(watcher) {
     this.watchers.set(watcher.name, watcher);
     return this;
@@ -485,22 +612,23 @@ var Lens = class {
     return this;
   }
   static async start(config = {
-    basePath: "lens",
+    path: "lens",
     appName: "Lens",
     enabled: true
   }) {
     if (!config.enabled) {
       return;
     }
-    await this.bindContainerDeps(config);
+    this.config = config;
+    await this.bindContainerDeps();
     let adapter = this.getAdapter();
     adapter.setWatchers(Array.from(this.watchers.values())).setup();
     const { apiRoutes } = this.getRoutes({
-      basePath: config.basePath
+      path: config.path
     });
     adapter.registerRoutes(apiRoutes);
     const uiPath = this.getUiPath();
-    adapter.serveUI(uiPath, config.basePath, getUiConfig());
+    adapter.serveUI(uiPath, config.path, getUiConfig());
   }
   static setStore(store) {
     this.store = store;
@@ -523,7 +651,7 @@ var Lens = class {
     const { __dirname: __dirname2 } = getMeta(import.meta.url);
     return path2.resolve(this.normalizeDirName(__dirname2), "ui");
   }
-  static getRoutes({ basePath }) {
+  static getRoutes({ path: path4 }) {
     const apiRoutes = [
       {
         method: "GET",
@@ -532,71 +660,71 @@ var Lens = class {
       },
       {
         method: "GET",
-        path: `/${basePath}/api/requests`,
+        path: `/${path4}/api/requests`,
         handler: async (data) => await ApiController.getRequests(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/requests/:id`,
+        path: `/${path4}/api/requests/:id`,
         handler: async (data) => await ApiController.getRequest(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/queries`,
+        path: `/${path4}/api/queries`,
         handler: async (data) => await ApiController.getQueries(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/queries/:id`,
+        path: `/${path4}/api/queries/:id`,
         handler: async (data) => await ApiController.getQuery(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/cache`,
+        path: `/${path4}/api/cache`,
         handler: async (data) => await ApiController.getCacheEntries(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/cache/:id`,
+        path: `/${path4}/api/cache/:id`,
         handler: async (data) => await ApiController.getCacheEntry(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/exceptions`,
+        path: `/${path4}/api/exceptions`,
         handler: async (data) => await ApiController.getExceptions(data)
       },
       {
         method: "GET",
-        path: `/${basePath}/api/exceptions/:id`,
+        path: `/${path4}/api/exceptions/:id`,
         handler: async (data) => await ApiController.getException(data)
       },
       {
         method: "DELETE",
-        path: `/${basePath}/api/truncate`,
+        path: `/${path4}/api/truncate`,
         handler: async () => await ApiController.truncate()
       }
     ];
     return { apiRoutes };
   }
-  static async bindContainerDeps(config) {
+  static async bindContainerDeps() {
     const dbStore = await this.getStore();
     Container.singleton("store", () => dbStore);
     Container.singleton("uiConfig", () => {
       return {
-        appName: config.appName,
-        path: `/${config.basePath}`,
+        appName: this.config.appName,
+        path: `/${this.config.path}`,
         api: {
-          requests: `/${config.basePath}/api/requests`,
-          queries: `/${config.basePath}/api/queries`,
-          cache: `/${config.basePath}/api/cache`,
-          exceptions: `/${config.basePath}/api/exceptions`,
-          truncate: `/${config.basePath}/api/truncate`
+          requests: `/${this.config.path}/api/requests`,
+          queries: `/${this.config.path}/api/queries`,
+          cache: `/${this.config.path}/api/cache`,
+          exceptions: `/${this.config.path}/api/exceptions`,
+          truncate: `/${this.config.path}/api/truncate`
         }
       };
     });
   }
   static async getDefaultStore() {
-    const store = new BetterSqliteStore();
+    const store = new QueuedSqliteStore(this.config.storeQueueConfig);
     await store.initialize();
     return store;
   }
@@ -622,10 +750,19 @@ var QueryWatcher = class extends Watcher {
 };
 
 // src/watchers/request_watcher.ts
-var RequestWatcher = class extends Watcher {
+var RequestWatcher = class _RequestWatcher extends Watcher {
   name = "request" /* REQUEST */;
-  async log(data) {
-    await getStore().save({
+  static DEFAULT_HIDDEN_HEADERS = ["Authorization", "Basic"];
+  static DEFAULT_HIDDEN_BODY_PARAMS = [
+    "password",
+    "passwordConfirmation",
+    "secret",
+    "password_confirmation"
+  ];
+  async log(data, hidden = {}) {
+    const headersToHide = (hidden.headers?.length ? hidden.headers : _RequestWatcher.DEFAULT_HIDDEN_HEADERS).map((header) => header.toLowerCase());
+    const bodyParamsToHide = hidden.bodyParams?.length ? hidden.bodyParams : _RequestWatcher.DEFAULT_HIDDEN_BODY_PARAMS;
+    const payload = {
       id: data.request.id,
       type: this.name,
       minimal_data: {
@@ -639,9 +776,40 @@ var RequestWatcher = class extends Watcher {
       data: {
         ...data.request,
         user: data.user,
-        response: data.response
+        response: data.response || {}
       }
-    });
+    };
+    payload.data.headers = this.hideSensitive(
+      this.normalizeHeaders(payload.data.headers),
+      headersToHide
+    );
+    payload.data.response.headers = this.hideSensitive(
+      this.normalizeHeaders(payload.data.response?.headers),
+      headersToHide
+    );
+    payload.data.body = this.hideSensitive(payload.data.body, bodyParamsToHide, false);
+    await getStore().save(payload);
+  }
+  /**
+   * Normalize headers: converts keys to lowercase.
+   */
+  normalizeHeaders(headers) {
+    if (!headers || typeof headers !== "object") return {};
+    return Object.fromEntries(
+      Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])
+    );
+  }
+  hideSensitive(obj, keysToHide = [], isObjHeaders = true) {
+    if (!obj || typeof obj !== "object") {
+      return isObjHeaders ? {} : obj;
+    }
+    const clone = { ...obj };
+    for (const key of keysToHide) {
+      if (key in clone) {
+        clone[key] = "*******";
+      }
+    }
+    return clone;
   }
 };
 
@@ -871,6 +1039,7 @@ export {
   Store as LensStore,
   Watcher as LensWatcher,
   QueryWatcher,
+  QueuedSqliteStore,
   RequestWatcher,
   WatcherTypeEnum,
   createEmittery,

package/dist/mixins/queued_store.cjs

@@ -0,0 +1,114 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/mixins/queued_store.ts
+var queued_store_exports = {};
+__export(queued_store_exports, {
+  QueuedStore: () => QueuedStore
+});
+module.exports = __toCommonJS(queued_store_exports);
+var import_denque = __toESM(require("denque"), 1);
+function QueuedStore(Base) {
+  return class Queued extends Base {
+    queue;
+    processingInterval = null;
+    BATCH_SIZE;
+    PROCESS_INTERVAL_MS;
+    WARN_THRESHOLD;
+    PREALLOCATE;
+    constructor(...args) {
+      super(...args);
+      const config = args[0] || {};
+      this.storeConfig = config;
+      this.BATCH_SIZE = config.batchSize ?? 100;
+      this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
+      this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
+      this.PREALLOCATE = config.preallocate ?? true;
+      this.queue = this.PREALLOCATE ? new import_denque.default([], { capacity: this.WARN_THRESHOLD * 2 }) : new import_denque.default();
+    }
+    async initialize() {
+      if (super["initialize"]) {
+        await super["initialize"].call(this);
+      }
+      this.startProcessingQueue();
+      process.on("SIGINT", () => this.shutdown());
+      process.on("SIGTERM", () => this.shutdown());
+    }
+    async truncate() {
+      this.queue.clear();
+      if (super["truncate"]) {
+        await super["truncate"].call(this);
+      }
+    }
+    async save(entry) {
+      this.queue.push(entry);
+      if (this.queue.length > this.WARN_THRESHOLD) {
+        console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
+      }
+    }
+    startProcessingQueue() {
+      if (this.processingInterval) clearInterval(this.processingInterval);
+      this.processingInterval = setInterval(
+        () => this.processQueue(),
+        this.PROCESS_INTERVAL_MS
+      );
+    }
+    async processQueue() {
+      if (this.queue.isEmpty()) return;
+      const batchSize = Math.min(
+        this.BATCH_SIZE,
+        Math.max(10, Math.floor(this.queue.length / 10))
+      );
+      const entriesToProcess = this.queue.remove(0, batchSize);
+      if (!entriesToProcess.length) return;
+      for (const entry of entriesToProcess) {
+        super["save"]?.call(this, entry).catch((error) => {
+          console.error("Error saving queued entry:", error);
+        });
+      }
+    }
+    async stopProcessingQueue() {
+      if (this.processingInterval) {
+        clearInterval(this.processingInterval);
+        this.processingInterval = null;
+      }
+      if (!this.queue.isEmpty()) {
+        await this.processQueue();
+      }
+    }
+    async shutdown() {
+      await this.stopProcessingQueue();
+      process.exit(0);
+    }
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  QueuedStore
+});

package/dist/mixins/queued_store.d.cts

@@ -0,0 +1,38 @@
+import denque__default from 'denque';
+import { Constructor, WatcherTypeEnum } from '../types/index.cjs';
+import 'sql-formatter';
+
+declare function QueuedStore<TBase extends Constructor>(Base: TBase): {
+    new (...args: any[]): {
+        [x: string]: any;
+        queue: denque__default<{
+            id?: string;
+            data: Record<string, any>;
+            minimal_data?: Record<string, any>;
+            type: WatcherTypeEnum;
+            timestamp?: string;
+            requestId?: string;
+        }>;
+        processingInterval: NodeJS.Timeout | null;
+        readonly BATCH_SIZE: number;
+        readonly PROCESS_INTERVAL_MS: number;
+        readonly WARN_THRESHOLD: number;
+        readonly PREALLOCATE: boolean;
+        initialize(): Promise<void>;
+        truncate(): Promise<void>;
+        save(entry: {
+            id?: string;
+            data: Record<string, any>;
+            minimal_data?: Record<string, any>;
+            type: WatcherTypeEnum;
+            timestamp?: string;
+            requestId?: string;
+        }): Promise<void>;
+        startProcessingQueue(): void;
+        processQueue(): Promise<void>;
+        stopProcessingQueue(): Promise<void>;
+        shutdown(): Promise<void>;
+    };
+} & TBase;
+
+export { QueuedStore };

package/dist/mixins/queued_store.d.ts

@@ -0,0 +1,38 @@
+import denque__default from 'denque';
+import { Constructor, WatcherTypeEnum } from '../types/index.js';
+import 'sql-formatter';
+
+declare function QueuedStore<TBase extends Constructor>(Base: TBase): {
+    new (...args: any[]): {
+        [x: string]: any;
+        queue: denque__default<{
+            id?: string;
+            data: Record<string, any>;
+            minimal_data?: Record<string, any>;
+            type: WatcherTypeEnum;
+            timestamp?: string;
+            requestId?: string;
+        }>;
+        processingInterval: NodeJS.Timeout | null;
+        readonly BATCH_SIZE: number;
+        readonly PROCESS_INTERVAL_MS: number;
+        readonly WARN_THRESHOLD: number;
+        readonly PREALLOCATE: boolean;
+        initialize(): Promise<void>;
+        truncate(): Promise<void>;
+        save(entry: {
+            id?: string;
+            data: Record<string, any>;
+            minimal_data?: Record<string, any>;
+            type: WatcherTypeEnum;
+            timestamp?: string;
+            requestId?: string;
+        }): Promise<void>;
+        startProcessingQueue(): void;
+        processQueue(): Promise<void>;
+        stopProcessingQueue(): Promise<void>;
+        shutdown(): Promise<void>;
+    };
+} & TBase;
+
+export { QueuedStore };

package/dist/mixins/queued_store.js

@@ -0,0 +1,79 @@
+// src/mixins/queued_store.ts
+import Denque from "denque";
+function QueuedStore(Base) {
+  return class Queued extends Base {
+    queue;
+    processingInterval = null;
+    BATCH_SIZE;
+    PROCESS_INTERVAL_MS;
+    WARN_THRESHOLD;
+    PREALLOCATE;
+    constructor(...args) {
+      super(...args);
+      const config = args[0] || {};
+      this.storeConfig = config;
+      this.BATCH_SIZE = config.batchSize ?? 100;
+      this.PROCESS_INTERVAL_MS = config.processIntervalMs ?? 100;
+      this.WARN_THRESHOLD = config.warnThreshold ?? 1e5;
+      this.PREALLOCATE = config.preallocate ?? true;
+      this.queue = this.PREALLOCATE ? new Denque([], { capacity: this.WARN_THRESHOLD * 2 }) : new Denque();
+    }
+    async initialize() {
+      if (super["initialize"]) {
+        await super["initialize"].call(this);
+      }
+      this.startProcessingQueue();
+      process.on("SIGINT", () => this.shutdown());
+      process.on("SIGTERM", () => this.shutdown());
+    }
+    async truncate() {
+      this.queue.clear();
+      if (super["truncate"]) {
+        await super["truncate"].call(this);
+      }
+    }
+    async save(entry) {
+      this.queue.push(entry);
+      if (this.queue.length > this.WARN_THRESHOLD) {
+        console.warn(`\u26A0\uFE0F LensJs Queue size very large: ${this.queue.length}`);
+      }
+    }
+    startProcessingQueue() {
+      if (this.processingInterval) clearInterval(this.processingInterval);
+      this.processingInterval = setInterval(
+        () => this.processQueue(),
+        this.PROCESS_INTERVAL_MS
+      );
+    }
+    async processQueue() {
+      if (this.queue.isEmpty()) return;
+      const batchSize = Math.min(
+        this.BATCH_SIZE,
+        Math.max(10, Math.floor(this.queue.length / 10))
+      );
+      const entriesToProcess = this.queue.remove(0, batchSize);
+      if (!entriesToProcess.length) return;
+      for (const entry of entriesToProcess) {
+        super["save"]?.call(this, entry).catch((error) => {
+          console.error("Error saving queued entry:", error);
+        });
+      }
+    }
+    async stopProcessingQueue() {
+      if (this.processingInterval) {
+        clearInterval(this.processingInterval);
+        this.processingInterval = null;
+      }
+      if (!this.queue.isEmpty()) {
+        await this.processQueue();
+      }
+    }
+    async shutdown() {
+      await this.stopProcessingQueue();
+      process.exit(0);
+    }
+  };
+}
+export {
+  QueuedStore
+};