@semics-tech/mongolite 1.1.0 → 1.3.0

package/dist/cjs/cursors/findCursor.js

@@ -2,6 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.FindCursor = void 0;
 const projection_js_1 = require("../utils/projection.js");
+/**
+ * Matches compiled clauses that invoke SQLite's json1 functions, which are the
+ * ones that throw on malformed JSON. See `buildGuardedWhereClause`.
+ */
+const JSON1_FUNCTION_PATTERN = /\bjson_(?:extract|each)\s*\(/i;
 /**
  * Safely parses JSON data with fallback mechanisms for malformed JSON.
  * @param jsonString The JSON string to parse
@@ -155,17 +160,18 @@ class FindCursor {
         }
         // For non-Date values, handle both array and non-array fields
         const arrayPath = this.parseJsonPath(field);
-        const isArrayCheck = `json_type(json_extract(${elementPrefix}, ${arrayPath})) = 'array'`;
+        const isArrayCheck = this.buildIsArrayCheck(elementPrefix, arrayPath);
+        const eachArg = this.buildJsonEachArg(elementPrefix, arrayPath);
         const placeholders = value.map(() => '?').join(',');
         let arrayCondition;
         let nonArrayCondition;
         if (operator === '$in') {
-            arrayCondition = `EXISTS (SELECT 1 FROM json_each(json_extract(${elementPrefix}, ${arrayPath})) WHERE json_each.value IN (${placeholders}))`;
+            arrayCondition = `EXISTS (SELECT 1 FROM json_each(${eachArg}) WHERE json_each.value IN (${placeholders}))`;
             nonArrayCondition = `(${jsonPath} IN (${placeholders}))`;
         }
         else {
             // $nin
-            arrayCondition = `NOT EXISTS (SELECT 1 FROM json_each(json_extract(${elementPrefix}, ${arrayPath})) WHERE json_each.value IN (${placeholders}))`;
+            arrayCondition = `NOT EXISTS (SELECT 1 FROM json_each(${eachArg}) WHERE json_each.value IN (${placeholders}))`;
             nonArrayCondition = `(${jsonPath} NOT IN (${placeholders}) OR ${jsonPath} IS NULL)`;
         }
         // Push parameters for both array and non-array conditions
@@ -173,9 +179,48 @@ class FindCursor {
         params.push(...value.map((v) => toSQLiteValue(v))); // for non-array condition
         return `(
       (${isArrayCheck} AND ${arrayCondition}) OR
-      (json_type(json_extract(${elementPrefix}, ${arrayPath})) != 'array' AND ${nonArrayCondition})
+      (${this.buildIsNotArrayCheck(elementPrefix, arrayPath)} AND ${nonArrayCondition})
     )`;
     }
+    /**
+     * SQL testing whether the value at `arrayPath` is a JSON array.
+     *
+     * Uses `json_type`'s two-argument form. The one-argument form —
+     * `json_type(json_extract(root, path))` — re-parses the *extracted value* as
+     * JSON, which raises `malformed JSON` for any plain string field: for
+     * `{"role":"admin"}` `json_extract` yields the bare text `admin`, and
+     * `json_type('admin')` is not valid JSON. That made `$in`/`$nin`/`$all`/
+     * `$elemMatch` throw on every non-array field, on healthy data.
+     *
+     * The two-argument form also classifies correctly where the one-argument form
+     * lied: a string field holding `"[1,2]"` reported as `array` rather than `text`.
+     */
+    buildIsArrayCheck(root, arrayPath) {
+        return `(json_valid(${root}) AND json_type(${root}, ${arrayPath}) = 'array')`;
+    }
+    /**
+     * Negation of {@link buildIsArrayCheck} — but not just `!=`. When `arrayPath` doesn't exist
+     * in the document, `json_type` returns SQL NULL, and `NULL != 'array'` is itself NULL (falsy),
+     * not TRUE. That silently dropped missing-field documents from the "not array" branch of
+     * $in/$nin/$all, even though they're exactly the case `IS NULL` handling downstream expects to
+     * see. `IS NOT` treats NULL as distinct from 'array', so missing fields correctly count as
+     * "not an array" here.
+     */
+    buildIsNotArrayCheck(root, arrayPath) {
+        return `(json_valid(${root}) AND json_type(${root}, ${arrayPath}) IS NOT 'array')`;
+    }
+    /**
+     * Builds a safe argument for `json_each(...)`.
+     *
+     * SQLite evaluates a table-valued function inside a correlated subquery even
+     * when a preceding `AND` term (such as the is-array check) is false, so
+     * guarding *outside* the call isn't enough — `json_each('admin')` still runs
+     * and throws. The guard therefore has to live inside the argument: anything
+     * that isn't a valid JSON array becomes an empty array, which yields no rows.
+     */
+    buildJsonEachArg(root, arrayPath) {
+        return `CASE WHEN json_valid(${root}) AND json_type(${root}, ${arrayPath}) = 'array' THEN json_extract(${root}, ${arrayPath}) ELSE '[]' END`;
+    }
     /**
      * Helper method to generate a comparison condition
      */
@@ -376,7 +421,8 @@ class FindCursor {
      */
     buildArraySubquery(field, operator, value, params) {
         const arrayPath = this.parseJsonPath(field);
-        const arrayTypeCheck = `json_type(json_extract(data, ${arrayPath})) = 'array'`;
+        const arrayTypeCheck = this.buildIsArrayCheck('data', arrayPath);
+        const eachArg = this.buildJsonEachArg('data', arrayPath);
         if (operator === '$all') {
             if (Array.isArray(value) && value.length > 0) {
                 const allConditions = value
@@ -393,12 +439,12 @@ class FindCursor {
                             objectConditions.push(`json_extract(json_each.value, '$.${key}') = ?`);
                         }
                         const condition = objectConditions.join(' AND ');
-                        return `EXISTS (SELECT 1 FROM json_each(json_extract(data, ${arrayPath})) WHERE ${condition})`;
+                        return `EXISTS (SELECT 1 FROM json_each(${eachArg}) WHERE ${condition})`;
                     }
                     else {
                         // For simple values, use direct comparison
                         params.push(toSQLiteValue(item));
-                        return `EXISTS (SELECT 1 FROM json_each(json_extract(data, ${arrayPath})) WHERE json_each.value = ?)`;
+                        return `EXISTS (SELECT 1 FROM json_each(${eachArg}) WHERE json_each.value = ?)`;
                     }
                 })
                     .join(' AND ');
@@ -425,12 +471,12 @@ class FindCursor {
                                 objectConditions.push(`json_extract(json_each.value, '$.${key}') = ?`);
                             }
                             const condition = objectConditions.join(' AND ');
-                            return `EXISTS (SELECT 1 FROM json_each(json_extract(data, ${arrayPath})) WHERE ${condition})`;
+                            return `EXISTS (SELECT 1 FROM json_each(${eachArg}) WHERE ${condition})`;
                         }
                         else {
                             // For scalar values
                             params.push(toSQLiteValue(item));
-                            return `EXISTS (SELECT 1 FROM json_each(json_extract(data, ${arrayPath})) WHERE json_each.value = ?)`;
+                            return `EXISTS (SELECT 1 FROM json_each(${eachArg}) WHERE json_each.value = ?)`;
                         }
                     })
                         .join(' OR ');
@@ -446,7 +492,7 @@ class FindCursor {
         else if (operator === '$elemMatch') {
             let elemMatchSubquery = `EXISTS (
         SELECT 1 
-        FROM json_each(json_extract(data, ${arrayPath})) as array_elements 
+        FROM json_each(${eachArg}) as array_elements 
         WHERE `;
             if (typeof value === 'object' && value !== null) {
                 const conditions = this.buildObjectConditions(value, params, 'array_elements');
@@ -629,9 +675,40 @@ class FindCursor {
         }
         return conditions.length > 0 ? conditions.join(' AND ') : '1=1';
     }
+    /**
+     * Wraps a compiled WHERE clause so that a single corrupted row can't take out
+     * every filtered query on the collection.
+     *
+     * SQLite's json1 functions (`json_extract`, `json_each`) raise
+     * `SqliteError: malformed JSON` as soon as they evaluate a row whose `data`
+     * column isn't valid JSON — including rows that would never have matched the
+     * filter anyway. One bad document therefore fails *all* filtered reads against
+     * that collection, not just the ones that would have returned it.
+     *
+     * Prefixing with `json_valid(data)` lets SQLite short-circuit those rows before
+     * any json1 function touches them, so corruption degrades to "this document is
+     * missing from results" rather than "this collection is unqueryable". Use
+     * {@link MongoLiteCollection.findInvalidJson} to find and repair the offenders.
+     *
+     * Applied only at the top level, and only when json1 functions are genuinely
+     * present, both deliberately:
+     * - `$not`/`$nor` compile their sub-clauses into `NOT (...)`. A guard nested
+     *   inside one would invert to `NOT (0 AND …)` — making corrupt rows *match* a
+     *   negated filter.
+     * - An empty filter compiles to `1=1`, touches no json1 function, and so can't
+     *   throw. Leaving it unguarded keeps `deleteMany({})` able to purge corrupt
+     *   rows, which is the escape hatch for cleaning them up.
+     */
+    buildGuardedWhereClause(filter, params) {
+        const whereClause = this.buildWhereClause(filter, params);
+        if (!JSON1_FUNCTION_PATTERN.test(whereClause)) {
+            return whereClause;
+        }
+        return `json_valid(data) AND (${whereClause})`;
+    }
     buildSelectQuery(filter) {
         const params = [];
-        const whereClause = this.buildWhereClause(filter, params);
+        const whereClause = this.buildGuardedWhereClause(filter, params);
         const sql = `SELECT _id, data FROM "${this.collectionName}" WHERE ${whereClause}`;
         if (this.options.verbose) {
             console.log(`Building SQL query for collection "${this.collectionName}"`);
@@ -702,6 +779,25 @@ class FindCursor {
      * @returns A promise that resolves to an array of documents.
      */
     async toArray() {
+        const { sql: finalSql, params: finalParams } = this.toSQL();
+        // Handle malformed json in the database
+        const rows = await this.db.all(finalSql, finalParams);
+        return rows.map((row) => {
+            const parsedData = safeJsonParse(row.data, `findCursor toArray for document ${row._id}`);
+            const restoredData = restoreDates(parsedData);
+            const doc = { _id: row._id, ...restoredData };
+            return this.applyProjection(doc);
+        });
+    }
+    /**
+     * Returns the SQL this cursor will execute, along with its bound parameters.
+     *
+     * Useful for debugging and for tools that want to show how a MongoDB-style
+     * filter was translated. Reflects any `sort`, `limit` and `skip` applied so
+     * far; `project` is applied in JavaScript after the rows come back and so
+     * does not appear here.
+     */
+    toSQL() {
         let finalSql = this.queryParts.sql;
         const finalParams = [...this.queryParts.params];
         if (this.sortCriteria) {
@@ -728,14 +824,7 @@ class FindCursor {
             finalSql += ` OFFSET ?`;
             finalParams.push(this.skipCount);
         }
-        // Handle malformed json in the database
-        const rows = await this.db.all(finalSql, finalParams);
-        return rows.map((row) => {
-            const parsedData = safeJsonParse(row.data, `findCursor toArray for document ${row._id}`);
-            const restoredData = restoreDates(parsedData);
-            const doc = { _id: row._id, ...restoredData };
-            return this.applyProjection(doc);
-        });
+        return { sql: finalSql, params: finalParams };
     }
     /**
      *  Executes the query and returns the first matching document.

package/dist/cjs/db.js

@@ -3,8 +3,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SQLiteDB = void 0;
+exports.SQLiteDB = exports.DEFAULT_BUSY_TIMEOUT_MS = void 0;
+exports.sharedConnectionCount = sharedConnectionCount;
 const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const node_path_1 = __importDefault(require("node:path"));
+const connection_registry_js_1 = require("./utils/connection-registry.js");
+Object.defineProperty(exports, "DEFAULT_BUSY_TIMEOUT_MS", { enumerable: true, get: function () { return connection_registry_js_1.DEFAULT_BUSY_TIMEOUT_MS; } });
+/** Shared `better-sqlite3` handles for this process, keyed by connection identity. */
+const sharedConnections = new connection_registry_js_1.ConnectionRegistry();
+/** Exposed for tests and diagnostics. */
+function sharedConnectionCount() {
+    return sharedConnections.size;
+}
 /**
  * SQLiteDB class provides a wrapper around the better-sqlite3 library
  * to simplify database operations.
@@ -18,18 +28,34 @@ class SQLiteDB {
     constructor(dbPathOrOptions) {
         this.db = null;
         this.openPromise = null;
+        let shared;
         if (typeof dbPathOrOptions === 'string') {
             this.filePath = dbPathOrOptions;
             this.verbose = false;
             this.readOnly = false;
-            this.WAL = false;
+            this.WAL = true;
+            this.busyTimeout = connection_registry_js_1.DEFAULT_BUSY_TIMEOUT_MS;
+            shared = false;
         }
         else {
             this.filePath = dbPathOrOptions.filePath;
             this.verbose = dbPathOrOptions.verbose || false;
             this.readOnly = dbPathOrOptions.readOnly || false;
-            this.WAL = dbPathOrOptions.WAL || true;
+            this.WAL = dbPathOrOptions.WAL ?? true;
+            this.busyTimeout = dbPathOrOptions.busyTimeout ?? connection_registry_js_1.DEFAULT_BUSY_TIMEOUT_MS;
+            shared = dbPathOrOptions.shared ?? false;
         }
+        this.shareKey =
+            shared && (0, connection_registry_js_1.isShareablePath)(this.filePath)
+                ? (0, connection_registry_js_1.buildConnectionKey)({
+                    backend: 'better-sqlite3',
+                    filePath: node_path_1.default.resolve(this.filePath),
+                    readOnly: this.readOnly,
+                    WAL: this.WAL,
+                    busyTimeout: this.busyTimeout,
+                    verbose: this.verbose,
+                })
+                : null;
     }
     /**
      * Opens the database connection if it's not already open.
@@ -45,44 +71,57 @@ class SQLiteDB {
         }
         this.openPromise = new Promise((resolve, reject) => {
             try {
-                const options = {
-                    readonly: this.readOnly,
-                    verbose: this.verbose ? console.log : undefined,
-                };
-                this.db = new better_sqlite3_1.default(this.filePath, options);
-                // Enable Write-Ahead Logging if requested
-                if (this.WAL && !this.readOnly) {
-                    this.db.pragma('journal_mode = WAL');
-                }
-                // Register regexp UDF for $regex operator support.
-                // WARNING: Patterns are compiled via JavaScript RegExp. User-supplied patterns that are
-                // not validated for catastrophic backtracking (ReDoS) could block the event loop.
-                // Avoid using untrusted/unvalidated patterns with $regex in security-sensitive contexts.
-                this.db.function('regexp', { deterministic: true }, (pattern, value) => {
-                    if (typeof pattern !== 'string' || value === null || value === undefined)
-                        return 0;
-                    try {
-                        return new RegExp(pattern).test(String(value)) ? 1 : 0;
-                    }
-                    catch {
-                        return 0;
+                // Only runs on a genuinely new connection — when sharing, an existing
+                // handle already has its pragmas and UDFs applied.
+                const openConnection = () => {
+                    const options = {
+                        readonly: this.readOnly,
+                        verbose: this.verbose ? console.log : undefined,
+                    };
+                    const db = new better_sqlite3_1.default(this.filePath, options);
+                    // Enable Write-Ahead Logging if requested
+                    if (this.WAL && !this.readOnly) {
+                        db.pragma('journal_mode = WAL');
                     }
-                });
-                // Register regexp_flags UDF for $regex with $options support
-                this.db.function('regexp_flags', { deterministic: true }, (pattern, flags, value) => {
-                    if (typeof pattern !== 'string' || value === null || value === undefined)
-                        return 0;
-                    try {
-                        const f = typeof flags === 'string' ? flags : '';
-                        return new RegExp(pattern, f).test(String(value)) ? 1 : 0;
+                    // Wait rather than immediately throwing SQLITE_BUSY when another
+                    // connection (often another process) holds a lock.
+                    if (this.busyTimeout > 0) {
+                        db.pragma(`busy_timeout = ${this.busyTimeout}`);
                     }
-                    catch {
-                        return 0;
+                    // Register regexp UDF for $regex operator support.
+                    // WARNING: Patterns are compiled via JavaScript RegExp. User-supplied patterns that are
+                    // not validated for catastrophic backtracking (ReDoS) could block the event loop.
+                    // Avoid using untrusted/unvalidated patterns with $regex in security-sensitive contexts.
+                    db.function('regexp', { deterministic: true }, (pattern, value) => {
+                        if (typeof pattern !== 'string' || value === null || value === undefined)
+                            return 0;
+                        try {
+                            return new RegExp(pattern).test(String(value)) ? 1 : 0;
+                        }
+                        catch {
+                            return 0;
+                        }
+                    });
+                    // Register regexp_flags UDF for $regex with $options support
+                    db.function('regexp_flags', { deterministic: true }, (pattern, flags, value) => {
+                        if (typeof pattern !== 'string' || value === null || value === undefined)
+                            return 0;
+                        try {
+                            const f = typeof flags === 'string' ? flags : '';
+                            return new RegExp(pattern, f).test(String(value)) ? 1 : 0;
+                        }
+                        catch {
+                            return 0;
+                        }
+                    });
+                    if (this.verbose) {
+                        console.log(`SQLite database opened: ${this.filePath}`);
                     }
-                });
-                if (this.verbose) {
-                    console.log(`SQLite database opened: ${this.filePath}`);
-                }
+                    return db;
+                };
+                this.db = this.shareKey
+                    ? sharedConnections.acquire(this.shareKey, openConnection)
+                    : openConnection();
                 resolve();
             }
             catch (err) {
@@ -191,9 +230,17 @@ class SQLiteDB {
         }
         if (this.db) {
             try {
-                this.db.close();
+                // When shared, only the last holder actually closes the handle.
+                const isLastHolder = this.shareKey
+                    ? sharedConnections.release(this.shareKey, this.db)
+                    : true;
+                if (isLastHolder) {
+                    this.db.close();
+                }
                 if (this.verbose) {
-                    console.log(`SQLite database closed: ${this.filePath}`);
+                    console.log(isLastHolder
+                        ? `SQLite database closed: ${this.filePath}`
+                        : `SQLite database released (still in use): ${this.filePath}`);
                 }
             }
             catch (err) {

package/dist/cjs/utils/connection-registry.js

@@ -0,0 +1,122 @@
+"use strict";
+/**
+ * Process-wide registry of open SQLite handles, keyed by connection identity.
+ *
+ * Consumers frequently end up with several `MongoLite` instances pointing at the
+ * same database file — a module-level client here, a background job there, each
+ * calling `new MongoLite(path)` independently. Every one of those opens its own
+ * SQLite connection, which turns same-process reads/writes into needless lock
+ * contention and forces each consumer to hand-roll its own instance cache.
+ *
+ * When `shared` is enabled, adapters acquire their underlying handle through this
+ * registry instead of opening one directly, so N adapter instances against the
+ * same identity share a single connection. Each adapter still tracks its own
+ * open/closed state; the handle is only really closed once the last holder
+ * releases it (reference counting), so one consumer calling `close()` can't pull
+ * the connection out from under another.
+ *
+ * Note this only dedupes connections *within a process*. Separate processes (e.g.
+ * worker threads spawned as child processes, or a job runner forking workers)
+ * each get their own registry and therefore their own connection — cross-process
+ * contention is handled by WAL mode and `busy_timeout`, not by sharing.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConnectionRegistry = exports.DEFAULT_BUSY_TIMEOUT_MS = void 0;
+exports.buildConnectionKey = buildConnectionKey;
+exports.isShareablePath = isShareablePath;
+/**
+ * Default `busy_timeout`, in milliseconds, applied to new connections.
+ *
+ * Lives here rather than in `db.ts` so the `node:sqlite` adapter can read it
+ * without importing that module — `db.ts` pulls in `better-sqlite3`, which is an
+ * optional dependency and may not be installed.
+ */
+exports.DEFAULT_BUSY_TIMEOUT_MS = 5000;
+class ConnectionRegistry {
+    constructor() {
+        this.entries = new Map();
+    }
+    /**
+     * Returns the handle registered for `key`, opening one via `open()` on first
+     * use. Increments the reference count for existing entries.
+     */
+    acquire(key, open) {
+        const existing = this.entries.get(key);
+        if (existing) {
+            existing.refCount++;
+            return existing.handle;
+        }
+        const handle = open();
+        this.entries.set(key, { handle, refCount: 1 });
+        return handle;
+    }
+    /**
+     * Drops one reference to `key`.
+     * @returns `true` when the caller held the last reference and is therefore
+     * responsible for closing the handle; `false` while other holders remain.
+     */
+    release(key, handle) {
+        const entry = this.entries.get(key);
+        // Untracked handle (never shared, or already fully released) — the caller
+        // owns it outright and should close it.
+        if (!entry || entry.handle !== handle) {
+            return true;
+        }
+        entry.refCount--;
+        if (entry.refCount > 0) {
+            return false;
+        }
+        this.entries.delete(key);
+        return true;
+    }
+    /** Live reference count for `key`; 0 when nothing holds it. */
+    refCount(key) {
+        return this.entries.get(key)?.refCount ?? 0;
+    }
+    /** Number of distinct shared handles currently open. */
+    get size() {
+        return this.entries.size;
+    }
+    /**
+     * Forgets every entry without closing the handles.
+     * Intended for test isolation, not for production teardown.
+     */
+    clear() {
+        this.entries.clear();
+    }
+}
+exports.ConnectionRegistry = ConnectionRegistry;
+/**
+ * Builds a stable registry key. Connections may only be shared when every
+ * setting that affects the handle's behaviour matches, so each of these is part
+ * of the identity rather than just the file path.
+ */
+function buildConnectionKey(parts) {
+    return JSON.stringify([
+        parts.backend,
+        parts.filePath,
+        parts.readOnly,
+        parts.WAL,
+        parts.busyTimeout,
+        parts.verbose,
+    ]);
+}
+/**
+ * Whether a database path is eligible for sharing.
+ *
+ * `:memory:` databases are private to their connection — two `:memory:` opens are
+ * two unrelated databases, so sharing them would silently merge state that
+ * callers expect to be isolated. Anonymous temp databases (empty path) behave the
+ * same way.
+ */
+function isShareablePath(filePath) {
+    const normalised = filePath.trim().toLowerCase();
+    if (normalised === '' || normalised === ':memory:') {
+        return false;
+    }
+    // file: URIs opt into sharing only when not explicitly in-memory.
+    if (normalised.startsWith('file:') && normalised.includes('mode=memory')) {
+        return false;
+    }
+    return true;
+}

package/dist/cli/discover.d.ts

@@ -0,0 +1,40 @@
+import type { IDatabaseAdapter } from '../db.js';
+export interface DatabaseCandidate {
+    /** Absolute path to the file. */
+    path: string;
+    /** Path relative to the search root, for display. */
+    relativePath: string;
+    sizeBytes: number;
+    modified: Date;
+}
+export interface FindDatabasesOptions {
+    /** How many directory levels below the root to search. Default 2. */
+    depth?: number;
+    /** Stop after this many files. Default 50. */
+    limit?: number;
+}
+/**
+ * Walk `root` looking for SQLite-looking files, skipping the usual noisy
+ * directories. Results are newest-first, since the file you just wrote is
+ * almost always the one you want.
+ */
+export declare function findDatabaseFiles(root?: string, options?: FindDatabasesOptions): DatabaseCandidate[];
+export declare function formatBytes(bytes: number): string;
+export declare function formatRelativeTime(date: Date, now?: Date): string;
+export interface CollectionInfo {
+    name: string;
+    /** Number of documents (rows) in the collection. */
+    count: number;
+    /** False for tables that are not shaped like a MongoLite collection. */
+    isMongoLite: boolean;
+}
+/**
+ * List the tables in the database, flagging which ones look like MongoLite
+ * collections (an `_id` column plus a `data` column). Foreign tables are still
+ * reported so a user pointed at the wrong file gets a useful message rather
+ * than an empty list.
+ */
+export declare function listCollectionInfo(db: IDatabaseAdapter): Promise<CollectionInfo[]>;
+/** Resolve a user-supplied database path, allowing `:memory:` and `~` shorthand. */
+export declare function resolveDatabasePath(input: string): string;
+export declare function databaseExists(dbPath: string): boolean;