@semics-tech/mongolite 1.3.0 → 1.4.0

package/dist/cjs/sync/mongoSink.js

@@ -0,0 +1,161 @@
+"use strict";
+/**
+ * A {@link SyncSink} that writes to a MongoDB deployment using the official
+ * `mongodb` driver.
+ *
+ * The driver is an **optional peer dependency**, imported dynamically the first
+ * time a connection is opened. Projects that never replicate upstream do not
+ * need it installed, and bundlers that never reach this module do not pull it in.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MongoUpstreamSink = exports.UPDATED_AT_FIELD = exports.VERSION_FIELD = void 0;
+const commands_js_1 = require("./commands.js");
+const mongoExecutor_js_1 = require("./mongoExecutor.js");
+var commands_js_2 = require("./commands.js");
+Object.defineProperty(exports, "VERSION_FIELD", { enumerable: true, get: function () { return commands_js_2.VERSION_FIELD; } });
+Object.defineProperty(exports, "UPDATED_AT_FIELD", { enumerable: true, get: function () { return commands_js_2.UPDATED_AT_FIELD; } });
+class MongoUpstreamSink {
+    constructor(options) {
+        this.options = options;
+        this.name = 'mongodb';
+        this.client = null;
+        this.db = null;
+        this.driver = null;
+        if (!options.connectionString) {
+            throw new Error('MongoUpstreamSink requires a `connectionString`.');
+        }
+        this.idMapping = options.idMapping ?? 'auto';
+        this.versioning = options.versioning ?? true;
+        this.driver = options.driver ?? null;
+    }
+    async connect() {
+        if (this.client && this.db)
+            return;
+        const driver = await this.loadDriver();
+        const client = new driver.MongoClient(this.options.connectionString, this.clientOptions());
+        await client.connect();
+        this.client = client;
+        this.db = client.db(this.options.database);
+    }
+    async apply(operations) {
+        if (!this.db)
+            await this.connect();
+        const db = this.db;
+        if (!db)
+            throw new Error('MongoUpstreamSink is not connected.');
+        const writes = operations.map((op, index) => ({
+            index,
+            op,
+            command: (0, commands_js_1.buildWriteCommand)(op, this.commandOptions()),
+        }));
+        return (0, mongoExecutor_js_1.executeWrites)(db, writes, {
+            writeConcern: this.options.writeConcern ?? { w: 'majority' },
+            versioning: this.versioning,
+            idMapping: this.idMapping,
+            ObjectId: this.driver?.ObjectId,
+        });
+    }
+    /**
+     * Command-building settings. The driver's own `ObjectId` is used when available so
+     * BSON instances match the connection that will serialise them.
+     */
+    commandOptions() {
+        return {
+            idMapping: this.idMapping,
+            versioning: this.versioning,
+            ObjectId: this.driver?.ObjectId,
+        };
+    }
+    async fetch(collection, documentIds) {
+        if (documentIds.length === 0)
+            return [];
+        if (!this.db)
+            await this.connect();
+        const db = this.db;
+        if (!db)
+            throw new Error('MongoUpstreamSink is not connected.');
+        return (0, mongoExecutor_js_1.fetchDocuments)(db, collection, documentIds, this.idMapping, this.driver?.ObjectId);
+    }
+    async close() {
+        const client = this.client;
+        this.client = null;
+        this.db = null;
+        if (client)
+            await client.close();
+    }
+    // ---------------------------------------------------------------- internals
+    async loadDriver() {
+        if (this.driver)
+            return this.driver;
+        try {
+            // Indirect specifier: `mongodb` is optional, and a literal import would
+            // make it a hard build-time dependency for everyone compiling this package.
+            const specifier = 'mongodb';
+            const mod = (await Promise.resolve(`${specifier}`).then(s => __importStar(require(s))));
+            this.driver = mod;
+            return mod;
+        }
+        catch (err) {
+            throw new Error('MongoDB sync requires the `mongodb` driver. Install it with `npm install mongodb`, ' +
+                `or pass a \`driver\` explicitly. Original error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    clientOptions() {
+        const { authSource, authMechanism, authMechanismProperties, username, password, tls, tlsCAFile, tlsCertificateKeyFile, tlsCertificateKeyFilePassword, tlsCRLFile, tlsAllowInvalidHostnames, tlsAllowInvalidCertificates, driverOptions, } = this.options;
+        const auth = username !== undefined || password !== undefined ? { username, password } : undefined;
+        // Undefined keys are stripped so they never override a setting that the
+        // connection string already specified.
+        return stripUndefined({
+            authSource,
+            authMechanism,
+            authMechanismProperties,
+            auth,
+            tls,
+            tlsCAFile,
+            tlsCertificateKeyFile,
+            tlsCertificateKeyFilePassword,
+            tlsCRLFile,
+            tlsAllowInvalidHostnames,
+            tlsAllowInvalidCertificates,
+            ...driverOptions,
+        });
+    }
+}
+exports.MongoUpstreamSink = MongoUpstreamSink;
+/** Drops unset keys so they never override what the connection string specified. */
+function stripUndefined(value) {
+    return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
+}

package/dist/cjs/sync/outbox.js

@@ -0,0 +1,358 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SyncOutbox = exports.SYNC_OBJECT_PREFIX = exports.DEAD_LETTER_TABLE = exports.STATE_TABLE = exports.OUTBOX_TABLE = void 0;
+exports.OUTBOX_TABLE = '__mongolite_sync_outbox__';
+exports.STATE_TABLE = '__mongolite_sync_state__';
+exports.DEAD_LETTER_TABLE = '__mongolite_sync_deadletter__';
+/** Prefix shared by every table and trigger this module creates. */
+exports.SYNC_OBJECT_PREFIX = '__mongolite_sync';
+/** Quotes a SQL identifier, escaping any embedded double quotes. */
+function quoteIdent(name) {
+    return `"${name.replace(/"/g, '""')}"`;
+}
+/** Quotes a SQL string literal, escaping any embedded single quotes. */
+function quoteLiteral(value) {
+    return `'${value.replace(/'/g, "''")}'`;
+}
+/** Millisecond epoch as a SQL expression — portable across SQLite builds. */
+const NOW_MS_SQL = `CAST((julianday('now') - 2440587.5) * 86400000.0 AS INTEGER)`;
+function triggerName(collection, operation) {
+    return `${exports.SYNC_OBJECT_PREFIX}_${operation}__${collection}`;
+}
+/**
+ * Owns the outbox tables, the capture triggers, and every replicator's
+ * checkpoint. A single outbox serves any number of replicators: each keeps its
+ * own checkpoint row, and rows are pruned only once *every* replicator has
+ * passed them.
+ */
+class SyncOutbox {
+    constructor(db) {
+        this.db = db;
+        this.schemaReady = null;
+        this.installedTriggers = new Set();
+    }
+    /** Creates the outbox, checkpoint and dead-letter tables. Idempotent. */
+    async ensureSchema() {
+        if (!this.schemaReady) {
+            this.schemaReady = this.createSchema().catch((err) => {
+                // Let a later call retry rather than caching the failure forever.
+                this.schemaReady = null;
+                throw err;
+            });
+        }
+        return this.schemaReady;
+    }
+    async createSchema() {
+        await this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${exports.OUTBOX_TABLE} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        collection_name TEXT NOT NULL,
+        document_id TEXT NOT NULL,
+        operation TEXT NOT NULL,
+        full_document TEXT,
+        captured_at INTEGER NOT NULL
+      );
+    `);
+        await this.db.exec(`
+      CREATE INDEX IF NOT EXISTS ${exports.SYNC_OBJECT_PREFIX}_outbox_key
+      ON ${exports.OUTBOX_TABLE} (collection_name, document_id, id);
+    `);
+        await this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${exports.STATE_TABLE} (
+        name TEXT PRIMARY KEY,
+        checkpoint INTEGER NOT NULL DEFAULT 0,
+        backfilled TEXT NOT NULL DEFAULT '[]',
+        updated_at INTEGER NOT NULL DEFAULT 0
+      );
+    `);
+        await this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${exports.DEAD_LETTER_TABLE} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        replicator TEXT NOT NULL,
+        collection_name TEXT NOT NULL,
+        document_id TEXT NOT NULL,
+        operation TEXT NOT NULL,
+        full_document TEXT,
+        error TEXT NOT NULL,
+        failed_at INTEGER NOT NULL
+      );
+    `);
+    }
+    /**
+     * Installs the insert/update/delete capture triggers for a collection.
+     *
+     * Triggers deliberately outlive the replicator that created them: changes made
+     * while replication is stopped still need to be captured, or restarting would
+     * silently skip them. Use {@link SyncOutbox.removeTriggers} to opt out.
+     */
+    async ensureTriggers(collection) {
+        if (this.installedTriggers.has(collection))
+            return;
+        await this.ensureSchema();
+        const table = quoteIdent(collection);
+        const literal = quoteLiteral(collection);
+        const insert = (op, row, doc) => `
+      CREATE TRIGGER IF NOT EXISTS ${quoteIdent(triggerName(collection, op))}
+      AFTER ${op === 'insert' ? 'INSERT' : op === 'update' ? 'UPDATE' : 'DELETE'} ON ${table}
+      FOR EACH ROW
+      BEGIN
+        INSERT INTO ${exports.OUTBOX_TABLE}
+          (collection_name, document_id, operation, full_document, captured_at)
+        VALUES
+          (${literal}, ${row}._id, ${quoteLiteral(op)}, ${doc}, ${NOW_MS_SQL});
+      END;
+    `;
+        await this.db.exec(insert('insert', 'NEW', 'NEW.data'));
+        await this.db.exec(insert('update', 'NEW', 'NEW.data'));
+        await this.db.exec(insert('delete', 'OLD', 'NULL'));
+        this.installedTriggers.add(collection);
+    }
+    /**
+     * Removes the capture triggers for a collection. Changes to it stop being
+     * recorded — existing queued rows are untouched.
+     */
+    async removeTriggers(collection) {
+        for (const op of ['insert', 'update', 'delete']) {
+            await this.db.exec(`DROP TRIGGER IF EXISTS ${quoteIdent(triggerName(collection, op))}`);
+        }
+        this.installedTriggers.delete(collection);
+    }
+    /** Lists user collections, excluding MongoLite's own bookkeeping tables. */
+    async listCollections() {
+        const rows = await this.db.all(`SELECT name FROM sqlite_master
+       WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__mongolite%'
+       ORDER BY name`);
+        return rows.map((row) => row.name);
+    }
+    /**
+     * Reconciles the in-memory "already installed" cache with what is actually in
+     * the schema.
+     *
+     * Dropping a table drops its triggers with it, so a collection that was
+     * dropped and recreated would otherwise be silently uncaptured — the cache
+     * would insist its triggers were still there.
+     */
+    async refreshTriggerCache() {
+        const rows = await this.db.all(`SELECT name FROM sqlite_master WHERE type = 'trigger' AND name LIKE '${exports.SYNC_OBJECT_PREFIX}\\_%' ESCAPE '\\'`);
+        const present = new Set(rows.map((row) => row.name));
+        for (const collection of [...this.installedTriggers]) {
+            const complete = ['insert', 'update', 'delete'].every((op) => present.has(triggerName(collection, op)));
+            if (!complete)
+                this.installedTriggers.delete(collection);
+        }
+    }
+    /** The newest outbox id, or 0 when the log is empty. */
+    async currentSequence() {
+        const row = await this.db.get(`SELECT MAX(id) AS seq FROM ${exports.OUTBOX_TABLE}`);
+        return row?.seq ?? 0;
+    }
+    /** Reads the persisted checkpoint for a replicator, creating its state row if absent. */
+    async loadState(name) {
+        await this.ensureSchema();
+        const row = await this.db.get(`SELECT checkpoint, backfilled FROM ${exports.STATE_TABLE} WHERE name = ?`, [name]);
+        if (row) {
+            let backfilled = [];
+            try {
+                const parsed = JSON.parse(row.backfilled);
+                if (Array.isArray(parsed))
+                    backfilled = parsed.filter((v) => typeof v === 'string');
+            }
+            catch {
+                // Corrupt bookkeeping should not wedge replication; treat as "nothing backfilled".
+            }
+            return { checkpoint: Number(row.checkpoint) || 0, backfilled };
+        }
+        // A brand new replicator starts from the current tail. Anything already in
+        // the collections is handled by the backfill, not by replaying the log.
+        const checkpoint = await this.currentSequence();
+        await this.db.run(`INSERT INTO ${exports.STATE_TABLE} (name, checkpoint, backfilled, updated_at)
+       VALUES (?, ?, '[]', ${NOW_MS_SQL})`, [name, checkpoint]);
+        return { checkpoint, backfilled: [] };
+    }
+    /**
+     * Advances a replicator's checkpoint.
+     *
+     * This is the acknowledgement, and it is a single atomic statement: a crash
+     * either leaves the old checkpoint (the batch is replayed, which is safe
+     * because upstream writes are idempotent) or the new one.
+     */
+    async saveCheckpoint(name, checkpoint) {
+        await this.db.run(`UPDATE ${exports.STATE_TABLE} SET checkpoint = ?, updated_at = ${NOW_MS_SQL} WHERE name = ?`, [checkpoint, name]);
+    }
+    /** Records that a collection's existing contents have been backfilled. */
+    async markBackfilled(name, collections) {
+        await this.db.run(`UPDATE ${exports.STATE_TABLE} SET backfilled = ?, updated_at = ${NOW_MS_SQL} WHERE name = ?`, [JSON.stringify(collections), name]);
+    }
+    /** Forgets a replicator, releasing the outbox rows its checkpoint was pinning. */
+    async unregister(name) {
+        await this.ensureSchema();
+        await this.db.run(`DELETE FROM ${exports.STATE_TABLE} WHERE name = ?`, [name]);
+    }
+    /** Reads the next batch of changes after `checkpoint`, oldest first. */
+    async readBatch(checkpoint, limit, collections) {
+        const params = [checkpoint];
+        let filter = '';
+        if (collections) {
+            if (collections.length === 0)
+                return [];
+            filter = ` AND collection_name IN (${collections.map(() => '?').join(', ')})`;
+            params.push(...collections);
+        }
+        params.push(limit);
+        const rows = await this.db.all(`SELECT id, collection_name, document_id, operation, full_document, captured_at
+       FROM ${exports.OUTBOX_TABLE}
+       WHERE id > ?${filter}
+       ORDER BY id ASC
+       LIMIT ?`, params);
+        return rows.map((row) => ({
+            id: Number(row.id),
+            collection: row.collection_name,
+            documentId: row.document_id,
+            operation: row.operation,
+            document: parseDocument(row.full_document, row.document_id),
+            capturedAt: Number(row.captured_at),
+        }));
+    }
+    /** Count of rows a replicator still has to apply. */
+    async pendingCount(checkpoint, collections) {
+        const params = [checkpoint];
+        let filter = '';
+        if (collections) {
+            if (collections.length === 0)
+                return 0;
+            filter = ` AND collection_name IN (${collections.map(() => '?').join(', ')})`;
+            params.push(...collections);
+        }
+        const row = await this.db.get(`SELECT COUNT(*) AS total FROM ${exports.OUTBOX_TABLE} WHERE id > ?${filter}`, params);
+        return Number(row?.total ?? 0);
+    }
+    /**
+     * Deletes rows every registered replicator has already applied.
+     * @returns Number of rows removed.
+     */
+    async prune() {
+        const row = await this.db.get(`SELECT MIN(checkpoint) AS floor FROM ${exports.STATE_TABLE}`);
+        const floor = row?.floor;
+        if (floor === null || floor === undefined)
+            return 0;
+        const result = await this.db.run(`DELETE FROM ${exports.OUTBOX_TABLE} WHERE id <= ?`, [floor]);
+        return result.changes;
+    }
+    /**
+     * Drops superseded revisions, keeping only the newest row per document.
+     *
+     * Safe because replication carries whole documents rather than deltas: the
+     * newest row alone reproduces the same upstream state. Rows at or below the
+     * slowest replicator's checkpoint are left to {@link SyncOutbox.prune}.
+     *
+     * @returns Number of rows removed.
+     */
+    async compact() {
+        const result = await this.db.run(`DELETE FROM ${exports.OUTBOX_TABLE}
+       WHERE id NOT IN (
+         SELECT MAX(id) FROM ${exports.OUTBOX_TABLE} GROUP BY collection_name, document_id
+       )
+       AND id > COALESCE((SELECT MIN(checkpoint) FROM ${exports.STATE_TABLE}), 0)`);
+        return result.changes;
+    }
+    /**
+     * Appends a change back onto the log so it is retried on a later drain.
+     *
+     * Used when a conflict is resolved in favour of the local version: the shadow has
+     * since been refreshed, so the replayed push diffs against current upstream state and
+     * carries the version it actually lost to.
+     */
+    async requeue(collection, documentId, document) {
+        await this.ensureSchema();
+        // `_id` lives in its own column locally and is re-attached on read, so it is
+        // stripped here to match what the capture triggers store.
+        let payload = null;
+        if (document) {
+            const rest = { ...document };
+            delete rest._id;
+            payload = JSON.stringify(rest);
+        }
+        await this.db.run(`INSERT INTO ${exports.OUTBOX_TABLE}
+         (collection_name, document_id, operation, full_document, captured_at)
+       VALUES (?, ?, ?, ?, ${NOW_MS_SQL})`, [collection, documentId, document ? 'update' : 'delete', payload]);
+    }
+    /** Moves a permanently-rejected change out of the stream so it cannot stall replication. */
+    async deadLetter(replicator, record, error) {
+        await this.db.run(`INSERT INTO ${exports.DEAD_LETTER_TABLE}
+         (replicator, collection_name, document_id, operation, full_document, error, failed_at)
+       VALUES (?, ?, ?, ?, ?, ?, ${NOW_MS_SQL})`, [
+            replicator,
+            record.collection,
+            record.documentId,
+            record.operation,
+            record.document === null || record.document === undefined
+                ? null
+                : JSON.stringify(record.document),
+            error,
+        ]);
+    }
+    /** Reads dead-lettered changes, newest first. */
+    async listDeadLetters(replicator, limit = 100) {
+        await this.ensureSchema();
+        const where = replicator ? 'WHERE replicator = ?' : '';
+        const params = replicator ? [replicator, limit] : [limit];
+        const rows = await this.db.all(`SELECT id, collection_name, document_id, operation, full_document, error, failed_at
+       FROM ${exports.DEAD_LETTER_TABLE} ${where} ORDER BY id DESC LIMIT ?`, params);
+        return rows.map((row) => ({
+            id: Number(row.id),
+            collection: row.collection_name,
+            documentId: row.document_id,
+            operation: row.operation,
+            document: parseDocument(row.full_document, row.document_id),
+            error: row.error,
+            failedAt: Number(row.failed_at),
+        }));
+    }
+    /** Removes dead-lettered rows, e.g. after replaying them by hand. */
+    async clearDeadLetters(replicator) {
+        await this.ensureSchema();
+        const result = replicator
+            ? await this.db.run(`DELETE FROM ${exports.DEAD_LETTER_TABLE} WHERE replicator = ?`, [replicator])
+            : await this.db.run(`DELETE FROM ${exports.DEAD_LETTER_TABLE}`);
+        return result.changes;
+    }
+    /**
+     * Streams a collection's current contents in `_id` order, for the initial
+     * backfill. Keyset pagination keeps memory flat on large collections.
+     */
+    async *scanCollection(collection, chunkSize) {
+        // `null` on the first pass so an empty-string `_id` is not skipped.
+        let after = null;
+        for (;;) {
+            const rows = await this.db.all(`SELECT _id, data FROM ${quoteIdent(collection)}
+         ${after === null ? '' : 'WHERE _id > ?'}
+         ORDER BY _id ASC LIMIT ?`, after === null ? [chunkSize] : [after, chunkSize]);
+            if (rows.length === 0)
+                return;
+            yield rows.map((row) => ({
+                documentId: row._id,
+                document: parseDocument(row.data, row._id),
+            }));
+            after = rows[rows.length - 1]._id;
+        }
+    }
+}
+exports.SyncOutbox = SyncOutbox;
+/**
+ * Parses a stored document, re-attaching the `_id` that lives in its own column.
+ * Returns `null` for deletes and for rows whose JSON is unreadable — the caller
+ * dead-letters the latter rather than crashing the replicator.
+ */
+function parseDocument(json, documentId) {
+    if (json === null || json === undefined)
+        return null;
+    try {
+        const parsed = JSON.parse(json);
+        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
+            return null;
+        return { _id: documentId, ...parsed };
+    }
+    catch {
+        return null;
+    }
+}