@semics-tech/mongolite 1.3.0 → 1.4.1

package/dist/cjs/sync/outbox.js

@@ -0,0 +1,358 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SyncOutbox = exports.SYNC_OBJECT_PREFIX = exports.DEAD_LETTER_TABLE = exports.STATE_TABLE = exports.OUTBOX_TABLE = void 0;
+exports.OUTBOX_TABLE = '__mongolite_sync_outbox__';
+exports.STATE_TABLE = '__mongolite_sync_state__';
+exports.DEAD_LETTER_TABLE = '__mongolite_sync_deadletter__';
+/** Prefix shared by every table and trigger this module creates. */
+exports.SYNC_OBJECT_PREFIX = '__mongolite_sync';
+/** Quotes a SQL identifier, escaping any embedded double quotes. */
+function quoteIdent(name) {
+    return `"${name.replace(/"/g, '""')}"`;
+}
+/** Quotes a SQL string literal, escaping any embedded single quotes. */
+function quoteLiteral(value) {
+    return `'${value.replace(/'/g, "''")}'`;
+}
+/** Millisecond epoch as a SQL expression — portable across SQLite builds. */
+const NOW_MS_SQL = `CAST((julianday('now') - 2440587.5) * 86400000.0 AS INTEGER)`;
+function triggerName(collection, operation) {
+    return `${exports.SYNC_OBJECT_PREFIX}_${operation}__${collection}`;
+}
+/**
+ * Owns the outbox tables, the capture triggers, and every replicator's
+ * checkpoint. A single outbox serves any number of replicators: each keeps its
+ * own checkpoint row, and rows are pruned only once *every* replicator has
+ * passed them.
+ */
+class SyncOutbox {
+    constructor(db) {
+        this.db = db;
+        this.schemaReady = null;
+        this.installedTriggers = new Set();
+    }
+    /** Creates the outbox, checkpoint and dead-letter tables. Idempotent. */
+    async ensureSchema() {
+        if (!this.schemaReady) {
+            this.schemaReady = this.createSchema().catch((err) => {
+                // Let a later call retry rather than caching the failure forever.
+                this.schemaReady = null;
+                throw err;
+            });
+        }
+        return this.schemaReady;
+    }
+    async createSchema() {
+        await this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${exports.OUTBOX_TABLE} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        collection_name TEXT NOT NULL,
+        document_id TEXT NOT NULL,
+        operation TEXT NOT NULL,
+        full_document TEXT,
+        captured_at INTEGER NOT NULL
+      );
+    `);
+        await this.db.exec(`
+      CREATE INDEX IF NOT EXISTS ${exports.SYNC_OBJECT_PREFIX}_outbox_key
+      ON ${exports.OUTBOX_TABLE} (collection_name, document_id, id);
+    `);
+        await this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${exports.STATE_TABLE} (
+        name TEXT PRIMARY KEY,
+        checkpoint INTEGER NOT NULL DEFAULT 0,
+        backfilled TEXT NOT NULL DEFAULT '[]',
+        updated_at INTEGER NOT NULL DEFAULT 0
+      );
+    `);
+        await this.db.exec(`
+      CREATE TABLE IF NOT EXISTS ${exports.DEAD_LETTER_TABLE} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        replicator TEXT NOT NULL,
+        collection_name TEXT NOT NULL,
+        document_id TEXT NOT NULL,
+        operation TEXT NOT NULL,
+        full_document TEXT,
+        error TEXT NOT NULL,
+        failed_at INTEGER NOT NULL
+      );
+    `);
+    }
+    /**
+     * Installs the insert/update/delete capture triggers for a collection.
+     *
+     * Triggers deliberately outlive the replicator that created them: changes made
+     * while replication is stopped still need to be captured, or restarting would
+     * silently skip them. Use {@link SyncOutbox.removeTriggers} to opt out.
+     */
+    async ensureTriggers(collection) {
+        if (this.installedTriggers.has(collection))
+            return;
+        await this.ensureSchema();
+        const table = quoteIdent(collection);
+        const literal = quoteLiteral(collection);
+        const insert = (op, row, doc) => `
+      CREATE TRIGGER IF NOT EXISTS ${quoteIdent(triggerName(collection, op))}
+      AFTER ${op === 'insert' ? 'INSERT' : op === 'update' ? 'UPDATE' : 'DELETE'} ON ${table}
+      FOR EACH ROW
+      BEGIN
+        INSERT INTO ${exports.OUTBOX_TABLE}
+          (collection_name, document_id, operation, full_document, captured_at)
+        VALUES
+          (${literal}, ${row}._id, ${quoteLiteral(op)}, ${doc}, ${NOW_MS_SQL});
+      END;
+    `;
+        await this.db.exec(insert('insert', 'NEW', 'NEW.data'));
+        await this.db.exec(insert('update', 'NEW', 'NEW.data'));
+        await this.db.exec(insert('delete', 'OLD', 'NULL'));
+        this.installedTriggers.add(collection);
+    }
+    /**
+     * Removes the capture triggers for a collection. Changes to it stop being
+     * recorded — existing queued rows are untouched.
+     */
+    async removeTriggers(collection) {
+        for (const op of ['insert', 'update', 'delete']) {
+            await this.db.exec(`DROP TRIGGER IF EXISTS ${quoteIdent(triggerName(collection, op))}`);
+        }
+        this.installedTriggers.delete(collection);
+    }
+    /** Lists user collections, excluding MongoLite's own bookkeeping tables. */
+    async listCollections() {
+        const rows = await this.db.all(`SELECT name FROM sqlite_master
+       WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__mongolite%'
+       ORDER BY name`);
+        return rows.map((row) => row.name);
+    }
+    /**
+     * Reconciles the in-memory "already installed" cache with what is actually in
+     * the schema.
+     *
+     * Dropping a table drops its triggers with it, so a collection that was
+     * dropped and recreated would otherwise be silently uncaptured — the cache
+     * would insist its triggers were still there.
+     */
+    async refreshTriggerCache() {
+        const rows = await this.db.all(`SELECT name FROM sqlite_master WHERE type = 'trigger' AND name LIKE '${exports.SYNC_OBJECT_PREFIX}\\_%' ESCAPE '\\'`);
+        const present = new Set(rows.map((row) => row.name));
+        for (const collection of [...this.installedTriggers]) {
+            const complete = ['insert', 'update', 'delete'].every((op) => present.has(triggerName(collection, op)));
+            if (!complete)
+                this.installedTriggers.delete(collection);
+        }
+    }
+    /** The newest outbox id, or 0 when the log is empty. */
+    async currentSequence() {
+        const row = await this.db.get(`SELECT MAX(id) AS seq FROM ${exports.OUTBOX_TABLE}`);
+        return row?.seq ?? 0;
+    }
+    /** Reads the persisted checkpoint for a replicator, creating its state row if absent. */
+    async loadState(name) {
+        await this.ensureSchema();
+        const row = await this.db.get(`SELECT checkpoint, backfilled FROM ${exports.STATE_TABLE} WHERE name = ?`, [name]);
+        if (row) {
+            let backfilled = [];
+            try {
+                const parsed = JSON.parse(row.backfilled);
+                if (Array.isArray(parsed))
+                    backfilled = parsed.filter((v) => typeof v === 'string');
+            }
+            catch {
+                // Corrupt bookkeeping should not wedge replication; treat as "nothing backfilled".
+            }
+            return { checkpoint: Number(row.checkpoint) || 0, backfilled };
+        }
+        // A brand new replicator starts from the current tail. Anything already in
+        // the collections is handled by the backfill, not by replaying the log.
+        const checkpoint = await this.currentSequence();
+        await this.db.run(`INSERT INTO ${exports.STATE_TABLE} (name, checkpoint, backfilled, updated_at)
+       VALUES (?, ?, '[]', ${NOW_MS_SQL})`, [name, checkpoint]);
+        return { checkpoint, backfilled: [] };
+    }
+    /**
+     * Advances a replicator's checkpoint.
+     *
+     * This is the acknowledgement, and it is a single atomic statement: a crash
+     * either leaves the old checkpoint (the batch is replayed, which is safe
+     * because upstream writes are idempotent) or the new one.
+     */
+    async saveCheckpoint(name, checkpoint) {
+        await this.db.run(`UPDATE ${exports.STATE_TABLE} SET checkpoint = ?, updated_at = ${NOW_MS_SQL} WHERE name = ?`, [checkpoint, name]);
+    }
+    /** Records that a collection's existing contents have been backfilled. */
+    async markBackfilled(name, collections) {
+        await this.db.run(`UPDATE ${exports.STATE_TABLE} SET backfilled = ?, updated_at = ${NOW_MS_SQL} WHERE name = ?`, [JSON.stringify(collections), name]);
+    }
+    /** Forgets a replicator, releasing the outbox rows its checkpoint was pinning. */
+    async unregister(name) {
+        await this.ensureSchema();
+        await this.db.run(`DELETE FROM ${exports.STATE_TABLE} WHERE name = ?`, [name]);
+    }
+    /** Reads the next batch of changes after `checkpoint`, oldest first. */
+    async readBatch(checkpoint, limit, collections) {
+        const params = [checkpoint];
+        let filter = '';
+        if (collections) {
+            if (collections.length === 0)
+                return [];
+            filter = ` AND collection_name IN (${collections.map(() => '?').join(', ')})`;
+            params.push(...collections);
+        }
+        params.push(limit);
+        const rows = await this.db.all(`SELECT id, collection_name, document_id, operation, full_document, captured_at
+       FROM ${exports.OUTBOX_TABLE}
+       WHERE id > ?${filter}
+       ORDER BY id ASC
+       LIMIT ?`, params);
+        return rows.map((row) => ({
+            id: Number(row.id),
+            collection: row.collection_name,
+            documentId: row.document_id,
+            operation: row.operation,
+            document: parseDocument(row.full_document, row.document_id),
+            capturedAt: Number(row.captured_at),
+        }));
+    }
+    /** Count of rows a replicator still has to apply. */
+    async pendingCount(checkpoint, collections) {
+        const params = [checkpoint];
+        let filter = '';
+        if (collections) {
+            if (collections.length === 0)
+                return 0;
+            filter = ` AND collection_name IN (${collections.map(() => '?').join(', ')})`;
+            params.push(...collections);
+        }
+        const row = await this.db.get(`SELECT COUNT(*) AS total FROM ${exports.OUTBOX_TABLE} WHERE id > ?${filter}`, params);
+        return Number(row?.total ?? 0);
+    }
+    /**
+     * Deletes rows every registered replicator has already applied.
+     * @returns Number of rows removed.
+     */
+    async prune() {
+        const row = await this.db.get(`SELECT MIN(checkpoint) AS floor FROM ${exports.STATE_TABLE}`);
+        const floor = row?.floor;
+        if (floor === null || floor === undefined)
+            return 0;
+        const result = await this.db.run(`DELETE FROM ${exports.OUTBOX_TABLE} WHERE id <= ?`, [floor]);
+        return result.changes;
+    }
+    /**
+     * Drops superseded revisions, keeping only the newest row per document.
+     *
+     * Safe because replication carries whole documents rather than deltas: the
+     * newest row alone reproduces the same upstream state. Rows at or below the
+     * slowest replicator's checkpoint are left to {@link SyncOutbox.prune}.
+     *
+     * @returns Number of rows removed.
+     */
+    async compact() {
+        const result = await this.db.run(`DELETE FROM ${exports.OUTBOX_TABLE}
+       WHERE id NOT IN (
+         SELECT MAX(id) FROM ${exports.OUTBOX_TABLE} GROUP BY collection_name, document_id
+       )
+       AND id > COALESCE((SELECT MIN(checkpoint) FROM ${exports.STATE_TABLE}), 0)`);
+        return result.changes;
+    }
+    /**
+     * Appends a change back onto the log so it is retried on a later drain.
+     *
+     * Used when a conflict is resolved in favour of the local version: the shadow has
+     * since been refreshed, so the replayed push diffs against current upstream state and
+     * carries the version it actually lost to.
+     */
+    async requeue(collection, documentId, document) {
+        await this.ensureSchema();
+        // `_id` lives in its own column locally and is re-attached on read, so it is
+        // stripped here to match what the capture triggers store.
+        let payload = null;
+        if (document) {
+            const rest = { ...document };
+            delete rest._id;
+            payload = JSON.stringify(rest);
+        }
+        await this.db.run(`INSERT INTO ${exports.OUTBOX_TABLE}
+         (collection_name, document_id, operation, full_document, captured_at)
+       VALUES (?, ?, ?, ?, ${NOW_MS_SQL})`, [collection, documentId, document ? 'update' : 'delete', payload]);
+    }
+    /** Moves a permanently-rejected change out of the stream so it cannot stall replication. */
+    async deadLetter(replicator, record, error) {
+        await this.db.run(`INSERT INTO ${exports.DEAD_LETTER_TABLE}
+         (replicator, collection_name, document_id, operation, full_document, error, failed_at)
+       VALUES (?, ?, ?, ?, ?, ?, ${NOW_MS_SQL})`, [
+            replicator,
+            record.collection,
+            record.documentId,
+            record.operation,
+            record.document === null || record.document === undefined
+                ? null
+                : JSON.stringify(record.document),
+            error,
+        ]);
+    }
+    /** Reads dead-lettered changes, newest first. */
+    async listDeadLetters(replicator, limit = 100) {
+        await this.ensureSchema();
+        const where = replicator ? 'WHERE replicator = ?' : '';
+        const params = replicator ? [replicator, limit] : [limit];
+        const rows = await this.db.all(`SELECT id, collection_name, document_id, operation, full_document, error, failed_at
+       FROM ${exports.DEAD_LETTER_TABLE} ${where} ORDER BY id DESC LIMIT ?`, params);
+        return rows.map((row) => ({
+            id: Number(row.id),
+            collection: row.collection_name,
+            documentId: row.document_id,
+            operation: row.operation,
+            document: parseDocument(row.full_document, row.document_id),
+            error: row.error,
+            failedAt: Number(row.failed_at),
+        }));
+    }
+    /** Removes dead-lettered rows, e.g. after replaying them by hand. */
+    async clearDeadLetters(replicator) {
+        await this.ensureSchema();
+        const result = replicator
+            ? await this.db.run(`DELETE FROM ${exports.DEAD_LETTER_TABLE} WHERE replicator = ?`, [replicator])
+            : await this.db.run(`DELETE FROM ${exports.DEAD_LETTER_TABLE}`);
+        return result.changes;
+    }
+    /**
+     * Streams a collection's current contents in `_id` order, for the initial
+     * backfill. Keyset pagination keeps memory flat on large collections.
+     */
+    async *scanCollection(collection, chunkSize) {
+        // `null` on the first pass so an empty-string `_id` is not skipped.
+        let after = null;
+        for (;;) {
+            const rows = await this.db.all(`SELECT _id, data FROM ${quoteIdent(collection)}
+         ${after === null ? '' : 'WHERE _id > ?'}
+         ORDER BY _id ASC LIMIT ?`, after === null ? [chunkSize] : [after, chunkSize]);
+            if (rows.length === 0)
+                return;
+            yield rows.map((row) => ({
+                documentId: row._id,
+                document: parseDocument(row.data, row._id),
+            }));
+            after = rows[rows.length - 1]._id;
+        }
+    }
+}
+exports.SyncOutbox = SyncOutbox;
+/**
+ * Parses a stored document, re-attaching the `_id` that lives in its own column.
+ * Returns `null` for deletes and for rows whose JSON is unreadable — the caller
+ * dead-letters the latter rather than crashing the replicator.
+ */
+function parseDocument(json, documentId) {
+    if (json === null || json === undefined)
+        return null;
+    try {
+        const parsed = JSON.parse(json);
+        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
+            return null;
+        return { _id: documentId, ...parsed };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/cjs/sync/protocol.js

@@ -0,0 +1,177 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SyncProtocolError = exports.SYNC_CONTENT_TYPE = exports.SYNC_PROTOCOL_VERSION = void 0;
+exports.encodeBody = encodeBody;
+exports.decodeBody = decodeBody;
+exports.toWireOperation = toWireOperation;
+exports.fromWireOperation = fromWireOperation;
+exports.parseApplyRequest = parseApplyRequest;
+exports.parseFetchRequest = parseFetchRequest;
+/**
+ * The wire format shared by the HTTP sink and the receiver that applies its messages.
+ *
+ * Both halves live in this package on purpose: a client and a receiver that disagree
+ * about the protocol lose data quietly, so they are versioned and released together.
+ *
+ * Bodies are **relaxed Extended JSON**, not plain JSON. Plain JSON would flatten
+ * `ObjectId` to a string and `Date` to an ISO string in transit — degrading exactly the
+ * BSON types the sync design goes to some trouble to preserve, and doing it invisibly.
+ *
+ * Relaxed rather than canonical, because canonical encodes every number as `$numberInt`
+ * or `$numberDouble` and revives it as a BSON wrapper object. A local `age: 31` would
+ * then be written upstream as an `Int32` over HTTP but as a double over a direct
+ * connection — the same change producing different stored types depending on transport.
+ * Relaxed keeps plain numbers plain while still preserving `Date`, `ObjectId`, `Binary`
+ * and `Decimal128`. The cost is that 64-bit integers beyond 2^53 lose precision, which
+ * the local store could not represent anyway.
+ */
+const bson_1 = require("bson");
+/**
+ * Protocol version, sent with every request.
+ *
+ * A receiver rejects a version it does not understand rather than guessing, so an
+ * upgraded client talking to an old server fails loudly instead of silently writing
+ * something unintended.
+ */
+exports.SYNC_PROTOCOL_VERSION = 1;
+/** Content type used for request and response bodies. */
+exports.SYNC_CONTENT_TYPE = 'application/json';
+/** Raised when a payload is malformed or speaks a version we do not implement. */
+class SyncProtocolError extends Error {
+    constructor(message, 
+    /** Suggested HTTP status for a receiver to return. */
+    status = 400) {
+        super(message);
+        this.status = status;
+        this.name = 'SyncProtocolError';
+    }
+}
+exports.SyncProtocolError = SyncProtocolError;
+/** Serialises a body to relaxed Extended JSON. */
+function encodeBody(value) {
+    return bson_1.EJSON.stringify(value, { relaxed: true });
+}
+/** Parses a relaxed Extended JSON body back into BSON-typed values. */
+function decodeBody(raw) {
+    if (typeof raw !== 'string') {
+        // Some frameworks parse JSON before we see it. Re-encoding is the only way to get
+        // Extended JSON revived into real BSON types.
+        return bson_1.EJSON.parse(JSON.stringify(raw), { relaxed: true });
+    }
+    try {
+        return bson_1.EJSON.parse(raw, { relaxed: true });
+    }
+    catch (err) {
+        throw new SyncProtocolError(`Malformed sync payload: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+/** Converts an internal operation into its wire form. */
+function toWireOperation(op, command) {
+    const wire = {
+        collection: op.collection,
+        documentId: op.documentId,
+        type: op.type,
+    };
+    if (op.baseVersion !== undefined)
+        wire.baseVersion = op.baseVersion;
+    if (op.diff)
+        wire.diff = op.diff;
+    if (op.document)
+        wire.document = op.document;
+    if (command)
+        wire.command = command;
+    return wire;
+}
+/**
+ * Validates a wire operation and converts it back into an internal one.
+ *
+ * Everything reaching a receiver is untrusted input, so this checks shape rather than
+ * assuming it. `index` and `sourceCollection` are reconstructed locally — a client does
+ * not get to choose them.
+ */
+function fromWireOperation(value, index) {
+    if (typeof value !== 'object' || value === null) {
+        throw new SyncProtocolError(`operations[${index}] is not an object`);
+    }
+    const wire = value;
+    if (typeof wire.collection !== 'string' || wire.collection.length === 0) {
+        throw new SyncProtocolError(`operations[${index}].collection must be a non-empty string`);
+    }
+    if (typeof wire.documentId !== 'string' || wire.documentId.length === 0) {
+        throw new SyncProtocolError(`operations[${index}].documentId must be a non-empty string`);
+    }
+    if (wire.type !== 'upsert' && wire.type !== 'delete') {
+        throw new SyncProtocolError(`operations[${index}].type must be "upsert" or "delete"`);
+    }
+    if (wire.baseVersion !== undefined &&
+        wire.baseVersion !== null &&
+        typeof wire.baseVersion !== 'number') {
+        throw new SyncProtocolError(`operations[${index}].baseVersion must be a number or null`);
+    }
+    const op = {
+        collection: wire.collection,
+        sourceCollection: wire.collection,
+        documentId: wire.documentId,
+        type: wire.type,
+        outboxId: index,
+        baseVersion: wire.baseVersion ?? null,
+    };
+    if (isPlainRecord(wire.document))
+        op.document = wire.document;
+    if (wire.diff !== undefined) {
+        if (!isPlainRecord(wire.diff)) {
+            throw new SyncProtocolError(`operations[${index}].diff must be an object`);
+        }
+        const set = wire.diff.set;
+        const unset = wire.diff.unset;
+        if (!isPlainRecord(set) || !Array.isArray(unset) || unset.some((p) => typeof p !== 'string')) {
+            throw new SyncProtocolError(`operations[${index}].diff must be { set: object, unset: string[] }`);
+        }
+        op.diff = { set, unset: unset };
+    }
+    if (op.type === 'upsert' && !op.diff && !op.document) {
+        throw new SyncProtocolError(`operations[${index}] must carry a diff or a document`);
+    }
+    return op;
+}
+/** Validates the envelope of an apply request and returns its operations. */
+function parseApplyRequest(body) {
+    const request = requireEnvelope(body);
+    const operations = request.operations;
+    if (!Array.isArray(operations)) {
+        throw new SyncProtocolError('`operations` must be an array');
+    }
+    return {
+        replicator: typeof request.replicator === 'string' ? request.replicator : 'unknown',
+        operations: operations.map((value, index) => fromWireOperation(value, index)),
+    };
+}
+/** Validates the envelope of a fetch request. */
+function parseFetchRequest(body) {
+    const request = requireEnvelope(body);
+    if (typeof request.collection !== 'string' || request.collection.length === 0) {
+        throw new SyncProtocolError('`collection` must be a non-empty string');
+    }
+    if (!Array.isArray(request.documentIds) ||
+        request.documentIds.some((id) => typeof id !== 'string')) {
+        throw new SyncProtocolError('`documentIds` must be an array of strings');
+    }
+    return { collection: request.collection, documentIds: request.documentIds };
+}
+function requireEnvelope(body) {
+    if (!isPlainRecord(body)) {
+        throw new SyncProtocolError('Request body must be an object');
+    }
+    // Coerced rather than compared strictly: a proxy that re-encodes the body could hand
+    // us the version as a string or a BSON number wrapper, and refusing on that would be
+    // a confusing failure for something entirely cosmetic.
+    if (Number(body.protocol) !== exports.SYNC_PROTOCOL_VERSION) {
+        throw new SyncProtocolError(`Unsupported sync protocol ${String(body.protocol)}; this receiver implements ${exports.SYNC_PROTOCOL_VERSION}`, 
+        // 426 tells the client the fix is to upgrade, not to retry.
+        426);
+    }
+    return body;
+}
+function isPlainRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}