@powersync/service-module-mongodb-storage 0.0.0-dev-20250108073049

package/dist/storage/MongoBucketStorage.js

@@ -0,0 +1,426 @@
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import { LRUCache } from 'lru-cache/min';
+import * as mongo from 'mongodb';
+import * as timers from 'timers/promises';
+import { storage, sync, utils } from '@powersync/service-core';
+import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+import { v4 as uuid } from 'uuid';
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
+import { generateSlotName } from './implementation/util.js';
+export class MongoBucketStorage extends DisposableObserver {
+    constructor(db, options) {
+        super();
+        this.storageCache = new LRUCache({
+            max: 3,
+            fetchMethod: async (id) => {
+                const doc2 = await this.db.sync_rules.findOne({
+                    _id: id
+                }, { limit: 1 });
+                if (doc2 == null) {
+                    // Deleted in the meantime?
+                    return undefined;
+                }
+                const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
+                return this.getInstance(rules);
+            },
+            dispose: (storage) => {
+                storage[Symbol.dispose]();
+            }
+        });
+        // Nothing is done here until a subscriber starts to iterate
+        this.sharedIter = new sync.BroadcastIterable((signal) => {
+            return this.watchActiveCheckpoint(signal);
+        });
+        this.client = db.client;
+        this.db = db;
+        this.session = this.client.startSession();
+        this.slot_name_prefix = options.slot_name_prefix;
+    }
+    getInstance(options) {
+        let { id, slot_name } = options;
+        if (typeof id == 'bigint') {
+            id = Number(id);
+        }
+        const storage = new MongoSyncBucketStorage(this, id, options, slot_name);
+        this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+        storage.registerListener({
+            batchStarted: (batch) => {
+                // This nested listener will be automatically disposed when the storage is disposed
+                batch.registerManagedListener(storage, {
+                    replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload))
+                });
+            }
+        });
+        return storage;
+    }
+    async configureSyncRules(sync_rules, options) {
+        const next = await this.getNextSyncRulesContent();
+        const active = await this.getActiveSyncRulesContent();
+        if (next?.sync_rules_content == sync_rules) {
+            logger.info('Sync rules from configuration unchanged');
+            return { updated: false };
+        }
+        else if (next == null && active?.sync_rules_content == sync_rules) {
+            logger.info('Sync rules from configuration unchanged');
+            return { updated: false };
+        }
+        else {
+            logger.info('Sync rules updated from configuration');
+            const persisted_sync_rules = await this.updateSyncRules({
+                content: sync_rules,
+                lock: options?.lock
+            });
+            return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
+        }
+    }
+    async slotRemoved(slot_name) {
+        const next = await this.getNextSyncRulesContent();
+        const active = await this.getActiveSyncRulesContent();
+        // In both the below cases, we create a new sync rules instance.
+        // The current one will continue erroring until the next one has finished processing.
+        // TODO: Update
+        if (next != null && next.slot_name == slot_name) {
+            // We need to redo the "next" sync rules
+            await this.updateSyncRules({
+                content: next.sync_rules_content
+            });
+            // Pro-actively stop replicating
+            await this.db.sync_rules.updateOne({
+                _id: next.id,
+                state: storage.SyncRuleState.PROCESSING
+            }, {
+                $set: {
+                    state: storage.SyncRuleState.STOP
+                }
+            });
+        }
+        else if (next == null && active?.slot_name == slot_name) {
+            // Slot removed for "active" sync rules, while there is no "next" one.
+            await this.updateSyncRules({
+                content: active.sync_rules_content
+            });
+            // Pro-actively stop replicating
+            await this.db.sync_rules.updateOne({
+                _id: active.id,
+                state: storage.SyncRuleState.ACTIVE
+            }, {
+                $set: {
+                    state: storage.SyncRuleState.STOP
+                }
+            });
+        }
+    }
+    async updateSyncRules(options) {
+        // Parse and validate before applying any changes
+        const parsed = SqlSyncRules.fromYaml(options.content, {
+            // No schema-based validation at this point
+            schema: undefined,
+            defaultSchema: 'not_applicable', // Not needed for validation
+            throwOnError: true
+        });
+        let rules = undefined;
+        await this.session.withTransaction(async () => {
+            // Only have a single set of sync rules with PROCESSING.
+            await this.db.sync_rules.updateMany({
+                state: storage.SyncRuleState.PROCESSING
+            }, { $set: { state: storage.SyncRuleState.STOP } });
+            const id_doc = await this.db.op_id_sequence.findOneAndUpdate({
+                _id: 'sync_rules'
+            }, {
+                $inc: {
+                    op_id: 1n
+                }
+            }, {
+                upsert: true,
+                returnDocument: 'after'
+            });
+            const id = Number(id_doc.op_id);
+            const slot_name = generateSlotName(this.slot_name_prefix, id);
+            const doc = {
+                _id: id,
+                content: options.content,
+                last_checkpoint: null,
+                last_checkpoint_lsn: null,
+                no_checkpoint_before: null,
+                keepalive_op: null,
+                snapshot_done: false,
+                state: storage.SyncRuleState.PROCESSING,
+                slot_name: slot_name,
+                last_checkpoint_ts: null,
+                last_fatal_error: null,
+                last_keepalive_ts: null
+            };
+            await this.db.sync_rules.insertOne(doc);
+            rules = new MongoPersistedSyncRulesContent(this.db, doc);
+            if (options.lock) {
+                const lock = await rules.lock();
+            }
+        });
+        return rules;
+    }
+    async getActiveSyncRulesContent() {
+        const doc = await this.db.sync_rules.findOne({
+            state: storage.SyncRuleState.ACTIVE
+        }, { sort: { _id: -1 }, limit: 1 });
+        if (doc == null) {
+            return null;
+        }
+        return new MongoPersistedSyncRulesContent(this.db, doc);
+    }
+    async getActiveSyncRules(options) {
+        const content = await this.getActiveSyncRulesContent();
+        return content?.parsed(options) ?? null;
+    }
+    async getNextSyncRulesContent() {
+        const doc = await this.db.sync_rules.findOne({
+            state: storage.SyncRuleState.PROCESSING
+        }, { sort: { _id: -1 }, limit: 1 });
+        if (doc == null) {
+            return null;
+        }
+        return new MongoPersistedSyncRulesContent(this.db, doc);
+    }
+    async getNextSyncRules(options) {
+        const content = await this.getNextSyncRulesContent();
+        return content?.parsed(options) ?? null;
+    }
+    async getReplicatingSyncRules() {
+        const docs = await this.db.sync_rules
+            .find({
+            $or: [{ state: storage.SyncRuleState.ACTIVE }, { state: storage.SyncRuleState.PROCESSING }]
+        })
+            .toArray();
+        return docs.map((doc) => {
+            return new MongoPersistedSyncRulesContent(this.db, doc);
+        });
+    }
+    async getStoppedSyncRules() {
+        const docs = await this.db.sync_rules
+            .find({
+            state: storage.SyncRuleState.STOP
+        })
+            .toArray();
+        return docs.map((doc) => {
+            return new MongoPersistedSyncRulesContent(this.db, doc);
+        });
+    }
+    async getActiveCheckpoint() {
+        const doc = await this.db.sync_rules.findOne({
+            state: storage.SyncRuleState.ACTIVE
+        }, {
+            sort: { _id: -1 },
+            limit: 1,
+            projection: { _id: 1, last_checkpoint: 1, last_checkpoint_lsn: 1 }
+        });
+        return this.makeActiveCheckpoint(doc);
+    }
+    async getStorageMetrics() {
+        const ignoreNotExiting = (e) => {
+            if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') {
+                // Collection doesn't exist - return 0
+                return [{ storageStats: { size: 0 } }];
+            }
+            else {
+                return Promise.reject(e);
+            }
+        };
+        const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
+        if (active_sync_rules == null) {
+            return {
+                operations_size_bytes: 0,
+                parameters_size_bytes: 0,
+                replication_size_bytes: 0
+            };
+        }
+        const operations_aggregate = await this.db.bucket_data
+            .aggregate([
+            {
+                $collStats: {
+                    storageStats: {}
+                }
+            }
+        ])
+            .toArray()
+            .catch(ignoreNotExiting);
+        const parameters_aggregate = await this.db.bucket_parameters
+            .aggregate([
+            {
+                $collStats: {
+                    storageStats: {}
+                }
+            }
+        ])
+            .toArray()
+            .catch(ignoreNotExiting);
+        const replication_aggregate = await this.db.current_data
+            .aggregate([
+            {
+                $collStats: {
+                    storageStats: {}
+                }
+            }
+        ])
+            .toArray()
+            .catch(ignoreNotExiting);
+        return {
+            operations_size_bytes: Number(operations_aggregate[0].storageStats.size),
+            parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size),
+            replication_size_bytes: Number(replication_aggregate[0].storageStats.size)
+        };
+    }
+    async getPowerSyncInstanceId() {
+        let instance = await this.db.instance.findOne({
+            _id: { $exists: true }
+        });
+        if (!instance) {
+            const manager = new lib_mongo.locks.MongoLockManager({
+                collection: this.db.locks,
+                name: `instance-id-insertion-lock`
+            });
+            await manager.lock(async () => {
+                await this.db.instance.insertOne({
+                    _id: uuid()
+                });
+            });
+            instance = await this.db.instance.findOne({
+                _id: { $exists: true }
+            });
+        }
+        return instance._id;
+    }
+    makeActiveCheckpoint(doc) {
+        return {
+            checkpoint: utils.timestampToOpId(doc?.last_checkpoint ?? 0n),
+            lsn: doc?.last_checkpoint_lsn ?? null,
+            hasSyncRules() {
+                return doc != null;
+            },
+            getBucketStorage: async () => {
+                if (doc == null) {
+                    return null;
+                }
+                return (await this.storageCache.fetch(doc._id)) ?? null;
+            }
+        };
+    }
+    /**
+     * Instance-wide watch on the latest available checkpoint (op_id + lsn).
+     */
+    async *watchActiveCheckpoint(signal) {
+        const pipeline = [
+            {
+                $match: {
+                    'fullDocument.state': 'ACTIVE',
+                    operationType: { $in: ['insert', 'update'] }
+                }
+            },
+            {
+                $project: {
+                    operationType: 1,
+                    'fullDocument._id': 1,
+                    'fullDocument.last_checkpoint': 1,
+                    'fullDocument.last_checkpoint_lsn': 1
+                }
+            }
+        ];
+        // Use this form instead of (doc: SyncRuleDocument | null = null),
+        // otherwise we get weird "doc: never" issues.
+        let doc = null;
+        let clusterTime = null;
+        await this.client.withSession(async (session) => {
+            doc = await this.db.sync_rules.findOne({
+                state: storage.SyncRuleState.ACTIVE
+            }, {
+                session,
+                sort: { _id: -1 },
+                limit: 1,
+                projection: {
+                    _id: 1,
+                    last_checkpoint: 1,
+                    last_checkpoint_lsn: 1
+                }
+            });
+            const time = session.clusterTime?.clusterTime ?? null;
+            clusterTime = time;
+        });
+        if (clusterTime == null) {
+            throw new Error('Could not get clusterTime');
+        }
+        if (signal.aborted) {
+            return;
+        }
+        if (doc) {
+            yield this.makeActiveCheckpoint(doc);
+        }
+        const stream = this.db.sync_rules.watch(pipeline, {
+            fullDocument: 'updateLookup',
+            // Start at the cluster time where we got the initial doc, to make sure
+            // we don't skip any updates.
+            // This may result in the first operation being a duplicate, but we filter
+            // it out anyway.
+            startAtOperationTime: clusterTime
+        });
+        signal.addEventListener('abort', () => {
+            stream.close();
+        }, { once: true });
+        let lastOp = null;
+        for await (const update of stream.stream()) {
+            if (signal.aborted) {
+                break;
+            }
+            if (update.operationType != 'insert' && update.operationType != 'update') {
+                continue;
+            }
+            const doc = update.fullDocument;
+            if (doc == null) {
+                continue;
+            }
+            const op = this.makeActiveCheckpoint(doc);
+            // Check for LSN / checkpoint changes - ignore other metadata changes
+            if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
+                lastOp = op;
+                yield op;
+            }
+        }
+    }
+    /**
+     * User-specific watch on the latest checkpoint and/or write checkpoint.
+     */
+    async *watchWriteCheckpoint(user_id, signal) {
+        let lastCheckpoint = null;
+        let lastWriteCheckpoint = null;
+        const iter = wrapWithAbort(this.sharedIter, signal);
+        for await (const cp of iter) {
+            const { checkpoint, lsn } = cp;
+            // lsn changes are not important by itself.
+            // What is important is:
+            // 1. checkpoint (op_id) changes.
+            // 2. write checkpoint changes for the specific user
+            const bucketStorage = await cp.getBucketStorage();
+            if (!bucketStorage) {
+                continue;
+            }
+            const lsnFilters = lsn ? { 1: lsn } : {};
+            const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({
+                user_id,
+                heads: {
+                    ...lsnFilters
+                }
+            });
+            if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
+                // No change - wait for next one
+                // In some cases, many LSNs may be produced in a short time.
+                // Add a delay to throttle the write checkpoint lookup a bit.
+                await timers.setTimeout(20 + 10 * Math.random());
+                continue;
+            }
+            lastWriteCheckpoint = currentWriteCheckpoint;
+            lastCheckpoint = checkpoint;
+            yield { base: cp, writeCheckpoint: currentWriteCheckpoint };
+        }
+    }
+}
+//# sourceMappingURL=MongoBucketStorage.js.map

package/dist/storage/MongoBucketStorage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,yCAAyC,CAAC;AACxE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAE1C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC/E,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAG5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,MAAM,OAAO,kBACX,SAAQ,kBAAwD;IA+BhE,YACE,EAAkB,EAClB,OAEC;QAED,KAAK,EAAE,CAAC;QA7BO,iBAAY,GAAG,IAAI,QAAQ,CAAiC;YAC3E,GAAG,EAAE,CAAC;YACN,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;gBACxB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC3C;oBACE,GAAG,EAAE,EAAE;iBACR,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;gBACF,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,2BAA2B;oBAC3B,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,MAAM,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;gBACnB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,CAAC;SACF,CAAC,CAAC;QAobH,4DAA4D;QAC3C,eAAU,GAAG,IAAI,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,EAAE;YAClE,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QA5aD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,WAAW,CAAC,OAA0C;QACpD,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAChC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACzE,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,mFAAmF;gBACnF,KAAK,CAAC,uBAAuB,CAAC,OAAO,EAAE;oBACrC,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,UAAkB,EAAE,OAA4B;QACvE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,UAAU,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,UAAU,EAAE,CAAC;YACpE,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACrD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC;gBACtD,OAAO,EAAE,UAAU;gBACnB,IAAI,EAAE,OAAO,EAAE,IAAI;aACpB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,gEAAgE;QAChE,qFAAqF;QACrF,eAAe;QACf,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;YAChD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;aACjC,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;QACJ,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,SAAS,IAAI,SAAS,EAAE,CAAC;YAC1D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,kBAAkB;aACnC,CAAC,CAAC;YAEH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,iDAAiD;QACjD,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE;YACpD,2CAA2C;YAC3C,MAAM,EAAE,SAAS;YACjB,aAAa,EAAE,gBAAgB,EAAE,4BAA4B;YAC7D,YAAY,EAAE,IAAI;SACnB,CAAC,CAAC;QAEH,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAChD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,KAAK;gBACpB,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;SACpC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAsC;QAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;SACxC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAsC;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;SAC5F,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;SAClC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;SACpC,EACD;YACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;YACjB,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE;SACnE,CACF,CAAC;QAEF,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,gBAAgB,GAAG,CAAC,CAAU,EAAE,EAAE;YACtC,IAAI,CAAC,YAAY,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBAC7E,sCAAsC;gBACtC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAE3B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAE3B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAE3B,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,sBAAsB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC5C,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK;gBACzB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/B,GAAG,EAAE,IAAI,EAAE;iBACZ,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAS,CAAC,GAAG,CAAC;IACvB,CAAC;IAEO,oBAAoB,CAAC,GAA4B;QACvD,OAAO;YACL,UAAU,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,EAAE,eAAe,IAAI,EAAE,CAAC;YAC7D,GAAG,EAAE,GAAG,EAAE,mBAAmB,IAAI,IAAI;YACrC,YAAY;gBACV,OAAO,GAAG,IAAI,IAAI,CAAC;YACrB,CAAC;YACD,gBAAgB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;YAC1D,CAAC;SACiC,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,CAAC,qBAAqB,CAAC,MAAmB;QACtD,MAAM,QAAQ,GAAqB;YACjC;gBACE,MAAM,EAAE;oBACN,oBAAoB,EAAE,QAAQ;oBAC9B,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE;iBAC7C;aACF;YACD;gBACE,QAAQ,EAAE;oBACR,aAAa,EAAE,CAAC;oBAChB,kBAAkB,EAAE,CAAC;oBACrB,8BAA8B,EAAE,CAAC;oBACjC,kCAAkC,EAAE,CAAC;iBACtC;aACF;SACF,CAAC;QAEF,kEAAkE;QAClE,8CAA8C;QAC9C,IAAI,GAAG,GAAG,IAA+B,CAAC;QAC1C,IAAI,WAAW,GAAG,IAA8B,CAAC;QAEjD,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9C,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CACpC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,OAAO;gBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,eAAe,EAAE,CAAC;oBAClB,mBAAmB,EAAE,CAAC;iBACvB;aACF,CACF,CAAC;YACF,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,IAAI,IAAI,CAAC;YACtD,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE;YAChD,YAAY,EAAE,cAAc;YAC5B,uEAAuE;YACvE,6BAA6B;YAC7B,0EAA0E;YAC1E,iBAAiB;YACjB,oBAAoB,EAAE,WAAW;SAClC,CAAC,CAAC;QAEH,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QAEF,IAAI,MAAM,GAAoC,IAAI,CAAC;QAEnD,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,EAAE,CAAC;gBACzE,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,CAAC,YAAa,CAAC;YACjC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC1C,qEAAqE;YACrE,IAAI,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACjF,MAAM,GAAG,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAOD;;OAEG;IACH,KAAK,CAAC,CAAC,oBAAoB,CAAC,OAAe,EAAE,MAAmB;QAC9D,IAAI,cAAc,GAAsB,IAAI,CAAC;QAC7C,IAAI,mBAAmB,GAAkB,IAAI,CAAC;QAE9C,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;YAE/B,2CAA2C;YAC3C,wBAAwB;YACxB,iCAAiC;YACjC,oDAAoD;YACpD,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAClD,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAA2B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAEjE,MAAM,sBAAsB,GAAG,MAAM,aAAa,CAAC,mBAAmB,CAAC;gBACrE,OAAO;gBACP,KAAK,EAAE;oBACL,GAAG,UAAU;iBACd;aACF,CAAC,CAAC;YAEH,IAAI,sBAAsB,IAAI,mBAAmB,IAAI,UAAU,IAAI,cAAc,EAAE,CAAC;gBAClF,gCAAgC;gBAChC,4DAA4D;gBAC5D,6DAA6D;gBAC7D,MAAM,MAAM,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,SAAS;YACX,CAAC;YAED,mBAAmB,GAAG,sBAAsB,CAAC;YAC7C,cAAc,GAAG,UAAU,CAAC;YAE5B,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,eAAe,EAAE,sBAAsB,EAAE,CAAC;QAC9D,CAAC;IACH,CAAC;CACF"}

package/dist/storage/implementation/MongoBucketBatch.d.ts

@@ -0,0 +1,72 @@
+import { SqlEventDescriptor, SqlSyncRules } from '@powersync/service-sync-rules';
+import * as mongo from 'mongodb';
+import { DisposableObserver } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import { PowerSyncMongo } from './db.js';
+import { CurrentBucket } from './models.js';
+export interface MongoBucketBatchOptions {
+    db: PowerSyncMongo;
+    syncRules: SqlSyncRules;
+    groupId: number;
+    slotName: string;
+    lastCheckpointLsn: string | null;
+    keepaliveOp: string | null;
+    noCheckpointBeforeLsn: string;
+    storeCurrentData: boolean;
+    /**
+     * Set to true for initial replication.
+     */
+    skipExistingRows: boolean;
+}
+export declare class MongoBucketBatch extends DisposableObserver<storage.BucketBatchStorageListener> implements storage.BucketStorageBatch {
+    private readonly client;
+    readonly db: PowerSyncMongo;
+    readonly session: mongo.ClientSession;
+    private readonly sync_rules;
+    private readonly group_id;
+    private readonly slot_name;
+    private readonly storeCurrentData;
+    private readonly skipExistingRows;
+    private batch;
+    private write_checkpoint_batch;
+    /**
+     * Last LSN received associated with a checkpoint.
+     *
+     * This could be either:
+     * 1. A commit LSN.
+     * 2. A keepalive message LSN.
+     */
+    private last_checkpoint_lsn;
+    private no_checkpoint_before_lsn;
+    private persisted_op;
+    /**
+     * For tests only - not for persistence logic.
+     */
+    last_flushed_op: bigint | null;
+    constructor(options: MongoBucketBatchOptions);
+    addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void;
+    get lastCheckpointLsn(): string | null;
+    flush(): Promise<storage.FlushedResult | null>;
+    private flushInner;
+    private replicateBatch;
+    private saveOperation;
+    private withTransaction;
+    private withReplicationTransaction;
+    [Symbol.asyncDispose](): Promise<void>;
+    private lastWaitingLogThottled;
+    commit(lsn: string): Promise<boolean>;
+    keepalive(lsn: string): Promise<boolean>;
+    save(record: storage.SaveOptions): Promise<storage.FlushedResult | null>;
+    /**
+     * Drop is equivalent to TRUNCATE, plus removing our record of the table.
+     */
+    drop(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
+    truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null>;
+    truncateSingle(sourceTable: storage.SourceTable): Promise<bigint>;
+    markSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn: string): Promise<storage.SourceTable[]>;
+    /**
+     * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
+     */
+    protected getTableEvents(table: storage.SourceTable): SqlEventDescriptor[];
+}
+export declare function currentBucketKey(b: CurrentBucket): string;