@powersync/service-core 0.0.2

package/dist/storage/MongoBucketStorage.js

@@ -0,0 +1,388 @@
+import * as timers from 'timers/promises';
+import { LRUCache } from 'lru-cache/min';
+import * as micro from '@journeyapps-platform/micro';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import * as replication from '../replication/replication-index.js';
+import * as sync from '../sync/sync-index.js';
+import * as util from '../util/util-index.js';
+import { MongoPersistedSyncRulesContent } from './mongo/MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage } from './mongo/MongoSyncBucketStorage.js';
+import { SyncRuleState } from './mongo/models.js';
+import { generateSlotName } from './mongo/util.js';
+export class MongoBucketStorage {
+    constructor(db, options) {
+        this.storageCache = new LRUCache({
+            max: 3,
+            fetchMethod: async (id) => {
+                const doc2 = await this.db.sync_rules.findOne({
+                    _id: id
+                }, { limit: 1 });
+                if (doc2 == null) {
+                    // Deleted in the meantime?
+                    return undefined;
+                }
+                const rules = new MongoPersistedSyncRulesContent(this.db, doc2);
+                const storage = this.getInstance(rules.parsed());
+                return storage;
+            }
+        });
+        // Nothing is done here until a subscriber starts to iterate
+        this.sharedIter = new sync.BroadcastIterable((signal) => {
+            return this.watchActiveCheckpoint(signal);
+        });
+        this.client = db.client;
+        this.db = db;
+        this.session = this.client.startSession();
+        this.slot_name_prefix = options.slot_name_prefix;
+    }
+    getInstance(options) {
+        let { id, sync_rules, slot_name } = options;
+        if (typeof id == 'bigint') {
+            id = Number(id);
+        }
+        return new MongoSyncBucketStorage(this, id, sync_rules, slot_name);
+    }
+    async configureSyncRules(sync_rules, options) {
+        const next = await this.getNextSyncRulesContent();
+        const active = await this.getActiveSyncRulesContent();
+        if (next?.sync_rules_content == sync_rules) {
+            micro.logger.info('Sync rules from configuration unchanged');
+            return { updated: false };
+        }
+        else if (next == null && active?.sync_rules_content == sync_rules) {
+            micro.logger.info('Sync rules from configuration unchanged');
+            return { updated: false };
+        }
+        else {
+            micro.logger.info('Sync rules updated from configuration');
+            const persisted_sync_rules = await this.updateSyncRules({
+                content: sync_rules,
+                lock: options?.lock
+            });
+            return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
+        }
+    }
+    async slotRemoved(slot_name) {
+        const next = await this.getNextSyncRulesContent();
+        const active = await this.getActiveSyncRulesContent();
+        // In both the below cases, we create a new sync rules instance.
+        // The current one will continue erroring until the next one has finished processing.
+        // TODO: Update
+        if (next != null && next.slot_name == slot_name) {
+            // We need to redo the "next" sync rules
+            await this.updateSyncRules({
+                content: next.sync_rules_content
+            });
+            // Pro-actively stop replicating
+            await this.db.sync_rules.updateOne({
+                _id: next.id,
+                state: SyncRuleState.PROCESSING
+            }, {
+                $set: {
+                    state: SyncRuleState.STOP
+                }
+            });
+        }
+        else if (next == null && active?.slot_name == slot_name) {
+            // Slot removed for "active" sync rules, while there is no "next" one.
+            await this.updateSyncRules({
+                content: active.sync_rules_content
+            });
+            // Pro-actively stop replicating
+            await this.db.sync_rules.updateOne({
+                _id: active.id,
+                state: SyncRuleState.ACTIVE
+            }, {
+                $set: {
+                    state: SyncRuleState.STOP
+                }
+            });
+        }
+    }
+    async updateSyncRules(options) {
+        // Parse and validate before applying any changes
+        const parsed = SqlSyncRules.fromYaml(options.content);
+        let rules = undefined;
+        await this.session.withTransaction(async () => {
+            // Only have a single set of sync rules with PROCESSING.
+            await this.db.sync_rules.updateMany({
+                state: SyncRuleState.PROCESSING
+            }, { $set: { state: SyncRuleState.STOP } });
+            const id_doc = await this.db.op_id_sequence.findOneAndUpdate({
+                _id: 'sync_rules'
+            }, {
+                $inc: {
+                    op_id: 1n
+                }
+            }, {
+                upsert: true,
+                returnDocument: 'after'
+            });
+            const id = Number(id_doc.op_id);
+            const slot_name = generateSlotName(this.slot_name_prefix, id);
+            const doc = {
+                _id: id,
+                content: options.content,
+                last_checkpoint: null,
+                last_checkpoint_lsn: null,
+                no_checkpoint_before: null,
+                snapshot_done: false,
+                state: SyncRuleState.PROCESSING,
+                slot_name: slot_name,
+                last_checkpoint_ts: null,
+                last_fatal_error: null,
+                last_keepalive_ts: null
+            };
+            await this.db.sync_rules.insertOne(doc);
+            rules = new MongoPersistedSyncRulesContent(this.db, doc);
+            if (options.lock) {
+                const lock = await rules.lock();
+            }
+        });
+        return rules;
+    }
+    async getActiveSyncRulesContent() {
+        const doc = await this.db.sync_rules.findOne({
+            state: SyncRuleState.ACTIVE
+        }, { sort: { _id: -1 }, limit: 1 });
+        if (doc == null) {
+            return null;
+        }
+        return new MongoPersistedSyncRulesContent(this.db, doc);
+    }
+    async getActiveSyncRules() {
+        const content = await this.getActiveSyncRulesContent();
+        return content?.parsed() ?? null;
+    }
+    async getNextSyncRulesContent() {
+        const doc = await this.db.sync_rules.findOne({
+            state: SyncRuleState.PROCESSING
+        }, { sort: { _id: -1 }, limit: 1 });
+        if (doc == null) {
+            return null;
+        }
+        return new MongoPersistedSyncRulesContent(this.db, doc);
+    }
+    async getNextSyncRules() {
+        const content = await this.getNextSyncRulesContent();
+        return content?.parsed() ?? null;
+    }
+    async getReplicatingSyncRules() {
+        const docs = await this.db.sync_rules
+            .find({
+            $or: [{ state: SyncRuleState.ACTIVE }, { state: SyncRuleState.PROCESSING }]
+        })
+            .toArray();
+        return docs.map((doc) => {
+            return new MongoPersistedSyncRulesContent(this.db, doc);
+        });
+    }
+    async getStoppedSyncRules() {
+        const docs = await this.db.sync_rules
+            .find({
+            state: SyncRuleState.STOP
+        })
+            .toArray();
+        return docs.map((doc) => {
+            return new MongoPersistedSyncRulesContent(this.db, doc);
+        });
+    }
+    async createWriteCheckpoint(user_id, lsns) {
+        const doc = await this.db.write_checkpoints.findOneAndUpdate({
+            user_id: user_id
+        }, {
+            $set: {
+                lsns: lsns
+            },
+            $inc: {
+                client_id: 1n
+            }
+        }, { upsert: true, returnDocument: 'after' });
+        return doc.client_id;
+    }
+    async lastWriteCheckpoint(user_id, lsn) {
+        const lastWriteCheckpoint = await this.db.write_checkpoints.findOne({
+            user_id: user_id,
+            'lsns.1': { $lte: lsn }
+        });
+        return lastWriteCheckpoint?.client_id ?? null;
+    }
+    async getActiveCheckpoint() {
+        const doc = await this.db.sync_rules.findOne({
+            state: SyncRuleState.ACTIVE
+        }, {
+            sort: { _id: -1 },
+            limit: 1,
+            projection: { _id: 1, last_checkpoint: 1, last_checkpoint_lsn: 1 }
+        });
+        return this.makeActiveCheckpoint(doc);
+    }
+    async getStorageMetrics() {
+        const active_sync_rules = await this.getActiveSyncRules();
+        if (active_sync_rules == null) {
+            return {
+                operations_size_bytes: 0,
+                parameters_size_bytes: 0,
+                replication_size_bytes: 0
+            };
+        }
+        const operations_aggregate = await this.db.bucket_data
+            .aggregate([
+            {
+                $collStats: {
+                    storageStats: {},
+                    count: {}
+                }
+            }
+        ])
+            .toArray();
+        const parameters_aggregate = await this.db.bucket_parameters
+            .aggregate([
+            {
+                $collStats: {
+                    storageStats: {},
+                    count: {}
+                }
+            }
+        ])
+            .toArray();
+        const replication_aggregate = await this.db.current_data
+            .aggregate([
+            {
+                $collStats: {
+                    storageStats: {},
+                    count: {}
+                }
+            }
+        ])
+            .toArray();
+        return {
+            operations_size_bytes: operations_aggregate[0].storageStats.size,
+            parameters_size_bytes: parameters_aggregate[0].storageStats.size,
+            replication_size_bytes: replication_aggregate[0].storageStats.size
+        };
+    }
+    makeActiveCheckpoint(doc) {
+        return {
+            checkpoint: util.timestampToOpId(doc?.last_checkpoint ?? 0n),
+            lsn: doc?.last_checkpoint_lsn ?? replication.ZERO_LSN,
+            hasSyncRules() {
+                return doc != null;
+            },
+            getBucketStorage: async () => {
+                if (doc == null) {
+                    return null;
+                }
+                return (await this.storageCache.fetch(doc._id)) ?? null;
+            }
+        };
+    }
+    /**
+     * Instance-wide watch on the latest available checkpoint (op_id + lsn).
+     */
+    async *watchActiveCheckpoint(signal) {
+        const pipeline = [
+            {
+                $match: {
+                    'fullDocument.state': 'ACTIVE',
+                    operationType: { $in: ['insert', 'update'] }
+                }
+            },
+            {
+                $project: {
+                    operationType: 1,
+                    'fullDocument._id': 1,
+                    'fullDocument.last_checkpoint': 1,
+                    'fullDocument.last_checkpoint_lsn': 1
+                }
+            }
+        ];
+        // Use this form instead of (doc: SyncRuleDocument | null = null),
+        // otherwise we get weird "doc: never" issues.
+        let doc = null;
+        let clusterTime = null;
+        await this.client.withSession(async (session) => {
+            doc = await this.db.sync_rules.findOne({
+                state: SyncRuleState.ACTIVE
+            }, {
+                session,
+                sort: { _id: -1 },
+                limit: 1,
+                projection: {
+                    _id: 1,
+                    last_checkpoint: 1,
+                    last_checkpoint_lsn: 1
+                }
+            });
+            const time = session.clusterTime?.clusterTime ?? null;
+            clusterTime = time;
+        });
+        if (clusterTime == null) {
+            throw new Error('Could not get clusterTime');
+        }
+        if (signal.aborted) {
+            return;
+        }
+        if (doc) {
+            yield this.makeActiveCheckpoint(doc);
+        }
+        const stream = this.db.sync_rules.watch(pipeline, {
+            fullDocument: 'updateLookup',
+            // Start at the cluster time where we got the initial doc, to make sure
+            // we don't skip any updates.
+            // This may result in the first operation being a duplicate, but we filter
+            // it out anyway.
+            startAtOperationTime: clusterTime
+        });
+        signal.addEventListener('abort', () => {
+            stream.close();
+        }, { once: true });
+        let lastOp = null;
+        for await (const update of stream.stream()) {
+            if (signal.aborted) {
+                break;
+            }
+            if (update.operationType != 'insert' && update.operationType != 'update') {
+                continue;
+            }
+            const doc = update.fullDocument;
+            if (doc == null) {
+                continue;
+            }
+            const op = this.makeActiveCheckpoint(doc);
+            // Check for LSN / checkpoint changes - ignore other metadata changes
+            if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
+                lastOp = op;
+                yield op;
+            }
+        }
+    }
+    /**
+     * User-specific watch on the latest checkpoint and/or write checkpoint.
+     */
+    async *watchWriteCheckpoint(user_id, signal) {
+        let lastCheckpoint = null;
+        let lastWriteCheckpoint = null;
+        const iter = wrapWithAbort(this.sharedIter, signal);
+        for await (const cp of iter) {
+            const { checkpoint, lsn } = cp;
+            // lsn changes are not important by itself.
+            // What is important is:
+            // 1. checkpoint (op_id) changes.
+            // 2. write checkpoint changes for the specific user
+            const currentWriteCheckpoint = await this.lastWriteCheckpoint(user_id, lsn ?? '');
+            if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) {
+                // No change - wait for next one
+                // In some cases, many LSNs may be produced in a short time.
+                // Add a delay to throttle the write checkpoint lookup a bit.
+                await timers.setTimeout(20 + 10 * Math.random());
+                continue;
+            }
+            lastWriteCheckpoint = currentWriteCheckpoint;
+            lastCheckpoint = checkpoint;
+            yield { base: cp, writeCheckpoint: currentWriteCheckpoint };
+        }
+    }
+}
+//# sourceMappingURL=MongoBucketStorage.js.map

package/dist/storage/MongoBucketStorage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,yCAAyC,CAAC;AAExE,OAAO,KAAK,WAAW,MAAM,oCAAoC,CAAC;AAClE,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAC7C,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAW7C,OAAO,EAAE,8BAA8B,EAAE,MAAM,2CAA2C,CAAC;AAC3F,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAE3E,OAAO,EAAoB,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAInD,MAAM,OAAO,kBAAkB;IA0B7B,YAAY,EAAkB,EAAE,OAAqC;QArBpD,iBAAY,GAAG,IAAI,QAAQ,CAAiC;YAC3E,GAAG,EAAE,CAAC;YACN,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;gBACxB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC3C;oBACE,GAAG,EAAE,EAAE;iBACR,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;gBACF,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;oBACjB,2BAA2B;oBAC3B,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,MAAM,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAChE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,OAAO,OAAO,CAAC;YACjB,CAAC;SACF,CAAC,CAAC;QAqZH,4DAA4D;QAC3C,eAAU,GAAG,IAAI,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,EAAE;YAClE,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAnZD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,WAAW,CAAC,OAA2B;QACrC,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC5C,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,UAAkB,EAAE,OAA4B;QACvE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,UAAU,EAAE,CAAC;YAC3C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YAC7D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,UAAU,EAAE,CAAC;YACpE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YAC7D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YAC3D,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC;gBACtD,OAAO,EAAE,UAAU;gBACnB,IAAI,EAAE,OAAO,EAAE,IAAI;aACpB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,gEAAgE;QAChE,qFAAqF;QACrF,eAAe;QACf,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;YAChD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;aACjC,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,aAAa,CAAC,UAAU;aAChC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,aAAa,CAAC,IAAI;iBAC1B;aACF,CACF,CAAC;QACJ,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,SAAS,IAAI,SAAS,EAAE,CAAC;YAC1D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,kBAAkB;aACnC,CAAC,CAAC;YAEH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,aAAa,CAAC,MAAM;aAC5B,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,aAAa,CAAC,IAAI;iBAC1B;aACF,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAA+B;QACnD,iDAAiD;QACjD,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEtD,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,aAAa,CAAC,UAAU;aAChC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,EAAE,EAAE,CACxC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,aAAa,EAAE,KAAK;gBACpB,KAAK,EAAE,aAAa,CAAC,UAAU;gBAC/B,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,aAAa,CAAC,MAAM;SAC5B,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,aAAa,CAAC,UAAU;SAChC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,UAAU,EAAE,CAAC;SAC5E,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,aAAa,CAAC,IAAI;SAC1B,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,OAAe,EAAE,IAA4B;QACvE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC1D;YACE,OAAO,EAAE,OAAO;SACjB,EACD;YACE,IAAI,EAAE;gBACJ,IAAI,EAAE,IAAI;aACX;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,EAAE;aACd;SACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;QACF,OAAO,GAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAAe,EAAE,GAAW;QACpD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC;YAClE,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,aAAa,CAAC,MAAM;SAC5B,EACD;YACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;YACjB,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC,EAAE;SACnE,CACF,CAAC;QAEF,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1D,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;oBAChB,KAAK,EAAE,EAAE;iBACV;aACF;SACF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;oBAChB,KAAK,EAAE,EAAE;iBACV;aACF;SACF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;oBAChB,KAAK,EAAE,EAAE;iBACV;aACF;SACF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO;YACL,qBAAqB,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI;YAChE,qBAAqB,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI;YAChE,sBAAsB,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI;SACnE,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,GAA4B;QACvD,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,eAAe,IAAI,EAAE,CAAC;YAC5D,GAAG,EAAE,GAAG,EAAE,mBAAmB,IAAI,WAAW,CAAC,QAAQ;YACrD,YAAY;gBACV,OAAO,GAAG,IAAI,IAAI,CAAC;YACrB,CAAC;YACD,gBAAgB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;YAC1D,CAAC;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,CAAC,qBAAqB,CAAC,MAAmB;QACtD,MAAM,QAAQ,GAAqB;YACjC;gBACE,MAAM,EAAE;oBACN,oBAAoB,EAAE,QAAQ;oBAC9B,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE;iBAC7C;aACF;YACD;gBACE,QAAQ,EAAE;oBACR,aAAa,EAAE,CAAC;oBAChB,kBAAkB,EAAE,CAAC;oBACrB,8BAA8B,EAAE,CAAC;oBACjC,kCAAkC,EAAE,CAAC;iBACtC;aACF;SACF,CAAC;QAEF,kEAAkE;QAClE,8CAA8C;QAC9C,IAAI,GAAG,GAAG,IAA+B,CAAC;QAC1C,IAAI,WAAW,GAAG,IAA8B,CAAC;QAEjD,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9C,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CACpC;gBACE,KAAK,EAAE,aAAa,CAAC,MAAM;aAC5B,EACD;gBACE,OAAO;gBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,eAAe,EAAE,CAAC;oBAClB,mBAAmB,EAAE,CAAC;iBACvB;aACF,CACF,CAAC;YACF,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,WAAW,IAAI,IAAI,CAAC;YACtD,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE;YAChD,YAAY,EAAE,cAAc;YAC5B,uEAAuE;YACvE,6BAA6B;YAC7B,0EAA0E;YAC1E,iBAAiB;YACjB,oBAAoB,EAAE,WAAW;SAClC,CAAC,CAAC;QAEH,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QAEF,IAAI,MAAM,GAA4B,IAAI,CAAC;QAE3C,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,IAAI,MAAM,CAAC,aAAa,IAAI,QAAQ,EAAE,CAAC;gBACzE,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,CAAC,YAAa,CAAC;YACjC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAC1C,qEAAqE;YACrE,IAAI,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACjF,MAAM,GAAG,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAOD;;OAEG;IACH,KAAK,CAAC,CAAC,oBAAoB,CAAC,OAAe,EAAE,MAAmB;QAC9D,IAAI,cAAc,GAAqB,IAAI,CAAC;QAC5C,IAAI,mBAAmB,GAAkB,IAAI,CAAC;QAE9C,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;YAE/B,2CAA2C;YAC3C,wBAAwB;YACxB,iCAAiC;YACjC,oDAAoD;YAEpD,MAAM,sBAAsB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;YAElF,IAAI,sBAAsB,IAAI,mBAAmB,IAAI,UAAU,IAAI,cAAc,EAAE,CAAC;gBAClF,gCAAgC;gBAChC,4DAA4D;gBAC5D,6DAA6D;gBAC7D,MAAM,MAAM,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBACjD,SAAS;YACX,CAAC;YAED,mBAAmB,GAAG,sBAAsB,CAAC;YAC7C,cAAc,GAAG,UAAU,CAAC;YAE5B,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,eAAe,EAAE,sBAAsB,EAAE,CAAC;QAC9D,CAAC;IACH,CAAC;CACF"}

package/dist/storage/SourceTable.d.ts

@@ -0,0 +1,39 @@
+import * as replication from '../replication/replication-index.js';
+export declare class SourceTable {
+    readonly id: any;
+    readonly connectionTag: string;
+    readonly relationId: number;
+    readonly schema: string;
+    readonly table: string;
+    readonly replicaIdColumns: replication.ReplicationColumn[];
+    readonly snapshotComplete: boolean;
+    static readonly DEFAULT_SCHEMA = "public";
+    static readonly DEFAULT_TAG = "default";
+    /**
+     * True if the table is used in sync rules for data queries.
+     *
+     * This value is resolved externally, and cached here.
+     *
+     * Defaults to true for tests.
+     */
+    syncData: boolean;
+    /**
+     * True if the table is used in sync rules for data queries.
+     *
+     * This value is resolved externally, and cached here.
+     *
+     * Defaults to true for tests.
+     */
+    syncParameters: boolean;
+    constructor(id: any, connectionTag: string, relationId: number, schema: string, table: string, replicaIdColumns: replication.ReplicationColumn[], snapshotComplete: boolean);
+    get hasReplicaIdentity(): boolean;
+    /**
+     * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]})
+     */
+    get qualifiedName(): string;
+    /**
+     * Usage: db.query(`SELECT * FROM ${table.escapedIdentifier}`)
+     */
+    get escapedIdentifier(): string;
+    get syncAny(): boolean;
+}

package/dist/storage/SourceTable.js

@@ -0,0 +1,50 @@
+import { DEFAULT_SCHEMA, DEFAULT_TAG } from '@powersync/service-sync-rules';
+import * as util from '../util/util-index.js';
+export class SourceTable {
+    constructor(id, connectionTag, relationId, schema, table, replicaIdColumns, snapshotComplete) {
+        this.id = id;
+        this.connectionTag = connectionTag;
+        this.relationId = relationId;
+        this.schema = schema;
+        this.table = table;
+        this.replicaIdColumns = replicaIdColumns;
+        this.snapshotComplete = snapshotComplete;
+        /**
+         * True if the table is used in sync rules for data queries.
+         *
+         * This value is resolved externally, and cached here.
+         *
+         * Defaults to true for tests.
+         */
+        this.syncData = true;
+        /**
+         * True if the table is used in sync rules for data queries.
+         *
+         * This value is resolved externally, and cached here.
+         *
+         * Defaults to true for tests.
+         */
+        this.syncParameters = true;
+    }
+    get hasReplicaIdentity() {
+        return this.replicaIdColumns.length > 0;
+    }
+    /**
+     * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]})
+     */
+    get qualifiedName() {
+        return this.escapedIdentifier;
+    }
+    /**
+     * Usage: db.query(`SELECT * FROM ${table.escapedIdentifier}`)
+     */
+    get escapedIdentifier() {
+        return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.table)}`;
+    }
+    get syncAny() {
+        return this.syncData || this.syncParameters;
+    }
+}
+SourceTable.DEFAULT_SCHEMA = DEFAULT_SCHEMA;
+SourceTable.DEFAULT_TAG = DEFAULT_TAG;
+//# sourceMappingURL=SourceTable.js.map

package/dist/storage/SourceTable.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAG5E,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,MAAM,OAAO,WAAW;IAsBtB,YACkB,EAAO,EACP,aAAqB,EACrB,UAAkB,EAClB,MAAc,EACd,KAAa,EAEb,gBAAiD,EACjD,gBAAyB;QAPzB,OAAE,GAAF,EAAE,CAAK;QACP,kBAAa,GAAb,aAAa,CAAQ;QACrB,eAAU,GAAV,UAAU,CAAQ;QAClB,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAAQ;QAEb,qBAAgB,GAAhB,gBAAgB,CAAiC;QACjD,qBAAgB,GAAhB,gBAAgB,CAAS;QA1B3C;;;;;;WAMG;QACI,aAAQ,GAAG,IAAI,CAAC;QAEvB;;;;;;WAMG;QACI,mBAAc,GAAG,IAAI,CAAC;IAW1B,CAAC;IAEJ,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,iBAAiB;QACnB,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;IACtF,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,CAAC;IAC9C,CAAC;;AApDe,0BAAc,GAAG,cAAc,AAAjB,CAAkB;AAChC,uBAAW,GAAG,WAAW,AAAd,CAAe"}

package/dist/storage/mongo/MongoBucketBatch.d.ts

@@ -0,0 +1,48 @@
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import * as mongo from 'mongodb';
+import { BucketStorageBatch, FlushedResult, SaveOptions } from '../BucketStorage.js';
+import { SourceTable } from '../SourceTable.js';
+import { PowerSyncMongo } from './db.js';
+import { CurrentBucket } from './models.js';
+export declare class MongoBucketBatch implements BucketStorageBatch {
+    private readonly client;
+    readonly db: PowerSyncMongo;
+    readonly session: mongo.ClientSession;
+    private readonly sync_rules;
+    private readonly group_id;
+    private readonly slot_name;
+    private batch;
+    /**
+     * Last LSN received associated with a checkpoint.
+     *
+     * This could be either:
+     * 1. A commit LSN.
+     * 2. A keepalive message LSN.
+     */
+    private last_checkpoint_lsn;
+    private no_checkpoint_before_lsn;
+    private persisted_op;
+    /**
+     * For tests only - not for persistence logic.
+     */
+    last_flushed_op: bigint | null;
+    constructor(db: PowerSyncMongo, sync_rules: SqlSyncRules, group_id: number, slot_name: string, last_checkpoint_lsn: string | null, no_checkpoint_before_lsn: string | null);
+    flush(): Promise<FlushedResult | null>;
+    private flushInner;
+    private replicateBatch;
+    private saveOperation;
+    private withTransaction;
+    private withReplicationTransaction;
+    abort(): Promise<void>;
+    commit(lsn: string): Promise<boolean>;
+    keepalive(lsn: string): Promise<boolean>;
+    save(record: SaveOptions): Promise<FlushedResult | null>;
+    /**
+     * Drop is equivalent to TRUNCATE, plus removing our record of the table.
+     */
+    drop(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
+    truncate(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
+    truncateSingle(sourceTable: SourceTable): Promise<bigint>;
+    markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
+}
+export declare function currentBucketKey(b: CurrentBucket): string;