@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/dist/storage/implementation/v1/MongoBucketBatchV1.js

@@ -0,0 +1,407 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { mongoTableId } from '../../../utils/util.js';
+import { calculateCheckpointState } from '../CheckpointState.js';
+import { MongoBucketBatch } from '../MongoBucketBatch.js';
+import { PersistedBatchV1 } from './PersistedBatchV1.js';
+import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
+export class MongoBucketBatchV1 extends MongoBucketBatch {
+    store;
+    needsActivation = true;
+    lastWaitingLogThrottled = 0;
+    constructor(options) {
+        super(options);
+        this.store = new SourceRecordStoreV1(this.db, this.group_id);
+    }
+    createPersistedBatch(writtenSize) {
+        return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, {
+            logger: this.logger
+        });
+    }
+    get sourceRecordStore() {
+        return this.store;
+    }
+    async cleanupDroppedSourceTables(_tables) {
+        // No-op for V1: source records live in a shared collection.
+    }
+    async resolveTables(options) {
+        const syncRules = options.syncRules ?? this.sync_rules;
+        const { connection_id, source } = options;
+        const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
+        const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
+            name: column.name,
+            type: column.type,
+            type_oid: column.typeId
+        }));
+        let result = null;
+        await this.db.client.withSession(async (session) => {
+            const col = this.db.commonSourceTables(this.group_id);
+            const filter = {
+                group_id: this.group_id,
+                connection_id,
+                schema_name: schema,
+                table_name: name,
+                replica_id_columns2: normalizedReplicaIdColumns
+            };
+            if (objectId != null) {
+                filter.relation_id = objectId;
+            }
+            let doc = await col.findOne(filter, { session });
+            if (doc == null) {
+                doc = {
+                    _id: options.idGenerator ? options.idGenerator() : new bson.ObjectId(),
+                    group_id: this.group_id,
+                    connection_id,
+                    relation_id: objectId,
+                    schema_name: schema,
+                    table_name: name,
+                    replica_id_columns: null,
+                    replica_id_columns2: normalizedReplicaIdColumns,
+                    snapshot_done: false,
+                    snapshot_status: undefined
+                };
+                await col.insertOne(doc, { session });
+            }
+            const sourceTable = new storage.SourceTable({
+                id: doc._id,
+                ref: source,
+                objectId,
+                replicaIdColumns,
+                snapshotComplete: doc.snapshot_done ?? true,
+                ...syncRules.getMatchingSources(source)
+            });
+            sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
+            sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
+            sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
+            sourceTable.storeCurrentData = sendsCompleteRows !== true;
+            sourceTable.snapshotStatus =
+                doc.snapshot_status == null
+                    ? undefined
+                    : {
+                        lastKey: doc.snapshot_status.last_key?.buffer ?? null,
+                        totalEstimatedCount: doc.snapshot_status.total_estimated_count,
+                        replicatedCount: doc.snapshot_status.replicated_count
+                    };
+            const truncateFilter = [{ schema_name: schema, table_name: name }];
+            if (objectId != null) {
+                truncateFilter.push({ relation_id: objectId });
+            }
+            const truncate = await col
+                .find({
+                group_id: this.group_id,
+                connection_id,
+                _id: { $ne: doc._id },
+                $or: truncateFilter
+            }, { session })
+                .toArray();
+            const dropTables = truncate.map((dropDoc) => {
+                const ref = {
+                    connectionTag,
+                    schema: dropDoc.schema_name,
+                    name: dropDoc.table_name
+                };
+                const table = new storage.SourceTable({
+                    id: dropDoc._id,
+                    ref,
+                    objectId: dropDoc.relation_id,
+                    replicaIdColumns: dropDoc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
+                    snapshotComplete: dropDoc.snapshot_done ?? true,
+                    ...syncRules.getMatchingSources(ref)
+                });
+                table.syncEvent = syncRules.tableTriggersEvent(ref);
+                table.syncData = table.bucketDataSources.length > 0;
+                table.syncParameters = table.parameterLookupSources.length > 0;
+                return table;
+            });
+            result = { tables: [sourceTable], dropTables };
+        });
+        return result;
+    }
+    async getSourceTableStatus(table) {
+        const doc = (await this.db.commonSourceTables(this.group_id).findOne({
+            group_id: this.group_id,
+            _id: mongoTableId(table.id)
+        }, { session: this.session }));
+        if (doc == null) {
+            return null;
+        }
+        const ref = {
+            connectionTag: table.ref.connectionTag,
+            schema: doc.schema_name,
+            name: doc.table_name
+        };
+        const sourceTable = new storage.SourceTable({
+            id: doc._id,
+            ref,
+            objectId: doc.relation_id,
+            replicaIdColumns: doc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
+            snapshotComplete: doc.snapshot_done ?? true,
+            ...this.sync_rules.getMatchingSources(ref)
+        });
+        sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
+        sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
+        sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
+        sourceTable.snapshotStatus =
+            doc.snapshot_status == null
+                ? undefined
+                : {
+                    lastKey: doc.snapshot_status.last_key?.buffer ?? null,
+                    totalEstimatedCount: doc.snapshot_status.total_estimated_count,
+                    replicatedCount: doc.snapshot_status.replicated_count
+                };
+        return sourceTable;
+    }
+    async commit(lsn, options) {
+        const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
+        await this.flush(options);
+        const now = new Date();
+        await this.db.write_checkpoints.updateMany({
+            processed_at_lsn: null,
+            'lsns.1': { $lte: lsn }
+        }, {
+            $set: {
+                processed_at_lsn: lsn
+            }
+        }, {
+            session: this.session
+        });
+        const can_checkpoint = {
+            $and: [
+                { $eq: ['$snapshot_done', true] },
+                {
+                    $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
+                },
+                {
+                    $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
+                }
+            ]
+        };
+        const new_keepalive_op = {
+            $cond: [
+                can_checkpoint,
+                { $literal: null },
+                {
+                    $toString: {
+                        $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
+                    }
+                }
+            ]
+        };
+        const new_last_checkpoint = {
+            $cond: [
+                can_checkpoint,
+                {
+                    $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
+                },
+                '$last_checkpoint'
+            ]
+        };
+        const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate({ _id: this.group_id }, [
+            {
+                $set: {
+                    _can_checkpoint: can_checkpoint,
+                    _not_empty: createEmptyCheckpoints
+                        ? true
+                        : {
+                            $or: [
+                                { $literal: createEmptyCheckpoints },
+                                { $ne: ['$keepalive_op', new_keepalive_op] },
+                                { $ne: ['$last_checkpoint', new_last_checkpoint] }
+                            ]
+                        }
+                }
+            },
+            {
+                $set: {
+                    last_checkpoint_lsn: {
+                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
+                    },
+                    last_checkpoint_ts: {
+                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
+                    },
+                    last_keepalive_ts: { $literal: now },
+                    last_fatal_error: { $literal: null },
+                    last_fatal_error_ts: { $literal: null },
+                    keepalive_op: new_keepalive_op,
+                    last_checkpoint: new_last_checkpoint,
+                    snapshot_lsn: {
+                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
+                    }
+                }
+            },
+            {
+                $unset: ['_can_checkpoint', '_not_empty']
+            }
+        ], {
+            session: this.session,
+            returnDocument: 'before',
+            projection: {
+                snapshot_done: 1,
+                last_checkpoint_lsn: 1,
+                no_checkpoint_before: 1,
+                keepalive_op: 1,
+                last_checkpoint: 1
+            }
+        }));
+        if (preUpdateDocument == null) {
+            throw new ReplicationAssertionError('Failed to update checkpoint - no matching sync_rules document for _id: ' + this.group_id);
+        }
+        const checkpointState = calculateCheckpointState({
+            lsn,
+            snapshotDone: preUpdateDocument.snapshot_done === true,
+            lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
+            noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
+            keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
+            lastCheckpoint: preUpdateDocument.last_checkpoint,
+            persistedOp: this.persisted_op,
+            createEmptyCheckpoints
+        });
+        if (checkpointState.checkpointBlocked) {
+            if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
+                this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
+                    snapshot_done: preUpdateDocument.snapshot_done,
+                    last_checkpoint_lsn: preUpdateDocument.last_checkpoint_lsn,
+                    no_checkpoint_before: preUpdateDocument.no_checkpoint_before
+                })}`);
+                this.lastWaitingLogThrottled = Date.now();
+            }
+        }
+        else {
+            if (checkpointState.checkpointCreated) {
+                this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`);
+            }
+            await this.autoActivate(lsn);
+            await this.db.notifyCheckpoint();
+            this.persisted_op = null;
+            this.last_checkpoint_lsn = lsn;
+            if (checkpointState.newLastCheckpoint != null) {
+                await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
+            }
+        }
+        return {
+            checkpointBlocked: checkpointState.checkpointBlocked,
+            checkpointCreated: checkpointState.checkpointCreated
+        };
+    }
+    async keepalive(lsn) {
+        return await this.commit(lsn, { createEmptyCheckpoints: true });
+    }
+    async setResumeLsn(lsn) {
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id
+        }, {
+            $set: {
+                snapshot_lsn: lsn
+            }
+        }, { session: this.session });
+    }
+    async markAllSnapshotDone(no_checkpoint_before_lsn) {
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id
+        }, {
+            $set: {
+                snapshot_done: true,
+                last_keepalive_ts: new Date()
+            },
+            $max: {
+                no_checkpoint_before: no_checkpoint_before_lsn
+            }
+        }, { session: this.session });
+    }
+    async markSnapshotDone(no_checkpoint_before_lsn, options) {
+        await this.withTransaction(async () => {
+            // Protect against race conditions
+            const count = await this.db.commonSourceTables(this.group_id).countDocuments({
+                group_id: this.group_id,
+                snapshot_done: false
+            }, { session: this.session });
+            if (count > 0) {
+                if (options?.throwOnConflict ?? true) {
+                    throw new ReplicationAssertionError(`Cannot mark snapshot done while ${count} source table${count == 1 ? '' : 's'} still require snapshotting`);
+                }
+                else {
+                    return;
+                }
+            }
+            await this.markAllSnapshotDone(no_checkpoint_before_lsn);
+        });
+    }
+    async markTableSnapshotRequired(_table) {
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id
+        }, {
+            $set: {
+                snapshot_done: false
+            }
+        }, { session: this.session });
+    }
+    async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
+        const session = this.session;
+        const ids = tables.map((table) => mongoTableId(table.id));
+        await this.withTransaction(async () => {
+            await this.db.commonSourceTables(this.group_id).updateMany({ _id: { $in: ids } }, {
+                $set: {
+                    snapshot_done: true
+                },
+                $unset: {
+                    snapshot_status: 1
+                }
+            }, { session });
+            if (no_checkpoint_before_lsn != null) {
+                await this.db.sync_rules.updateOne({
+                    _id: this.group_id
+                }, {
+                    $set: {
+                        last_keepalive_ts: new Date()
+                    },
+                    $max: {
+                        no_checkpoint_before: no_checkpoint_before_lsn
+                    }
+                }, { session: this.session });
+            }
+        });
+        return tables.map((table) => {
+            const copy = table.clone();
+            copy.snapshotComplete = true;
+            return copy;
+        });
+    }
+    async autoActivate(lsn) {
+        if (!this.needsActivation) {
+            return;
+        }
+        const session = this.session;
+        let activated = false;
+        await session.withTransaction(async () => {
+            const doc = (await this.db.sync_rules.findOne({ _id: this.group_id }, { session }));
+            if (doc && doc.state == storage.SyncRuleState.PROCESSING && doc.snapshot_done && doc.last_checkpoint != null) {
+                await this.db.sync_rules.updateOne({
+                    _id: this.group_id
+                }, {
+                    $set: {
+                        state: storage.SyncRuleState.ACTIVE
+                    }
+                }, { session });
+                await this.db.sync_rules.updateMany({
+                    _id: { $ne: this.group_id },
+                    state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
+                }, {
+                    $set: {
+                        state: storage.SyncRuleState.STOP
+                    }
+                }, { session });
+                activated = true;
+            }
+            else if (doc?.state != storage.SyncRuleState.PROCESSING) {
+                this.needsActivation = false;
+            }
+        });
+        if (activated) {
+            this.logger.info(`Activated new replication stream at ${lsn}`);
+            await this.db.notifyCheckpoint();
+            this.needsActivation = false;
+        }
+    }
+}
+//# sourceMappingURL=MongoBucketBatchV1.js.map

package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoBucketBatchV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoBucketBatchV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAiC,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACjF,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAA2B,MAAM,wBAAwB,CAAC;AAGnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAI/D,MAAM,OAAO,kBAAmB,SAAQ,gBAAgB;IAGrC,KAAK,CAAoB;IAClC,eAAe,GAAG,IAAI,CAAC;IACvB,uBAAuB,GAAG,CAAC,CAAC;IAEpC,YAAY,OAAgC;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAES,oBAAoB,CAAC,WAAmB;QAChD,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE;YAC7E,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;IAED,IAAc,iBAAiB;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAAsB;QAC/D,4DAA4D;IAC9D,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAqC;QACvD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC;QACvD,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAC1C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAAC;QAE9F,MAAM,0BAA0B,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnE,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,MAAM;SACxB,CAAC,CAAC,CAAC;QAEJ,IAAI,MAAM,GAAuC,IAAI,CAAC;QACtD,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtD,MAAM,MAAM,GAAQ;gBAClB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa;gBACb,WAAW,EAAE,MAAM;gBACnB,UAAU,EAAE,IAAI;gBAChB,mBAAmB,EAAE,0BAA0B;aAChD,CAAC;YACF,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC;YAChC,CAAC;YAED,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,GAAG,GAAG;oBACJ,GAAG,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAE,OAAO,CAAC,WAAW,EAAoB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;oBACzF,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,aAAa;oBACb,WAAW,EAAE,QAAQ;oBACrB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,IAAI;oBAChB,kBAAkB,EAAE,IAAI;oBACxB,mBAAmB,EAAE,0BAA0B;oBAC/C,aAAa,EAAE,KAAK;oBACpB,eAAe,EAAE,SAAS;iBAC3B,CAAC;gBACF,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;gBAC1C,EAAE,EAAE,GAAG,CAAC,GAAG;gBACX,GAAG,EAAE,MAAM;gBACX,QAAQ;gBACR,gBAAgB;gBAChB,gBAAgB,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;gBAC3C,GAAG,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC;aACxC,CAAC,CAAC;YACH,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC7D,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;YAChE,WAAW,CAAC,cAAc,GAAG,WAAW,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3E,WAAW,CAAC,gBAAgB,GAAG,iBAAiB,KAAK,IAAI,CAAC;YAC1D,WAAW,CAAC,cAAc;gBACxB,GAAG,CAAC,eAAe,IAAI,IAAI;oBACzB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC;wBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;wBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;wBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;qBACtD,CAAC;YAER,MAAM,cAAc,GAAG,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAU,CAAC;YAC5E,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,cAAc,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,GAAG;iBACvB,IAAI,CACH;gBACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa;gBACb,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;gBACrB,GAAG,EAAE,cAAc;aACpB,EACD,EAAE,OAAO,EAAE,CACZ;iBACA,OAAO,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC1C,MAAM,GAAG,GAAG;oBACV,aAAa;oBACb,MAAM,EAAE,OAAO,CAAC,WAAW;oBAC3B,IAAI,EAAE,OAAO,CAAC,UAAU;iBACzB,CAAC;gBACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;oBACpC,EAAE,EAAE,OAAO,CAAC,GAAG;oBACf,GAAG;oBACH,QAAQ,EAAE,OAAO,CAAC,WAAW;oBAC7B,gBAAgB,EACd,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF,IAAI,EAAE;oBACT,gBAAgB,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;oBAC/C,GAAG,SAAS,CAAC,kBAAkB,CAAC,GAAG,CAAC;iBACrC,CAAC,CAAC;gBACH,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACpD,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;gBACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC/D,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,GAAG,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,OAAO,MAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,KAA0B;QACnD,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAClE;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;SAC5B,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAiC,CAAC;QACnC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG;YACV,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa;YACtC,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,GAAG;YACH,QAAQ,EAAE,GAAG,CAAC,WAAW;YACzB,gBAAgB,EACd,GAAG,CAAC,mBAAmB,EAAE,GAAG,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF,IAAI,EAAE;YACT,gBAAgB,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YAC3C,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC;SAC3C,CAAC,CAAC;QACH,WAAW,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAChE,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,WAAW,CAAC,cAAc,GAAG,WAAW,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3E,WAAW,CAAC,cAAc;YACxB,GAAG,CAAC,eAAe,IAAI,IAAI;gBACzB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC;oBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;oBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;oBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;iBACtD,CAAC;QACR,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,OAA0C;QAClE,MAAM,EAAE,sBAAsB,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,mCAAmC,EAAE,GAAG,OAAO,EAAE,CAAC;QAElG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE1B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CACxC;YACE,gBAAgB,EAAE,IAAI;YACtB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,EACD;YACE,IAAI,EAAE;gBACJ,gBAAgB,EAAE,GAAG;aACtB;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CACF,CAAC;QAEF,MAAM,cAAc,GAAG;YACrB,IAAI,EAAE;gBACJ,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE,IAAI,CAAC,EAAE;gBACjC;oBACE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBACtG;gBACD;oBACE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBACxG;aACF;SACF,CAAC;QAEF,MAAM,gBAAgB,GAAG;YACvB,KAAK,EAAE;gBACL,cAAc;gBACd,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAClB;oBACE,SAAS,EAAE;wBACT,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,CAAC;qBAC1E;iBACF;aACF;SACF,CAAC;QAEF,MAAM,mBAAmB,GAAG;YAC1B,KAAK,EAAE;gBACL,cAAc;gBACd;oBACE,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;iBAC9F;gBACD,kBAAkB;aACnB;SACF,CAAC;QAEF,MAAM,iBAAiB,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAClE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EACtB;YACE;gBACE,IAAI,EAAE;oBACJ,eAAe,EAAE,cAAc;oBAC/B,UAAU,EAAE,sBAAsB;wBAChC,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC;4BACE,GAAG,EAAE;gCACH,EAAE,QAAQ,EAAE,sBAAsB,EAAE;gCACpC,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,gBAAgB,CAAC,EAAE;gCAC5C,EAAE,GAAG,EAAE,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,EAAE;6BACnD;yBACF;iBACN;aACF;YACD;gBACE,IAAI,EAAE;oBACJ,mBAAmB,EAAE;wBACnB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC;qBAClG;oBACD,kBAAkB,EAAE;wBAClB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,qBAAqB,CAAC;qBACjG;oBACD,iBAAiB,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE;oBACpC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACpC,mBAAmB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACvC,YAAY,EAAE,gBAAgB;oBAC9B,eAAe,EAAE,mBAAmB;oBACpC,YAAY,EAAE;wBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,eAAe,CAAC;qBAC5F;iBACF;aACF;YACD;gBACE,MAAM,EAAE,CAAC,iBAAiB,EAAE,YAAY,CAAC;aAC1C;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,cAAc,EAAE,QAAQ;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAC;gBACtB,oBAAoB,EAAE,CAAC;gBACvB,YAAY,EAAE,CAAC;gBACf,eAAe,EAAE,CAAC;aACnB;SACF,CACF,CAAuB,CAAC;QAEzB,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,yBAAyB,CACjC,yEAAyE,GAAG,IAAI,CAAC,QAAQ,CAC1F,CAAC;QACJ,CAAC;QAED,MAAM,eAAe,GAAG,wBAAwB,CAAC;YAC/C,GAAG;YACH,YAAY,EAAE,iBAAiB,CAAC,aAAa,KAAK,IAAI;YACtD,iBAAiB,EAAE,iBAAiB,CAAC,mBAAmB;YACxD,kBAAkB,EAAE,iBAAiB,CAAC,oBAAoB;YAC1D,WAAW,EAAE,iBAAiB,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC;YACnG,cAAc,EAAE,iBAAiB,CAAC,eAAe;YACjD,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,sBAAsB;SACvB,CAAC,CAAC;QACH,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,uBAAuB,GAAG,KAAK,EAAE,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oDAAoD,GAAG,MAAM,eAAe,CAAC,cAAc,oBAAoB,IAAI,CAAC,SAAS,CAC3H;oBACE,aAAa,EAAE,iBAAiB,CAAC,aAAa;oBAC9C,mBAAmB,EAAE,iBAAiB,CAAC,mBAAmB;oBAC1D,oBAAoB,EAAE,iBAAiB,CAAC,oBAAoB;iBAC7D,CACF,EAAE,CACJ,CAAC;gBACF,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,MAAM,eAAe,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;YAC/B,IAAI,eAAe,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAC9C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,eAAe,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;QAED,OAAO;YACL,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;YACpD,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;SACrD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,YAAY,EAAE,GAAG;aAClB;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,wBAAgC;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,IAAI;gBACnB,iBAAiB,EAAE,IAAI,IAAI,EAAE;aAC9B;YACD,IAAI,EAAE;gBACJ,oBAAoB,EAAE,wBAAwB;aAC/C;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,wBAAgC,EAAE,OAAuC;QAC9F,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACpC,kCAAkC;YAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,cAAc,CAC1E;gBACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa,EAAE,KAAK;aACrB,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;YACF,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,IAAI,OAAO,EAAE,eAAe,IAAI,IAAI,EAAE,CAAC;oBACrC,MAAM,IAAI,yBAAyB,CACjC,mCAAmC,KAAK,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,6BAA6B,CAC3G,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO;gBACT,CAAC;YACH,CAAC;YAED,MAAM,IAAI,CAAC,mBAAmB,CAAC,wBAAwB,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAA2B;QACzD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,KAAK;aACrB;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,MAA6B,EAC7B,wBAAiC;QAEjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAE1D,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACpC,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,UAAU,CACxD,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EACrB;gBACE,IAAI,EAAE;oBACJ,aAAa,EAAE,IAAI;iBACpB;gBACD,MAAM,EAAE;oBACN,eAAe,EAAE,CAAC;iBACnB;aACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YAEF,IAAI,wBAAwB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;oBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;iBACnB,EACD;oBACE,IAAI,EAAE;wBACJ,iBAAiB,EAAE,IAAI,IAAI,EAAE;qBAC9B;oBACD,IAAI,EAAE;wBACJ,oBAAoB,EAAE,wBAAwB;qBAC/C;iBACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACvC,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAuB,CAAC;YAC1G,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC7G,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;oBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;iBACnB,EACD;oBACE,IAAI,EAAE;wBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;qBACpC;iBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;oBACE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3B,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;iBAC9E,EACD;oBACE,IAAI,EAAE;wBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;qBAClC;iBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,GAAG,EAAE,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC1D,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts

@@ -0,0 +1,12 @@
+import { BucketChecksum, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap } from '@powersync/service-core';
+import { FetchPartialBucketChecksumByBucket, MongoChecksums } from '../MongoChecksums.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoChecksumsV1 extends MongoChecksums {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    computePartialChecksumsDirectByBucket(batch: FetchPartialBucketChecksumByBucket[]): Promise<PartialChecksumMap>;
+    protected fetchPreStates(batch: FetchPartialBucketChecksum[]): Promise<Map<string, {
+        opId: InternalOpId;
+        checksum: BucketChecksum;
+    }>>;
+    protected computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
+}

package/dist/storage/implementation/v1/MongoChecksumsV1.js

@@ -0,0 +1,56 @@
+import { bson } from '@powersync/service-core';
+import { MongoChecksums } from '../MongoChecksums.js';
+export class MongoChecksumsV1 extends MongoChecksums {
+    async computePartialChecksumsDirectByBucket(batch) {
+        return this.computePartialChecksumsForCollection(batch, this.db.bucketDataV1, (request) => ({
+            _id: {
+                $gt: {
+                    g: this.group_id,
+                    b: request.bucket,
+                    o: request.start ?? new bson.MinKey()
+                },
+                $lte: {
+                    g: this.group_id,
+                    b: request.bucket,
+                    o: request.end
+                }
+            }
+        }));
+    }
+    async fetchPreStates(batch) {
+        const preFilters = batch
+            .filter((request) => request.start == null)
+            .map((request) => ({
+            _id: {
+                g: this.group_id,
+                b: request.bucket
+            },
+            'compacted_state.op_id': { $exists: true, $lte: request.end }
+        }));
+        const preStates = new Map();
+        if (preFilters.length == 0) {
+            return preStates;
+        }
+        const states = await this.db.bucketStateV1
+            .find({
+            $or: preFilters
+        })
+            .toArray();
+        for (const state of states) {
+            const compactedState = state.compacted_state;
+            preStates.set(state._id.b, {
+                opId: compactedState.op_id,
+                checksum: {
+                    bucket: state._id.b,
+                    checksum: Number(compactedState.checksum),
+                    count: compactedState.count
+                }
+            });
+        }
+        return preStates;
+    }
+    async computePartialChecksumsInternal(batch) {
+        return this.computePartialChecksumsDirectByBucket(batch);
+    }
+}
+//# sourceMappingURL=MongoChecksumsV1.js.map

package/dist/storage/implementation/v1/MongoChecksumsV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoChecksumsV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoChecksumsV1.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAKL,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAsC,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG1F,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAGlD,KAAK,CAAC,qCAAqC,CACzC,KAA2C;QAE3C,OAAO,IAAI,CAAC,oCAAoC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC1F,GAAG,EAAE;gBACH,GAAG,EAAE;oBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,GAAG;iBACf;aACF;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,cAAc,CAC5B,KAAmC;QAEnC,MAAM,UAAU,GAAG,KAAK;aACrB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;aAC1C,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjB,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;aAClB;YACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;SAC9D,CAAC,CAAC,CAAC;QAEN,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QACtF,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa;aACvC,IAAI,CAAC;YACJ,GAAG,EAAE,UAAU;SAChB,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;YAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,QAAQ,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;oBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;oBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;iBAC5B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAES,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QACjF,OAAO,IAAI,CAAC,qCAAqC,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoCompactorV1.d.ts

@@ -0,0 +1,23 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
+import { SingleBucketStore } from '../common/SingleBucketStore.js';
+import { BucketStateDocumentBase } from '../models.js';
+import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
+import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoCompactorV1 extends MongoCompactor {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    protected readonly storage: MongoSyncBucketStorageV1;
+    dirtyBucketBatches(options: {
+        minBucketChanges: number;
+        minChangeRatio: number;
+    }): AsyncGenerator<DirtyBucket[]>;
+    dirtyBucketBatchForChecksums(options: {
+        minBucketChanges: number;
+    }): Promise<DirtyBucket[]>;
+    protected writeBucketStateUpdates(): Promise<void>;
+    protected computeChecksumsForBuckets(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<storage.PartialChecksumMap>;
+    protected bucketStateFilter(bucket: string, _definitionId: BucketDefinitionId | null): mongo.Filter<BucketStateDocumentBase>;
+    protected getBucketDataContext(bucket: string, _definitionId: BucketDefinitionId | null): Promise<SingleBucketStore | null>;
+}

package/dist/storage/implementation/v1/MongoCompactorV1.js

@@ -0,0 +1,52 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
+import { MongoCompactor } from '../MongoCompactor.js';
+import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
+export class MongoCompactorV1 extends MongoCompactor {
+    async *dirtyBucketBatches(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes.
+        // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is
+        // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting.
+        yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options, () => null);
+    }
+    async dirtyBucketBatchForChecksums(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
+        // estimate_since_compact as it progresses.
+        return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketStateV1, {
+            '_id.g': this.group_id,
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        }, () => null);
+    }
+    async writeBucketStateUpdates() {
+        await this.db.bucketStateV1.bulkWrite(this.bucketStateUpdates, { ordered: false });
+    }
+    async computeChecksumsForBuckets(buckets) {
+        return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
+            bucket,
+            end: this.maxOpId
+        })));
+    }
+    bucketStateFilter(bucket, _definitionId) {
+        return {
+            _id: {
+                g: this.group_id,
+                b: bucket
+            }
+        };
+    }
+    async getBucketDataContext(bucket, _definitionId) {
+        return new SingleBucketStoreV1(this.db, {
+            replicationStreamId: this.group_id,
+            definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
+            bucket
+        });
+    }
+}
+//# sourceMappingURL=MongoCompactorV1.js.map

package/dist/storage/implementation/v1/MongoCompactorV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAI9E,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,aAAwC;QAExC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts

@@ -0,0 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoParameterCompactorV1 extends MongoParameterCompactor {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+    protected collectionFilter(): mongo.Document;
+    protected deleteFilter(doc: mongo.Document): mongo.Document;
+}

package/dist/storage/implementation/v1/MongoParameterCompactorV1.js

@@ -0,0 +1,20 @@
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+export class MongoParameterCompactorV1 extends MongoParameterCompactor {
+    async getCollections() {
+        return [this.db.parameterIndexV1];
+    }
+    collectionFilter() {
+        return {
+            'key.g': this.group_id
+        };
+    }
+    deleteFilter(doc) {
+        return {
+            'key.g': doc.key.g,
+            lookup: doc.lookup,
+            _id: { $lte: doc._id },
+            key: doc.key
+        };
+    }
+}
+//# sourceMappingURL=MongoParameterCompactorV1.js.map

package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,gBAA+D,CAAC,CAAC;IACnF,CAAC;IAES,gBAAgB;QACxB,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC;IACJ,CAAC;IAES,YAAY,CAAC,GAAmB;QACxC,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAW;YAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG,EAAE,GAAG,CAAC,GAAG;SACb,CAAC;IACJ,CAAC;CACF"}