@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/dist/storage/implementation/v1/MongoBucketBatchV1.js

@@ -1,14 +1,32 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { mongoTableId } from '../../../utils/util.js';
+import { calculateCheckpointState } from '../CheckpointState.js';
 import { MongoBucketBatch } from '../MongoBucketBatch.js';
 import { PersistedBatchV1 } from './PersistedBatchV1.js';
 import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
 export class MongoBucketBatchV1 extends MongoBucketBatch {
     store;
+    needsActivation = true;
+    lastWaitingLogThrottled = 0;
+    /**
+     * The last persisted op that is not yet covered by a checkpoint, if any.
+     *
+     * Seeded from the persisted keepalive_op, so that ops persisted before a restart can still be
+     * included in the next checkpoint.
+     */
+    persisted_op = null;
     constructor(options) {
         super(options);
-        this.store = new SourceRecordStoreV1(this.db, this.group_id);
+        this.persisted_op = options.keepaliveOp ?? null;
+        this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
+    }
+    recordPersistedOp(lastOp) {
+        this.persisted_op = lastOp;
     }
     createPersistedBatch(writtenSize) {
-        return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, {
+        return new PersistedBatchV1(this.db, this.replicationStreamId, this.mapping, writtenSize, {
             logger: this.logger
         });
     }
@@ -18,5 +36,388 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     async cleanupDroppedSourceTables(_tables) {
         // No-op for V1: source records live in a shared collection.
     }
+    async resolveTables(options) {
+        const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
+        const { connection_id, source } = options;
+        const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
+        const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
+            name: column.name,
+            type: column.type,
+            type_oid: column.typeId
+        }));
+        let result = null;
+        await this.db.client.withSession(async (session) => {
+            const col = this.db.sourceTablesV1(this.replicationStreamId);
+            const filter = {
+                group_id: this.replicationStreamId,
+                connection_id,
+                schema_name: schema,
+                table_name: name,
+                replica_id_columns2: normalizedReplicaIdColumns
+            };
+            if (objectId != null) {
+                filter.relation_id = objectId;
+            }
+            let doc = await col.findOne(filter, { session });
+            if (doc == null) {
+                doc = {
+                    _id: options.idGenerator ? options.idGenerator() : new bson.ObjectId(),
+                    group_id: this.replicationStreamId,
+                    connection_id,
+                    relation_id: objectId,
+                    schema_name: schema,
+                    table_name: name,
+                    replica_id_columns: null,
+                    replica_id_columns2: normalizedReplicaIdColumns,
+                    snapshot_done: false,
+                    snapshot_status: undefined
+                };
+                await col.insertOne(doc, { session });
+            }
+            const sourceTable = new storage.SourceTable({
+                id: doc._id,
+                ref: source,
+                objectId,
+                replicaIdColumns,
+                snapshotComplete: doc.snapshot_done ?? true,
+                ...syncRules.getMatchingSources(source)
+            });
+            sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
+            sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
+            sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
+            sourceTable.storeCurrentData = sendsCompleteRows !== true;
+            sourceTable.snapshotStatus =
+                doc.snapshot_status == null
+                    ? undefined
+                    : {
+                        lastKey: doc.snapshot_status.last_key?.buffer ?? null,
+                        totalEstimatedCount: doc.snapshot_status.total_estimated_count,
+                        replicatedCount: doc.snapshot_status.replicated_count
+                    };
+            const truncateFilter = [{ schema_name: schema, table_name: name }];
+            if (objectId != null) {
+                truncateFilter.push({ relation_id: objectId });
+            }
+            const truncate = await col
+                .find({
+                group_id: this.replicationStreamId,
+                connection_id,
+                _id: { $ne: doc._id },
+                $or: truncateFilter
+            }, { session })
+                .toArray();
+            const dropTables = truncate.map((dropDoc) => {
+                const ref = {
+                    connectionTag,
+                    schema: dropDoc.schema_name,
+                    name: dropDoc.table_name
+                };
+                const table = new storage.SourceTable({
+                    id: dropDoc._id,
+                    ref,
+                    objectId: dropDoc.relation_id,
+                    replicaIdColumns: dropDoc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
+                    snapshotComplete: dropDoc.snapshot_done ?? true,
+                    ...syncRules.getMatchingSources(ref)
+                });
+                table.syncEvent = syncRules.tableTriggersEvent(ref);
+                table.syncData = table.bucketDataSources.length > 0;
+                table.syncParameters = table.parameterLookupSources.length > 0;
+                return table;
+            });
+            result = { tables: [sourceTable], dropTables };
+        });
+        return result;
+    }
+    async getSourceTableStatus(table) {
+        const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne({
+            group_id: this.replicationStreamId,
+            _id: mongoTableId(table.id)
+        }, { session: this.session });
+        if (doc == null) {
+            return null;
+        }
+        const ref = {
+            connectionTag: table.ref.connectionTag,
+            schema: doc.schema_name,
+            name: doc.table_name
+        };
+        const sourceTable = new storage.SourceTable({
+            id: doc._id,
+            ref,
+            objectId: doc.relation_id,
+            replicaIdColumns: doc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
+            snapshotComplete: doc.snapshot_done ?? true,
+            ...this.sync_rules.getMatchingSources(ref)
+        });
+        sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
+        sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
+        sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
+        sourceTable.snapshotStatus =
+            doc.snapshot_status == null
+                ? undefined
+                : {
+                    lastKey: doc.snapshot_status.last_key?.buffer ?? null,
+                    totalEstimatedCount: doc.snapshot_status.total_estimated_count,
+                    replicatedCount: doc.snapshot_status.replicated_count
+                };
+        return sourceTable;
+    }
+    async commit(lsn, options) {
+        const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
+        await this.flush(options);
+        const now = new Date();
+        await this.db.write_checkpoints.updateMany({
+            processed_at_lsn: null,
+            'lsns.1': { $lte: lsn }
+        }, {
+            $set: {
+                processed_at_lsn: lsn
+            }
+        }, {
+            session: this.session
+        });
+        const can_checkpoint = {
+            $and: [
+                { $eq: ['$snapshot_done', true] },
+                {
+                    $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
+                },
+                {
+                    $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
+                }
+            ]
+        };
+        const new_keepalive_op = {
+            $cond: [
+                can_checkpoint,
+                { $literal: null },
+                {
+                    $toString: {
+                        $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
+                    }
+                }
+            ]
+        };
+        const new_last_checkpoint = {
+            $cond: [
+                can_checkpoint,
+                {
+                    $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
+                },
+                '$last_checkpoint'
+            ]
+        };
+        const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate({ _id: this.replicationStreamId }, [
+            {
+                $set: {
+                    _can_checkpoint: can_checkpoint,
+                    _not_empty: createEmptyCheckpoints
+                        ? true
+                        : {
+                            $or: [
+                                { $literal: createEmptyCheckpoints },
+                                { $ne: ['$keepalive_op', new_keepalive_op] },
+                                { $ne: ['$last_checkpoint', new_last_checkpoint] }
+                            ]
+                        }
+                }
+            },
+            {
+                $set: {
+                    last_checkpoint_lsn: {
+                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
+                    },
+                    last_checkpoint_ts: {
+                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
+                    },
+                    last_keepalive_ts: { $literal: now },
+                    last_fatal_error: { $literal: null },
+                    last_fatal_error_ts: { $literal: null },
+                    keepalive_op: new_keepalive_op,
+                    last_checkpoint: new_last_checkpoint,
+                    snapshot_lsn: {
+                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
+                    }
+                }
+            },
+            {
+                $unset: ['_can_checkpoint', '_not_empty']
+            }
+        ], {
+            session: this.session,
+            returnDocument: 'before',
+            projection: {
+                snapshot_done: 1,
+                last_checkpoint_lsn: 1,
+                no_checkpoint_before: 1,
+                keepalive_op: 1,
+                last_checkpoint: 1
+            }
+        }));
+        if (preUpdateDocument == null) {
+            throw new ReplicationAssertionError('Failed to update checkpoint - no matching sync_rules document for _id: ' + this.replicationStreamId);
+        }
+        const checkpointState = calculateCheckpointState({
+            lsn,
+            snapshotDone: preUpdateDocument.snapshot_done === true,
+            lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
+            noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
+            keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
+            lastCheckpoint: preUpdateDocument.last_checkpoint,
+            persistedOp: this.persisted_op,
+            createEmptyCheckpoints
+        });
+        if (checkpointState.checkpointBlocked) {
+            if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
+                this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
+                    snapshot_done: preUpdateDocument.snapshot_done,
+                    last_checkpoint_lsn: preUpdateDocument.last_checkpoint_lsn,
+                    no_checkpoint_before: preUpdateDocument.no_checkpoint_before
+                })}`);
+                this.lastWaitingLogThrottled = Date.now();
+            }
+        }
+        else {
+            if (checkpointState.checkpointCreated) {
+                this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`);
+            }
+            await this.autoActivate(lsn);
+            await this.db.notifyCheckpoint();
+            this.persisted_op = null;
+            if (checkpointState.newLastCheckpoint != null) {
+                await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
+            }
+        }
+        return {
+            checkpointBlocked: checkpointState.checkpointBlocked,
+            checkpointCreated: checkpointState.checkpointCreated
+        };
+    }
+    async keepalive(lsn) {
+        return await this.commit(lsn, { createEmptyCheckpoints: true });
+    }
+    async setResumeLsn(lsn) {
+        await this.db.sync_rules.updateOne({
+            _id: this.replicationStreamId
+        }, {
+            $set: {
+                snapshot_lsn: lsn
+            }
+        }, { session: this.session });
+    }
+    async markAllSnapshotDone(no_checkpoint_before_lsn) {
+        await this.db.sync_rules.updateOne({
+            _id: this.replicationStreamId
+        }, {
+            $set: {
+                snapshot_done: true,
+                last_keepalive_ts: new Date()
+            },
+            $max: {
+                no_checkpoint_before: no_checkpoint_before_lsn
+            }
+        }, { session: this.session });
+    }
+    async markSnapshotDone(no_checkpoint_before_lsn, options) {
+        await this.withTransaction(async () => {
+            // Protect against race conditions
+            const blockingTables = await this.db
+                .sourceTablesV1(this.replicationStreamId)
+                .find({
+                group_id: this.replicationStreamId,
+                snapshot_done: false
+            }, {
+                session: this.session,
+                projection: { schema_name: 1, table_name: 1 }
+            })
+                .toArray();
+            if (blockingTables.length > 0) {
+                if (options?.throwOnConflict ?? true) {
+                    throw new ReplicationAssertionError(`Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`);
+                }
+                else {
+                    return;
+                }
+            }
+            await this.markAllSnapshotDone(no_checkpoint_before_lsn);
+        });
+    }
+    async markTableSnapshotRequired(_table) {
+        await this.db.sync_rules.updateOne({
+            _id: this.replicationStreamId
+        }, {
+            $set: {
+                snapshot_done: false
+            }
+        }, { session: this.session });
+    }
+    async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
+        const session = this.session;
+        const ids = tables.map((table) => mongoTableId(table.id));
+        await this.withTransaction(async () => {
+            await this.db.sourceTablesV1(this.replicationStreamId).updateMany({ _id: { $in: ids } }, {
+                $set: {
+                    snapshot_done: true
+                },
+                $unset: {
+                    snapshot_status: 1
+                }
+            }, { session });
+            if (no_checkpoint_before_lsn != null) {
+                await this.db.sync_rules.updateOne({
+                    _id: this.replicationStreamId
+                }, {
+                    $set: {
+                        last_keepalive_ts: new Date()
+                    },
+                    $max: {
+                        no_checkpoint_before: no_checkpoint_before_lsn
+                    }
+                }, { session: this.session });
+            }
+        });
+        return tables.map((table) => {
+            const copy = table.clone();
+            copy.snapshotComplete = true;
+            return copy;
+        });
+    }
+    async autoActivate(lsn) {
+        if (!this.needsActivation) {
+            return;
+        }
+        const session = this.session;
+        let activated = false;
+        await session.withTransaction(async () => {
+            const doc = (await this.db.sync_rules.findOne({ _id: this.replicationStreamId }, { session }));
+            if (doc && doc.state == storage.SyncRuleState.PROCESSING && doc.snapshot_done && doc.last_checkpoint != null) {
+                await this.db.sync_rules.updateOne({
+                    _id: this.replicationStreamId
+                }, {
+                    $set: {
+                        state: storage.SyncRuleState.ACTIVE
+                    }
+                }, { session });
+                await this.db.sync_rules.updateMany({
+                    _id: { $ne: this.replicationStreamId },
+                    state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
+                }, {
+                    $set: {
+                        state: storage.SyncRuleState.STOP
+                    }
+                }, { session });
+                activated = true;
+            }
+            else if (doc?.state != storage.SyncRuleState.PROCESSING) {
+                this.needsActivation = false;
+            }
+        });
+        if (activated) {
+            this.logger.info(`Activated new replication stream at ${lsn}`);
+            await this.db.notifyCheckpoint();
+            this.needsActivation = false;
+        }
+    }
 }
 //# sourceMappingURL=MongoBucketBatchV1.js.map

package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoBucketBatchV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoBucketBatchV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAA2B,MAAM,wBAAwB,CAAC;AAGnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,kBAAmB,SAAQ,gBAAgB;IAGrC,KAAK,CAAoB;IAE1C,YAAY,OAAgC;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAES,oBAAoB,CAAC,WAAmB;QAChD,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE;YAC7E,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;IAED,IAAc,iBAAiB;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAAsB;QAC/D,4DAA4D;IAC9D,CAAC;CACF"}
+{"version":3,"file":"MongoBucketBatchV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoBucketBatchV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAA+C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAC/F,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAA2B,MAAM,wBAAwB,CAAC;AAGnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAI/D,MAAM,OAAO,kBAAmB,SAAQ,gBAAgB;IAGrC,KAAK,CAAoB;IAClC,eAAe,GAAG,IAAI,CAAC;IACvB,uBAAuB,GAAG,CAAC,CAAC;IAEpC;;;;;OAKG;IACK,YAAY,GAAwB,IAAI,CAAC;IAEjD,YAAY,OAAgC;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC;QAChD,IAAI,CAAC,KAAK,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC1E,CAAC;IAEkB,iBAAiB,CAAC,MAAoB;QACvD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAES,oBAAoB,CAAC,WAAmB;QAChD,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE;YACxF,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;IAED,IAAc,iBAAiB;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IACS,KAAK,CAAC,0BAA0B,CAAC,OAAsB;QAC/D,4DAA4D;IAC9D,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAqC;QACvD,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,EAAE,kBAAkB,IAAI,IAAI,CAAC,UAAU,CAAC;QAClF,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAC1C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAAC;QAE9F,MAAM,0BAA0B,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnE,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,MAAM;SACxB,CAAC,CAAC,CAAC;QAEJ,IAAI,MAAM,GAAuC,IAAI,CAAC;QACtD,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAQ;gBAClB,QAAQ,EAAE,IAAI,CAAC,mBAAmB;gBAClC,aAAa;gBACb,WAAW,EAAE,MAAM;gBACnB,UAAU,EAAE,IAAI;gBAChB,mBAAmB,EAAE,0BAA0B;aAChD,CAAC;YACF,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC;YAChC,CAAC;YAED,IAAI,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,GAAG,GAAG;oBACJ,GAAG,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAE,OAAO,CAAC,WAAW,EAAoB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;oBACzF,QAAQ,EAAE,IAAI,CAAC,mBAAmB;oBAClC,aAAa;oBACb,WAAW,EAAE,QAAQ;oBACrB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,IAAI;oBAChB,kBAAkB,EAAE,IAAI;oBACxB,mBAAmB,EAAE,0BAA0B;oBAC/C,aAAa,EAAE,KAAK;oBACpB,eAAe,EAAE,SAAS;iBAC3B,CAAC;gBACF,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;gBAC1C,EAAE,EAAE,GAAG,CAAC,GAAG;gBACX,GAAG,EAAE,MAAM;gBACX,QAAQ;gBACR,gBAAgB;gBAChB,gBAAgB,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;gBAC3C,GAAG,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC;aACxC,CAAC,CAAC;YACH,WAAW,CAAC,SAAS,GAAG,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC7D,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;YAChE,WAAW,CAAC,cAAc,GAAG,WAAW,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3E,WAAW,CAAC,gBAAgB,GAAG,iBAAiB,KAAK,IAAI,CAAC;YAC1D,WAAW,CAAC,cAAc;gBACxB,GAAG,CAAC,eAAe,IAAI,IAAI;oBACzB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC;wBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;wBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;wBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;qBACtD,CAAC;YAER,MAAM,cAAc,GAAG,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAU,CAAC;YAC5E,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,cAAc,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,GAAG;iBACvB,IAAI,CACH;gBACE,QAAQ,EAAE,IAAI,CAAC,mBAAmB;gBAClC,aAAa;gBACb,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;gBACrB,GAAG,EAAE,cAAc;aACpB,EACD,EAAE,OAAO,EAAE,CACZ;iBACA,OAAO,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC1C,MAAM,GAAG,GAAG;oBACV,aAAa;oBACb,MAAM,EAAE,OAAO,CAAC,WAAW;oBAC3B,IAAI,EAAE,OAAO,CAAC,UAAU;iBACzB,CAAC;gBACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;oBACpC,EAAE,EAAE,OAAO,CAAC,GAAG;oBACf,GAAG;oBACH,QAAQ,EAAE,OAAO,CAAC,WAAW;oBAC7B,gBAAgB,EACd,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF,IAAI,EAAE;oBACT,gBAAgB,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;oBAC/C,GAAG,SAAS,CAAC,kBAAkB,CAAC,GAAG,CAAC;iBACrC,CAAC,CAAC;gBACH,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACpD,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;gBACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC/D,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,GAAG,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,OAAO,MAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,KAA0B;QACnD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,OAAO,CACxE;YACE,QAAQ,EAAE,IAAI,CAAC,mBAAmB;YAClC,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;SAC5B,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG;YACV,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa;YACtC,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,GAAG;YACH,QAAQ,EAAE,GAAG,CAAC,WAAW;YACzB,gBAAgB,EACd,GAAG,CAAC,mBAAmB,EAAE,GAAG,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF,IAAI,EAAE;YACT,gBAAgB,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YAC3C,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC;SAC3C,CAAC,CAAC;QACH,WAAW,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAChE,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,WAAW,CAAC,cAAc,GAAG,WAAW,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3E,WAAW,CAAC,cAAc;YACxB,GAAG,CAAC,eAAe,IAAI,IAAI;gBACzB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC;oBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;oBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;oBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;iBACtD,CAAC;QACR,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,OAA0C;QAClE,MAAM,EAAE,sBAAsB,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,mCAAmC,EAAE,GAAG,OAAO,EAAE,CAAC;QAElG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE1B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CACxC;YACE,gBAAgB,EAAE,IAAI;YACtB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,EACD;YACE,IAAI,EAAE;gBACJ,gBAAgB,EAAE,GAAG;aACtB;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CACF,CAAC;QAEF,MAAM,cAAc,GAAG;YACrB,IAAI,EAAE;gBACJ,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE,IAAI,CAAC,EAAE;gBACjC;oBACE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBACtG;gBACD;oBACE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBACxG;aACF;SACF,CAAC;QAEF,MAAM,gBAAgB,GAAG;YACvB,KAAK,EAAE;gBACL,cAAc;gBACd,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAClB;oBACE,SAAS,EAAE;wBACT,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,CAAC;qBAC1E;iBACF;aACF;SACF,CAAC;QAEF,MAAM,mBAAmB,GAAG;YAC1B,KAAK,EAAE;gBACL,cAAc;gBACd;oBACE,IAAI,EAAE,CAAC,kBAAkB,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;iBAC9F;gBACD,kBAAkB;aACnB;SACF,CAAC;QAEF,MAAM,iBAAiB,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAClE,EAAE,GAAG,EAAE,IAAI,CAAC,mBAAmB,EAAE,EACjC;YACE;gBACE,IAAI,EAAE;oBACJ,eAAe,EAAE,cAAc;oBAC/B,UAAU,EAAE,sBAAsB;wBAChC,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC;4BACE,GAAG,EAAE;gCACH,EAAE,QAAQ,EAAE,sBAAsB,EAAE;gCACpC,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,gBAAgB,CAAC,EAAE;gCAC5C,EAAE,GAAG,EAAE,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,EAAE;6BACnD;yBACF;iBACN;aACF;YACD;gBACE,IAAI,EAAE;oBACJ,mBAAmB,EAAE;wBACnB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC;qBAClG;oBACD,kBAAkB,EAAE;wBAClB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,qBAAqB,CAAC;qBACjG;oBACD,iBAAiB,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE;oBACpC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACpC,mBAAmB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACvC,YAAY,EAAE,gBAAgB;oBAC9B,eAAe,EAAE,mBAAmB;oBACpC,YAAY,EAAE;wBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,eAAe,CAAC;qBAC5F;iBACF;aACF;YACD;gBACE,MAAM,EAAE,CAAC,iBAAiB,EAAE,YAAY,CAAC;aAC1C;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,cAAc,EAAE,QAAQ;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAC;gBACtB,oBAAoB,EAAE,CAAC;gBACvB,YAAY,EAAE,CAAC;gBACf,eAAe,EAAE,CAAC;aACnB;SACF,CACF,CAAuB,CAAC;QAEzB,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,yBAAyB,CACjC,yEAAyE,GAAG,IAAI,CAAC,mBAAmB,CACrG,CAAC;QACJ,CAAC;QAED,MAAM,eAAe,GAAG,wBAAwB,CAAC;YAC/C,GAAG;YACH,YAAY,EAAE,iBAAiB,CAAC,aAAa,KAAK,IAAI;YACtD,iBAAiB,EAAE,iBAAiB,CAAC,mBAAmB;YACxD,kBAAkB,EAAE,iBAAiB,CAAC,oBAAoB;YAC1D,WAAW,EAAE,iBAAiB,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC;YACnG,cAAc,EAAE,iBAAiB,CAAC,eAAe;YACjD,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,sBAAsB;SACvB,CAAC,CAAC;QACH,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,uBAAuB,GAAG,KAAK,EAAE,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oDAAoD,GAAG,MAAM,eAAe,CAAC,cAAc,oBAAoB,IAAI,CAAC,SAAS,CAC3H;oBACE,aAAa,EAAE,iBAAiB,CAAC,aAAa;oBAC9C,mBAAmB,EAAE,iBAAiB,CAAC,mBAAmB;oBAC1D,oBAAoB,EAAE,iBAAiB,CAAC,oBAAoB;iBAC7D,CACF,EAAE,CACJ,CAAC;gBACF,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,MAAM,eAAe,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,eAAe,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAC9C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,eAAe,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;QAED,OAAO;YACL,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;YACpD,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;SACrD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,mBAAmB;SAC9B,EACD;YACE,IAAI,EAAE;gBACJ,YAAY,EAAE,GAAG;aAClB;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,wBAAgC;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,mBAAmB;SAC9B,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,IAAI;gBACnB,iBAAiB,EAAE,IAAI,IAAI,EAAE;aAC9B;YACD,IAAI,EAAE;gBACJ,oBAAoB,EAAE,wBAAwB;aAC/C;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,wBAAgC,EAAE,OAAuC;QAC9F,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACpC,kCAAkC;YAClC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE;iBACjC,cAAc,CAAC,IAAI,CAAC,mBAAmB,CAAC;iBACxC,IAAI,CACH;gBACE,QAAQ,EAAE,IAAI,CAAC,mBAAmB;gBAClC,aAAa,EAAE,KAAK;aACrB,EACD;gBACE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;aAC9C,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,IAAI,OAAO,EAAE,eAAe,IAAI,IAAI,EAAE,CAAC;oBACrC,MAAM,IAAI,yBAAyB,CACjC,qFAAqF,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChK,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO;gBACT,CAAC;YACH,CAAC;YAED,MAAM,IAAI,CAAC,mBAAmB,CAAC,wBAAwB,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAA2B;QACzD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,mBAAmB;SAC9B,EACD;YACE,IAAI,EAAE;gBACJ,aAAa,EAAE,KAAK;aACrB;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,MAA6B,EAC7B,wBAAiC;QAEjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAE1D,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACpC,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAC/D,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EACrB;gBACE,IAAI,EAAE;oBACJ,aAAa,EAAE,IAAI;iBACpB;gBACD,MAAM,EAAE;oBACN,eAAe,EAAE,CAAC;iBACnB;aACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YAEF,IAAI,wBAAwB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;oBACE,GAAG,EAAE,IAAI,CAAC,mBAAmB;iBAC9B,EACD;oBACE,IAAI,EAAE;wBACJ,iBAAiB,EAAE,IAAI,IAAI,EAAE;qBAC9B;oBACD,IAAI,EAAE;wBACJ,oBAAoB,EAAE,wBAAwB;qBAC/C;iBACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACvC,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC3C,EAAE,GAAG,EAAE,IAAI,CAAC,mBAAmB,EAAE,EACjC,EAAE,OAAO,EAAE,CACZ,CAAuB,CAAC;YACzB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC7G,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;oBACE,GAAG,EAAE,IAAI,CAAC,mBAAmB;iBAC9B,EACD;oBACE,IAAI,EAAE;wBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;qBACpC;iBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;oBACE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,mBAAmB,EAAE;oBACtC,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;iBAC9E,EACD;oBACE,IAAI,EAAE;wBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;qBAClC;iBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,GAAG,EAAE,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC1D,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoChecksumsV1.js

@@ -1,8 +1,10 @@
-import { bson } from '@powersync/service-core';
-import { MongoChecksums } from '../MongoChecksums.js';
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { addPartialChecksums, bson, isPartialChecksum } from '@powersync/service-core';
+import { checksumFromAggregate, DEFAULT_OPERATION_BATCH_LIMIT, MongoChecksums } from '../MongoChecksums.js';
 export class MongoChecksumsV1 extends MongoChecksums {
     async computePartialChecksumsDirectByBucket(batch) {
-        return this.computePartialChecksumsForCollection(batch, this.db.bucketDataV1, (request) => ({
+        const collection = this.db.bucketDataV1;
+        const createFilter = (request) => ({
             _id: {
                 $gt: {
                     g: this.group_id,
@@ -15,6 +17,115 @@ export class MongoChecksumsV1 extends MongoChecksums {
                     o: request.end
                 }
             }
+        });
+        const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT;
+        // Map requests by bucket. We adjust this as we get partial results.
+        let requests = new Map();
+        for (let request of batch) {
+            requests.set(request.bucket, request);
+        }
+        const partialChecksums = new Map();
+        while (requests.size > 0) {
+            const filters = Array.from(requests.values(), createFilter);
+            // Historically, checksum may be stored as 'int' or 'double'.
+            // More recently, this should be a 'long'.
+            // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
+            const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
+            // Aggregate over a max of `batchLimit` operations at a time.
+            // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
+            // Then we'll do three batches:
+            // 1. Query: A[1-end], B[1-end], C[1-end]
+            //    Returns: A[1-10], B[1-2]
+            // 2. Query: B[3-end], C[1-end]
+            //    Returns: B[3-10], C[1-4]
+            // 3. Query: C[5-end]
+            //    Returns: C[5-10]
+            const aggregate = await collection
+                .aggregate([
+                {
+                    $match: {
+                        $or: filters
+                    }
+                },
+                // sort and limit _before_ grouping
+                { $sort: { _id: 1 } },
+                { $limit: batchLimit },
+                {
+                    $group: {
+                        _id: '$_id.b',
+                        checksum_total: { $sum: checksumLong },
+                        count: { $sum: 1 },
+                        has_clear_op: {
+                            $max: {
+                                $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
+                            }
+                        },
+                        last_op: { $max: '$_id.o' }
+                    }
+                },
+                // Sort the aggregated results (100 max, so should be fast).
+                // This is important to identify which buckets we have partial data for.
+                { $sort: { _id: 1 } }
+            ], {
+                session: undefined,
+                readConcern: 'snapshot',
+                maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS
+            })
+                .toArray()
+                .catch((e) => {
+                throw lib_mongo.mapQueryError(e, 'while reading checksums');
+            });
+            let batchCount = 0;
+            let limitReached = false;
+            for (let doc of aggregate) {
+                const bucket = doc._id;
+                const checksum = checksumFromAggregate(doc);
+                const existing = partialChecksums.get(bucket);
+                if (existing != null) {
+                    partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum));
+                }
+                else {
+                    partialChecksums.set(bucket, checksum);
+                }
+                batchCount += doc.count;
+                if (batchCount == batchLimit) {
+                    // Limit reached. Request more in the next batch.
+                    // Note that this only affects the _last_ bucket in a batch.
+                    limitReached = true;
+                    const req = requests.get(bucket);
+                    requests.set(bucket, {
+                        ...req,
+                        start: doc.last_op
+                    });
+                }
+                else {
+                    // All done for this bucket
+                    requests.delete(bucket);
+                }
+            }
+            if (!limitReached) {
+                break;
+            }
+        }
+        return new Map(batch.map((request) => {
+            const bucket = request.bucket;
+            let partialChecksum = partialChecksums.get(bucket);
+            // Could be null if we got no data
+            if (partialChecksum == null) {
+                partialChecksum = {
+                    bucket,
+                    partialCount: 0,
+                    partialChecksum: 0
+                };
+            }
+            if (request.start == null && isPartialChecksum(partialChecksum)) {
+                partialChecksum = {
+                    bucket,
+                    count: partialChecksum.partialCount,
+                    checksum: partialChecksum.partialChecksum
+                };
+            }
+            return [bucket, partialChecksum];
         }));
     }
     async fetchPreStates(batch) {

package/dist/storage/implementation/v1/MongoChecksumsV1.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoChecksumsV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoChecksumsV1.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAKL,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAsC,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG1F,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAGlD,KAAK,CAAC,qCAAqC,CACzC,KAA2C;QAE3C,OAAO,IAAI,CAAC,oCAAoC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC1F,GAAG,EAAE;gBACH,GAAG,EAAE;oBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,GAAG;iBACf;aACF;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,cAAc,CAC5B,KAAmC;QAEnC,MAAM,UAAU,GAAG,KAAK;aACrB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;aAC1C,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjB,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;aAClB;YACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;SAC9D,CAAC,CAAC,CAAC;QAEN,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QACtF,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa;aACvC,IAAI,CAAC;YACJ,GAAG,EAAE,UAAU;SAChB,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;YAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,QAAQ,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;oBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;oBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;iBAC5B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAES,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QACjF,OAAO,IAAI,CAAC,qCAAqC,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;CACF"}
+{"version":3,"file":"MongoChecksumsV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoChecksumsV1.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAIJ,iBAAiB,EAGlB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,qBAAqB,EACrB,6BAA6B,EAE7B,cAAc,EACf,MAAM,sBAAsB,CAAC;AAG9B,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAGlD,KAAK,CAAC,qCAAqC,CACzC,KAA2C;QAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC;QACxC,MAAM,YAAY,GAAG,CAAC,OAA2C,EAAE,EAAE,CAAC,CAAC;YACrE,GAAG,EAAE;gBACH,GAAG,EAAE;oBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,GAAG;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAA8C,CAAC;QACrE,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;YAE5D,6DAA6D;YAC7D,0CAA0C;YAC1C,8FAA8F;YAC9F,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAE/F,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,UAAU;iBAC/B,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD;gBACE,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,SAAS,CAAC,yBAAyB;aAC/C,CACF;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,GAAG,GAAI;wBACP,KAAK,EAAE,GAAG,CAAC,OAAO;qBACnB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,kCAAkC;YAClC,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,cAAc,CAC5B,KAAmC;QAEnC,MAAM,UAAU,GAAG,KAAK;aACrB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;aAC1C,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjB,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;aAClB;YACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;SAC9D,CAAC,CAAC,CAAC;QAEN,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QACtF,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa;aACvC,IAAI,CAAC;YACJ,GAAG,EAAE,UAAU;SAChB,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;YAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,QAAQ,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;oBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;oBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;iBAC5B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAES,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QACjF,OAAO,IAAI,CAAC,qCAAqC,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoCompactorV1.d.ts

@@ -1,6 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage } from '@powersync/service-core';
-import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { SingleBucketStore } from '../common/SingleBucketStore.js';
 import { BucketStateDocumentBase } from '../models.js';
 import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';