@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

package/dist/storage/implementation/v3/MongoBucketBatchV3.js

@@ -0,0 +1,463 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+import * as bson from 'bson';
+import { mongoTableId } from '../../../utils/util.js';
+import { calculateCheckpointState } from '../CheckpointState.js';
+import { MongoBucketBatch } from '../MongoBucketBatch.js';
+import { syncRuleStateUpdatePipeline } from '../SyncRuleStateUpdate.js';
+import { PersistedBatchV3 } from './PersistedBatchV3.js';
+import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js';
+function sameStringArray(left, right) {
+    return left.length == right.length && left.every((value, index) => value == right[index]);
+}
+export class MongoBucketBatchV3 extends MongoBucketBatch {
+    store;
+    syncConfigId;
+    needsActivationV3 = true;
+    lastWaitingLogThrottledV3 = 0;
+    constructor(options) {
+        super(options);
+        if (options.syncConfigId == null) {
+            throw new ReplicationAssertionError('Missing sync config id for v3 batch');
+        }
+        this.syncConfigId = options.syncConfigId;
+        this.store = new SourceRecordStoreV3(this.db, this.group_id, this.mapping);
+    }
+    createPersistedBatch(writtenSize) {
+        return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, {
+            logger: this.logger
+        });
+    }
+    get sourceRecordStore() {
+        return this.store;
+    }
+    async cleanupDroppedSourceTables(sourceTables) {
+        for (const table of sourceTables) {
+            await this.db
+                .sourceRecordsV3(this.group_id, mongoTableId(table.id))
+                .drop()
+                .catch((error) => {
+                if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+                    return;
+                }
+                throw error;
+            });
+        }
+    }
+    async resolveTables(options) {
+        const ref = options.source;
+        const syncRules = options.syncRules ?? this.sync_rules;
+        const matchingSources = syncRules.getMatchingSources(ref);
+        const { connection_id, source } = options;
+        const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
+        const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
+            name: column.name,
+            type: column.type,
+            type_oid: column.typeId
+        }));
+        let result = null;
+        const initializeSourceRecordsFor = [];
+        await this.db.client.withSession(async (session) => {
+            const col = this.db.commonSourceTables(this.group_id);
+            const exactFilter = {
+                connection_id,
+                schema_name: schema,
+                table_name: name,
+                replica_id_columns2: normalizedReplicaIdColumns
+            };
+            if (objectId != null) {
+                exactFilter.relation_id = objectId;
+            }
+            const exactDocs = (await col.find(exactFilter, { session }).toArray());
+            const bucketSourceById = new Map(matchingSources.bucketDataSources.map((source) => [this.mapping.bucketSourceId(source), source]));
+            const parameterLookupSourceById = new Map(matchingSources.parameterLookupSources.map((source) => [this.mapping.parameterLookupId(source), source]));
+            const desiredBucketIds = new Set(bucketSourceById.keys());
+            const desiredLookupIds = new Set(parameterLookupSourceById.keys());
+            const desiredHasMembership = desiredBucketIds.size > 0 || desiredLookupIds.size > 0;
+            const triggersEvent = syncRules.tableTriggersEvent(ref);
+            const coveredBucketIds = new Set();
+            const coveredLookupIds = new Set();
+            const retainedDocIds = [];
+            const tables = [];
+            let retainedEventOnlyTable = false;
+            for (const doc of exactDocs) {
+                const bucketDataSourceIds = doc.bucket_data_source_ids.filter((id) => desiredBucketIds.has(id));
+                const parameterLookupSourceIds = doc.parameter_lookup_source_ids.filter((id) => desiredLookupIds.has(id));
+                const coversDesiredMembership = bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0;
+                const coversEventOnlyTable = !desiredHasMembership && triggersEvent && !retainedEventOnlyTable;
+                for (const id of bucketDataSourceIds) {
+                    coveredBucketIds.add(id);
+                }
+                for (const id of parameterLookupSourceIds) {
+                    coveredLookupIds.add(id);
+                }
+                const updates = {};
+                if (!sameStringArray(doc.bucket_data_source_ids, bucketDataSourceIds) ||
+                    !sameStringArray(doc.parameter_lookup_source_ids, parameterLookupSourceIds)) {
+                    updates.bucket_data_source_ids = bucketDataSourceIds;
+                    updates.parameter_lookup_source_ids = parameterLookupSourceIds;
+                }
+                if (Object.keys(updates).length > 0) {
+                    await col.updateOne({ _id: doc._id }, { $set: updates }, { session });
+                }
+                if (coversDesiredMembership || coversEventOnlyTable) {
+                    if (coversEventOnlyTable) {
+                        retainedEventOnlyTable = true;
+                    }
+                    retainedDocIds.push(doc._id);
+                    const table = this.sourceTableFromDocument({
+                        ...doc,
+                        bucket_data_source_ids: bucketDataSourceIds,
+                        parameter_lookup_source_ids: parameterLookupSourceIds
+                    }, connectionTag, syncRules, {
+                        bucketDataSources: bucketDataSourceIds.map((id) => bucketSourceById.get(id)),
+                        parameterLookupSources: parameterLookupSourceIds.map((id) => parameterLookupSourceById.get(id))
+                    });
+                    table.storeCurrentData = sendsCompleteRows !== true;
+                    tables.push(table);
+                }
+            }
+            const uncoveredBucketIds = [...desiredBucketIds].filter((id) => !coveredBucketIds.has(id));
+            const uncoveredLookupIds = [...desiredLookupIds].filter((id) => !coveredLookupIds.has(id));
+            if (uncoveredBucketIds.length > 0 || uncoveredLookupIds.length > 0 || (triggersEvent && tables.length == 0)) {
+                const id = options.idGenerator ? options.idGenerator() : new bson.ObjectId();
+                const sourceTable = new storage.SourceTable({
+                    id,
+                    ref,
+                    objectId,
+                    replicaIdColumns,
+                    snapshotComplete: false,
+                    bucketDataSources: uncoveredBucketIds.map((id) => bucketSourceById.get(id)),
+                    parameterLookupSources: uncoveredLookupIds.map((id) => parameterLookupSourceById.get(id))
+                });
+                sourceTable.syncData = uncoveredBucketIds.length > 0;
+                sourceTable.syncParameters = uncoveredLookupIds.length > 0;
+                sourceTable.syncEvent = triggersEvent;
+                sourceTable.storeCurrentData = sendsCompleteRows !== true;
+                const createDoc = {
+                    _id: id,
+                    connection_id,
+                    relation_id: objectId,
+                    schema_name: schema,
+                    table_name: name,
+                    replica_id_columns: null,
+                    replica_id_columns2: normalizedReplicaIdColumns,
+                    snapshot_done: false,
+                    snapshot_status: undefined,
+                    bucket_data_source_ids: uncoveredBucketIds,
+                    parameter_lookup_source_ids: uncoveredLookupIds
+                };
+                await col.insertOne(createDoc, { session });
+                initializeSourceRecordsFor.push(createDoc._id);
+                retainedDocIds.push(createDoc._id);
+                tables.push(sourceTable);
+            }
+            const conflictFilter = [{ schema_name: schema, table_name: name }];
+            if (objectId != null) {
+                conflictFilter.push({ relation_id: objectId });
+            }
+            const dropTables = await col
+                .find({
+                connection_id,
+                _id: { $nin: retainedDocIds },
+                $or: conflictFilter
+            }, { session })
+                .toArray();
+            result = {
+                tables,
+                dropTables: dropTables.map((doc) => this.sourceTableFromDocument(doc, connectionTag, syncRules))
+            };
+        });
+        for (const sourceTableId of initializeSourceRecordsFor) {
+            await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId);
+        }
+        return result;
+    }
+    sourceTableFromDocument(doc, connectionTag, syncRules, memberships) {
+        const resolvedMemberships = memberships ?? this.sourceTableMembershipsFromDocument(doc, syncRules);
+        const table = new storage.SourceTable({
+            id: doc._id,
+            ref: {
+                connectionTag,
+                schema: doc.schema_name,
+                name: doc.table_name
+            },
+            objectId: doc.relation_id,
+            replicaIdColumns: doc.replica_id_columns2.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })),
+            snapshotComplete: doc.snapshot_done ?? true,
+            bucketDataSources: resolvedMemberships.bucketDataSources,
+            parameterLookupSources: resolvedMemberships.parameterLookupSources
+        });
+        table.syncData = table.bucketDataSources.length > 0;
+        table.syncParameters = table.parameterLookupSources.length > 0;
+        table.syncEvent = syncRules.tableTriggersEvent(table.ref);
+        table.snapshotStatus =
+            doc.snapshot_status == null
+                ? undefined
+                : {
+                    lastKey: doc.snapshot_status.last_key?.buffer ?? null,
+                    totalEstimatedCount: doc.snapshot_status.total_estimated_count,
+                    replicatedCount: doc.snapshot_status.replicated_count
+                };
+        return table;
+    }
+    sourceTableMembershipsFromDocument(doc, syncRules) {
+        const bucketDataSourceIds = new Set(doc.bucket_data_source_ids);
+        const parameterLookupSourceIds = new Set(doc.parameter_lookup_source_ids);
+        return {
+            bucketDataSources: syncRules.bucketDataSources.filter((source) => bucketDataSourceIds.has(this.mapping.bucketSourceId(source))),
+            parameterLookupSources: syncRules.bucketParameterLookupSources.filter((source) => parameterLookupSourceIds.has(this.mapping.parameterLookupId(source)))
+        };
+    }
+    async getSourceTableStatus(table) {
+        const doc = (await this.db
+            .commonSourceTables(this.group_id)
+            .findOne({ _id: mongoTableId(table.id) }, { session: this.session }));
+        if (doc == null) {
+            return null;
+        }
+        return this.sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules);
+    }
+    async commit(lsn, options) {
+        const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
+        await this.flush(options);
+        const now = new Date();
+        await this.db.write_checkpoints.updateMany({
+            processed_at_lsn: null,
+            'lsns.1': { $lte: lsn }
+        }, {
+            $set: {
+                processed_at_lsn: lsn
+            }
+        }, {
+            session: this.session
+        });
+        const preUpdateDocument = await this.db.sync_rules.findOne({
+            _id: this.group_id,
+            'sync_configs._id': this.syncConfigId
+        }, {
+            session: this.session,
+            projection: {
+                snapshot_lsn: 1,
+                sync_configs: {
+                    $elemMatch: {
+                        _id: this.syncConfigId
+                    }
+                }
+            }
+        });
+        const state = preUpdateDocument?.sync_configs?.[0];
+        if (state == null) {
+            throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.group_id}/${this.syncConfigId.toHexString()}`);
+        }
+        const checkpointState = calculateCheckpointState({
+            lsn,
+            snapshotDone: state.snapshot_done === true,
+            lastCheckpointLsn: state.last_checkpoint_lsn,
+            noCheckpointBefore: state.no_checkpoint_before,
+            keepaliveOp: state.keepalive_op == null ? null : BigInt(state.keepalive_op),
+            lastCheckpoint: state.last_checkpoint,
+            persistedOp: this.persisted_op,
+            createEmptyCheckpoints
+        });
+        const updateSet = {
+            last_keepalive_ts: now,
+            last_fatal_error: null,
+            last_fatal_error_ts: null,
+            'sync_configs.$[config].keepalive_op': checkpointState.newKeepaliveOp,
+            'sync_configs.$[config].last_checkpoint': checkpointState.newLastCheckpoint
+        };
+        if (checkpointState.checkpointCreated) {
+            updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
+            updateSet['snapshot_lsn'] = null;
+            updateSet['last_checkpoint_ts'] = now;
+        }
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id,
+            'sync_configs._id': this.syncConfigId
+        }, {
+            $set: updateSet
+        }, {
+            session: this.session,
+            arrayFilters: [{ 'config._id': this.syncConfigId }]
+        });
+        if (checkpointState.checkpointBlocked) {
+            if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) {
+                this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
+                    snapshot_done: state.snapshot_done,
+                    last_checkpoint_lsn: state.last_checkpoint_lsn,
+                    no_checkpoint_before: state.no_checkpoint_before
+                })}`);
+                this.lastWaitingLogThrottledV3 = Date.now();
+            }
+        }
+        else {
+            if (checkpointState.checkpointCreated) {
+                this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`);
+            }
+            await this.autoActivateV3(lsn);
+            await this.db.notifyCheckpoint();
+            this.persisted_op = null;
+            this.last_checkpoint_lsn = lsn;
+            if (checkpointState.newLastCheckpoint != null) {
+                await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
+            }
+        }
+        return {
+            checkpointBlocked: checkpointState.checkpointBlocked,
+            checkpointCreated: checkpointState.checkpointCreated
+        };
+    }
+    async keepalive(lsn) {
+        return await this.commit(lsn, { createEmptyCheckpoints: true });
+    }
+    async setResumeLsn(lsn) {
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id,
+            'sync_configs._id': this.syncConfigId
+        }, {
+            $set: {
+                snapshot_lsn: lsn
+            }
+        }, { session: this.session });
+    }
+    async autoActivateV3(lsn) {
+        if (!this.needsActivationV3) {
+            return;
+        }
+        const session = this.session;
+        let activated = false;
+        await session.withTransaction(async () => {
+            const doc = await this.db.sync_rules.findOne({
+                _id: this.group_id,
+                'sync_configs._id': this.syncConfigId
+            }, {
+                session,
+                projection: {
+                    state: 1,
+                    sync_configs: {
+                        $elemMatch: {
+                            _id: this.syncConfigId
+                        }
+                    }
+                }
+            });
+            const state = doc?.sync_configs?.[0];
+            if (doc &&
+                doc.state == storage.SyncRuleState.PROCESSING &&
+                state?.state == storage.SyncRuleState.PROCESSING &&
+                state.snapshot_done &&
+                state.last_checkpoint != null) {
+                await this.db.sync_rules.updateOne({
+                    _id: this.group_id,
+                    'sync_configs._id': this.syncConfigId
+                }, {
+                    $set: {
+                        state: storage.SyncRuleState.ACTIVE,
+                        'sync_configs.$[config].state': storage.SyncRuleState.ACTIVE
+                    }
+                }, {
+                    session,
+                    arrayFilters: [{ 'config._id': this.syncConfigId }]
+                });
+                await this.db.sync_rules.updateMany({
+                    _id: { $ne: this.group_id },
+                    state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
+                }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
+                activated = true;
+            }
+            else if (doc?.state != storage.SyncRuleState.PROCESSING) {
+                this.needsActivationV3 = false;
+            }
+        });
+        if (activated) {
+            this.logger.info(`Activated new replication stream at ${lsn}`);
+            await this.db.notifyCheckpoint();
+            this.needsActivationV3 = false;
+        }
+    }
+    async markAllSnapshotDone(no_checkpoint_before_lsn) {
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id,
+            'sync_configs._id': this.syncConfigId
+        }, {
+            $set: {
+                'sync_configs.$[config].snapshot_done': true,
+                last_keepalive_ts: new Date()
+            },
+            $max: {
+                'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn
+            }
+        }, {
+            session: this.session,
+            arrayFilters: [{ 'config._id': this.syncConfigId }]
+        });
+    }
+    async markSnapshotDone(no_checkpoint_before_lsn, options) {
+        await this.withTransaction(async () => {
+            // Protect against race conditions
+            const count = await this.db.sourceTablesV3(this.group_id).countDocuments({
+                snapshot_done: false
+            }, { session: this.session });
+            if (count > 0) {
+                if (options?.throwOnConflict ?? true) {
+                    throw new ReplicationAssertionError(`Cannot mark snapshot done while ${count} source table${count == 1 ? '' : 's'} still require snapshotting`);
+                }
+                else {
+                    return;
+                }
+            }
+            await this.markAllSnapshotDone(no_checkpoint_before_lsn);
+        });
+    }
+    async markTableSnapshotRequired(_table) {
+        await this.db.sync_rules.updateOne({
+            _id: this.group_id,
+            'sync_configs._id': this.syncConfigId
+        }, {
+            $set: {
+                'sync_configs.$[config].snapshot_done': false
+            }
+        }, {
+            session: this.session,
+            arrayFilters: [{ 'config._id': this.syncConfigId }]
+        });
+    }
+    async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
+        const session = this.session;
+        const ids = tables.map((table) => mongoTableId(table.id));
+        await this.withTransaction(async () => {
+            await this.db.commonSourceTables(this.group_id).updateMany({ _id: { $in: ids } }, {
+                $set: {
+                    snapshot_done: true
+                },
+                $unset: {
+                    snapshot_status: 1
+                }
+            }, { session });
+            if (no_checkpoint_before_lsn != null) {
+                await this.db.sync_rules.updateOne({
+                    _id: this.group_id,
+                    'sync_configs._id': this.syncConfigId
+                }, {
+                    $set: {
+                        last_keepalive_ts: new Date()
+                    },
+                    $max: {
+                        'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn
+                    }
+                }, {
+                    session: this.session,
+                    arrayFilters: [{ 'config._id': this.syncConfigId }]
+                });
+            }
+        });
+        return tables.map((table) => {
+            const copy = table.clone();
+            copy.snapshotComplete = true;
+            return copy;
+        });
+    }
+}
+//# sourceMappingURL=MongoBucketBatchV3.js.map

package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoBucketBatchV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/MongoBucketBatchV3.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAoB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEpE,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAA2B,MAAM,wBAAwB,CAAC;AACnF,OAAO,EAAE,2BAA2B,EAAE,MAAM,2BAA2B,CAAC;AAGxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAI/D,SAAS,eAAe,CAAC,IAAc,EAAE,KAAe;IACtD,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,MAAM,OAAO,kBAAmB,SAAQ,gBAAgB;IAGrC,KAAK,CAAoB;IACzB,YAAY,CAAgB;IACrC,iBAAiB,GAAG,IAAI,CAAC;IACzB,yBAAyB,GAAG,CAAC,CAAC;IAEtC,YAAY,OAAgC;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,OAAO,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,IAAI,yBAAyB,CAAC,qCAAqC,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,KAAK,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7E,CAAC;IAES,oBAAoB,CAAC,WAAmB;QAChD,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE;YAC7E,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;IAED,IAAc,iBAAiB;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,YAAmC;QAC5E,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,EAAE;iBACV,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACtD,IAAI,EAAE;iBACN,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,SAAS,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,KAAK,mBAAmB,EAAE,CAAC;oBAClF,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAAqC;QACvD,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;QAC3B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC;QACvD,MAAM,eAAe,GAAG,SAAS,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAE1D,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAC1C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAAC;QAC9F,MAAM,0BAA0B,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnE,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,QAAQ,EAAE,MAAM,CAAC,MAAM;SACxB,CAAC,CAAC,CAAC;QAEJ,IAAI,MAAM,GAAuC,IAAI,CAAC;QACtD,MAAM,0BAA0B,GAAoB,EAAE,CAAC;QAEvD,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtD,MAAM,WAAW,GAA4B;gBAC3C,aAAa;gBACb,WAAW,EAAE,MAAM;gBACnB,UAAU,EAAE,IAAI;gBAChB,mBAAmB,EAAE,0BAA0B;aAChD,CAAC;YACF,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,WAAW,CAAC,WAAW,GAAG,QAAQ,CAAC;YACrC,CAAC;YAED,MAAM,SAAS,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAA4B,CAAC;YAClG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CAC1G,CAAC;YACF,MAAM,yBAAyB,GAAG,IAAI,GAAG,CACvC,eAAe,CAAC,sBAAsB,CAAC,GAAG,CACxC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CACtE,CACF,CAAC;YACF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,oBAAoB,GAAG,gBAAgB,CAAC,IAAI,GAAG,CAAC,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAC;YACpF,MAAM,aAAa,GAAG,SAAS,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAExD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;YAC3C,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;YAC3C,MAAM,cAAc,GAAoB,EAAE,CAAC;YAC3C,MAAM,MAAM,GAA0B,EAAE,CAAC;YACzC,IAAI,sBAAsB,GAAG,KAAK,CAAC;YAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,mBAAmB,GAAG,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBAChG,MAAM,wBAAwB,GAAG,GAAG,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1G,MAAM,uBAAuB,GAAG,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,CAAC;gBACtG,MAAM,oBAAoB,GAAG,CAAC,oBAAoB,IAAI,aAAa,IAAI,CAAC,sBAAsB,CAAC;gBAE/F,KAAK,MAAM,EAAE,IAAI,mBAAmB,EAAE,CAAC;oBACrC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC3B,CAAC;gBACD,KAAK,MAAM,EAAE,IAAI,wBAAwB,EAAE,CAAC;oBAC1C,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC3B,CAAC;gBAED,MAAM,OAAO,GAAmC,EAAE,CAAC;gBACnD,IACE,CAAC,eAAe,CAAC,GAAG,CAAC,sBAAsB,EAAE,mBAAmB,CAAC;oBACjE,CAAC,eAAe,CAAC,GAAG,CAAC,2BAA2B,EAAE,wBAAwB,CAAC,EAC3E,CAAC;oBACD,OAAO,CAAC,sBAAsB,GAAG,mBAAmB,CAAC;oBACrD,OAAO,CAAC,2BAA2B,GAAG,wBAAwB,CAAC;gBACjE,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxE,CAAC;gBAED,IAAI,uBAAuB,IAAI,oBAAoB,EAAE,CAAC;oBACpD,IAAI,oBAAoB,EAAE,CAAC;wBACzB,sBAAsB,GAAG,IAAI,CAAC;oBAChC,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,uBAAuB,CACxC;wBACE,GAAG,GAAG;wBACN,sBAAsB,EAAE,mBAAmB;wBAC3C,2BAA2B,EAAE,wBAAwB;qBACtD,EACD,aAAa,EACb,SAAS,EACT;wBACE,iBAAiB,EAAE,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;wBAC7E,sBAAsB,EAAE,wBAAwB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;qBACjG,CACF,CAAC;oBACF,KAAK,CAAC,gBAAgB,GAAG,iBAAiB,KAAK,IAAI,CAAC;oBACpD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,MAAM,kBAAkB,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC3F,MAAM,kBAAkB,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAE3F,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC5G,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAE,OAAO,CAAC,WAAW,EAAoB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChG,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;oBAC1C,EAAE;oBACF,GAAG;oBACH,QAAQ;oBACR,gBAAgB;oBAChB,gBAAgB,EAAE,KAAK;oBACvB,iBAAiB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;oBAC5E,sBAAsB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;iBAC3F,CAAC,CAAC;gBACH,WAAW,CAAC,QAAQ,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;gBACrD,WAAW,CAAC,cAAc,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC3D,WAAW,CAAC,SAAS,GAAG,aAAa,CAAC;gBACtC,WAAW,CAAC,gBAAgB,GAAG,iBAAiB,KAAK,IAAI,CAAC;gBAE1D,MAAM,SAAS,GAA0B;oBACvC,GAAG,EAAE,EAAE;oBACP,aAAa;oBACb,WAAW,EAAE,QAAQ;oBACrB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,IAAI;oBAChB,kBAAkB,EAAE,IAAI;oBACxB,mBAAmB,EAAE,0BAA0B;oBAC/C,aAAa,EAAE,KAAK;oBACpB,eAAe,EAAE,SAAS;oBAC1B,sBAAsB,EAAE,kBAAkB;oBAC1C,2BAA2B,EAAE,kBAAkB;iBAChD,CAAC;gBAEF,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5C,0BAA0B,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC/C,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC3B,CAAC;YAED,MAAM,cAAc,GAAG,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAA8B,CAAC;YAChG,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACrB,cAAc,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,GAAG;iBACzB,IAAI,CACH;gBACE,aAAa;gBACb,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE;gBAC7B,GAAG,EAAE,cAAc;aACpB,EACD,EAAE,OAAO,EAAE,CACZ;iBACA,OAAO,EAAE,CAAC;YAEb,MAAM,GAAG;gBACP,MAAM;gBACN,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,IAAI,CAAC,uBAAuB,CAAC,GAA4B,EAAE,aAAa,EAAE,SAAS,CAAC,CACrF;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,0BAA0B,EAAE,CAAC;YACvD,MAAM,IAAI,CAAC,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAChF,CAAC;QAED,OAAO,MAAO,CAAC;IACjB,CAAC;IAEO,uBAAuB,CAC7B,GAA0B,EAC1B,aAAqB,EACrB,SAA6B,EAC7B,WAA6B;QAE7B,MAAM,mBAAmB,GAAG,WAAW,IAAI,IAAI,CAAC,kCAAkC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACnG,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;YACpC,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,GAAG,EAAE;gBACH,aAAa;gBACb,MAAM,EAAE,GAAG,CAAC,WAAW;gBACvB,IAAI,EAAE,GAAG,CAAC,UAAU;aACrB;YACD,QAAQ,EAAE,GAAG,CAAC,WAAW;YACzB,gBAAgB,EAAE,GAAG,CAAC,mBAAoB,CAAC,GAAG,CAC5C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF;YACD,gBAAgB,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YAC3C,iBAAiB,EAAE,mBAAmB,CAAC,iBAAiB;YACxD,sBAAsB,EAAE,mBAAmB,CAAC,sBAAsB;SACnE,CAAC,CAAC;QACH,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;QACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;QAC/D,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1D,KAAK,CAAC,cAAc;YAClB,GAAG,CAAC,eAAe,IAAI,IAAI;gBACzB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC;oBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;oBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;oBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;iBACtD,CAAC;QACR,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,kCAAkC,CACxC,GAA0B,EAC1B,SAA6B;QAE7B,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QAChE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QAE1E,OAAO;YACL,iBAAiB,EAAE,SAAS,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/D,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAC7D;YACD,sBAAsB,EAAE,SAAS,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/E,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CACrE;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,KAA0B;QACnD,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE;aACvB,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC;aACjC,OAAO,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAiC,CAAC;QACxG,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACrF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,OAA0C;QAClE,MAAM,EAAE,sBAAsB,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,mCAAmC,EAAE,GAAG,OAAO,EAAE,CAAC;QAElG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE1B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CACxC;YACE,gBAAgB,EAAE,IAAI;YACtB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,EACD;YACE,IAAI,EAAE;gBACJ,gBAAgB,EAAE,GAAG;aACtB;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CACF,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CACxD;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;SACtC,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE;gBACV,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE;oBACZ,UAAU,EAAE;wBACV,GAAG,EAAE,IAAI,CAAC,YAAY;qBACvB;iBACF;aACF;SACF,CACF,CAAC;QAEF,MAAM,KAAK,GAAI,iBAAiD,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;QACpF,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,yBAAyB,CACjC,kEAAkE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,EAAE,CACrH,CAAC;QACJ,CAAC;QAED,MAAM,eAAe,GAAG,wBAAwB,CAAC;YAC/C,GAAG;YACH,YAAY,EAAE,KAAK,CAAC,aAAa,KAAK,IAAI;YAC1C,iBAAiB,EAAE,KAAK,CAAC,mBAAmB;YAC5C,kBAAkB,EAAE,KAAK,CAAC,oBAAoB;YAC9C,WAAW,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;YAC3E,cAAc,EAAE,KAAK,CAAC,eAAe;YACrC,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,sBAAsB;SACvB,CAAC,CAAC;QAEH,MAAM,SAAS,GAAwB;YACrC,iBAAiB,EAAE,GAAG;YACtB,gBAAgB,EAAE,IAAI;YACtB,mBAAmB,EAAE,IAAI;YACzB,qCAAqC,EAAE,eAAe,CAAC,cAAc;YACrE,wCAAwC,EAAE,eAAe,CAAC,iBAAiB;SAC5E,CAAC;QACF,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;YACtC,SAAS,CAAC,4CAA4C,CAAC,GAAG,GAAG,CAAC;YAC9D,SAAS,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC;YACjC,SAAS,CAAC,oBAAoB,CAAC,GAAG,GAAG,CAAC;QACxC,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;SACtC,EACD;YACE,IAAI,EAAE,SAAS;SAChB,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;SACpD,CACF,CAAC;QAEF,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,yBAAyB,GAAG,KAAK,EAAE,CAAC;gBACxD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oDAAoD,GAAG,MAAM,eAAe,CAAC,cAAc,oBAAoB,IAAI,CAAC,SAAS,CAC3H;oBACE,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;iBACjD,CACF,EAAE,CACJ,CAAC;gBACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,eAAe,CAAC,iBAAiB,EAAE,CAAC;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,MAAM,eAAe,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;YAC/B,IAAI,eAAe,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAC9C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,eAAe,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;QACD,OAAO;YACL,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;YACpD,iBAAiB,EAAE,eAAe,CAAC,iBAAiB;SACrD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;SACtC,EACD;YACE,IAAI,EAAE;gBACJ,YAAY,EAAE,GAAG;aAClB;SACF,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,GAAW;QACtC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACvC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;gBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;aACtC,EACD;gBACE,OAAO;gBACP,UAAU,EAAE;oBACV,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE;wBACZ,UAAU,EAAE;4BACV,GAAG,EAAE,IAAI,CAAC,YAAY;yBACvB;qBACF;iBACF;aACF,CACF,CAAC;YACF,MAAM,KAAK,GAAI,GAAmC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;YACtE,IACE,GAAG;gBACH,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU;gBAC7C,KAAK,EAAE,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU;gBAChD,KAAK,CAAC,aAAa;gBACnB,KAAK,CAAC,eAAe,IAAI,IAAI,EAC7B,CAAC;gBACD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;oBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;oBAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;iBACtC,EACD;oBACE,IAAI,EAAE;wBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;wBACnC,8BAA8B,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;qBAC7D;iBACF,EACD;oBACE,OAAO;oBACP,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;iBACpD,CACF,CAAC;gBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;oBACE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE;oBAC3B,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;iBAC9E,EACD,2BAA2B,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,EACvD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,GAAG,EAAE,KAAK,IAAI,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;gBAC1D,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,GAAG,EAAE,CAAC,CAAC;YAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QACjC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,wBAAgC;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;SACtC,EACD;YACE,IAAI,EAAE;gBACJ,sCAAsC,EAAE,IAAI;gBAC5C,iBAAiB,EAAE,IAAI,IAAI,EAAE;aAC9B;YACD,IAAI,EAAE;gBACJ,6CAA6C,EAAE,wBAAwB;aACxE;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;SACpD,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,wBAAgC,EAAE,OAAuC;QAC9F,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACpC,kCAAkC;YAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,cAAc,CACtE;gBACE,aAAa,EAAE,KAAK;aACrB,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAC1B,CAAC;YACF,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,IAAI,OAAO,EAAE,eAAe,IAAI,IAAI,EAAE,CAAC;oBACrC,MAAM,IAAI,yBAAyB,CACjC,mCAAmC,KAAK,gBAAgB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,6BAA6B,CAC3G,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO;gBACT,CAAC;YACH,CAAC;YAED,MAAM,IAAI,CAAC,mBAAmB,CAAC,wBAAwB,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAA2B;QACzD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;YACE,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;SACtC,EACD;YACE,IAAI,EAAE;gBACJ,sCAAsC,EAAE,KAAK;aAC9C;SACF,EACD;YACE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;SACpD,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,MAA6B,EAC7B,wBAAiC;QAEjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAE1D,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YACpC,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,UAAU,CACxD,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EACrB;gBACE,IAAI,EAAE;oBACJ,aAAa,EAAE,IAAI;iBACpB;gBACD,MAAM,EAAE;oBACN,eAAe,EAAE,CAAC;iBACnB;aACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;YAEF,IAAI,wBAAwB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;oBACE,GAAG,EAAE,IAAI,CAAC,QAAQ;oBAClB,kBAAkB,EAAE,IAAI,CAAC,YAAY;iBACtC,EACD;oBACE,IAAI,EAAE;wBACJ,iBAAiB,EAAE,IAAI,IAAI,EAAE;qBAC9B;oBACD,IAAI,EAAE;wBACJ,6CAA6C,EAAE,wBAAwB;qBACxE;iBACF,EACD;oBACE,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;iBACpD,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts

@@ -0,0 +1,15 @@
+import { BucketChecksum, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap } from '@powersync/service-core';
+import { FetchPartialBucketChecksumV3, MongoChecksumOptions, MongoChecksums } from '../MongoChecksums.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+export declare class MongoChecksumsV3 extends MongoChecksums {
+    protected readonly db: VersionedPowerSyncMongoV3;
+    private readonly mapping;
+    constructor(db: VersionedPowerSyncMongoV3, group_id: number, options: MongoChecksumOptions);
+    private normalizeBatch;
+    computePartialChecksumsDirectByDefinition(batch: FetchPartialBucketChecksumV3[]): Promise<PartialChecksumMap>;
+    protected fetchPreStates(batch: FetchPartialBucketChecksum[]): Promise<Map<string, {
+        opId: InternalOpId;
+        checksum: BucketChecksum;
+    }>>;
+    protected computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
+}

package/dist/storage/implementation/v3/MongoChecksumsV3.js

@@ -0,0 +1,84 @@
+import { bson } from '@powersync/service-core';
+import { emptyChecksumForRequest, MongoChecksums } from '../MongoChecksums.js';
+export class MongoChecksumsV3 extends MongoChecksums {
+    mapping;
+    constructor(db, group_id, options) {
+        super(db, group_id, options);
+        this.mapping = options.mapping;
+    }
+    normalizeBatch(batch) {
+        return batch.map((request) => ({
+            bucket: request.bucket,
+            definitionId: this.mapping.bucketSourceId(request.source),
+            start: request.start,
+            end: request.end
+        }));
+    }
+    async computePartialChecksumsDirectByDefinition(batch) {
+        const results = new Map();
+        const requestsByDefinition = new Map();
+        for (const request of batch) {
+            const existing = requestsByDefinition.get(request.definitionId) ?? [];
+            existing.push(request);
+            requestsByDefinition.set(request.definitionId, existing);
+        }
+        for (const [definitionId, requests] of requestsByDefinition.entries()) {
+            const groupResults = await this.computePartialChecksumsForCollection(requests, this.db.bucketDataV3(this.group_id, definitionId), createV3BucketFilter);
+            for (const checksum of groupResults.values()) {
+                results.set(checksum.bucket, checksum);
+            }
+        }
+        return new Map(batch.map((request) => [request.bucket, results.get(request.bucket) ?? emptyChecksumForRequest(request)]));
+    }
+    async fetchPreStates(batch) {
+        const preFilters = this.normalizeBatch(batch)
+            .filter((request) => request.start == null)
+            .map((request) => ({
+            _id: {
+                d: request.definitionId,
+                b: request.bucket
+            },
+            'compacted_state.op_id': { $exists: true, $lte: request.end }
+        }));
+        const preStates = new Map();
+        if (preFilters.length == 0) {
+            return preStates;
+        }
+        const states = await this.db
+            .bucketStateV3(this.group_id)
+            .find({
+            $or: preFilters
+        })
+            .toArray();
+        for (const state of states) {
+            const compactedState = state.compacted_state;
+            preStates.set(state._id.b, {
+                opId: compactedState.op_id,
+                checksum: {
+                    bucket: state._id.b,
+                    checksum: Number(compactedState.checksum),
+                    count: compactedState.count
+                }
+            });
+        }
+        return preStates;
+    }
+    async computePartialChecksumsInternal(batch) {
+        return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch));
+    }
+}
+function createV3BucketFilter(request) {
+    return {
+        _id: {
+            $gt: {
+                b: request.bucket,
+                o: request.start ?? new bson.MinKey()
+            },
+            $lte: {
+                b: request.bucket,
+                o: request.end
+            }
+        }
+    };
+}
+//# sourceMappingURL=MongoChecksumsV3.js.map

package/dist/storage/implementation/v3/MongoChecksumsV3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoChecksumsV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/MongoChecksumsV3.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAML,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,uBAAuB,EAGvB,cAAc,EACf,MAAM,sBAAsB,CAAC;AAG9B,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAEjC,OAAO,CAA0B;IAElD,YAAY,EAA6B,EAAE,QAAgB,EAAE,OAA6B;QACxF,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAQ,CAAC;IAClC,CAAC;IAEO,cAAc,CAAC,KAAmC;QACxD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,yCAAyC,CAAC,KAAqC;QACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACzD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAA0C,CAAC;QAE/E,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;YACtE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,KAAK,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,oBAAoB,CAAC,OAAO,EAAE,EAAE,CAAC;YACtE,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,oCAAoC,CAClE,QAAQ,EACR,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EACjD,oBAAoB,CACrB,CAAC;YACF,KAAK,MAAM,QAAQ,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC7C,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,CAC1G,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,cAAc,CAC5B,KAAmC;QAEnC,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC;aAC1C,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;aAC1C,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjB,GAAG,EAAE;gBACH,CAAC,EAAE,OAAO,CAAC,YAAY;gBACvB,CAAC,EAAE,OAAO,CAAC,MAAM;aAClB;YACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;SAC9D,CAAC,CAAC,CAAC;QAEN,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QACtF,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE;aACzB,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC5B,IAAI,CAAC;YACJ,GAAG,EAAE,UAAU;SAChB,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;YAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,QAAQ,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;oBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;oBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;iBAC5B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAES,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QACjF,OAAO,IAAI,CAAC,yCAAyC,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IACpF,CAAC;CACF;AAED,SAAS,oBAAoB,CAAC,OAAuE;IACnG,OAAO;QACL,GAAG,EAAE;YACH,GAAG,EAAE;gBACH,CAAC,EAAE,OAAO,CAAC,MAAM;gBACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;aACtC;YACD,IAAI,EAAE;gBACJ,CAAC,EAAE,OAAO,CAAC,MAAM;gBACjB,CAAC,EAAE,OAAO,CAAC,GAAG;aACf;SACF;KACF,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/v3/MongoCompactorV3.d.ts

@@ -0,0 +1,23 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { BucketDefinitionId } from '@powersync/service-sync-rules';
+import { SingleBucketStore } from '../common/SingleBucketStore.js';
+import { BucketStateDocumentBase } from '../models.js';
+import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
+import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+export declare class MongoCompactorV3 extends MongoCompactor {
+    protected readonly db: VersionedPowerSyncMongoV3;
+    protected readonly storage: MongoSyncBucketStorageV3;
+    dirtyBucketBatches(options: {
+        minBucketChanges: number;
+        minChangeRatio: number;
+    }): AsyncGenerator<DirtyBucket[]>;
+    dirtyBucketBatchForChecksums(options: {
+        minBucketChanges: number;
+    }): Promise<DirtyBucket[]>;
+    protected writeBucketStateUpdates(): Promise<void>;
+    protected computeChecksumsForBuckets(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<storage.PartialChecksumMap>;
+    protected bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Filter<BucketStateDocumentBase>;
+    protected getBucketDataContext(bucket: string, definitionId: BucketDefinitionId | null): Promise<SingleBucketStore | null>;
+}