@powersync/service-module-mongodb-storage 0.13.1 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,46 @@
 # @powersync/service-module-mongodb-storage
 
+## 0.14.0
+
+### Minor Changes
+
+- 8a4c34e: Refactor `BucketStorageFactory` and `PersistedSyncRulesContent` to be abstract classes instead of interfaces.
+- 8bd83e8: Introduce storage versions.
+- 83989b2: Store compiled sync plans in bucket storage.
+
+### Patch Changes
+
+- 354dac6: [MongoDB Storage] Fix query timeout on bucket_state during compact process in some extreme cases.
+- Updated dependencies [15aea77]
+- Updated dependencies [0998251]
+- Updated dependencies [65f3c89]
+- Updated dependencies [1c45667]
+- Updated dependencies [8785a3f]
+- Updated dependencies [8a4c34e]
+- Updated dependencies [b440093]
+- Updated dependencies [d7ff4ad]
+- Updated dependencies [c683322]
+- Updated dependencies [8bd83e8]
+- Updated dependencies [83989b2]
+- Updated dependencies [79a9729]
+- Updated dependencies [5edd95f]
+  - @powersync/lib-service-mongodb@0.6.20
+  - @powersync/service-core@1.20.0
+  - @powersync/service-types@0.15.0
+  - @powersync/service-sync-rules@0.32.0
+  - @powersync/lib-services-framework@0.8.3
+
+## 0.13.2
+
+### Patch Changes
+
+- e21db9b: [MongoDB Storage] Revert logic for bucket checksum pre-calculations from 1.19.1
+- Updated dependencies [a04252d]
+  - @powersync/service-sync-rules@0.31.1
+  - @powersync/lib-services-framework@0.8.2
+  - @powersync/service-core@1.19.2
+  - @powersync/lib-service-mongodb@0.6.19
+
 ## 0.13.1
 
 ### Patch Changes

package/dist/migrations/db/migrations/1770213298299-storage-version.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1770213298299-storage-version.js

@@ -0,0 +1,29 @@
+import { storage as core_storage } from '@powersync/service-core';
+import * as mongo_storage from '../../../storage/storage-index.js';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = mongo_storage.createPowerSyncMongo(configuration.storage);
+    try {
+        await db.sync_rules.updateMany({ storage_version: { $exists: false } }, { $set: { storage_version: core_storage.LEGACY_STORAGE_VERSION } });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = mongo_storage.createPowerSyncMongo(configuration.storage);
+    try {
+        const newRules = await db.sync_rules
+            .find({ storage_version: { $gt: core_storage.LEGACY_STORAGE_VERSION } })
+            .toArray();
+        if (newRules.length > 0) {
+            throw new Error(`Cannot revert migration due to newer storage versions in use: ${newRules.map((r) => `${r._id}: v${r.storage_version}`).join(', ')}`);
+        }
+        await db.sync_rules.updateMany({ storage_version: core_storage.LEGACY_STORAGE_VERSION }, { $unset: { storage_version: 1 } });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+//# sourceMappingURL=1770213298299-storage-version.js.map

package/dist/migrations/db/migrations/1770213298299-storage-version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1770213298299-storage-version.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1770213298299-storage-version.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,OAAO,IAAI,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,aAAa,MAAM,mCAAmC,CAAC;AAGnE,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,aAAa,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B,EAAE,eAAe,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EACvC,EAAE,IAAI,EAAE,EAAE,eAAe,EAAE,YAAY,CAAC,sBAAsB,EAAE,EAAE,CACnE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,aAAa,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAA6B,CAAC,CAAC;IAE3F,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,UAAU;aACjC,IAAI,CAAC,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,YAAY,CAAC,sBAAsB,EAAE,EAAE,CAAC;aACvE,OAAO,EAAE,CAAC;QACb,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,iEAAiE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrI,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B,EAAE,eAAe,EAAE,YAAY,CAAC,sBAAsB,EAAE,EACxD,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,EAAE,CACnC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}

package/dist/storage/MongoBucketStorage.d.ts

@@ -1,9 +1,12 @@
 import { GetIntanceOptions, storage } from '@powersync/service-core';
-import { BaseObserver } from '@powersync/lib-services-framework';
 import { PowerSyncMongo } from './implementation/db.js';
 import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
-import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './implementation/MongoSyncBucketStorage.js';
-export declare class MongoBucketStorage extends BaseObserver<storage.BucketStorageFactoryListener> implements storage.BucketStorageFactory {
+import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
+import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
+export interface MongoBucketStorageOptions {
+    checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+}
+export declare class MongoBucketStorage extends storage.BucketStorageFactory {
     private internalOptions?;
     private readonly client;
     private readonly session;
@@ -12,25 +15,14 @@ export declare class MongoBucketStorage extends BaseObserver<storage.BucketStora
     readonly db: PowerSyncMongo;
     constructor(db: PowerSyncMongo, options: {
         slot_name_prefix: string;
-    }, internalOptions?: MongoSyncBucketStorageOptions | undefined);
+    }, internalOptions?: MongoBucketStorageOptions | undefined);
     [Symbol.asyncDispose](): Promise<void>;
     getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage;
     getSystemIdentifier(): Promise<storage.BucketStorageSystemIdentifier>;
-    configureSyncRules(options: storage.UpdateSyncRulesOptions): Promise<{
-        updated: boolean;
-        persisted_sync_rules?: undefined;
-        lock?: undefined;
-    } | {
-        updated: boolean;
-        persisted_sync_rules: MongoPersistedSyncRulesContent;
-        lock: import("./storage-index.js").MongoSyncRulesLock | undefined;
-    }>;
     restartReplication(sync_rules_group_id: number): Promise<void>;
     updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent>;
     getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null>;
-    getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null>;
     getNextSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null>;
-    getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise<storage.PersistedSyncRules | null>;
     getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
     getStoppedSyncRules(): Promise<storage.PersistedSyncRulesContent[]>;
     getActiveStorage(): Promise<MongoSyncBucketStorage | null>;

package/dist/storage/MongoBucketStorage.js

@@ -1,12 +1,11 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
 import { storage } from '@powersync/service-core';
-import { BaseObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
 import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
 import { generateSlotName } from '../utils/util.js';
-export class MongoBucketStorage extends BaseObserver {
+export class MongoBucketStorage extends storage.BucketStorageFactory {
     internalOptions;
     client;
     session;
@@ -30,7 +29,11 @@ export class MongoBucketStorage extends BaseObserver {
         if (typeof id == 'bigint') {
             id = Number(id);
         }
-        const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, this.internalOptions);
+        const storageConfig = syncRules.getStorageConfig();
+        const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name, undefined, {
+            ...this.internalOptions,
+            storageConfig
+        });
         if (!options?.skipLifecycleHooks) {
             this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
         }
@@ -55,32 +58,12 @@ export class MongoBucketStorage extends BaseObserver {
             type: lib_mongo.MONGO_CONNECTION_TYPE
         };
     }
-    async configureSyncRules(options) {
-        const next = await this.getNextSyncRulesContent();
-        const active = await this.getActiveSyncRulesContent();
-        if (next?.sync_rules_content == options.content) {
-            logger.info('Sync rules from configuration unchanged');
-            return { updated: false };
-        }
-        else if (next == null && active?.sync_rules_content == options.content) {
-            logger.info('Sync rules from configuration unchanged');
-            return { updated: false };
-        }
-        else {
-            logger.info('Sync rules updated from configuration');
-            const persisted_sync_rules = await this.updateSyncRules(options);
-            return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined };
-        }
-    }
     async restartReplication(sync_rules_group_id) {
         const next = await this.getNextSyncRulesContent();
         const active = await this.getActiveSyncRulesContent();
         if (next != null && next.id == sync_rules_group_id) {
             // We need to redo the "next" sync rules
-            await this.updateSyncRules({
-                content: next.sync_rules_content,
-                validate: false
-            });
+            await this.updateSyncRules(next.asUpdateOptions());
             // Pro-actively stop replicating
             await this.db.sync_rules.updateOne({
                 _id: next.id,
@@ -94,10 +77,7 @@ export class MongoBucketStorage extends BaseObserver {
         }
         else if (next == null && active?.id == sync_rules_group_id) {
             // Slot removed for "active" sync rules, while there is no "next" one.
-            await this.updateSyncRules({
-                content: active.sync_rules_content,
-                validate: false
-            });
+            await this.updateSyncRules(active.asUpdateOptions());
             // In this case we keep the old one as active for clients, so that that existing clients
             // can still get the latest data while we replicate the new ones.
             // It will however not replicate anymore.
@@ -125,19 +105,6 @@ export class MongoBucketStorage extends BaseObserver {
         }
     }
     async updateSyncRules(options) {
-        if (options.validate) {
-            // Parse and validate before applying any changes
-            SqlSyncRules.fromYaml(options.content, {
-                // No schema-based validation at this point
-                schema: undefined,
-                defaultSchema: 'not_applicable', // Not needed for validation
-                throwOnError: true
-            });
-        }
-        else {
-            // We do not validate sync rules at this point.
-            // That is done when using the sync rules, so that the diagnostics API can report the errors.
-        }
         let rules = undefined;
         await this.session.withTransaction(async () => {
             // Only have a single set of sync rules with PROCESSING.
@@ -156,9 +123,12 @@ export class MongoBucketStorage extends BaseObserver {
             });
             const id = Number(id_doc.op_id);
             const slot_name = generateSlotName(this.slot_name_prefix, id);
+            const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
             const doc = {
                 _id: id,
-                content: options.content,
+                storage_version: storageVersion,
+                content: options.config.yaml,
+                serialized_plan: options.config.plan,
                 last_checkpoint: null,
                 last_checkpoint_lsn: null,
                 no_checkpoint_before: null,
@@ -190,10 +160,6 @@ export class MongoBucketStorage extends BaseObserver {
         }
         return new MongoPersistedSyncRulesContent(this.db, doc);
     }
-    async getActiveSyncRules(options) {
-        const content = await this.getActiveSyncRulesContent();
-        return content?.parsed(options) ?? null;
-    }
     async getNextSyncRulesContent() {
         const doc = await this.db.sync_rules.findOne({
             state: storage.SyncRuleState.PROCESSING
@@ -203,10 +169,6 @@ export class MongoBucketStorage extends BaseObserver {
         }
         return new MongoPersistedSyncRulesContent(this.db, doc);
     }
-    async getNextSyncRules(options) {
-        const content = await this.getNextSyncRulesContent();
-        return content?.parsed(options) ?? null;
-    }
     async getReplicatingSyncRules() {
         const docs = await this.db.sync_rules
             .find({

package/dist/storage/MongoBucketStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAE7D,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAK5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAiC,MAAM,4CAA4C,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,MAAM,OAAO,kBACX,SAAQ,YAAkD;IAiBhD;IAdO,MAAM,CAAoB;IAC1B,OAAO,CAAsB;IAC9C,iEAAiE;IACjD,gBAAgB,CAAS;IAEjC,kBAAkB,CAAqC;IAE/C,EAAE,CAAiB;IAEnC,YACE,EAAkB,EAClB,OAEC,EACO,eAA+C;QAEvD,KAAK,EAAE,CAAC;QAFA,oBAAe,GAAf,eAAe,CAAgC;QAGvD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;IAED,WAAW,CAAC,SAA4C,EAAE,OAA2B;QACnF,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAClC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5G,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/C,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,qBAAqB;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAuC;QAC9D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,kBAAkB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACrD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;QACvG,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,mBAAmB,EAAE,CAAC;YACnD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,IAAI,CAAC,kBAAkB;gBAChC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,kBAAkB;gBAClC,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YAEH,wFAAwF;YACxF,iEAAiE;YACjE,yCAAyC;YAEzC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,6EAA6E;YAE7E,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,iDAAiD;YACjD,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE;gBACrC,2CAA2C;gBAC3C,MAAM,EAAE,SAAS;gBACjB,aAAa,EAAE,gBAAgB,EAAE,4BAA4B;gBAC7D,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,+CAA+C;YAC/C,6FAA6F;QAC/F,CAAC;QAED,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAChD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,SAAS;gBACvB,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB,EAAE,IAAI;gBACzB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;SAC9E,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAsC;QAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;SACxC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAsC;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACrD,OAAO,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;SACjF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;SAClC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gDAAgD;QAChD,+FAA+F;QAC/F,wBAAwB;QACxB,IAAI,IAAI,CAAC,kBAAkB,EAAE,QAAQ,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,iBAAiB,GAAG,CAAC,CAAU,EAAE,EAAE;YACvC,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBACzE,sCAAsC;gBACtC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,sBAAsB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC5C,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK;gBACzB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/B,GAAG,EAAE,IAAI,EAAE;iBACZ,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAS,CAAC,GAAG,CAAC;IACvB,CAAC;CACF"}
+{"version":3,"file":"MongoBucketStorage.js","sourceRoot":"","sources":["../../src/storage/MongoBucketStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAElC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAK5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,oDAAoD,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAOpD,MAAM,OAAO,kBAAmB,SAAQ,OAAO,CAAC,oBAAoB;IAexD;IAdO,MAAM,CAAoB;IAC1B,OAAO,CAAsB;IAC9C,iEAAiE;IACjD,gBAAgB,CAAS;IAEjC,kBAAkB,CAAqC;IAE/C,EAAE,CAAiB;IAEnC,YACE,EAAkB,EAClB,OAEC,EACO,eAA2C;QAEnD,KAAK,EAAE,CAAC;QAFA,oBAAe,GAAf,eAAe,CAA4B;QAGnD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,QAAQ;IACV,CAAC;IAED,WAAW,CAAC,SAA4C,EAAE,OAA2B;QACnF,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAClC,IAAK,OAAO,EAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,aAAa,GAAI,SAA4C,CAAC,gBAAgB,EAAE,CAAC;QACvF,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE;YACpF,GAAG,IAAI,CAAC,eAAe;YACvB,aAAa;SACd,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,CAAC,gBAAgB,CAAC;YACvB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,KAAK,CAAC,gBAAgB,CAAC;oBACrB,gBAAgB,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,CAAC;iBAC7F,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;YAC/C,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,oEAAoE,CACrE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,qBAAqB;SACtC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,mBAA2B;QAClD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEtD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,EAAE,IAAI,mBAAmB,EAAE,CAAC;YACnD,wCAAwC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;YACnD,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,IAAI,CAAC,EAAE;gBACZ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;iBAClC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,sEAAsE;YACtE,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;YAErD,wFAAwF;YACxF,iEAAiE;YACjE,yCAAyC;YAEzC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,EAAE,EAAE,IAAI,mBAAmB,EAAE,CAAC;YAC7D,6EAA6E;YAE7E,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAChC;gBACE,GAAG,EAAE,MAAM,CAAC,EAAE;gBACd,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM;aACpC,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO;iBACrC;aACF,CACF,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAuC;QAC3D,IAAI,KAAK,GAA+C,SAAS,CAAC;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,IAAI,EAAE;YAC5C,wDAAwD;YACxD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CACjC;gBACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;aACxC,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAChD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAC1D;gBACE,GAAG,EAAE,YAAY;aAClB,EACD;gBACE,IAAI,EAAE;oBACJ,KAAK,EAAE,EAAE;iBACV;aACF,EACD;gBACE,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;aACxB,CACF,CAAC;YAEF,MAAM,EAAE,GAAG,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE9D,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,uBAAuB,CAAC;YACjF,MAAM,GAAG,GAAqB;gBAC5B,GAAG,EAAE,EAAE;gBACP,eAAe,EAAE,cAAc;gBAC/B,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI;gBAC5B,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI;gBACpC,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,YAAY,EAAE,IAAI;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,SAAS;gBACvB,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB,EAAE,IAAI;gBACzB,iBAAiB,EAAE,IAAI;aACxB,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,KAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,yBAAyB;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;SAC9E,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAC1C;YACE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,UAAU;SACxC,EACD,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAChC,CAAC;QACF,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;SACjF,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU;aAClC,IAAI,CAAC;YACJ,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;SAClC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtB,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACvD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,gDAAgD;QAChD,+FAA+F;QAC/F,wBAAwB;QACxB,IAAI,IAAI,CAAC,kBAAkB,EAAE,QAAQ,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,iBAAiB,GAAG,CAAC,CAAU,EAAE,EAAE;YACvC,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBACzE,sCAAsC;gBACtC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC9B,OAAO;gBACL,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;gBACxB,sBAAsB,EAAE,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;aAEnD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aACzD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;aACrD,SAAS,CAAC;YACT;gBACE,UAAU,EAAE;oBACV,YAAY,EAAE,EAAE;iBACjB;aACF;SACF,CAAC;aACD,OAAO,EAAE;aACT,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAE5B,OAAO;YACL,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;YACxE,sBAAsB,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC5C,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC;gBACnD,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK;gBACzB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/B,GAAG,EAAE,IAAI,EAAE;iBACZ,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACxC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAS,CAAC,GAAG,CAAC;IACvB,CAAC;CACF"}

package/dist/storage/implementation/MongoChecksums.d.ts

@@ -1,5 +1,6 @@
 import { ChecksumMap, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
+import { StorageConfig } from './models.js';
 /**
  * Checksum calculation options, primarily for tests.
  */
@@ -12,6 +13,7 @@ export interface MongoChecksumOptions {
      * Limit on the number of documents to calculate a checksum on at a time.
      */
     operationBatchLimit?: number;
+    storageConfig: StorageConfig;
 }
 /**
  * Checksum query implementation.
@@ -25,9 +27,10 @@ export interface MongoChecksumOptions {
 export declare class MongoChecksums {
     private db;
     private group_id;
-    private options?;
+    private options;
     private _cache;
-    constructor(db: PowerSyncMongo, group_id: number, options?: MongoChecksumOptions | undefined);
+    private readonly storageConfig;
+    constructor(db: PowerSyncMongo, group_id: number, options: MongoChecksumOptions);
     /**
      * Lazy-instantiated cache.
      *

package/dist/storage/implementation/MongoChecksums.js

@@ -16,10 +16,12 @@ export class MongoChecksums {
     group_id;
     options;
     _cache;
+    storageConfig;
     constructor(db, group_id, options) {
         this.db = db;
         this.group_id = group_id;
         this.options = options;
+        this.storageConfig = options.storageConfig;
     }
     /**
      * Lazy-instantiated cache.
@@ -174,6 +176,10 @@ export class MongoChecksums {
                     }
                 });
             }
+            // Historically, checksum may be stored as 'int' or 'double'.
+            // More recently, this should be a 'long'.
+            // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
+            const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
             // Aggregate over a max of `batchLimit` operations at a time.
             // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
             // Then we'll do three batches:
@@ -196,10 +202,7 @@ export class MongoChecksums {
                 {
                     $group: {
                         _id: '$_id.b',
-                        // Historically, checksum may be stored as 'int' or 'double'.
-                        // More recently, this should be a 'long'.
-                        // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-                        checksum_total: { $sum: { $toLong: '$checksum' } },
+                        checksum_total: { $sum: checksumLong },
                         count: { $sum: 1 },
                         has_clear_op: {
                             $max: {

package/dist/storage/implementation/MongoChecksums.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAEJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAkBjC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IAIf;IACA;IACA;IALF,MAAM,CAA4B;IAE1C,YACU,EAAkB,EAClB,QAAgB,EAChB,OAA8B;QAF9B,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAuB;IACrC,CAAC;IAEJ;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAC;YAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC7C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAiB;QAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBAC1B,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;qBAClB;oBACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;iBACtC,IAAI,CAAC;gBACJ,GAAG,EAAE,UAAU;aAChB,CAAC;iBACD,OAAO,EAAE,CAAC;YACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;gBAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;oBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;oBAC1B,QAAQ,EAAE;wBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;wBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;wBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;qBAC5B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC7D,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAU,EAAE,CAAC;YAC1B,KAAK,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE;wBACH,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;yBACtC;wBACD,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,GAAG;yBACf;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YAED,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACxC,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,6DAA6D;wBAC7D,0CAA0C;wBAC1C,8FAA8F;wBAC9F,cAAc,EAAE,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;wBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,MAAM;wBACN,KAAK,EAAE,GAAG,CAAC,OAAO;wBAClB,GAAG,EAAE,GAAI,CAAC,GAAG;qBACd,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}
+{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAEJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAqBjC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IAKf;IACA;IACA;IANF,MAAM,CAA4B;IACzB,aAAa,CAAgB;IAE9C,YACU,EAAkB,EAClB,QAAgB,EAChB,OAA6B;QAF7B,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAsB;QAErC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAC;YAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC7C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAiB;QAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBAC1B,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;qBAClB;oBACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;iBACtC,IAAI,CAAC;gBACJ,GAAG,EAAE,UAAU;aAChB,CAAC;iBACD,OAAO,EAAE,CAAC;YACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;gBAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;oBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;oBAC1B,QAAQ,EAAE;wBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;wBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;wBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;qBAC5B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC7D,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAU,EAAE,CAAC;YAC1B,KAAK,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE;wBACH,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;yBACtC;wBACD,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,GAAG;yBACf;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YAED,6DAA6D;YAC7D,0CAA0C;YAC1C,8FAA8F;YAC9F,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAE/F,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACxC,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,MAAM;wBACN,KAAK,EAAE,GAAG,CAAC,OAAO;wBAClB,GAAG,EAAE,GAAI,CAAC,GAAG;qBACd,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}

package/dist/storage/implementation/MongoCompactor.d.ts

@@ -58,5 +58,17 @@ export declare class MongoCompactor {
      * minChangeRatio: minimum ratio of changes to total ops for a bucket to be included in the results, number between 0 and 1.
      */
     private dirtyBucketBatches;
+    /**
+     * Returns a batch of dirty buckets - buckets with most changes first.
+     *
+     * This cannot be used to iterate on its own - the client is expected to process these buckets and
+     * set estimate_since_compact.count: 0 when done, before fetching the next batch.
+     *
+     * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart,
+     * since it is used as the last step for initial replication.
+     *
+     * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing.
+     */
+    private dirtyBucketBatchForChecksums;
     private updateChecksumsBatch;
 }

package/dist/storage/implementation/MongoCompactor.js

@@ -7,6 +7,7 @@ const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
 const DEFAULT_MIN_CHANGE_RATIO = 0.1;
+const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
 export class MongoCompactor {
@@ -381,18 +382,13 @@ export class MongoCompactor {
      */
     async populateChecksums(options) {
         let count = 0;
-        for await (let buckets of this.dirtyBucketBatches({
-            minBucketChanges: options.minBucketChanges,
-            minChangeRatio: 0
-        })) {
-            if (this.signal?.aborted) {
+        while (!this.signal?.aborted) {
+            const buckets = await this.dirtyBucketBatchForChecksums(options);
+            if (buckets.length == 0 || this.signal?.aborted) {
+                // All done
                 break;
             }
-            if (buckets.length == 0) {
-                continue;
-            }
             const start = Date.now();
-            logger.info(`Calculating checksums for batch of ${buckets.length} buckets, starting at ${buckets[0].bucket}`);
             // Filter batch by estimated bucket size, to reduce possibility of timeouts
             let checkBuckets = [];
             let totalCountEstimate = 0;
@@ -403,9 +399,10 @@ export class MongoCompactor {
                     break;
                 }
             }
+            logger.info(`Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`);
             await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
             logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
-            count += buckets.length;
+            count += checkBuckets.length;
         }
         return { buckets: count };
     }
@@ -429,28 +426,52 @@ export class MongoCompactor {
         let lastId = { g: this.group_id, b: new mongo.MinKey() };
         const maxId = { g: this.group_id, b: new mongo.MaxKey() };
         while (true) {
-            const batch = await this.db.bucket_state
-                .find({
-                _id: { $gt: lastId, $lt: maxId },
-                'estimate_since_compact.count': { $gte: options.minBucketChanges }
-            }, {
-                projection: {
-                    _id: 1,
-                    estimate_since_compact: 1,
-                    compacted_state: 1
+            // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline
+            // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting
+            // on the output number.
+            const [result] = await this.db.bucket_state
+                .aggregate([
+                {
+                    $match: {
+                        _id: { $gt: lastId, $lt: maxId }
+                    }
+                },
+                {
+                    $sort: { _id: 1 }
                 },
-                sort: {
-                    _id: 1
+                {
+                    // Scan a fixed number of docs each query so sparse matches don't block progress.
+                    $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE
                 },
-                limit: 2000,
-                maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
-            })
+                {
+                    $facet: {
+                        // This is the results for the batch
+                        buckets: [
+                            {
+                                $match: {
+                                    'estimate_since_compact.count': { $gte: options.minBucketChanges }
+                                }
+                            },
+                            {
+                                $project: {
+                                    _id: 1,
+                                    estimate_since_compact: 1,
+                                    compacted_state: 1
+                                }
+                            }
+                        ],
+                        // This is used for the next query.
+                        cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
+                    }
+                }
+            ], { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS })
                 .toArray();
-            if (batch.length == 0) {
+            const cursor = result?.cursor?.[0];
+            if (cursor == null) {
                 break;
             }
-            lastId = batch[batch.length - 1]._id;
-            const mapped = batch.map((b) => {
+            lastId = cursor._id;
+            const mapped = (result?.buckets ?? []).map((b) => {
                 const updatedCount = b.estimate_since_compact?.count ?? 0;
                 const totalCount = (b.compacted_state?.count ?? 0) + updatedCount;
                 const updatedBytes = b.estimate_since_compact?.bytes ?? 0;
@@ -467,6 +488,44 @@ export class MongoCompactor {
             yield filtered;
         }
     }
+    /**
+     * Returns a batch of dirty buckets - buckets with most changes first.
+     *
+     * This cannot be used to iterate on its own - the client is expected to process these buckets and
+     * set estimate_since_compact.count: 0 when done, before fetching the next batch.
+     *
+     * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart,
+     * since it is used as the last step for initial replication.
+     *
+     * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing.
+     */
+    async dirtyBucketBatchForChecksums(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // We make use of an index on {_id.g: 1, 'estimate_since_compact.count': -1}
+        const dirtyBuckets = await this.db.bucket_state
+            .find({
+            '_id.g': this.group_id,
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        }, {
+            projection: {
+                _id: 1,
+                estimate_since_compact: 1,
+                compacted_state: 1
+            },
+            sort: {
+                'estimate_since_compact.count': -1
+            },
+            limit: 200,
+            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+        })
+            .toArray();
+        return dirtyBuckets.map((bucket) => ({
+            bucket: bucket._id.b,
+            estimatedCount: bucket.estimate_since_compact.count + (bucket.compacted_state?.count ?? 0)
+        }));
+    }
     async updateChecksumsBatch(buckets) {
         const checksums = await this.storage.checksums.computePartialChecksumsDirect(buckets.map((bucket) => {
             return {