@powersync/service-module-mongodb-storage 0.0.0-dev-20250828090417 → 0.0.0-dev-20250828134335

package/CHANGELOG.md

@@ -1,10 +1,19 @@
 # @powersync/service-module-mongodb-storage
 
-## 0.0.0-dev-20250828090417
+## 0.0.0-dev-20250828134335
+
+### Patch Changes
+
+- d74d874: Fix pre-computing of checksums after intial replication causing replication timeouts
+- Updated dependencies [d74d874]
+  - @powersync/service-core@0.0.0-dev-20250828134335
+  - @powersync/lib-service-mongodb@0.0.0-dev-20250828134335
+
+## 0.12.0
 
 ### Minor Changes
 
-- c821420: Create a persisted checksum cache when compacting buckets.
+- 6d4a4d1: Create a persisted checksum cache when compacting buckets.
 
 ### Patch Changes
 
@@ -12,7 +21,7 @@
 - 060b829: Update license abbreviation to FSL-1.1-ALv2.
 - d49bebe: - Hooked up the MySQL binlog heartbeat events with the bucket batch keepalive mechanism.
   Heartbeat events will now update the latest keepalive timestamp in the sync rules.
-- Updated dependencies [c821420]
+- Updated dependencies [6d4a4d1]
 - Updated dependencies [b0b8ae9]
 - Updated dependencies [d2be184]
 - Updated dependencies [29a368e]
@@ -26,12 +35,12 @@
 - Updated dependencies [86807d0]
 - Updated dependencies [060b829]
 - Updated dependencies [d49bebe]
-  - @powersync/service-core@0.0.0-dev-20250828090417
-  - @powersync/service-sync-rules@0.0.0-dev-20250828090417
-  - @powersync/service-types@0.0.0-dev-20250828090417
-  - @powersync/lib-service-mongodb@0.0.0-dev-20250828090417
-  - @powersync/lib-services-framework@0.0.0-dev-20250828090417
-  - @powersync/service-jsonbig@0.0.0-dev-20250828090417
+  - @powersync/service-core@1.15.0
+  - @powersync/service-sync-rules@0.29.0
+  - @powersync/service-types@0.13.0
+  - @powersync/lib-service-mongodb@0.6.4
+  - @powersync/lib-services-framework@0.7.3
+  - @powersync/service-jsonbig@0.17.11
 
 ## 0.11.0
 

package/dist/storage/implementation/MongoCompactor.d.ts

@@ -1,13 +1,14 @@
 import { storage } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
+import { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
 /**
  * Additional options, primarily for testing.
  */
 export interface MongoCompactOptions extends storage.CompactOptions {
 }
 export declare class MongoCompactor {
+    private storage;
     private db;
-    private group_id;
     private updates;
     private bucketStateUpdates;
     private idLimitBytes;
@@ -17,7 +18,8 @@ export declare class MongoCompactor {
     private maxOpId;
     private buckets;
     private signal?;
-    constructor(db: PowerSyncMongo, group_id: number, options?: MongoCompactOptions);
+    private group_id;
+    constructor(storage: MongoSyncBucketStorage, db: PowerSyncMongo, options?: MongoCompactOptions);
     /**
      * Compact buckets by converting operations into MOVE and/or CLEAR operations.
      *
@@ -38,4 +40,9 @@ export declare class MongoCompactor {
      * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
      */
     private clearBucket;
+    /**
+     * Subset of compact, only populating checksums where relevant.
+     */
+    populateChecksums(): Promise<void>;
+    private updateChecksumsBatch;
 }

package/dist/storage/implementation/MongoCompactor.js

@@ -1,6 +1,6 @@
-import { mongo } from '@powersync/lib-service-mongodb';
+import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
 import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { addChecksums, utils } from '@powersync/service-core';
+import { addChecksums, isPartialChecksum, utils } from '@powersync/service-core';
 import { cacheKey } from './OperationBatch.js';
 import { readSingleBatch } from './util.js';
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
@@ -9,8 +9,8 @@ const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
 export class MongoCompactor {
+    storage;
     db;
-    group_id;
     updates = [];
     bucketStateUpdates = [];
     idLimitBytes;
@@ -20,9 +20,11 @@ export class MongoCompactor {
     maxOpId;
     buckets;
     signal;
-    constructor(db, group_id, options) {
+    group_id;
+    constructor(storage, db, options) {
+        this.storage = storage;
         this.db = db;
-        this.group_id = group_id;
+        this.group_id = storage.group_id;
         this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
         this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
         this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
@@ -383,5 +385,81 @@ export class MongoCompactor {
             await session.endSession();
         }
     }
+    /**
+     * Subset of compact, only populating checksums where relevant.
+     */
+    async populateChecksums() {
+        let lastId = null;
+        while (!this.signal?.aborted) {
+            // By filtering buckets, we effectively make this "resumeable".
+            let filter = {
+                compacted_state: { $exists: false }
+            };
+            if (lastId) {
+                filter._id = { $gt: lastId };
+            }
+            const bucketsWithoutChecksums = await this.db.bucket_state
+                .find(filter, {
+                projection: {
+                    _id: 1
+                },
+                sort: {
+                    _id: 1
+                },
+                limit: 5_000,
+                maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+            })
+                .toArray();
+            if (bucketsWithoutChecksums.length == 0) {
+                // All done
+                break;
+            }
+            logger.info(`Calculating checksums for batch of ${bucketsWithoutChecksums.length} buckets`);
+            await this.updateChecksumsBatch(bucketsWithoutChecksums.map((b) => b._id.b));
+            lastId = bucketsWithoutChecksums[bucketsWithoutChecksums.length - 1]._id;
+        }
+    }
+    async updateChecksumsBatch(buckets) {
+        const checksums = await this.storage.queryPartialChecksums(buckets.map((bucket) => {
+            return {
+                bucket,
+                end: this.maxOpId
+            };
+        }));
+        for (let bucketChecksum of checksums.values()) {
+            if (isPartialChecksum(bucketChecksum)) {
+                // Should never happen since we don't specify `start`
+                throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`);
+            }
+            this.bucketStateUpdates.push({
+                updateOne: {
+                    filter: {
+                        _id: {
+                            g: this.group_id,
+                            b: bucketChecksum.bucket
+                        }
+                    },
+                    update: {
+                        $set: {
+                            compacted_state: {
+                                op_id: this.maxOpId,
+                                count: bucketChecksum.count,
+                                checksum: BigInt(bucketChecksum.checksum),
+                                bytes: 0 // We don't calculate that here
+                            }
+                        },
+                        $setOnInsert: {
+                            // Only set this if we're creating the document.
+                            // In all other cases, the replication process will have a set a more accurate id.
+                            last_op: this.maxOpId
+                        }
+                    },
+                    // We generally expect this to have been created before, but do handle cases of old unchanged buckets
+                    upsert: true
+                }
+            });
+        }
+        await this.flush();
+    }
 }
 //# sourceMappingURL=MongoCompactor.js.map

package/dist/storage/implementation/MongoCompactor.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC7G,OAAO,EAAE,YAAY,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIrF,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AA6C5C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAaf;IACA;IAbF,OAAO,GAAsD,EAAE,CAAC;IAChE,kBAAkB,GAAuD,EAAE,CAAC;IAE5E,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAS;IAChB,OAAO,CAAuB;IAC9B,MAAM,CAAe;IAE7B,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAAkC,CAAC;QACvC,IAAI,WAAkC,CAAC;QAEvC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAiD;gBAC3F;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEtD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;wBACzB,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;4BACrE,0CAA0C;4BAC1C,wDAAwD;4BACxD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;4BAEF,qCAAqC;4BACrC,YAAa,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;4BAE3B,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;wBACvC,CAAC;wBAED,uDAAuD;wBACvD,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;oBAC3C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;wBAEd,QAAQ,EAAE,CAAC;wBACX,OAAO,EAAE,CAAC;wBACV,OAAO,EAAE,CAAC;qBACX,CAAC;gBACJ,CAAC;gBAED,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;wBAEH,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;oBACnF,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,8BAA8B;YAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,+DAA+D;YAC/D,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC3C,CAAC;QACD,mDAAmD;QACnD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAyB;QACrD,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAC3B,SAAS,EAAE;gBACT,MAAM,EAAE;oBACN,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,KAAK,CAAC,MAAM;qBAChB;iBACF;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,IAAI,CAAC,OAAO;4BACnB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,sGAAsG;4BACtG,sCAAsC;4BACtC,oEAAoE;4BACpE,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;oBACD,YAAY,EAAE;wBACZ,gDAAgD;wBAChD,kFAAkF;wBAClF,OAAO,EAAE,IAAI,CAAC,OAAO;qBACtB;iBACF;gBACD,qGAAqG;gBACrG,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5D,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;iBAC7B;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,OAAO;iBACX;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACtC,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CACvE,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;6BAC7B;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,yFAAyF;gBACzF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC7G,OAAO,EAAE,YAAY,EAAgB,iBAAiB,EAAW,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAKxG,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AA6C5C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAcf;IACA;IAdF,OAAO,GAAsD,EAAE,CAAC;IAChE,kBAAkB,GAAuD,EAAE,CAAC;IAE5E,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAS;IAChB,OAAO,CAAuB;IAC9B,MAAM,CAAe;IACrB,QAAQ,CAAS;IAEzB,YACU,OAA+B,EAC/B,EAAkB,EAC1B,OAA6B;QAFrB,YAAO,GAAP,OAAO,CAAwB;QAC/B,OAAE,GAAF,EAAE,CAAgB;QAG1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAAkC,CAAC;QACvC,IAAI,WAAkC,CAAC;QAEvC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAiD;gBAC3F;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEtD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;wBACzB,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;4BACrE,0CAA0C;4BAC1C,wDAAwD;4BACxD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;4BAEF,qCAAqC;4BACrC,YAAa,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;4BAE3B,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;wBACvC,CAAC;wBAED,uDAAuD;wBACvD,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;oBAC3C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;wBAEd,QAAQ,EAAE,CAAC;wBACX,OAAO,EAAE,CAAC;wBACV,OAAO,EAAE,CAAC;qBACX,CAAC;gBACJ,CAAC;gBAED,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;wBAEH,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;oBACnF,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,8BAA8B;YAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,+DAA+D;YAC/D,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC3C,CAAC;QACD,mDAAmD;QACnD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAyB;QACrD,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAC3B,SAAS,EAAE;gBACT,MAAM,EAAE;oBACN,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,KAAK,CAAC,MAAM;qBAChB;iBACF;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,IAAI,CAAC,OAAO;4BACnB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,sGAAsG;4BACtG,sCAAsC;4BACtC,oEAAoE;4BACpE,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;oBACD,YAAY,EAAE;wBACZ,gDAAgD;wBAChD,kFAAkF;wBAClF,OAAO,EAAE,IAAI,CAAC,OAAO;qBACtB;iBACF;gBACD,qGAAqG;gBACrG,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5D,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;iBAC7B;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,OAAO;iBACX;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACtC,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CACvE,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;6BAC7B;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,yFAAyF;gBACzF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,IAAI,MAAM,GAAsC,IAAI,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,+DAA+D;YAC/D,IAAI,MAAM,GAAsC;gBAC9C,eAAe,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;aACpC,CAAC;YACF,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,CAAC,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;YAC/B,CAAC;YAED,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;iBACvD,IAAI,CAAC,MAAM,EAAE;gBACZ,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;iBACP;gBACD,IAAI,EAAE;oBACJ,GAAG,EAAE,CAAC;iBACP;gBACD,KAAK,EAAE,KAAK;gBACZ,SAAS,EAAE,0BAA0B;aACtC,CAAC;iBACD,OAAO,EAAE,CAAC;YACb,IAAI,uBAAuB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxC,WAAW;gBACX,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,sCAAsC,uBAAuB,CAAC,MAAM,UAAU,CAAC,CAAC;YAE5F,MAAM,IAAI,CAAC,oBAAoB,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAE7E,MAAM,GAAG,uBAAuB,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAiB;QAClD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACrB,OAAO;gBACL,MAAM;gBACN,GAAG,EAAE,IAAI,CAAC,OAAO;aAClB,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,KAAK,IAAI,cAAc,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,qDAAqD;gBACrD,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC3B,SAAS,EAAE;oBACT,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,cAAc,CAAC,MAAM;yBACzB;qBACF;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,eAAe,EAAE;gCACf,KAAK,EAAE,IAAI,CAAC,OAAO;gCACnB,KAAK,EAAE,cAAc,CAAC,KAAK;gCAC3B,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;gCACzC,KAAK,EAAE,CAAC,CAAC,+BAA+B;6BACzC;yBACF;wBACD,YAAY,EAAE;4BACZ,gDAAgD;4BAChD,kFAAkF;4BAClF,OAAO,EAAE,IAAI,CAAC,OAAO;yBACtB;qBACF;oBACD,qGAAqG;oBACrG,MAAM,EAAE,IAAI;iBACb;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}

package/dist/storage/implementation/MongoSyncBucketStorage.d.ts

@@ -27,13 +27,22 @@ export declare class MongoSyncBucketStorage extends BaseObserver<storage.SyncRul
     getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap>;
     clearChecksumCache(): void;
     private getChecksumsInternal;
+    queryPartialChecksums(batch: storage.FetchPartialBucketChecksum[]): Promise<storage.PartialChecksumMap>;
+    private queryPartialChecksumsInternal;
+    /**
+     * Checksums for large buckets can run over the query timeout.
+     * To avoid this, we query in batches.
+     * This version can handle larger amounts of data, but is slower, especially for many buckets.
+     */
+    queryPartialChecksumsFallback(batch: storage.FetchPartialBucketChecksum[]): Promise<storage.PartialChecksumMap>;
+    private slowChecksum;
     terminate(options?: storage.TerminateOptions): Promise<void>;
     getStatus(): Promise<storage.SyncRuleStatus>;
     clear(options?: storage.ClearStorageOptions): Promise<void>;
     private clearIteration;
     reportError(e: any): Promise<void>;
     compact(options?: storage.CompactOptions): Promise<void>;
-    populatePersistentChecksumCache(options: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
+    populatePersistentChecksumCache(options: Required<Pick<CompactOptions, 'signal' | 'maxOpId'>>): Promise<void>;
     /**
      * Instance-wide watch on the latest available checkpoint (op_id + lsn).
      */

package/dist/storage/implementation/MongoSyncBucketStorage.js

@@ -52,7 +52,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
 });
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { BaseObserver, logger, ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { addPartialChecksums, BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, deserializeParameterLookup, internalToExternalOpId, maxLsn, storage } from '@powersync/service-core';
+import { addPartialChecksums, BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, deserializeParameterLookup, internalToExternalOpId, isPartialChecksum, maxLsn, storage } from '@powersync/service-core';
 import { JSONBig } from '@powersync/service-jsonbig';
 import * as bson from 'bson';
 import { LRUCache } from 'lru-cache';
@@ -61,7 +61,7 @@ import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
-import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js';
+import { CHECKSUM_QUERY_GROUP_STAGE, checksumFromAggregate, idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js';
 export class MongoSyncBucketStorage extends BaseObserver {
     factory;
     group_id;
@@ -498,8 +498,7 @@ export class MongoSyncBucketStorage extends BaseObserver {
                 });
             }
         }
-        const filters = [];
-        for (let request of batch) {
+        const mappedRequests = batch.map((request) => {
             let start = request.start;
             if (start == null) {
                 const preState = preStates.get(request.bucket);
@@ -507,12 +506,44 @@ export class MongoSyncBucketStorage extends BaseObserver {
                     start = preState.opId;
                 }
             }
+            return {
+                ...request,
+                start
+            };
+        });
+        const queriedChecksums = await this.queryPartialChecksums(mappedRequests);
+        return new Map(batch.map((request) => {
+            const bucket = request.bucket;
+            // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available
+            const preState = preStates.get(bucket);
+            // Could be null if we got no data
+            const partialChecksum = queriedChecksums.get(bucket);
+            const merged = addPartialChecksums(bucket, preState?.checksum ?? null, partialChecksum ?? null);
+            return [bucket, merged];
+        }));
+    }
+    async queryPartialChecksums(batch) {
+        try {
+            return await this.queryPartialChecksumsInternal(batch);
+        }
+        catch (e) {
+            if (e.codeName == 'MaxTimeMSExpired') {
+                logger.warn(`Checksum query timed out; falling back to slower version`, e);
+                // Timeout - try the slower but more robust version
+                return await this.queryPartialChecksumsFallback(batch);
+            }
+            throw lib_mongo.mapQueryError(e, 'while reading checksums');
+        }
+    }
+    async queryPartialChecksumsInternal(batch) {
+        const filters = [];
+        for (let request of batch) {
             filters.push({
                 _id: {
                     $gt: {
                         g: this.group_id,
                         b: request.bucket,
-                        o: start ?? new bson.MinKey()
+                        o: request.start ?? new bson.MinKey()
                     },
                     $lte: {
                         g: this.group_id,
@@ -529,56 +560,106 @@ export class MongoSyncBucketStorage extends BaseObserver {
                     $or: filters
                 }
             },
-            {
-                $group: {
-                    _id: '$_id.b',
-                    // Historically, checksum may be stored as 'int' or 'double'.
-                    // More recently, this should be a 'long'.
-                    // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-                    checksum_total: { $sum: { $toLong: '$checksum' } },
-                    count: { $sum: 1 },
-                    has_clear_op: {
-                        $max: {
-                            $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
-                        }
-                    }
-                }
-            }
-        ], { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.db.MONGO_CHECKSUM_TIMEOUT_MS })
-            .toArray()
-            .catch((e) => {
-            throw lib_mongo.mapQueryError(e, 'while reading checksums');
-        });
+            CHECKSUM_QUERY_GROUP_STAGE
+        ], { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS })
+            // Don't map the error here - we want to keep timeout errors as-is
+            .toArray();
         const partialChecksums = new Map(aggregate.map((doc) => {
-            const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff;
             const bucket = doc._id;
-            return [
-                bucket,
-                doc.has_clear_op == 1
-                    ? {
-                        // full checksum - replaces any previous one
-                        bucket,
-                        checksum: partialChecksum,
-                        count: doc.count
-                    }
-                    : {
-                        // partial checksum - is added to a previous one
-                        bucket,
-                        partialCount: doc.count,
-                        partialChecksum
-                    }
-            ];
+            return [bucket, checksumFromAggregate(doc)];
         }));
         return new Map(batch.map((request) => {
             const bucket = request.bucket;
-            // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available
-            const preState = preStates.get(bucket);
             // Could be null if we got no data
-            const partialChecksum = partialChecksums.get(bucket);
-            const merged = addPartialChecksums(bucket, preState?.checksum ?? null, partialChecksum ?? null);
-            return [bucket, merged];
+            let partialChecksum = partialChecksums.get(bucket);
+            if (partialChecksum == null) {
+                partialChecksum = {
+                    bucket,
+                    partialCount: 0,
+                    partialChecksum: 0
+                };
+            }
+            if (request.start == null && isPartialChecksum(partialChecksum)) {
+                partialChecksum = {
+                    bucket,
+                    count: partialChecksum.partialCount,
+                    checksum: partialChecksum.partialChecksum
+                };
+            }
+            return [bucket, partialChecksum];
         }));
     }
+    /**
+     * Checksums for large buckets can run over the query timeout.
+     * To avoid this, we query in batches.
+     * This version can handle larger amounts of data, but is slower, especially for many buckets.
+     */
+    async queryPartialChecksumsFallback(batch) {
+        const partialChecksums = new Map();
+        for (let request of batch) {
+            const checksum = await this.slowChecksum(request);
+            partialChecksums.set(request.bucket, checksum);
+        }
+        return partialChecksums;
+    }
+    async slowChecksum(request) {
+        const batchLimit = 50_000;
+        let lowerBound = 0n;
+        const bucket = request.bucket;
+        let runningChecksum = {
+            bucket,
+            partialCount: 0,
+            partialChecksum: 0
+        };
+        if (request.start == null) {
+            runningChecksum = {
+                bucket,
+                count: 0,
+                checksum: 0
+            };
+        }
+        while (true) {
+            const filter = {
+                _id: {
+                    $gt: {
+                        g: this.group_id,
+                        b: bucket,
+                        o: lowerBound
+                    },
+                    $lte: {
+                        g: this.group_id,
+                        b: bucket,
+                        o: request.end
+                    }
+                }
+            };
+            const docs = await this.db.bucket_data
+                .aggregate([
+                {
+                    $match: filter
+                },
+                // sort and limit _before_ grouping
+                { $sort: { _id: 1 } },
+                { $limit: batchLimit },
+                CHECKSUM_QUERY_GROUP_STAGE
+            ], { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS })
+                .toArray();
+            const doc = docs[0];
+            if (doc == null) {
+                return runningChecksum;
+            }
+            const partial = checksumFromAggregate(doc);
+            runningChecksum = addPartialChecksums(bucket, runningChecksum, partial);
+            const isFinal = doc.count != batchLimit;
+            if (isFinal) {
+                break;
+            }
+            else {
+                lowerBound = doc.last_op;
+            }
+        }
+        return runningChecksum;
+    }
     async terminate(options) {
         // Default is to clear the storage except when explicitly requested not to.
         if (!options || options?.clearStorage) {
@@ -687,21 +768,22 @@ export class MongoSyncBucketStorage extends BaseObserver {
             const checkpoint = await this.getCheckpointInternal();
             maxOpId = checkpoint?.checkpoint ?? undefined;
         }
-        await new MongoCompactor(this.db, this.group_id, { ...options, maxOpId }).compact();
+        await new MongoCompactor(this, this.db, { ...options, maxOpId }).compact();
         if (maxOpId != null && options?.compactParameterData) {
             await new MongoParameterCompactor(this.db, this.group_id, maxOpId, options).compact();
         }
     }
     async populatePersistentChecksumCache(options) {
+        logger.info(`Populating persistent checksum cache...`);
         const start = Date.now();
-        // We do a minimal compact, primarily to populate the checksum cache
-        await this.compact({
+        // We do a minimal compact here.
+        // We can optimize this in the future.
+        const compactor = new MongoCompactor(this, this.db, {
             ...options,
-            // Skip parameter data
-            compactParameterData: false,
             // Don't track updates for MOVE compacting
             memoryLimitMB: 0
         });
+        await compactor.populateChecksums();
         const duration = Date.now() - start;
         logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
     }