@powersync/service-module-mongodb-storage 0.10.4 → 0.12.0

package/dist/storage/implementation/MongoCompactor.js

@@ -1,7 +1,8 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { logger, ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { utils } from '@powersync/service-core';
+import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import { addChecksums, utils } from '@powersync/service-core';
 import { cacheKey } from './OperationBatch.js';
+import { readSingleBatch } from './util.js';
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
@@ -11,12 +12,14 @@ export class MongoCompactor {
     db;
     group_id;
     updates = [];
+    bucketStateUpdates = [];
     idLimitBytes;
     moveBatchLimit;
     moveBatchQueryLimit;
     clearBatchLimit;
     maxOpId;
     buckets;
+    signal;
     constructor(db, group_id, options) {
         this.db = db;
         this.group_id = group_id;
@@ -24,8 +27,9 @@ export class MongoCompactor {
         this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
         this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
         this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
-        this.maxOpId = options?.maxOpId;
+        this.maxOpId = options?.maxOpId ?? 0n;
         this.buckets = options?.compactBuckets;
+        this.signal = options?.signal;
     }
     /**
      * Compact buckets by converting operations into MOVE and/or CLEAR operations.
@@ -76,28 +80,33 @@ export class MongoCompactor {
             b: bucketUpper,
             o: new mongo.MaxKey()
         };
-        while (true) {
+        while (!this.signal?.aborted) {
             // Query one batch at a time, to avoid cursor timeouts
-            const batch = await this.db.bucket_data
-                .find({
-                _id: {
-                    $gte: lowerBound,
-                    $lt: upperBound
-                }
-            }, {
-                projection: {
-                    _id: 1,
-                    op: 1,
-                    table: 1,
-                    row_id: 1,
-                    source_table: 1,
-                    source_key: 1
+            const cursor = this.db.bucket_data.aggregate([
+                {
+                    $match: {
+                        _id: {
+                            $gte: lowerBound,
+                            $lt: upperBound
+                        }
+                    }
                 },
-                limit: this.moveBatchQueryLimit,
-                sort: { _id: -1 },
-                singleBatch: true
-            })
-                .toArray();
+                { $sort: { _id: -1 } },
+                { $limit: this.moveBatchQueryLimit },
+                {
+                    $project: {
+                        _id: 1,
+                        op: 1,
+                        table: 1,
+                        row_id: 1,
+                        source_table: 1,
+                        source_key: 1,
+                        checksum: 1,
+                        size: { $bsonSize: '$$ROOT' }
+                    }
+                }
+            ]);
+            const { data: batch } = await readSingleBatch(cursor);
             if (batch.length == 0) {
                 // We've reached the end
                 break;
@@ -106,28 +115,37 @@ export class MongoCompactor {
             upperBound = batch[batch.length - 1]._id;
             for (let doc of batch) {
                 if (currentState == null || doc._id.b != currentState.bucket) {
-                    if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
-                        // Important to flush before clearBucket()
-                        await this.flush();
-                        logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
-                        const bucket = currentState.bucket;
-                        const clearOp = currentState.lastNotPut;
-                        // Free memory before clearing bucket
-                        currentState = null;
-                        await this.clearBucket(bucket, clearOp);
+                    if (currentState != null) {
+                        if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
+                            // Important to flush before clearBucket()
+                            // Does not have to happen before flushBucketChecksums()
+                            await this.flush();
+                            logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
+                            // Free memory before clearing bucket
+                            currentState.seen.clear();
+                            await this.clearBucket(currentState);
+                        }
+                        // Should happen after clearBucket() for accurate stats
+                        this.updateBucketChecksums(currentState);
                     }
                     currentState = {
                         bucket: doc._id.b,
                         seen: new Map(),
                         trackingSize: 0,
                         lastNotPut: null,
-                        opsSincePut: 0
+                        opsSincePut: 0,
+                        checksum: 0,
+                        opCount: 0,
+                        opBytes: 0
                     };
                 }
-                if (this.maxOpId != null && doc._id.o > this.maxOpId) {
+                if (doc._id.o > this.maxOpId) {
                     continue;
                 }
+                currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum));
+                currentState.opCount += 1;
                 let isPersistentPut = doc.op == 'PUT';
+                currentState.opBytes += Number(doc.size);
                 if (doc.op == 'REMOVE' || doc.op == 'PUT') {
                     const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table, doc.source_key)}`;
                     const targetOp = currentState.seen.get(key);
@@ -154,6 +172,7 @@ export class MongoCompactor {
                                 }
                             }
                         });
+                        currentState.opBytes += 200 - Number(doc.size); // TODO: better estimate for this
                     }
                     else {
                         if (currentState.trackingSize >= idLimitBytes) {
@@ -181,21 +200,66 @@ export class MongoCompactor {
                     }
                     currentState.opsSincePut += 1;
                 }
-                if (this.updates.length >= this.moveBatchLimit) {
+                if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) {
                     await this.flush();
                 }
             }
         }
-        await this.flush();
         currentState?.seen.clear();
         if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
             logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
-            const bucket = currentState.bucket;
-            const clearOp = currentState.lastNotPut;
-            // Free memory before clearing bucket
-            currentState = null;
-            await this.clearBucket(bucket, clearOp);
+            // Need flush() before clear()
+            await this.flush();
+            await this.clearBucket(currentState);
+        }
+        if (currentState != null) {
+            // Do this _after_ clearBucket so that we have accurate counts.
+            this.updateBucketChecksums(currentState);
         }
+        // Need another flush after updateBucketChecksums()
+        await this.flush();
+    }
+    /**
+     * Call when done with a bucket.
+     */
+    updateBucketChecksums(state) {
+        if (state.opCount < 0) {
+            throw new ServiceAssertionError(`Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`);
+        }
+        this.bucketStateUpdates.push({
+            updateOne: {
+                filter: {
+                    _id: {
+                        g: this.group_id,
+                        b: state.bucket
+                    }
+                },
+                update: {
+                    $set: {
+                        compacted_state: {
+                            op_id: this.maxOpId,
+                            count: state.opCount,
+                            checksum: BigInt(state.checksum),
+                            bytes: state.opBytes
+                        },
+                        estimate_since_compact: {
+                            // Note: There could have been a whole bunch of new operations added to the bucket _while_ compacting,
+                            // which we don't currently cater for.
+                            // We could potentially query for that, but that could add overhead.
+                            count: 0,
+                            bytes: 0
+                        }
+                    },
+                    $setOnInsert: {
+                        // Only set this if we're creating the document.
+                        // In all other cases, the replication process will have a set a more accurate id.
+                        last_op: this.maxOpId
+                    }
+                },
+                // We generally expect this to have been created before, but do handle cases of old unchanged buckets
+                upsert: true
+            }
+        });
     }
     async flush() {
         if (this.updates.length > 0) {
@@ -209,14 +273,24 @@ export class MongoCompactor {
             });
             this.updates = [];
         }
+        if (this.bucketStateUpdates.length > 0) {
+            logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
+            await this.db.bucket_state.bulkWrite(this.bucketStateUpdates, {
+                ordered: false
+            });
+            this.bucketStateUpdates = [];
+        }
     }
     /**
      * Perform a CLEAR compact for a bucket.
      *
+     *
      * @param bucket bucket name
      * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
      */
-    async clearBucket(bucket, op) {
+    async clearBucket(currentState) {
+        const bucket = currentState.bucket;
+        const clearOp = currentState.lastNotPut;
         const opFilter = {
             _id: {
                 $gte: {
@@ -227,14 +301,15 @@ export class MongoCompactor {
                 $lte: {
                     g: this.group_id,
                     b: bucket,
-                    o: op
+                    o: clearOp
                 }
             }
         };
         const session = this.db.client.startSession();
         try {
             let done = false;
-            while (!done) {
+            while (!done && !this.signal?.aborted) {
+                let opCountDiff = 0;
                 // Do the CLEAR operation in batches, with each batch a separate transaction.
                 // The state after each batch is fully consistent.
                 // We need a transaction per batch to make sure checksums stay consistent.
@@ -295,10 +370,13 @@ export class MongoCompactor {
                         data: null,
                         target_op: targetOp
                     }, { session });
+                    opCountDiff = -numberOfOpsToClear + 1;
                 }, {
                     writeConcern: { w: 'majority' },
                     readConcern: { level: 'snapshot' }
                 });
+                // Update _outside_ the transaction, since the transaction can be retried multiple times.
+                currentState.opCount += opCountDiff;
             }
         }
         finally {

package/dist/storage/implementation/MongoCompactor.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIvE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AA8B/C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAWf;IACA;IAXF,OAAO,GAAsD,EAAE,CAAC;IAEhE,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAqB;IAC5B,OAAO,CAAuB;IAEtC,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAAkC,CAAC;QACvC,IAAI,WAAkC,CAAC;QAEvC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,OAAO,IAAI,EAAE,CAAC;YACZ,sDAAsD;YACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACpC,IAAI,CACH;gBACE,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB;aACF,EACD;gBACE,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,CAAC;iBACd;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB;gBAC/B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,WAAW,EAAE,IAAI;aAClB,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;wBAC7F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC1C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;gBACJ,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAgB;QACxD,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;iBAC7B;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,EAAE;iBACN;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE,CAAC;gBACb,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CACvE,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;6BAC7B;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC7G,OAAO,EAAE,YAAY,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIrF,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AA6C5C,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IAaf;IACA;IAbF,OAAO,GAAsD,EAAE,CAAC;IAChE,kBAAkB,GAAuD,EAAE,CAAC;IAE5E,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAS;IAChB,OAAO,CAAuB;IAC9B,MAAM,CAAe;IAE7B,YACU,EAAkB,EAClB,QAAgB,EACxB,OAA6B;QAFrB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAAkC,CAAC;QACvC,IAAI,WAAkC,CAAC;QAEvC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,WAAW,GAAG,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,WAAqB;YACxB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;SAC7B,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC7B,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAiD;gBAC3F;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,IAAI,EAAE,UAAU;4BAChB,GAAG,EAAE,UAAU;yBAChB;qBACF;iBACF;gBACD,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE;gBACtB,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACpC;oBACE,QAAQ,EAAE;wBACR,GAAG,EAAE,CAAC;wBACN,EAAE,EAAE,CAAC;wBACL,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,CAAC;wBACT,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;qBAC9B;iBACF;aACF,CAAC,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEtD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC7D,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;wBACzB,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;4BACrE,0CAA0C;4BAC1C,wDAAwD;4BACxD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;4BAEF,qCAAqC;4BACrC,YAAa,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;4BAE3B,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;wBACvC,CAAC;wBAED,uDAAuD;wBACvD,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;oBAC3C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;wBAEd,QAAQ,EAAE,CAAC;wBACX,OAAO,EAAE,CAAC;wBACV,OAAO,EAAE,CAAC;qBACX,CAAC;gBACJ,CAAC;gBAED,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAClF,YAAY,CAAC,OAAO,IAAI,CAAC,CAAC;gBAE1B,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACzF,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;wBAEH,YAAY,CAAC,OAAO,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iCAAiC;oBACnF,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAChF,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,8BAA8B;YAC9B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,+DAA+D;YAC/D,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC3C,CAAC;QACD,mDAAmD;QACnD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,KAAyB;QACrD,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAC3B,SAAS,EAAE;gBACT,MAAM,EAAE;oBACN,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,KAAK,CAAC,MAAM;qBAChB;iBACF;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,IAAI,CAAC,OAAO;4BACnB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,sGAAsG;4BACtG,sCAAsC;4BACtC,oEAAoE;4BACpE,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;oBACD,YAAY,EAAE;wBACZ,gDAAgD;wBAChD,kFAAkF;wBAClF,OAAO,EAAE,IAAI,CAAC,OAAO;qBACtB;iBACF;gBACD,qGAAqG;gBACrG,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5D,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,YAAgC;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;QACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAW,CAAC;QAEzC,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;iBAC7B;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,OAAO;iBACX;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACtC,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;oBAC3B,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;4BAC7D,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAC7D,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,kBAAkB,IAAI,CAAC,CAAC;4BACxB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;gCACrB,OAAO,GAAG,IAAI,CAAC;4BACjB,CAAC;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;gCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;oCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;gCAC1B,CAAC;4BACH,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CACvE,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,kBAAkB,WAAW,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC9E,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS;6BAC7B;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;wBAC1B,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,WAAW,GAAG,CAAC,kBAAkB,GAAG,CAAC,CAAC;gBACxC,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;gBACF,yFAAyF;gBACzF,YAAY,CAAC,OAAO,IAAI,WAAW,CAAC;YACtC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;CACF"}

package/dist/storage/implementation/MongoParameterCompactor.d.ts

@@ -0,0 +1,17 @@
+import { CompactOptions, InternalOpId } from '@powersync/service-core';
+import { PowerSyncMongo } from './db.js';
+/**
+ * Compacts parameter lookup data (the bucket_parameters collection).
+ *
+ * This scans through the entire collection to find data to compact.
+ *
+ * For background, see the `/docs/parameters-lookups.md` file.
+ */
+export declare class MongoParameterCompactor {
+    private db;
+    private group_id;
+    private checkpoint;
+    private options;
+    constructor(db: PowerSyncMongo, group_id: number, checkpoint: InternalOpId, options: CompactOptions);
+    compact(): Promise<void>;
+}

package/dist/storage/implementation/MongoParameterCompactor.js

@@ -0,0 +1,92 @@
+import { logger } from '@powersync/lib-services-framework';
+import { bson } from '@powersync/service-core';
+import { LRUCache } from 'lru-cache';
+/**
+ * Compacts parameter lookup data (the bucket_parameters collection).
+ *
+ * This scans through the entire collection to find data to compact.
+ *
+ * For background, see the `/docs/parameters-lookups.md` file.
+ */
+export class MongoParameterCompactor {
+    db;
+    group_id;
+    checkpoint;
+    options;
+    constructor(db, group_id, checkpoint, options) {
+        this.db = db;
+        this.group_id = group_id;
+        this.checkpoint = checkpoint;
+        this.options = options;
+    }
+    async compact() {
+        logger.info(`Compacting parameters for group ${this.group_id} up to checkpoint ${this.checkpoint}`);
+        // This is the currently-active checkpoint.
+        // We do not remove any data that may be used by this checkpoint.
+        // snapshot queries ensure that if any clients are still using older checkpoints, they would
+        // not be affected by this compaction.
+        const checkpoint = this.checkpoint;
+        // Index on {'key.g': 1, lookup: 1, _id: 1}
+        // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup)
+        // in MongoDB already. However, that risks running into cases where MongoDB needs to process
+        // very large amounts of data before returning results, which could lead to timeouts.
+        const cursor = this.db.bucket_parameters.find({
+            'key.g': this.group_id
+        }, {
+            sort: { lookup: 1, _id: 1 },
+            batchSize: 10_000,
+            projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
+        });
+        // The index doesn't cover sorting by key, so we keep our own cache of the last seen key.
+        let lastByKey = new LRUCache({
+            max: this.options.compactParameterCacheLimit ?? 10_000
+        });
+        let removeIds = [];
+        let removeDeleted = [];
+        const flush = async (force) => {
+            if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
+                const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } });
+                logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
+                removeIds = [];
+            }
+            if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) {
+                const results = await this.db.bucket_parameters.bulkWrite(removeDeleted);
+                logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`);
+                removeDeleted = [];
+            }
+        };
+        while (await cursor.hasNext()) {
+            const batch = cursor.readBufferedDocuments();
+            for (let doc of batch) {
+                if (doc._id >= checkpoint) {
+                    continue;
+                }
+                const uniqueKey = bson.serialize({
+                    k: doc.key,
+                    l: doc.lookup
+                }).toString('base64');
+                const previous = lastByKey.get(uniqueKey);
+                if (previous != null && previous < doc._id) {
+                    // We have a newer entry for the same key, so we can remove the old one.
+                    removeIds.push(previous);
+                }
+                lastByKey.set(uniqueKey, doc._id);
+                if (doc.bucket_parameters?.length == 0) {
+                    // This is a delete operation, so we can remove it completely.
+                    // For this we cannot remove the operation itself only: There is a possibility that
+                    // there is still an earlier operation with the same key and lookup, that we don't have
+                    // in the cache due to cache size limits. So we need to explicitly remove all earlier operations.
+                    removeDeleted.push({
+                        deleteMany: {
+                            filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key }
+                        }
+                    });
+                }
+            }
+            await flush(false);
+        }
+        await flush(true);
+        logger.info('Parameter compaction completed');
+    }
+}
+//# sourceMappingURL=MongoParameterCompactor.js.map

package/dist/storage/implementation/MongoParameterCompactor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoParameterCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoParameterCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAgC,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAKrC;;;;;;GAMG;AACH,MAAM,OAAO,uBAAuB;IAExB;IACA;IACA;IACA;IAJV,YACU,EAAkB,EAClB,QAAgB,EAChB,UAAwB,EACxB,OAAuB;QAHvB,OAAE,GAAF,EAAE,CAAgB;QAClB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,eAAU,GAAV,UAAU,CAAc;QACxB,YAAO,GAAP,OAAO,CAAgB;IAC9B,CAAC;IAEJ,KAAK,CAAC,OAAO;QACX,MAAM,CAAC,IAAI,CAAC,mCAAmC,IAAI,CAAC,QAAQ,qBAAqB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACpG,2CAA2C;QAC3C,iEAAiE;QACjE,4FAA4F;QAC5F,sCAAsC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,2CAA2C;QAC3C,yFAAyF;QACzF,4FAA4F;QAC5F,qFAAqF;QACrF,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAC3C;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,EACD;YACE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;YAC3B,SAAS,EAAE,MAAM;YACjB,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE;SAChE,CACF,CAAC;QAEF,yFAAyF;QACzF,IAAI,SAAS,GAAG,IAAI,QAAQ,CAAuB;YACjD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,0BAA0B,IAAI,MAAM;SACvD,CAAC,CAAC;QACH,IAAI,SAAS,GAAmB,EAAE,CAAC;QACnC,IAAI,aAAa,GAA2D,EAAE,CAAC;QAE/E,MAAM,KAAK,GAAG,KAAK,EAAE,KAAc,EAAE,EAAE;YACrC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBAChE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,MAAM,gCAAgC,CAAC,CAAC;gBAClG,SAAS,GAAG,EAAE,CAAC;YACjB,CAAC;YAED,IAAI,aAAa,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,YAAY,KAAK,aAAa,CAAC,MAAM,6BAA6B,CAAC,CAAC;gBACnG,aAAa,GAAG,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC;QAEF,OAAO,MAAM,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;YAC7C,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,IAAI,GAAG,CAAC,GAAG,IAAI,UAAU,EAAE,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBACD,MAAM,SAAS,GACb,IAAI,CAAC,SAAS,CAAC;oBACb,CAAC,EAAE,GAAG,CAAC,GAAG;oBACV,CAAC,EAAE,GAAG,CAAC,MAAM;iBACd,CACF,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACrB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC1C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;oBAC3C,wEAAwE;oBACxE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;gBACD,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;gBAElC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;oBACvC,8DAA8D;oBAC9D,mFAAmF;oBACnF,uFAAuF;oBACvF,iGAAiG;oBACjG,aAAa,CAAC,IAAI,CAAC;wBACjB,UAAU,EAAE;4BACV,MAAM,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE;yBACzF;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAChD,CAAC;CACF"}

package/dist/storage/implementation/MongoSyncBucketStorage.d.ts

@@ -1,5 +1,6 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { BaseObserver } from '@powersync/lib-services-framework';
-import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
+import { CheckpointChanges, CompactOptions, GetCheckpointChangesOptions, InternalOpId, ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core';
 import { ParameterLookup, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 export declare class MongoSyncBucketStorage extends BaseObserver<storage.SyncRulesBucketStorageListener> implements storage.SyncRulesBucketStorage {
@@ -18,20 +19,21 @@ export declare class MongoSyncBucketStorage extends BaseObserver<storage.SyncRul
     lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
     getParsedSyncRules(options: storage.ParseSyncRulesOptions): SqlSyncRules;
     getCheckpoint(): Promise<storage.ReplicationCheckpoint>;
+    getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null>;
     startBatch(options: storage.StartBatchOptions, callback: (batch: storage.BucketStorageBatch) => Promise<void>): Promise<storage.FlushedResult | null>;
     resolveTable(options: storage.ResolveTableOptions): Promise<storage.ResolveTableResult>;
-    getParameterSets(checkpoint: utils.InternalOpId, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
+    getParameterSets(checkpoint: MongoReplicationCheckpoint, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
     getBucketDataBatch(checkpoint: utils.InternalOpId, dataBuckets: Map<string, InternalOpId>, options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
     getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap>;
+    clearChecksumCache(): void;
     private getChecksumsInternal;
     terminate(options?: storage.TerminateOptions): Promise<void>;
     getStatus(): Promise<storage.SyncRuleStatus>;
     clear(options?: storage.ClearStorageOptions): Promise<void>;
     private clearIteration;
-    autoActivate(): Promise<void>;
     reportError(e: any): Promise<void>;
     compact(options?: storage.CompactOptions): Promise<void>;
-    private makeActiveCheckpoint;
+    populatePersistentChecksumCache(options: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void>;
     /**
      * Instance-wide watch on the latest available checkpoint (op_id + lsn).
      */
@@ -58,4 +60,12 @@ interface InternalCheckpointChanges extends CheckpointChanges {
     updatedWriteCheckpoints: Map<string, bigint>;
     invalidateWriteCheckpoints: boolean;
 }
+declare class MongoReplicationCheckpoint implements ReplicationCheckpoint {
+    private storage;
+    readonly checkpoint: InternalOpId;
+    readonly lsn: string | null;
+    snapshotTime: mongo.Timestamp;
+    constructor(storage: MongoSyncBucketStorage, checkpoint: InternalOpId, lsn: string | null, snapshotTime: mongo.Timestamp);
+    getParameterSets(lookups: ParameterLookup[]): Promise<SqliteJsonRow[]>;
+}
 export {};