@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

package/dist/storage/implementation/MongoCompactor.js

@@ -1,15 +1,11 @@
 import * as timers from 'node:timers/promises';
-import { isMongoServerError, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
-import { logger as defaultLogger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { isPartialChecksum } from '@powersync/service-core';
+import { isMongoServerError } from '@powersync/lib-service-mongodb';
+import { logger as defaultLogger, ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { isRetryableObjectStorageError } from './v3/object-storage/ObjectStorage.js';
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MOVE_BATCH_BYTE_LIMIT = 16 * 1024 * 1024;
-const DEFAULT_MIN_BUCKET_CHANGES = 10;
-const DEFAULT_MIN_CHANGE_RATIO = 0.1;
-const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
 const COMPACTION_RETRY_LIMIT = 3;
@@ -20,22 +16,31 @@ export class ConcurrentCompactionError extends Error {
         this.name = 'ConcurrentCompactionError';
     }
 }
+/**
+ * A worker whose bucket lease has been replaced must stop immediately. Unlike
+ * a transactional replacement conflict, retrying it with the same compactor
+ * instance would still use the stale lease and could race the new owner.
+ */
+export class CompactionLeaseLostError extends ConcurrentCompactionError {
+    constructor(message) {
+        super(message);
+        this.name = 'CompactionLeaseLostError';
+    }
+}
 export class MongoCompactor {
     storage;
     db;
-    bucketStateUpdates = [];
     idLimitBytes;
     moveBatchLimit;
     moveBatchQueryLimit;
     moveBatchByteLimit;
     clearBatchLimit;
-    minBucketChanges;
-    minChangeRatio;
-    maxOpId;
     buckets;
     deleteCheckpointRequestsBefore;
     signal;
     group_id;
+    compactChunksOnly;
+    compactedBucketCount = 0;
     logger;
     constructor(storage, db, options) {
         this.storage = storage;
@@ -49,32 +54,12 @@ export class MongoCompactor {
         if (this.clearBatchLimit < 2) {
             throw new ReplicationAssertionError('clearBatchLimit must be >= 2');
         }
-        this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
-        this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
-        this.maxOpId = options.maxOpId ?? 0n;
         this.buckets = options.compactBuckets;
         this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
         this.signal = options.signal;
+        this.compactChunksOnly = options.compactChunksOnly ?? false;
         this.logger = options.logger ?? defaultLogger;
     }
-    /**
-     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
-     *
-     * See /docs/storage/compacting-operations.md for details.
-     */
-    async compact() {
-        await this.deleteOldCheckpointRequests();
-        if (this.buckets) {
-            for (const bucket of this.buckets) {
-                // We can make this more efficient later on by iterating through the buckets in a single query.
-                // That makes batching more tricky, so we leave for later.
-                await this.compactSingleBucketRetried(bucket);
-            }
-        }
-        else {
-            await this.compactDirtyBuckets();
-        }
-    }
     async deleteOldCheckpointRequests() {
         if (this.deleteCheckpointRequestsBefore == null) {
             return;
@@ -88,141 +73,16 @@ export class MongoCompactor {
             checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore },
             processed_at_lsn: { $ne: null }
         });
+        await this.deleteOldCustomCheckpointRequests();
+    }
+    async deleteOldCustomCheckpointRequests() {
+        if (this.deleteCheckpointRequestsBefore == null) {
+            return;
+        }
         await this.db.custom_write_checkpoints.deleteMany({
             checkpoint_requested_at: { $exists: true, $lt: this.deleteCheckpointRequestsBefore }
         });
     }
-    /**
-     * Subset of compact, only populating checksums where relevant.
-     */
-    async populateChecksums(options) {
-        let count = 0;
-        // Paginate through dirty buckets in batches until no more buckets meet the criteria.
-        while (true) {
-            this.signal?.throwIfAborted();
-            const buckets = await this.dirtyBucketBatchForChecksums(options);
-            if (buckets.length == 0) {
-                break;
-            }
-            this.signal?.throwIfAborted();
-            const start = Date.now();
-            // Filter batch by estimated bucket size, to reduce possibility of timeouts.
-            const checkBuckets = [];
-            let totalCountEstimate = 0;
-            for (const bucket of buckets) {
-                checkBuckets.push(bucket);
-                totalCountEstimate += bucket.estimatedCount;
-                if (totalCountEstimate > 50_000) {
-                    break;
-                }
-            }
-            this.logger.info(`Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`);
-            await this.updateChecksumsBatch(checkBuckets);
-            this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
-            count += checkBuckets.length;
-        }
-        return { buckets: count };
-    }
-    async *dirtyBucketBatchesForCollection(collection, lastId, maxId, options, getDefinitionId) {
-        // Paginate through the bucket state collection using cursor-based scanning.
-        while (true) {
-            // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline
-            // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria.
-            const [result] = await collection
-                .aggregate([
-                {
-                    $match: {
-                        _id: { $gt: lastId, $lt: maxId }
-                    }
-                },
-                {
-                    $sort: { _id: 1 }
-                },
-                {
-                    // Scan a fixed number of docs each query so sparse matches don't block progress.
-                    $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE
-                },
-                {
-                    $facet: {
-                        buckets: [
-                            {
-                                $match: {
-                                    'estimate_since_compact.count': { $gte: options.minBucketChanges }
-                                }
-                            },
-                            {
-                                $project: {
-                                    _id: 1,
-                                    estimate_since_compact: 1,
-                                    compacted_state: 1
-                                }
-                            }
-                        ],
-                        // This is used for the next query.
-                        cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
-                    }
-                }
-            ], { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS })
-                .toArray();
-            const cursor = result?.cursor?.[0];
-            if (cursor == null) {
-                break;
-            }
-            lastId = cursor._id;
-            const mapped = (result?.buckets ?? []).map((bucketState) => {
-                // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios.
-                // BigInt precision is not needed here since this is only an estimate.
-                const updatedCount = bucketState.estimate_since_compact?.count ?? 0;
-                const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount;
-                const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0);
-                const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes;
-                const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0;
-                const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0;
-                return {
-                    bucket: bucketState._id.b,
-                    definitionId: getDefinitionId(bucketState),
-                    estimatedCount: totalCount,
-                    dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes)
-                };
-            });
-            yield mapped.filter((bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio);
-        }
-    }
-    async dirtyBucketBatchForChecksumsForCollection(collection, filter, getDefinitionId) {
-        const dirtyBuckets = await collection
-            .find(filter, {
-            projection: {
-                _id: 1,
-                estimate_since_compact: 1,
-                compacted_state: 1
-            },
-            sort: {
-                'estimate_since_compact.count': -1
-            },
-            limit: 200,
-            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
-        })
-            .toArray();
-        return dirtyBuckets.map((bucket) => ({
-            bucket: bucket._id.b,
-            definitionId: getDefinitionId(bucket),
-            estimatedCount: Number(bucket.estimate_since_compact.count) + Number(bucket.compacted_state?.count ?? 0)
-        }));
-    }
-    async compactDirtyBuckets() {
-        for await (const buckets of this.dirtyBucketBatches({
-            minBucketChanges: this.minBucketChanges,
-            minChangeRatio: this.minChangeRatio
-        })) {
-            this.signal?.throwIfAborted();
-            if (buckets.length == 0) {
-                continue;
-            }
-            for (const { bucket, definitionId } of buckets) {
-                await this.compactSingleBucketRetried(bucket, definitionId);
-            }
-        }
-    }
     /**
      * Compaction for a single bucket, with retries on failure.
      *
@@ -231,14 +91,12 @@ export class MongoCompactor {
      * storage paths are also prepared with lifecycle markers, so a retry can
      * safely overwrite or eventually clean up uploads from the failed attempt.
      */
-    async compactSingleBucketRetried(bucket, definitionId = null) {
+    async retryCompaction(bucket, compact) {
         let retryCount = 0;
         while (true) {
             this.signal?.throwIfAborted();
-            // Do not carry queued bucket state writes from a failed attempt into the rescan.
-            this.bucketStateUpdates = [];
             try {
-                await this.compactSingleBucket(bucket, definitionId);
+                await compact();
                 return;
             }
             catch (e) {
@@ -260,80 +118,11 @@ export class MongoCompactor {
             }
         }
     }
-    collectBucketStateUpdates(state, compactedOpId) {
-        if (state.opCount < 0) {
-            throw new ServiceAssertionError(`Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`);
-        }
-        return {
-            updateOne: {
-                filter: this.bucketStateFilter(state.bucket, state.definitionId),
-                update: {
-                    $set: {
-                        compacted_state: {
-                            op_id: compactedOpId,
-                            count: state.opCount,
-                            checksum: BigInt(state.checksum),
-                            bytes: state.opBytes
-                        },
-                        estimate_since_compact: {
-                            // There could have been a whole bunch of new operations added to the bucket while compacting,
-                            // which we don't currently cater for. We could potentially query for that, but that adds overhead.
-                            count: 0,
-                            bytes: 0
-                        }
-                    }
-                },
-                // We generally expect this to have been created before.
-                // We don't create new ones here, to avoid issues with the unique index on bucket_updates.
-                upsert: false
-            }
-        };
-    }
-    updateBucketChecksums(state, compactedOpId) {
-        this.bucketStateUpdates.push(this.collectBucketStateUpdates(state, compactedOpId));
-    }
-    async flushBucketStateUpdates() {
-        if (this.bucketStateUpdates.length > 0) {
-            this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
-            await this.writeBucketStateUpdates();
-            this.bucketStateUpdates = [];
-        }
-    }
-    async updateChecksumsBatch(buckets) {
-        const checksums = await this.computeChecksumsForBuckets(buckets);
-        const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
-        for (const bucketChecksum of checksums.values()) {
-            if (isPartialChecksum(bucketChecksum)) {
-                // Should never happen since we don't specify `start`.
-                throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`);
-            }
-            this.bucketStateUpdates.push({
-                updateOne: {
-                    filter: this.bucketStateFilter(bucketChecksum.bucket, definitionIdByBucket.get(bucketChecksum.bucket) ?? null),
-                    update: {
-                        $set: {
-                            compacted_state: {
-                                op_id: this.maxOpId,
-                                count: bucketChecksum.count,
-                                checksum: BigInt(bucketChecksum.checksum),
-                                bytes: null
-                            },
-                            estimate_since_compact: {
-                                count: 0,
-                                bytes: 0
-                            }
-                        }
-                    },
-                    // We don't create new ones here - it gets tricky to get the last_op right with the unique index on
-                    // bucket_updates.
-                    upsert: false
-                }
-            });
-        }
-        await this.flushBucketStateUpdates();
-    }
 }
 function compactionRetryReason(error) {
+    if (error instanceof CompactionLeaseLostError) {
+        return null;
+    }
     if (error instanceof ConcurrentCompactionError) {
         return 'concurrent compaction';
     }

package/dist/storage/implementation/MongoCompactor.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,kBAAkB,EAAS,0BAA0B,EAAE,MAAM,gCAAgC,CAAC;AACvG,OAAO,EACL,MAAM,IAAI,aAAa,EAEvB,yBAAyB,EACzB,qBAAqB,EACtB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAgB,iBAAiB,EAAyC,MAAM,yBAAyB,CAAC;AAOjH,OAAO,EAAE,6BAA6B,EAAE,MAAM,sCAAsC,CAAC;AAsCrF,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAC9C,MAAM,6BAA6B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AACrC,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAC3C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IAClD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AASD,MAAM,OAAgB,cAAc;IAmBb;IACA;IAnBX,kBAAkB,GAA2D,EAAE,CAAC;IAEvE,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,kBAAkB,CAAS;IAC3B,eAAe,CAAS;IACxB,gBAAgB,CAAS;IACzB,cAAc,CAAS;IACvB,OAAO,CAAS;IAChB,OAAO,CAAuB;IAC9B,8BAA8B,CAAmB;IACjD,MAAM,CAAe;IACrB,QAAQ,CAAS;IAEjB,MAAM,CAAS;IAElC,YACqB,OAA+B,EAC/B,EAA2B,EAC9C,OAA4B;QAFT,YAAO,GAAP,OAAO,CAAwB;QAC/B,OAAE,GAAF,EAAE,CAAyB;QAG9C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,mBAAmB,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACrF,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,8BAA8B,CAAC;QACzF,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,6BAA6B,CAAC;QACtF,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;QAC5E,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,yBAAyB,CAAC,8BAA8B,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,0BAA0B,CAAC;QAC/E,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACzE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC;QACtC,IAAI,CAAC,8BAA8B,GAAG,OAAO,CAAC,8BAA8B,CAAC;QAC7E,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEzC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClC,+FAA+F;gBAC/F,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,2BAA2B;QACvC,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,EAAE,CAAC;YAChD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QAC9B,+DAA+D;QAC/D,6EAA6E;QAC7E,yFAAyF;QACzF,oBAAoB;QACpB,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC;YACzC,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,8BAA8B,EAAE;YACpF,gBAAgB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,UAAU,CAAC;YAChD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,8BAA8B,EAAE;SACrF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAAqC;QAC3D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,qFAAqF;QACrF,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;YACjE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,4EAA4E;YAC5E,MAAM,YAAY,GAAmB,EAAE,CAAC;YACxC,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1B,kBAAkB,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC5C,IAAI,kBAAkB,GAAG,MAAM,EAAE,CAAC;oBAChC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sCAAsC,OAAO,CAAC,MAAM,gCAAgC,kBAAkB,EAAE,CACzG,CAAC;YACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,YAAY,CAAC,MAAM,eAAe,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC;YAC7G,KAAK,IAAI,YAAY,CAAC,MAAM,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAES,KAAK,CAAC,CAAC,+BAA+B,CAC9C,UAAoD,EACpD,MAAqC,EACrC,KAAoC,EACpC,OAGC,EACD,eAA6E;QAE7E,4EAA4E;QAC5E,OAAO,IAAI,EAAE,CAAC;YACZ,iHAAiH;YACjH,8FAA8F;YAC9F,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU;iBAC9B,SAAS,CAIR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE;qBACjC;iBACF;gBACD;oBACE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBAClB;gBACD;oBACE,iFAAiF;oBACjF,MAAM,EAAE,4BAA4B;iBACrC;gBACD;oBACE,MAAM,EAAE;wBACN,OAAO,EAAE;4BACP;gCACE,MAAM,EAAE;oCACN,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;iCACnE;6BACF;4BACD;gCACE,QAAQ,EAAE;oCACR,GAAG,EAAE,CAAC;oCACN,sBAAsB,EAAE,CAAC;oCACzB,eAAe,EAAE,CAAC;iCACnB;6BACF;yBACF;wBACD,mCAAmC;wBACnC,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;qBAC1E;iBACF;aACF,EACD,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAC1C;iBACA,OAAO,EAAE,CAAC;YAEb,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC;YAEpB,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;gBACzD,yGAAyG;gBACzG,sEAAsE;gBACtE,MAAM,YAAY,GAAG,WAAW,CAAC,sBAAsB,EAAE,KAAK,IAAI,CAAC,CAAC;gBACpE,MAAM,UAAU,GAAG,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC;gBAC5E,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,sBAAsB,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;gBAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC;gBAClF,MAAM,iBAAiB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzE,MAAM,gBAAgB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxE,OAAO;oBACL,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;oBACzB,YAAY,EAAE,eAAe,CAAC,WAAW,CAAC;oBAC1C,cAAc,EAAE,UAAU;oBAC1B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;iBAC1D,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,MAAM,CACjB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,IAAI,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,UAAU,IAAI,OAAO,CAAC,cAAc,CAC7G,CAAC;QACJ,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yCAAyC,CACvD,UAA0C,EAC1C,MAAkC,EAClC,eAAiF;QAEjF,MAAM,YAAY,GAAG,MAAM,UAAU;aAClC,IAAI,CAAC,MAAM,EAAE;YACZ,UAAU,EAAE;gBACV,GAAG,EAAE,CAAC;gBACN,sBAAsB,EAAE,CAAC;gBACzB,eAAe,EAAE,CAAC;aACnB;YACD,IAAI,EAAE;gBACJ,8BAA8B,EAAE,CAAC,CAAC;aACnC;YACD,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,0BAA0B;SACtC,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,YAAY,EAAE,eAAe,CAAC,MAAM,CAAC;YACrC,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,sBAAuB,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,IAAI,CAAC,CAAC;SAC1G,CAAC,CAAC,CAAC;IACN,CAAC;IASS,KAAK,CAAC,mBAAmB;QACjC,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,IAAI,CAAC,kBAAkB,CAAC;YAClD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,cAAc,EAAE,IAAI,CAAC,cAAc;SACpC,CAAC,EAAE,CAAC;YACH,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACxB,SAAS;YACX,CAAC;YAED,KAAK,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,OAAO,EAAE,CAAC;gBAC/C,MAAM,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,0BAA0B,CAAC,MAAc,EAAE,eAA0C,IAAI;QACvG,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,iFAAiF;YACjF,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;gBACrD,OAAO;YACT,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBACzB,MAAM,CAAC,CAAC;gBACV,CAAC;gBAED,MAAM,WAAW,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAAC;gBAC7C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;oBACxB,MAAM,CAAC,CAAC;gBACV,CAAC;gBACD,IAAI,UAAU,IAAI,sBAAsB,EAAE,CAAC;oBACzC,MAAM,IAAI,KAAK,CACb,4BAA4B,MAAM,UAAU,UAAU,GAAG,CAAC,cAAc,WAAW,MAAM,YAAY,CAAC,CAAC,CAAC,EAAE,EAC1G,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;gBACJ,CAAC;gBAED,UAAU,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,yBAAyB,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;gBAChE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,2BAA2B,MAAM,KAAK,WAAW,kBAAkB,KAAK,KAAK;oBAC3E,YAAY,UAAU,GAAG,CAAC,IAAI,sBAAsB,GAAG,CAAC,GAAG,EAC7D,CAAC,CACF,CAAC;gBACF,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IAIS,yBAAyB,CACjC,KAAyB,EACzB,aAA2B;QAE3B,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAqB,CAC7B,oBAAoB,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,QAAQ,iBAAiB,KAAK,CAAC,WAAW,aAAa,IAAI,CAAC,OAAO,EAAE,CAC1H,CAAC;QACJ,CAAC;QACD,OAAO;YACL,SAAS,EAAE;gBACT,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC;gBAChE,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,eAAe,EAAE;4BACf,KAAK,EAAE,aAAa;4BACpB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;4BAChC,KAAK,EAAE,KAAK,CAAC,OAAO;yBACrB;wBACD,sBAAsB,EAAE;4BACtB,8FAA8F;4BAC9F,mGAAmG;4BACnG,KAAK,EAAE,CAAC;4BACR,KAAK,EAAE,CAAC;yBACT;qBACF;iBACoD;gBACvD,wDAAwD;gBACxD,0FAA0F;gBAC1F,MAAM,EAAE,KAAK;aACd;SACF,CAAC;IACJ,CAAC;IAES,qBAAqB,CAAC,KAAyB,EAAE,aAA2B;QACpF,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IACrF,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,gBAAgB,CAAC,CAAC;YAC7E,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACrC,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,OAAuD;QAC1F,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACjE,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEpG,KAAK,MAAM,cAAc,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,IAAI,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBACtC,sDAAsD;gBACtD,MAAM,IAAI,qBAAqB,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAC3B,SAAS,EAAE;oBACT,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAC5B,cAAc,CAAC,MAAM,EACrB,oBAAoB,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,IAAI,CACxD;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE;4BACJ,eAAe,EAAE;gCACf,KAAK,EAAE,IAAI,CAAC,OAAO;gCACnB,KAAK,EAAE,cAAc,CAAC,KAAK;gCAC3B,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;gCACzC,KAAK,EAAE,IAAI;6BACZ;4BACD,sBAAsB,EAAE;gCACtB,KAAK,EAAE,CAAC;gCACR,KAAK,EAAE,CAAC;6BACT;yBACF;qBACoD;oBACvD,mGAAmG;oBACnG,kBAAkB;oBAClB,MAAM,EAAE,KAAK;iBACd;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACvC,CAAC;CAOF;AAOD,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,KAAK,YAAY,yBAAyB,EAAE,CAAC;QAC/C,OAAO,uBAAuB,CAAC;IACjC,CAAC;IACD,IAAI,6BAA6B,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,OAAO,kCAAkC,CAAC;IAC5C,CAAC;IACD,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}
+{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,kBAAkB,EAAS,MAAM,gCAAgC,CAAC;AAC3E,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAM/G,OAAO,EAAE,6BAA6B,EAAE,MAAM,sCAAsC,CAAC;AAWrF,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAC9C,MAAM,6BAA6B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IAClD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,wBAAyB,SAAQ,yBAAyB;IACrE,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACzC,CAAC;CACF;AAED,MAAM,OAAgB,cAAc;IAgBb;IACA;IAhBF,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,kBAAkB,CAAS;IAC3B,eAAe,CAAS;IACxB,OAAO,CAAuB;IAC9B,8BAA8B,CAAmB;IACjD,MAAM,CAAe;IACrB,QAAQ,CAAS;IACjB,iBAAiB,CAAU;IACpC,oBAAoB,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IAElC,YACqB,OAA+B,EAC/B,EAA2B,EAC9C,OAA4B;QAFT,YAAO,GAAP,OAAO,CAAwB;QAC/B,OAAE,GAAF,EAAE,CAAyB;QAG9C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,mBAAmB,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACrF,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,8BAA8B,CAAC;QACzF,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,6BAA6B,CAAC;QACtF,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;QAC5E,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,yBAAyB,CAAC,8BAA8B,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC;QACtC,IAAI,CAAC,8BAA8B,GAAG,OAAO,CAAC,8BAA8B,CAAC;QAC7E,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,KAAK,CAAC;QAC5D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;IAChD,CAAC;IAIS,KAAK,CAAC,2BAA2B;QACzC,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,EAAE,CAAC;YAChD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QAC9B,+DAA+D;QAC/D,6EAA6E;QAC7E,yFAAyF;QACzF,oBAAoB;QACpB,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC;YACzC,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,8BAA8B,EAAE;YACpF,gBAAgB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,iCAAiC,EAAE,CAAC;IACjD,CAAC;IAES,KAAK,CAAC,iCAAiC;QAC/C,IAAI,IAAI,CAAC,8BAA8B,IAAI,IAAI,EAAE,CAAC;YAChD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,UAAU,CAAC;YAChD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,8BAA8B,EAAE;SACrF,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,eAAe,CAAC,MAAc,EAAE,OAA4B;QAC1E,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,OAAO,EAAE,CAAC;gBAChB,OAAO;YACT,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBACzB,MAAM,CAAC,CAAC;gBACV,CAAC;gBAED,MAAM,WAAW,GAAG,qBAAqB,CAAC,CAAC,CAAC,CAAC;gBAC7C,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;oBACxB,MAAM,CAAC,CAAC;gBACV,CAAC;gBACD,IAAI,UAAU,IAAI,sBAAsB,EAAE,CAAC;oBACzC,MAAM,IAAI,KAAK,CACb,4BAA4B,MAAM,UAAU,UAAU,GAAG,CAAC,cAAc,WAAW,MAAM,YAAY,CAAC,CAAC,CAAC,EAAE,EAC1G,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;gBACJ,CAAC;gBAED,UAAU,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,yBAAyB,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;gBAChE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,2BAA2B,MAAM,KAAK,WAAW,kBAAkB,KAAK,KAAK;oBAC3E,YAAY,UAAU,GAAG,CAAC,IAAI,sBAAsB,GAAG,CAAC,GAAG,EAC7D,CAAC,CACF,CAAC;gBACF,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAOD,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,KAAK,YAAY,wBAAwB,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,YAAY,yBAAyB,EAAE,CAAC;QAC/C,OAAO,uBAAuB,CAAC;IACjC,CAAC;IACD,IAAI,6BAA6B,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,OAAO,kCAAkC,CAAC;IAC5C,CAAC;IACD,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}

package/dist/storage/implementation/MongoParameterCompactor.d.ts

@@ -1,23 +1,96 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { CompactOptions, InternalOpId } from '@powersync/service-core';
+import { Logger } from '@powersync/lib-services-framework';
+import { bson, CompactOptions, InternalOpId } from '@powersync/service-core';
 import type { VersionedPowerSyncMongo } from './db.js';
+type ParameterCompactionReadDocument = {
+    _id: InternalOpId;
+    key: mongo.Document;
+    lookup: bson.Binary;
+    bucket_parameters?: unknown[] | null;
+};
+export type ParameterCompactionResult = {
+    collections: number;
+    scannedEntries: number;
+    deletedEntries: number;
+};
 /**
  * Compacts parameter lookup data (the bucket_parameters collection).
  *
- * This scans through the entire collection to find data to compact.
+ * Both storage versions persist a per-stream compaction cursor, so a run only scans entries in the
+ * un-compacted operation-id range. V1 scans its shared collection using only the `_id` index, so it
+ * additionally filters by stream in code.
+ *
+ * The cursor is a single value covering every collection of the stream, so it can only be advanced
+ * to a boundary that all collections have passed. To keep it moving during a long pass, collections
+ * are processed in lock-step rather than one after another - see {@link compactCollections}.
  *
  * For background, see the `/docs/storage/parameter-lookups.md` file.
  */
-export declare class MongoParameterCompactor {
+export declare abstract class MongoParameterCompactor {
+    #private;
     protected readonly db: VersionedPowerSyncMongo;
-    protected readonly group_id: number;
+    protected readonly replicationStreamId: number;
     protected readonly checkpoint: InternalOpId;
     protected readonly options: CompactOptions;
-    protected readonly getCollectionsCb?: (() => Promise<mongo.Collection<mongo.Document>[]>) | undefined;
-    constructor(db: VersionedPowerSyncMongo, group_id: number, checkpoint: InternalOpId, options: CompactOptions, getCollectionsCb?: (() => Promise<mongo.Collection<mongo.Document>[]>) | undefined);
+    protected readonly parameterCompactionBatchSize: number;
+    protected readonly parameterCompactionPersistIntervalMs: number;
+    protected readonly logger: Logger;
+    protected readonly signal?: AbortSignal;
+    constructor(db: VersionedPowerSyncMongo, replicationStreamId: number, checkpoint: InternalOpId, options: CompactOptions, parameterCompactionBatchSize?: number, parameterCompactionPersistIntervalMs?: number);
     compact(): Promise<void>;
-    protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
-    protected collectionFilter(): mongo.Document;
-    protected deleteFilter(doc: mongo.Document): mongo.Document;
-    protected compactCollection(collection: mongo.Collection<mongo.Document>): Promise<void>;
+    /**
+     * The exclusive operation-id boundary through which this stream's parameter indexes have all
+     * been compacted.
+     */
+    protected readCompactedBefore(): Promise<InternalOpId>;
+    protected persistCompactedBefore(compactedBefore: InternalOpId): Promise<void>;
+    /**
+     * Commits the checkpoint-change invalidation fence before the first delete of this pass.
+     *
+     * Checkpoint change detection finds changed lookups by querying parameter entries in
+     * (lastCheckpoint, nextCheckpoint]. Compaction physically removes entries in that range, so a
+     * checkpoint that can no longer see the full history must instead invalidate all parameter
+     * buckets. The fence records the boundary below which that history may be missing.
+     *
+     * The fence must be committed before the first delete: MongoDB snapshot ordering then
+     * guarantees that a checkpoint snapshot which observes a deletion also observes the fence.
+     *
+     * This deliberately isn't the same value as the compaction cursor. If the pass fails halfway,
+     * the fence only causes conservative invalidation, while an advanced cursor would skip
+     * deletion work that never completed.
+     */
+    private ensureInvalidationFence;
+    protected abstract getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+    protected abstract shouldCompactDocument(doc: ParameterCompactionReadDocument): boolean;
+    /** Deletes history preceding a batch for several identities sharing a lookup. */
+    protected abstract leadingHistoryDeleteFilter(lookup: bson.Binary, keys: mongo.Document[], before: InternalOpId): mongo.Document;
+    /**
+     * Processes every collection of the stream, in lock-step: each turn takes one batch from the
+     * collection that has processed the least so far.
+     *
+     * The persisted cursor is the minimum position over all collections, which is exactly the
+     * boundary that all of them have passed, so it can be advanced periodically during the pass.
+     * Always picking the collection that is furthest behind also keeps each of them within one batch
+     * of that boundary, bounding the work an interrupted pass has to repeat.
+     *
+     * V1 storage always has a single collection; V3 has a collection per defined index. So in V1 the
+     * ame process collapses to compacting the single collection in order, while V3 can alternate between
+     * collections.
+     */
+    private compactCollections;
+    /**
+     * The boundary that every scope has processed past, capped at the target checkpoint, and the
+     * scope sitting on it.
+     *
+     * The frontier is the furthest the cursor may be advanced. The scope is the one that has
+     * processed the least, or null once all of them have reached the target checkpoint.
+     */
+    private frontier;
+    /**
+     * Reads and processes one batch from the scope, and advances its position past that batch.
+     */
+    private compactBatch;
+    /** Deletes documents by `_id`, chunked to bound the command size. Returns the number deleted. */
+    private deleteByIds;
 }
+export {};