@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/dist/storage/implementation/MongoChecksums.d.ts

@@ -1,10 +1,9 @@
-import { BucketChecksum, BucketChecksumRequest, ChecksumMap, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap, PartialOrFullChecksum } from '@powersync/service-core';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { bson, BucketChecksum, BucketChecksumRequest, ChecksumMap, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap, PartialOrFullChecksum } from '@powersync/service-core';
 import type { VersionedPowerSyncMongo } from './db.js';
-import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { BucketDefinitionId } from '@powersync/service-sync-rules';
-import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
-import { BucketDataDocumentBase, StorageConfig } from './models.js';
-export interface FetchPartialBucketChecksumV3 {
+import { StorageConfig } from './models.js';
+export interface FetchPartialBucketChecksumByDefinition {
     bucket: string;
     definitionId: BucketDefinitionId;
     start?: InternalOpId;
@@ -27,15 +26,39 @@ export interface MongoChecksumOptions {
      * Limit on the number of documents to calculate a checksum on at a time.
      */
     operationBatchLimit?: number;
+    checksumCacheTtlMs?: number;
     storageConfig: StorageConfig;
-    mapping?: BucketDefinitionMapping;
 }
+/**
+ * Read options used in requests to the cache, used to construct a session.
+ *
+ * readConcern is always 'snapshot'.
+ */
+interface MongoChecksumCacheReadContext {
+    /**
+     * (Optional) Specify a specific Timestamp for the snapshot read.
+     * Must include clusterTime as well.
+     */
+    snapshotTime?: mongo.Timestamp;
+    clusterTime?: mongo.ClusterTime;
+    /**
+     * Read preference. Use undefined for the default,
+     */
+    readPreference: mongo.ReadPreference | undefined;
+}
+/**
+ * Read options passed into individual methods.
+ */
+export interface MongoChecksumSessionContext {
+    readOptions: Pick<mongo.CommandOperationOptions, 'session' | 'readPreference' | 'readConcern'>;
+}
+export declare const DEFAULT_OPERATION_BATCH_LIMIT = 50000;
 export declare abstract class MongoChecksums {
     protected readonly db: VersionedPowerSyncMongo;
     protected readonly group_id: number;
     protected readonly options: MongoChecksumOptions;
     private _cache;
-    private readonly storageConfig;
+    protected readonly storageConfig: StorageConfig;
     constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions);
     /**
      * Lazy-instantiated cache.
@@ -47,7 +70,7 @@ export declare abstract class MongoChecksums {
      * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
      * the database (bucket_state + bucket_data).
      */
-    getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap>;
+    getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[], options: MongoChecksumCacheReadContext): Promise<ChecksumMap>;
     clearCache(): void;
     /**
      * Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations).
@@ -59,6 +82,7 @@ export declare abstract class MongoChecksums {
      * can be slow.
      */
     private computePartialChecksums;
+    private computePartialChecksumsWithSession;
     /**
      * Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state.
      *
@@ -69,7 +93,7 @@ export declare abstract class MongoChecksums {
      * For large buckets, this can be slow, but should not time out as the underlying queries are performed in
      * smaller batches.
      */
-    computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
+    private computePartialChecksumsDirect;
     /**
      * Query a batch of checksums.
      *
@@ -77,11 +101,15 @@ export declare abstract class MongoChecksums {
      *
      * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
      */
-    protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
-    protected abstract fetchPreStates(batch: FetchPartialBucketChecksum[]): Promise<Map<string, {
+    protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[], context: MongoChecksumSessionContext): Promise<PartialChecksumMap>;
+    protected abstract fetchPreStates(batch: FetchPartialBucketChecksum[], context: MongoChecksumSessionContext): Promise<Map<string, {
         opId: InternalOpId;
         checksum: BucketChecksum;
     }>>;
-    protected computePartialChecksumsForCollection<TRequest extends FetchPartialBucketChecksumByBucket, TBucketDataDocument extends BucketDataDocumentBase>(batch: TRequest[], collection: lib_mongo.mongo.Collection<TBucketDataDocument>, createFilter: (request: TRequest) => any): Promise<PartialChecksumMap>;
 }
-export declare function emptyChecksumForRequest(request: Pick<FetchPartialBucketChecksum | FetchPartialBucketChecksumV3, 'bucket' | 'start'>): PartialOrFullChecksum;
+export declare function emptyChecksumForRequest(request: Pick<FetchPartialBucketChecksum | FetchPartialBucketChecksumByDefinition, 'bucket' | 'start'>): PartialOrFullChecksum;
+/**
+ * Convert output of the $group stage into a checksum.
+ */
+export declare function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum;
+export {};

package/dist/storage/implementation/MongoChecksums.js

@@ -1,7 +1,8 @@
-import { addPartialChecksums, ChecksumCache, isPartialChecksum } from '@powersync/service-core';
-import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { addPartialChecksums, ChecksumCache } from '@powersync/service-core';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { setSessionSnapshotTime } from '../../utils/util.js';
 const DEFAULT_BUCKET_BATCH_LIMIT = 200;
-const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
+export const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
 export class MongoChecksums {
     db;
     group_id;
@@ -21,8 +22,9 @@ export class MongoChecksums {
      */
     get cache() {
         this._cache ??= new ChecksumCache({
-            fetchChecksums: (batch) => {
-                return this.computePartialChecksums(batch);
+            ttlMs: this.options.checksumCacheTtlMs,
+            fetchChecksums: (batch, context) => {
+                return this.computePartialChecksums(batch, context);
             }
         });
         return this._cache;
@@ -31,8 +33,8 @@ export class MongoChecksums {
      * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from
      * the database (bucket_state + bucket_data).
      */
-    async getChecksums(checkpoint, buckets) {
-        return this.cache.getChecksumMap(checkpoint, buckets);
+    async getChecksums(checkpoint, buckets, options) {
+        return this.cache.getChecksumMap(checkpoint, buckets, options);
     }
     clearCache() {
         this.cache.clear();
@@ -46,11 +48,31 @@ export class MongoChecksums {
      * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state
      * can be slow.
      */
-    async computePartialChecksums(batch) {
+    async computePartialChecksums(batch, context) {
         if (batch.length == 0) {
             return new Map();
         }
-        const preStates = await this.fetchPreStates(batch);
+        if (context == null) {
+            throw new ServiceAssertionError(`context is required`);
+        }
+        const { snapshotTime, clusterTime, readPreference } = context;
+        return this.db.client.withSession({ snapshot: true }, async (session) => {
+            if (snapshotTime != null && clusterTime != null) {
+                setSessionSnapshotTime(session, snapshotTime);
+                session.advanceClusterTime(clusterTime);
+            }
+            return this.computePartialChecksumsWithSession(batch, {
+                readOptions: {
+                    // This is set on the session, but we set it on each operation as well just in case
+                    readConcern: 'snapshot',
+                    readPreference,
+                    session
+                }
+            });
+        });
+    }
+    async computePartialChecksumsWithSession(batch, context) {
+        const preStates = await this.fetchPreStates(batch, context);
         const mappedRequests = batch.map((request) => {
             let start = request.start;
             if (start == null) {
@@ -64,7 +86,7 @@ export class MongoChecksums {
                 start
             };
         });
-        const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests);
+        const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests, context);
         return new Map(batch.map((request) => {
             const bucket = request.bucket;
             // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available
@@ -85,131 +107,24 @@ export class MongoChecksums {
      * For large buckets, this can be slow, but should not time out as the underlying queries are performed in
      * smaller batches.
      */
-    async computePartialChecksumsDirect(batch) {
+    async computePartialChecksumsDirect(batch, context) {
         // Limit the number of buckets we query for at a time.
         const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT;
         if (batch.length <= bucketBatchLimit) {
             // Single batch - no need for splitting the batch and merging results
-            return await this.computePartialChecksumsInternal(batch);
+            return await this.computePartialChecksumsInternal(batch, context);
         }
         // Split the batch and merge results
         let results = new Map();
         for (let i = 0; i < batch.length; i += bucketBatchLimit) {
             const bucketBatch = batch.slice(i, i + bucketBatchLimit);
-            const batchResults = await this.computePartialChecksumsInternal(bucketBatch);
+            const batchResults = await this.computePartialChecksumsInternal(bucketBatch, context);
             for (let r of batchResults.values()) {
                 results.set(r.bucket, r);
             }
         }
         return results;
     }
-    async computePartialChecksumsForCollection(batch, collection, createFilter) {
-        const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT;
-        // Map requests by bucket. We adjust this as we get partial results.
-        let requests = new Map();
-        for (let request of batch) {
-            requests.set(request.bucket, request);
-        }
-        const partialChecksums = new Map();
-        while (requests.size > 0) {
-            const filters = Array.from(requests.values(), createFilter);
-            // Historically, checksum may be stored as 'int' or 'double'.
-            // More recently, this should be a 'long'.
-            // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-            const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
-            // Aggregate over a max of `batchLimit` operations at a time.
-            // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
-            // Then we'll do three batches:
-            // 1. Query: A[1-end], B[1-end], C[1-end]
-            //    Returns: A[1-10], B[1-2]
-            // 2. Query: B[3-end], C[1-end]
-            //    Returns: B[3-10], C[1-4]
-            // 3. Query: C[5-end]
-            //    Returns: C[5-10]
-            const aggregate = await collection
-                .aggregate([
-                {
-                    $match: {
-                        $or: filters
-                    }
-                },
-                // sort and limit _before_ grouping
-                { $sort: { _id: 1 } },
-                { $limit: batchLimit },
-                {
-                    $group: {
-                        _id: '$_id.b',
-                        checksum_total: { $sum: checksumLong },
-                        count: { $sum: 1 },
-                        has_clear_op: {
-                            $max: {
-                                $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
-                            }
-                        },
-                        last_op: { $max: '$_id.o' }
-                    }
-                },
-                // Sort the aggregated results (100 max, so should be fast).
-                // This is important to identify which buckets we have partial data for.
-                { $sort: { _id: 1 } }
-            ], { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS })
-                .toArray()
-                .catch((e) => {
-                throw lib_mongo.mapQueryError(e, 'while reading checksums');
-            });
-            let batchCount = 0;
-            let limitReached = false;
-            for (let doc of aggregate) {
-                const bucket = doc._id;
-                const checksum = checksumFromAggregate(doc);
-                const existing = partialChecksums.get(bucket);
-                if (existing != null) {
-                    partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum));
-                }
-                else {
-                    partialChecksums.set(bucket, checksum);
-                }
-                batchCount += doc.count;
-                if (batchCount == batchLimit) {
-                    // Limit reached. Request more in the next batch.
-                    // Note that this only affects the _last_ bucket in a batch.
-                    limitReached = true;
-                    const req = requests.get(bucket);
-                    requests.set(bucket, {
-                        ...req,
-                        start: doc.last_op
-                    });
-                }
-                else {
-                    // All done for this bucket
-                    requests.delete(bucket);
-                }
-            }
-            if (!limitReached) {
-                break;
-            }
-        }
-        return new Map(batch.map((request) => {
-            const bucket = request.bucket;
-            // Could be null if we got no data
-            let partialChecksum = partialChecksums.get(bucket);
-            if (partialChecksum == null) {
-                partialChecksum = {
-                    bucket,
-                    partialCount: 0,
-                    partialChecksum: 0
-                };
-            }
-            if (request.start == null && isPartialChecksum(partialChecksum)) {
-                partialChecksum = {
-                    bucket,
-                    count: partialChecksum.partialCount,
-                    checksum: partialChecksum.partialChecksum
-                };
-            }
-            return [bucket, partialChecksum];
-        }));
-    }
 }
 export function emptyChecksumForRequest(request) {
     if (request.start == null) {
@@ -220,7 +135,7 @@ export function emptyChecksumForRequest(request) {
 /**
  * Convert output of the $group stage into a checksum.
  */
-function checksumFromAggregate(doc) {
+export function checksumFromAggregate(doc) {
     const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff;
     const bucket = doc._id;
     if (doc.has_clear_op == 1) {

package/dist/storage/implementation/MongoChecksums.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EAInB,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAoC5D,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C,MAAM,OAAgB,cAAc;IAKb;IACA;IACA;IANb,MAAM,CAA4B;IACzB,aAAa,CAAgB;IAE9C,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAA6B;QAF7B,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAsB;QAEhD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAC;YAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC7C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAgC;QAC3E,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEnD,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAeS,KAAK,CAAC,oCAAoC,CAIlD,KAAiB,EACjB,UAA2D,EAC3D,YAAwC;QAExC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;YAE5D,6DAA6D;YAC7D,0CAA0C;YAC1C,8FAA8F;YAC9F,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAE/F,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,UAAU;iBAC/B,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,GAAG,GAAI;wBACP,KAAK,EAAE,GAAG,CAAC,OAAO;qBACnB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,uBAAuB,CACrC,OAA4F;IAE5F,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}
+{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AACA,OAAO,EACL,mBAAmB,EAInB,aAAa,EAOd,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AA6D7D,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,CAAC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAEpD,MAAM,OAAgB,cAAc;IAKb;IACA;IACA;IANb,MAAM,CAA2D;IACtD,aAAa,CAAgB;IAEhD,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAA6B;QAF7B,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAsB;QAEhD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAgC;YAC/D,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB;YACtC,cAAc,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBACjC,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACtD,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAChB,UAAwB,EACxB,OAAgC,EAChC,OAAsC;QAEtC,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACjE,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CACnC,KAAmC,EACnC,OAAkD;QAElD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,IAAI,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAC9D,OAAO,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACtE,IAAI,YAAY,IAAI,IAAI,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBAChD,sBAAsB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAC9C,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;YAC1C,CAAC;YACD,OAAO,IAAI,CAAC,kCAAkC,CAAC,KAAK,EAAE;gBACpD,WAAW,EAAE;oBACX,mFAAmF;oBACnF,WAAW,EAAE,UAAU;oBACvB,cAAc;oBACd,OAAO;iBACR;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,kCAAkC,CAC9C,KAAmC,EACnC,OAAoC;QAEpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE5D,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAE3F,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,6BAA6B,CACzC,KAAmC,EACnC,OAAoC;QAEpC,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACpE,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACtF,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CAkBF;AAED,MAAM,UAAU,uBAAuB,CACrC,OAAsG;IAEtG,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAkB;IACtD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}

package/dist/storage/implementation/MongoCompactor.d.ts

@@ -2,12 +2,11 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import { InternalOpId, PopulateChecksumCacheResults, storage } from '@powersync/service-core';
 import { BucketDefinitionId } from '@powersync/service-sync-rules';
-import { BucketDataDoc, BucketKey } from './common/BucketDataDoc.js';
-import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js';
+import { BucketKey } from './common/BucketDataDoc.js';
 import type { VersionedPowerSyncMongo } from './db.js';
 import { BucketStateDocumentBase } from './models.js';
 import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
-interface CurrentBucketState {
+export interface CurrentBucketState {
     /** Bucket name */
     bucket: string;
     definitionId: BucketDefinitionId;
@@ -42,6 +41,9 @@ interface CurrentBucketState {
 }
 export interface MongoCompactOptions extends storage.CompactOptions {
 }
+export declare class ConcurrentCompactionError extends Error {
+    constructor(message: string);
+}
 export interface DirtyBucket {
     bucket: string;
     definitionId: BucketDefinitionId | null;
@@ -51,16 +53,17 @@ export interface DirtyBucket {
 export declare abstract class MongoCompactor {
     protected readonly storage: MongoSyncBucketStorage;
     protected readonly db: VersionedPowerSyncMongo;
-    protected updates: mongo.AnyBulkWriteOperation<BucketDataDocumentGeneric>[];
     protected bucketStateUpdates: mongo.AnyBulkWriteOperation<BucketStateDocumentBase>[];
     protected readonly idLimitBytes: number;
     protected readonly moveBatchLimit: number;
     protected readonly moveBatchQueryLimit: number;
+    protected readonly moveBatchByteLimit: number;
     protected readonly clearBatchLimit: number;
     protected readonly minBucketChanges: number;
     protected readonly minChangeRatio: number;
     protected readonly maxOpId: bigint;
     protected readonly buckets: string[] | undefined;
+    protected readonly deleteCheckpointRequestsBefore: Date | undefined;
     protected readonly signal?: AbortSignal;
     protected readonly group_id: number;
     protected readonly logger: Logger;
@@ -68,9 +71,10 @@ export declare abstract class MongoCompactor {
     /**
      * Compact buckets by converting operations into MOVE and/or CLEAR operations.
      *
-     * See /docs/compacting-operations.md for details.
+     * See /docs/storage/compacting-operations.md for details.
      */
     compact(): Promise<void>;
+    private deleteOldCheckpointRequests;
     /**
      * Subset of compact, only populating checksums where relevant.
      */
@@ -93,28 +97,22 @@ export declare abstract class MongoCompactor {
     /**
      * Compaction for a single bucket, with retries on failure.
      *
-     * This covers against occasional network or other database errors during a long compact job.
+     * A compaction can race another compactor after its initial scan. Restarting
+     * the bucket is safe because each replacement is transactional. Object
+     * storage paths are also prepared with lifecycle markers, so a retry can
+     * safely overwrite or eventually clean up uploads from the failed attempt.
      */
     protected compactSingleBucketRetried(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
-    protected compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
-    protected updateBucketChecksums(state: CurrentBucketState): void;
-    protected flush(col: SingleBucketStore): Promise<void>;
-    private flushBucketStateUpdates;
-    /**
-     * Perform a CLEAR compact for a bucket.
-     *
-     * @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
-     */
-    protected clearBucket(currentState: CurrentBucketState, col: SingleBucketStore): Promise<void>;
+    protected abstract compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
+    protected collectBucketStateUpdates(state: CurrentBucketState, compactedOpId: InternalOpId): mongo.AnyBulkWriteOperation<BucketStateDocumentBase>;
+    protected updateBucketChecksums(state: CurrentBucketState, compactedOpId: InternalOpId): void;
+    protected flushBucketStateUpdates(): Promise<void>;
     protected updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<void>;
-    protected formatBucketDataKey(doc: Pick<BucketDataDoc, 'bucketKey' | 'o'>): string;
     protected abstract writeBucketStateUpdates(): Promise<void>;
     protected abstract computeChecksumsForBuckets(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<storage.PartialChecksumMap>;
     protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document;
-    protected abstract getBucketDataContext(bucket: string, definitionId: BucketDefinitionId | null): Promise<SingleBucketStore | null>;
 }
 export interface BucketDataCollectionContext<TBucketData extends mongo.Document> {
     bucketKey: BucketKey;
     collection: mongo.Collection<TBucketData>;
 }
-export {};