@powersync/service-module-mongodb-storage 0.18.2 → 0.19.0

package/src/storage/implementation/v3/MongoChecksumsV3.ts

@@ -1,7 +1,9 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   bson,
   BucketChecksum,
+  CheckpointChecksumInvalidatedError,
   FetchPartialBucketChecksum,
   InternalOpId,
   isPartialChecksum,
@@ -179,9 +181,42 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
       requests.set(request.bucket, request);
     }
 
-    const pipeline = this.buildPartialChecksumPipeline(requests);
-    const aggregate = await collection
-      .aggregate(pipeline, {
+    const aggregate = await this.aggregatePartialChecksums(collection, requests, context);
+    const { checksums, startStraddledBuckets } = this.normalizePartialChecksumResults(batch, aggregate);
+
+    if (startStraddledBuckets.size == 0) {
+      return checksums;
+    }
+
+    // A cached base ends inside a compaction-produced document. Its checksum cannot
+    // be safely combined with the document-level checksum, so recalculate this
+    // bucket from the beginning. A full result replaces the cached base upstream.
+    const fullRequests = batch
+      .filter((request) => startStraddledBuckets.has(request.bucket))
+      .map((request) => ({ ...request, start: undefined }));
+    const fullRequestMap = new Map(fullRequests.map((request) => [request.bucket, request]));
+    const fullAggregate = await this.aggregatePartialChecksums(collection, fullRequestMap, context);
+    const fullResults = this.normalizePartialChecksumResults(fullRequests, fullAggregate);
+
+    if (fullResults.startStraddledBuckets.size > 0) {
+      throw new ServiceAssertionError(
+        `Unexpected start-boundary straddle while recalculating bucket(s) ${[...fullResults.startStraddledBuckets].join(', ')}`
+      );
+    }
+    for (const [bucket, checksum] of fullResults.checksums) {
+      checksums.set(bucket, checksum);
+    }
+
+    return checksums;
+  }
+
+  private async aggregatePartialChecksums(
+    collection: lib_mongo.mongo.Collection<BucketDataDocumentV3>,
+    requests: Map<string, FetchPartialBucketChecksumByBucket>,
+    context: MongoChecksumSessionContext
+  ): Promise<bson.Document[]> {
+    return collection
+      .aggregate(this.buildPartialChecksumPipeline(requests), {
         ...context.readOptions,
         maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS
       })
@@ -189,8 +224,6 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
       .catch((e) => {
         throw lib_mongo.mapQueryError(e, 'while reading checksums');
       });
-
-    return this.normalizePartialChecksumResults(batch, aggregate);
   }
 
   private buildPartialChecksumPipeline(requests: Map<string, FetchPartialBucketChecksumByBucket>): bson.Document[] {
@@ -203,128 +236,64 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
           $or: filters
         }
       },
-      {
-        $addFields: {
-          bucket_start: {
-            $switch: {
-              branches: Array.from(requests.entries()).map(([bucket, req]) => ({
-                case: { $eq: ['$_id.b', bucket] },
-                then: req.start ?? new bson.MinKey()
-              })),
-              default: new bson.MinKey()
-            }
-          },
-          bucket_end: {
-            $switch: {
-              branches: Array.from(requests.entries()).map(([bucket, req]) => ({
-                case: { $eq: ['$_id.b', bucket] },
-                then: req.end
-              })),
-              default: new bson.MaxKey()
-            }
-          }
-        }
-      },
-      // Determine if document is fully included within [start, end] range
+      // Only document-level metadata is used below. Bucket-data operations may
+      // be offloaded, and checksum queries must never inspect the ops array.
       {
         $project: {
           _id: 1,
           min_op: 1,
           checksum: 1,
           count: 1,
-          ops: 1,
-          bucket_start: 1,
-          bucket_end: 1,
-          is_fully_included: {
-            $and: [{ $gt: ['$min_op', '$bucket_start'] }, { $lte: ['$_id.o', '$bucket_end'] }]
-          }
-        }
-      },
-      // Compute included checksum, count, and clear op detection
-      {
-        $project: {
-          _id: 1,
-          checksum_total: {
-            $cond: {
-              if: '$is_fully_included',
-              then: '$checksum',
-              else: {
-                $sum: {
-                  $map: {
-                    input: {
-                      $filter: {
-                        input: '$ops',
-                        cond: {
-                          $and: [{ $gt: ['$$this.o', '$bucket_start'] }, { $lte: ['$$this.o', '$bucket_end'] }]
-                        }
-                      }
-                    },
-                    in: '$$this.checksum'
-                  }
-                }
-              }
-            }
-          },
-          count_total: {
-            $cond: {
-              if: '$is_fully_included',
-              then: '$count',
-              else: {
-                $size: {
-                  $filter: {
-                    input: '$ops',
-                    cond: {
-                      $and: [{ $gt: ['$$this.o', '$bucket_start'] }, { $lte: ['$$this.o', '$bucket_end'] }]
-                    }
-                  }
-                }
-              }
-            }
-          },
-          has_clear_op: {
-            $max: {
-              $map: {
-                input: {
-                  $filter: {
-                    input: '$ops',
-                    cond: {
-                      $and: [{ $gt: ['$$this.o', '$bucket_start'] }, { $lte: ['$$this.o', '$bucket_end'] }]
-                    }
-                  }
-                },
-                in: { $cond: [{ $eq: ['$$this.op', 'CLEAR'] }, 1, 0] }
-              }
-            }
-          },
-          last_op: { $max: '$_id.o' }
+          has_clear_op: 1
         }
       },
-      // Group by bucket
+      // Aggregate document metadata per bucket. A CLEAR makes the result a full
+      // checksum that replaces any cached prefix; storage guarantees that
+      // documents preceding the CLEAR have already been removed.
       {
         $group: {
           _id: '$_id.b',
-          checksum_total: { $sum: '$checksum_total' },
-          count: { $sum: '$count_total' },
-          has_clear_op: { $max: '$has_clear_op' },
-          last_op: { $max: '$last_op' }
+          checksum_total: { $sum: '$checksum' },
+          count: { $sum: '$count' },
+          has_clear_op: {
+            $max: { $cond: ['$has_clear_op', 1, 0] }
+          },
+          min_op: { $min: '$min_op' },
+          max_op: { $max: '$_id.o' }
         }
-      },
-      // $sort results
-      { $sort: { _id: 1 } }
+      }
     ];
   }
 
   private normalizePartialChecksumResults(
     batch: FetchPartialBucketChecksumByBucket[],
     aggregate: bson.Document[]
-  ): PartialChecksumMap {
+  ): { checksums: PartialChecksumMap; startStraddledBuckets: Set<string> } {
+    const requests = new Map(batch.map((request) => [request.bucket, request]));
+    const startStraddledBuckets = new Set<string>();
     const partialChecksums = new Map<string, PartialOrFullChecksum>();
-    for (let doc of aggregate) {
-      const bucket = doc._id;
+
+    for (const doc of aggregate) {
+      const bucket = doc._id as string;
+      const request = requests.get(bucket)!;
+
+      // A maximum beyond end means a matched document straddles the checkpoint.
+      // Its document-level checksum includes later operations, so this checkpoint
+      // cannot be calculated from document metadata.
+      if (doc.max_op > request.end) {
+        throw new CheckpointChecksumInvalidatedError(request.end, bucket);
+      }
+      // The _id filter excludes documents ending at or before start. A minimum
+      // at or below start therefore means a matched document straddles it.
+      if (request.start != null && doc.min_op <= request.start) {
+        startStraddledBuckets.add(bucket);
+        continue;
+      }
+
       partialChecksums.set(bucket, checksumFromAggregate(doc));
     }
 
-    return new Map<string, PartialOrFullChecksum>(
+    const checksums = new Map<string, PartialOrFullChecksum>(
       batch.map((request) => {
         const bucket = request.bucket;
         let partialChecksum = partialChecksums.get(bucket);
@@ -346,6 +315,7 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
         return [bucket, partialChecksum];
       })
     );
+    return { checksums, startStraddledBuckets };
   }
 }