@powersync/service-module-mongodb-storage 0.18.3 → 0.20.0

package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts

@@ -16,7 +16,15 @@ import {
 import { JSONBig } from '@powersync/service-jsonbig';
 import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
+import {
+  clearCollectionInIdBatches,
+  clearCollectionInIdRanges,
+  idPrefixFilter,
+  mapOpEntry,
+  readSingleBatch,
+  retryOnMongoMaxTimeMSExpired,
+  setSessionSnapshotTime
+} from '../../../utils/util.js';
 import { MongoBucketStorage } from '../../MongoBucketStorage.js';
 import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
 import { SourceKey } from '../models.js';
@@ -214,64 +222,59 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
   }
 
   protected async clearBucketData(signal?: AbortSignal): Promise<void> {
-    await this.clearDeleteMany(
+    await clearCollectionInIdRanges(
+      this.logger,
       'bucket data',
-      () =>
-        this.db.bucket_data.deleteMany(
-          {
-            _id: idPrefixFilter<BucketDataKeyV1>({ g: this.replicationStreamId }, ['b', 'o'])
-          },
-          { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
-        ),
-      signal
+      this.db.bucket_data,
+      {
+        _id: idPrefixFilter<BucketDataKeyV1>({ g: this.replicationStreamId }, ['b', 'o'])
+      },
+      signal,
+      this.clearBatchThrottleRate
     );
   }
 
   protected async clearParameterIndexes(signal?: AbortSignal): Promise<void> {
-    await this.clearDeleteMany(
+    await clearCollectionInIdBatches(
+      this.logger,
       'parameter index',
-      () =>
-        this.db.parameterIndexV1.deleteMany(
-          {
-            'key.g': this.replicationStreamId
-          },
-          { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
-        ),
-      signal
+      this.db.parameterIndexV1,
+      {
+        'key.g': this.replicationStreamId
+      },
+      signal,
+      this.clearBatchThrottleRate
     );
   }
 
   protected async clearSourceRecords(signal?: AbortSignal): Promise<void> {
-    await this.clearDeleteMany(
+    await clearCollectionInIdRanges(
+      this.logger,
       'source records',
-      () =>
-        this.db.sourceRecordsV1.deleteMany(
-          {
-            _id: idPrefixFilter<SourceKey>({ g: this.replicationStreamId }, ['t', 'k'])
-          },
-          { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
-        ),
-      signal
+      this.db.sourceRecordsV1,
+      {
+        _id: idPrefixFilter<SourceKey>({ g: this.replicationStreamId }, ['t', 'k'])
+      },
+      signal,
+      this.clearBatchThrottleRate
     );
   }
 
   protected async clearBucketState(signal?: AbortSignal): Promise<void> {
-    await this.clearDeleteMany(
+    await clearCollectionInIdRanges(
+      this.logger,
       'bucket state',
-      () =>
-        this.db.bucketStateV1.deleteMany(
-          {
-            _id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.replicationStreamId }, ['b'])
-          },
-          { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
-        ),
-      signal
+      this.db.bucketStateV1,
+      {
+        _id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.replicationStreamId }, ['b'])
+      },
+      signal,
+      this.clearBatchThrottleRate
     );
   }
 
   protected async clearSourceTables(signal?: AbortSignal): Promise<void> {
-    await this.clearDeleteMany(
-      'source tables',
+    await retryOnMongoMaxTimeMSExpired(
       () =>
         this.db.sourceTablesV1(this.replicationStreamId).deleteMany(
           {
@@ -279,7 +282,17 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
           },
           { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
         ),
-      signal
+      {
+        signal,
+        abortMessage: 'Aborted clearing data',
+        // This is a fairly long delay - only expected to hit this when the storage database is under high load.
+        retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+        onRetry: () => {
+          this.logger.info(
+            `Clearing batch of source tables timed out after ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, retrying...`
+          );
+        }
+      }
     );
   }
 

package/src/storage/implementation/v1/SingleBucketStoreV1.ts

@@ -1,58 +1,50 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { InternalOpId } from '@powersync/service-core';
 import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
-import {
-  BucketDataDocumentGeneric,
-  BucketDataDocumentGenericId,
-  SingleBucketStore
-} from '../common/SingleBucketStore.js';
 import { BucketDataProperties } from '../models.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js';
 
-export class SingleBucketStoreV1 implements SingleBucketStore {
-  public readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
+export class SingleBucketStoreV1 {
+  public readonly collection: mongo.Collection<BucketDataDocumentV1>;
 
   constructor(
-    private db: VersionedPowerSyncMongoV1,
+    db: VersionedPowerSyncMongoV1,
     public readonly key: BucketKey
   ) {
-    this.collection = db.bucketDataV1 as unknown as mongo.Collection<BucketDataDocumentGeneric>;
+    this.collection = db.bucketDataV1;
   }
 
-  docId(o: InternalOpId): BucketDataDocumentGenericId {
-    // `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage
-    // `as anyt` is to allow casting to the interface virtual type
+  docId(o: InternalOpId): BucketDataKeyV1 {
     return {
       g: this.key.replicationStreamId,
       b: this.key.bucket,
       o
-    } satisfies BucketDataKeyV1 as any;
+    };
   }
 
-  get minId(): BucketDataDocumentGenericId {
+  get minId(): BucketDataKeyV1 {
     return {
       g: this.key.replicationStreamId,
       b: this.key.bucket,
-      o: new mongo.MinKey()
-    } as any;
+      o: new mongo.MinKey() as any
+    };
   }
 
-  get maxId(): BucketDataDocumentGenericId {
+  get maxId(): BucketDataKeyV1 {
     return {
       g: this.key.replicationStreamId,
       b: this.key.bucket,
-      o: new mongo.MaxKey()
-    } as any;
+      o: new mongo.MaxKey() as any
+    };
   }
 
-  toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric {
-    return serializeBucketDataV1({ bucketKey: this.key, ...source }) as unknown as BucketDataDocumentGeneric;
+  toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentV1 {
+    return serializeBucketDataV1({ bucketKey: this.key, ...source });
   }
 
-  fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc {
-    const document = doc as unknown as BucketDataDocumentV1;
-    const { _id, ...rest } = document;
+  fromPersistedDocument(doc: BucketDataDocumentV1): BucketDataDoc {
+    const { _id, ...rest } = doc;
     return {
       bucketKey: this.key,
       o: _id.o,
@@ -61,10 +53,9 @@ export class SingleBucketStoreV1 implements SingleBucketStore {
   }
 
   fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
-    doc: Pick<BucketDataDocumentGeneric, '_id' | T>
+    doc: Pick<BucketDataDocumentV1, '_id' | T>
   ): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
-    const document = doc as Pick<BucketDataDocumentV1, '_id' | T>;
-    const { _id, ...rest } = document;
+    const { _id, ...rest } = doc;
     return {
       bucketKey: this.key,
       o: _id.o,

package/src/storage/implementation/v3/BucketDataContextV3.ts

@@ -0,0 +1,32 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { InternalOpId } from '@powersync/service-core';
+import { BucketKey } from '../common/BucketDataDoc.js';
+import { BucketDataKey } from '../models.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+import { BucketDataDocumentV3 } from './models.js';
+
+export class BucketDataContextV3 {
+  public readonly collection: mongo.Collection<BucketDataDocumentV3>;
+
+  constructor(
+    db: VersionedPowerSyncMongoV3,
+    public readonly key: BucketKey
+  ) {
+    this.collection = db.bucketData(key.replicationStreamId, key.definitionId);
+  }
+
+  docId(o: InternalOpId): BucketDataKey {
+    return {
+      b: this.key.bucket,
+      o
+    };
+  }
+
+  get minId(): BucketDataKey {
+    // MongoDB's MinKey sentinel does not match the bigint type for _id.o.
+    return {
+      b: this.key.bucket,
+      o: new mongo.MinKey()
+    } as unknown as BucketDataKey;
+  }
+}

package/src/storage/implementation/v3/MongoBucketBatchV3.ts

@@ -43,7 +43,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
 
   protected createPersistedBatch(writtenSize: number): PersistedBatch {
     return new PersistedBatchV3(this.db, this.replicationStreamId, this.mapping, writtenSize, {
-      logger: this.logger
+      logger: this.logger,
+      objectStorage: this.options.objectStorage,
+      inlineThresholdBytes: this.options.inlineThresholdBytes
     });
   }
 
@@ -146,23 +148,16 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
     const mapping = parsedOverride?.mapping ?? this.mapping;
 
     const { connection_id, source } = options;
-    const context: SourceTableReconciliationContext = {
-      connectionId: connection_id,
-      connectionTag: source.connectionTag,
-      identity: {
-        schema: source.schema,
-        name: source.name,
-        objectId: source.objectId,
-        replicaIdColumns: source.replicaIdColumns.map((column) => ({
-          name: column.name,
-          type: column.type,
-          type_oid: column.typeId
-        }))
-      },
-      storeCurrentData: source.sendsCompleteRows !== true,
-      syncConfig,
-      mapping,
-      desired: sourceTableDesiredResolution(syncConfig, source, mapping)
+    const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
+    const identity = {
+      schema: source.schema,
+      name: source.name,
+      objectId: source.objectId,
+      replicaIdColumns: source.replicaIdColumns.map((column) => ({
+        name: column.name,
+        type: column.type,
+        type_oid: column.typeId
+      }))
     };
 
     let result: storage.ResolveTablesResult | null = null;
@@ -172,14 +167,36 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
     await session.withTransaction(async () => {
       const col = this.db.sourceTables(this.replicationStreamId);
 
-      // Fetch every persisted source-table doc that can overlap this physical table.
-      // Exact-identity docs are candidates for reuse; non-exact overlaps are possible drops.
+      // Find records that overlap by name or relation id.
       const candidateDocs = await col
-        .find(overlappingSourceTableFilter(connection_id, context.identity), { session })
+        .find(overlappingSourceTableFilter(connection_id, identity), { session })
         .toArray();
 
-      // Pure planning: which docs to retain (and narrow), whether a new doc is needed for
-      // uncovered memberships, and which docs conflict with the current identity.
+      const candidateTables = candidateDocs.map((doc) =>
+        sourceTableFromDocument(doc, source.connectionTag, syncConfig, mapping)
+      );
+      const candidates = candidateTables.map((table) => table.clone());
+      const resolution = await reconcile({ source, candidates });
+      storage.validateSourceTableCandidateResolution(candidates, resolution);
+
+      // Persist metadata from the reconciler without mutating the queried documents.
+      for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
+        await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
+      }
+
+      const context: SourceTableReconciliationContext = {
+        connectionId: connection_id,
+        connectionTag: source.connectionTag,
+        identity,
+        storeCurrentData: source.sendsCompleteRows !== true,
+        syncConfig,
+        mapping,
+        desired: sourceTableDesiredResolution(syncConfig, source, mapping),
+        sourceCompatibleTables: resolution.compatibleTables,
+        newTableSourceMetadata: resolution.newTableValues.sourceMetadata
+      };
+
+      // Plan record reuse, membership changes, creation, and removal.
       const plan = planSourceTableReconciliation(candidateDocs, context);
 
       // Persist narrowing for incomplete snapshots only. Snapshot-complete docs keep stale

package/src/storage/implementation/v3/MongoChecksumsV3.ts

@@ -1,7 +1,9 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   bson,
   BucketChecksum,
+  CheckpointChecksumInvalidatedError,
   FetchPartialBucketChecksum,
   InternalOpId,
   isPartialChecksum,
@@ -179,9 +181,42 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
       requests.set(request.bucket, request);
     }
 
-    const pipeline = this.buildPartialChecksumPipeline(requests);
-    const aggregate = await collection
-      .aggregate(pipeline, {
+    const aggregate = await this.aggregatePartialChecksums(collection, requests, context);
+    const { checksums, startStraddledBuckets } = this.normalizePartialChecksumResults(batch, aggregate);
+
+    if (startStraddledBuckets.size == 0) {
+      return checksums;
+    }
+
+    // A cached base ends inside a compaction-produced document. Its checksum cannot
+    // be safely combined with the document-level checksum, so recalculate this
+    // bucket from the beginning. A full result replaces the cached base upstream.
+    const fullRequests = batch
+      .filter((request) => startStraddledBuckets.has(request.bucket))
+      .map((request) => ({ ...request, start: undefined }));
+    const fullRequestMap = new Map(fullRequests.map((request) => [request.bucket, request]));
+    const fullAggregate = await this.aggregatePartialChecksums(collection, fullRequestMap, context);
+    const fullResults = this.normalizePartialChecksumResults(fullRequests, fullAggregate);
+
+    if (fullResults.startStraddledBuckets.size > 0) {
+      throw new ServiceAssertionError(
+        `Unexpected start-boundary straddle while recalculating bucket(s) ${[...fullResults.startStraddledBuckets].join(', ')}`
+      );
+    }
+    for (const [bucket, checksum] of fullResults.checksums) {
+      checksums.set(bucket, checksum);
+    }
+
+    return checksums;
+  }
+
+  private async aggregatePartialChecksums(
+    collection: lib_mongo.mongo.Collection<BucketDataDocumentV3>,
+    requests: Map<string, FetchPartialBucketChecksumByBucket>,
+    context: MongoChecksumSessionContext
+  ): Promise<bson.Document[]> {
+    return collection
+      .aggregate(this.buildPartialChecksumPipeline(requests), {
         ...context.readOptions,
         maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS
       })
@@ -189,8 +224,6 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
       .catch((e) => {
         throw lib_mongo.mapQueryError(e, 'while reading checksums');
       });
-
-    return this.normalizePartialChecksumResults(batch, aggregate);
   }
 
   private buildPartialChecksumPipeline(requests: Map<string, FetchPartialBucketChecksumByBucket>): bson.Document[] {
@@ -203,128 +236,64 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
           $or: filters
         }
       },
-      {
-        $addFields: {
-          bucket_start: {
-            $switch: {
-              branches: Array.from(requests.entries()).map(([bucket, req]) => ({
-                case: { $eq: ['$_id.b', bucket] },
-                then: req.start ?? new bson.MinKey()
-              })),
-              default: new bson.MinKey()
-            }
-          },
-          bucket_end: {
-            $switch: {
-              branches: Array.from(requests.entries()).map(([bucket, req]) => ({
-                case: { $eq: ['$_id.b', bucket] },
-                then: req.end
-              })),
-              default: new bson.MaxKey()
-            }
-          }
-        }
-      },
-      // Determine if document is fully included within [start, end] range
+      // Only document-level metadata is used below. Bucket-data operations may
+      // be offloaded, and checksum queries must never inspect the ops array.
       {
         $project: {
           _id: 1,
           min_op: 1,
           checksum: 1,
           count: 1,
-          ops: 1,
-          bucket_start: 1,
-          bucket_end: 1,
-          is_fully_included: {
-            $and: [{ $gt: ['$min_op', '$bucket_start'] }, { $lte: ['$_id.o', '$bucket_end'] }]
-          }
-        }
-      },
-      // Compute included checksum, count, and clear op detection
-      {
-        $project: {
-          _id: 1,
-          checksum_total: {
-            $cond: {
-              if: '$is_fully_included',
-              then: '$checksum',
-              else: {
-                $sum: {
-                  $map: {
-                    input: {
-                      $filter: {
-                        input: '$ops',
-                        cond: {
-                          $and: [{ $gt: ['$$this.o', '$bucket_start'] }, { $lte: ['$$this.o', '$bucket_end'] }]
-                        }
-                      }
-                    },
-                    in: '$$this.checksum'
-                  }
-                }
-              }
-            }
-          },
-          count_total: {
-            $cond: {
-              if: '$is_fully_included',
-              then: '$count',
-              else: {
-                $size: {
-                  $filter: {
-                    input: '$ops',
-                    cond: {
-                      $and: [{ $gt: ['$$this.o', '$bucket_start'] }, { $lte: ['$$this.o', '$bucket_end'] }]
-                    }
-                  }
-                }
-              }
-            }
-          },
-          has_clear_op: {
-            $max: {
-              $map: {
-                input: {
-                  $filter: {
-                    input: '$ops',
-                    cond: {
-                      $and: [{ $gt: ['$$this.o', '$bucket_start'] }, { $lte: ['$$this.o', '$bucket_end'] }]
-                    }
-                  }
-                },
-                in: { $cond: [{ $eq: ['$$this.op', 'CLEAR'] }, 1, 0] }
-              }
-            }
-          },
-          last_op: { $max: '$_id.o' }
+          has_clear_op: 1
         }
       },
-      // Group by bucket
+      // Aggregate document metadata per bucket. A CLEAR makes the result a full
+      // checksum that replaces any cached prefix; storage guarantees that
+      // documents preceding the CLEAR have already been removed.
       {
         $group: {
           _id: '$_id.b',
-          checksum_total: { $sum: '$checksum_total' },
-          count: { $sum: '$count_total' },
-          has_clear_op: { $max: '$has_clear_op' },
-          last_op: { $max: '$last_op' }
+          checksum_total: { $sum: '$checksum' },
+          count: { $sum: '$count' },
+          has_clear_op: {
+            $max: { $cond: ['$has_clear_op', 1, 0] }
+          },
+          min_op: { $min: '$min_op' },
+          max_op: { $max: '$_id.o' }
         }
-      },
-      // $sort results
-      { $sort: { _id: 1 } }
+      }
     ];
   }
 
   private normalizePartialChecksumResults(
     batch: FetchPartialBucketChecksumByBucket[],
     aggregate: bson.Document[]
-  ): PartialChecksumMap {
+  ): { checksums: PartialChecksumMap; startStraddledBuckets: Set<string> } {
+    const requests = new Map(batch.map((request) => [request.bucket, request]));
+    const startStraddledBuckets = new Set<string>();
     const partialChecksums = new Map<string, PartialOrFullChecksum>();
-    for (let doc of aggregate) {
-      const bucket = doc._id;
+
+    for (const doc of aggregate) {
+      const bucket = doc._id as string;
+      const request = requests.get(bucket)!;
+
+      // A maximum beyond end means a matched document straddles the checkpoint.
+      // Its document-level checksum includes later operations, so this checkpoint
+      // cannot be calculated from document metadata.
+      if (doc.max_op > request.end) {
+        throw new CheckpointChecksumInvalidatedError(request.end, bucket);
+      }
+      // The _id filter excludes documents ending at or before start. A minimum
+      // at or below start therefore means a matched document straddles it.
+      if (request.start != null && doc.min_op <= request.start) {
+        startStraddledBuckets.add(bucket);
+        continue;
+      }
+
       partialChecksums.set(bucket, checksumFromAggregate(doc));
     }
 
-    return new Map<string, PartialOrFullChecksum>(
+    const checksums = new Map<string, PartialOrFullChecksum>(
       batch.map((request) => {
         const bucket = request.bucket;
         let partialChecksum = partialChecksums.get(bucket);
@@ -346,6 +315,7 @@ export class MongoChecksumsV3 extends MongoChecksums implements DefinitionChecks
         return [bucket, partialChecksum];
       })
     );
+    return { checksums, startStraddledBuckets };
   }
 }