@powersync/service-module-mongodb-storage 0.0.0-dev-20250903124544 → 0.0.0-dev-20251015143910

package/src/storage/implementation/MongoChecksums.ts

@@ -247,7 +247,10 @@ export class MongoChecksums {
                 },
                 last_op: { $max: '$_id.o' }
               }
-            }
+            },
+            // Sort the aggregated results (100 max, so should be fast).
+            // This is important to identify which buckets we have partial data for.
+            { $sort: { _id: 1 } }
           ],
           { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS }
         )
@@ -284,7 +287,6 @@ export class MongoChecksums {
           // All done for this bucket
           requests.delete(bucket);
         }
-        batchCount++;
       }
       if (!limitReached) {
         break;

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -42,6 +42,17 @@ export interface MongoSyncBucketStorageOptions {
   checksumOptions?: MongoChecksumOptions;
 }
 
+/**
+ * Only keep checkpoints around for a minute, before fetching a fresh one.
+ *
+ * The reason is that we keep a MongoDB snapshot reference (clusterTime) with the checkpoint,
+ * and they expire after 5 minutes by default. This is an issue if the checkpoint stream is idle,
+ * but new clients connect and use an outdated checkpoint snapshot for parameter queries.
+ *
+ * These will be filtered out for existing clients, so should not create significant overhead.
+ */
+const CHECKPOINT_TIMEOUT_MS = 60_000;
+
 export class MongoSyncBucketStorage
   extends BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
@@ -681,25 +692,45 @@ export class MongoSyncBucketStorage
 
     // We only watch changes to the active sync rules.
     // If it changes to inactive, we abort and restart with the new sync rules.
-    let lastOp: storage.ReplicationCheckpoint | null = null;
+    try {
+      while (true) {
+        // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS)
+        // before we get another checkpoint, to avoid stale checkpoint snapshots.
+        const timeout = timers
+          .setTimeout(CHECKPOINT_TIMEOUT_MS, { done: false }, { signal })
+          .catch(() => ({ done: true }));
+        try {
+          const result = await Promise.race([stream.next(), timeout]);
+          if (result.done) {
+            break;
+          }
+        } catch (e) {
+          if (e.name == 'AbortError') {
+            break;
+          }
+          throw e;
+        }
 
-    for await (const _ of stream) {
-      if (signal.aborted) {
-        break;
-      }
+        if (signal.aborted) {
+          // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway
+          break;
+        }
 
-      const op = await this.getCheckpointInternal();
-      if (op == null) {
-        // Sync rules have changed - abort and restart.
-        // We do a soft close of the stream here - no error
-        break;
-      }
+        const op = await this.getCheckpointInternal();
+        if (op == null) {
+          // Sync rules have changed - abort and restart.
+          // We do a soft close of the stream here - no error
+          break;
+        }
 
-      // Check for LSN / checkpoint changes - ignore other metadata changes
-      if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
-        lastOp = op;
+        // Previously, we only yielded when the checkpoint or lsn changed.
+        // However, we always want to use the latest snapshotTime, so we skip that filtering here.
+        // That filtering could be added in the per-user streams if needed, but in general the capped collection
+        // should already only contain useful changes in most cases.
         yield op;
       }
+    } finally {
+      await stream.return(null);
     }
   }
 
@@ -715,7 +746,10 @@ export class MongoSyncBucketStorage
     let lastCheckpoint: ReplicationCheckpoint | null = null;
 
     const iter = this.sharedIter[Symbol.asyncIterator](options.signal);
+
     let writeCheckpoint: bigint | null = null;
+    // true if we queried the initial write checkpoint, even if it doesn't exist
+    let queriedInitialWriteCheckpoint = false;
 
     for await (const nextCheckpoint of iter) {
       // lsn changes are not important by itself.
@@ -723,14 +757,17 @@ export class MongoSyncBucketStorage
       // 1. checkpoint (op_id) changes.
       // 2. write checkpoint changes for the specific user
 
-      if (nextCheckpoint.lsn != null) {
-        writeCheckpoint ??= await this.writeCheckpointAPI.lastWriteCheckpoint({
+      if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) {
+        // Lookup the first write checkpoint for the user when we can.
+        // There will not actually be one in all cases.
+        writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
           sync_rules_id: this.group_id,
           user_id: options.user_id,
           heads: {
             '1': nextCheckpoint.lsn
           }
         });
+        queriedInitialWriteCheckpoint = true;
       }
 
       if (
@@ -740,12 +777,13 @@ export class MongoSyncBucketStorage
       ) {
         // No change - wait for next one
         // In some cases, many LSNs may be produced in a short time.
-        // Add a delay to throttle the write checkpoint lookup a bit.
+        // Add a delay to throttle the loop a bit.
         await timers.setTimeout(20 + 10 * Math.random());
         continue;
       }
 
       if (lastCheckpoint == null) {
+        // First message for this stream - "INVALIDATE_ALL" means it will lookup all data
         yield {
           base: nextCheckpoint,
           writeCheckpoint,
@@ -759,7 +797,9 @@ export class MongoSyncBucketStorage
 
         let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null;
         if (updates.invalidateWriteCheckpoints) {
-          updatedWriteCheckpoint ??= await this.writeCheckpointAPI.lastWriteCheckpoint({
+          // Invalidated means there were too many updates to track the individual ones,
+          // so we switch to "polling" (querying directly in each stream).
+          updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
             sync_rules_id: this.group_id,
             user_id: options.user_id,
             heads: {
@@ -769,6 +809,9 @@ export class MongoSyncBucketStorage
         }
         if (updatedWriteCheckpoint != null && (writeCheckpoint == null || updatedWriteCheckpoint > writeCheckpoint)) {
           writeCheckpoint = updatedWriteCheckpoint;
+          // If it happened that we haven't queried a write checkpoint at this point,
+          // then we don't need to anymore, since we got an updated one.
+          queriedInitialWriteCheckpoint = true;
         }
 
         yield {

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -12,12 +12,10 @@ export type MongoCheckpointAPIOptions = {
 export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
   readonly db: PowerSyncMongo;
   private _mode: storage.WriteCheckpointMode;
-  private sync_rules_id: number;
 
   constructor(options: MongoCheckpointAPIOptions) {
     this.db = options.db;
     this._mode = options.mode;
-    this.sync_rules_id = options.sync_rules_id;
   }
 
   get writeCheckpointMode() {