@powersync/service-module-mongodb-storage 0.0.0-dev-20251111070830 → 0.0.0-dev-20251120150014

package/src/storage/implementation/MongoCompactor.ts

@@ -61,6 +61,7 @@ export interface MongoCompactOptions extends storage.CompactOptions {}
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
+const DEFAULT_MIN_BUCKET_CHANGES = 10;
 
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
@@ -73,6 +74,7 @@ export class MongoCompactor {
   private moveBatchLimit: number;
   private moveBatchQueryLimit: number;
   private clearBatchLimit: number;
+  private minBucketChanges: number;
   private maxOpId: bigint;
   private buckets: string[] | undefined;
   private signal?: AbortSignal;
@@ -88,6 +90,7 @@ export class MongoCompactor {
     this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
     this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
     this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+    this.minBucketChanges = options?.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
     this.maxOpId = options?.maxOpId ?? 0n;
     this.buckets = options?.compactBuckets;
     this.signal = options?.signal;
@@ -113,14 +116,26 @@ export class MongoCompactor {
 
   private async compactDirtyBuckets() {
     while (!this.signal?.aborted) {
-      // Process all buckets with 1 or more changes since last time
-      const buckets = await this.dirtyBucketBatch({ minBucketChanges: 1 });
+      // Process all buckets with 10 or more changes since last time.
+      // We exclude the last 100 compacted buckets, to avoid repeatedly re-compacting the same buckets over and over
+      // if they are modified while compacting.
+      const TRACK_RECENTLY_COMPACTED_NUMBER = 100;
+
+      let recentlyCompacted: string[] = [];
+      const buckets = await this.dirtyBucketBatch({
+        minBucketChanges: this.minBucketChanges,
+        exclude: recentlyCompacted
+      });
       if (buckets.length == 0) {
         // All done
         break;
       }
-      for (let bucket of buckets) {
+      for (let { bucket } of buckets) {
         await this.compactSingleBucket(bucket);
+        recentlyCompacted.push(bucket);
+      }
+      if (recentlyCompacted.length > TRACK_RECENTLY_COMPACTED_NUMBER) {
+        recentlyCompacted = recentlyCompacted.slice(-TRACK_RECENTLY_COMPACTED_NUMBER);
       }
     }
   }
@@ -482,10 +497,20 @@ export class MongoCompactor {
         break;
       }
       const start = Date.now();
-      logger.info(`Calculating checksums for batch of ${buckets.length} buckets, starting at ${buckets[0]}`);
+      logger.info(`Calculating checksums for batch of ${buckets.length} buckets`);
 
-      await this.updateChecksumsBatch(buckets);
-      logger.info(`Updated checksums for batch of ${buckets.length} buckets in ${Date.now() - start}ms`);
+      // Filter batch by estimated bucket size, to reduce possibility of timeouts
+      let checkBuckets: typeof buckets = [];
+      let totalCountEstimate = 0;
+      for (let bucket of buckets) {
+        checkBuckets.push(bucket);
+        totalCountEstimate += bucket.estimatedCount;
+        if (totalCountEstimate > 50_000) {
+          break;
+        }
+      }
+      await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
+      logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
       count += buckets.length;
     }
     return { buckets: count };
@@ -497,7 +522,10 @@ export class MongoCompactor {
    * This cannot be used to iterate on its own - the client is expected to process these buckets and
    * set estimate_since_compact.count: 0 when done, before fetching the next batch.
    */
-  private async dirtyBucketBatch(options: { minBucketChanges: number }): Promise<string[]> {
+  private async dirtyBucketBatch(options: {
+    minBucketChanges: number;
+    exclude?: string[];
+  }): Promise<{ bucket: string; estimatedCount: number }[]> {
     if (options.minBucketChanges <= 0) {
       throw new ReplicationAssertionError('minBucketChanges must be >= 1');
     }
@@ -506,22 +534,28 @@ export class MongoCompactor {
       .find(
         {
           '_id.g': this.group_id,
-          'estimate_since_compact.count': { $gte: options.minBucketChanges }
+          'estimate_since_compact.count': { $gte: options.minBucketChanges },
+          '_id.b': { $nin: options.exclude ?? [] }
         },
         {
           projection: {
-            _id: 1
+            _id: 1,
+            estimate_since_compact: 1,
+            compacted_state: 1
           },
           sort: {
             'estimate_since_compact.count': -1
           },
-          limit: 5_000,
+          limit: 200,
           maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
         }
       )
       .toArray();
 
-    return dirtyBuckets.map((bucket) => bucket._id.b);
+    return dirtyBuckets.map((bucket) => ({
+      bucket: bucket._id.b,
+      estimatedCount: bucket.estimate_since_compact!.count + (bucket.compacted_state?.count ?? 0)
+    }));
   }
 
   private async updateChecksumsBatch(buckets: string[]) {

package/src/storage/implementation/MongoSyncRulesLock.ts

@@ -33,10 +33,19 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
     );
 
     if (doc == null) {
-      throw new ServiceError(
-        ErrorCode.PSYNC_S1003,
-        `Sync rules: ${sync_rules.id} have been locked by another process for replication.`
-      );
+      // Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
+      const heldLock = await db.sync_rules.findOne({ _id: sync_rules.id }, { projection: { lock: 1 } });
+      if (heldLock?.lock?.expires_at) {
+        throw new ServiceError(
+          ErrorCode.PSYNC_S1003,
+          `Sync rules: ${sync_rules.id} have been locked by another process for replication, expiring at ${heldLock.lock.expires_at.toISOString()}.`
+        );
+      } else {
+        throw new ServiceError(
+          ErrorCode.PSYNC_S1003,
+          `Sync rules: ${sync_rules.id} have been locked by another process for replication.`
+        );
+      }
     }
     return new MongoSyncRulesLock(db, sync_rules.id, lockId);
   }

package/src/storage/implementation/models.ts

@@ -197,6 +197,11 @@ export interface SyncRuleDocument {
   last_fatal_error: string | null;
 
   content: string;
+
+  lock?: {
+    id: string;
+    expires_at: Date;
+  } | null;
 }
 
 export interface CheckpointEventDocument {

package/src/utils/util.ts

@@ -125,24 +125,15 @@ export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
     if (!cursor) {
       return query;
     }
+    const connected_at = query.connected_at
+      ? { $lt: new Date(cursor), $gte: query.connected_at.$gte }
+      : { $lt: new Date(cursor) };
     return {
-      $and: [
-        query,
-        {
-          /** We are using the connected at date as the cursor so that the functionality works the same on Postgres implementation
-           * The id field in postgres is an uuid, this will work similarly to the ObjectId in Mongodb
-           * */
-          connected_at: {
-            $lt: new Date(cursor)
-          }
-        }
-      ]
+      ...query,
+      connected_at
     } as mongo.Filter<T>;
   };
 
-  /** cursor.count() deprecated */
-  const total = await collection.countDocuments(query);
-
   const findCursor = collection.find(createQuery(cursor), {
     sort: {
       /** We are sorting by connected at date descending to match cursor Postgres implementation */
@@ -152,12 +143,14 @@ export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
 
   const items = await findCursor.limit(limit).toArray();
   const count = items.length;
+  /** The returned total has been defaulted to 0 due to the overhead using documentCount from the mogo driver.
+   * cursor.count has been deprecated.
+   * */
   return {
     items,
-    total,
     count,
     /** Setting the cursor to the connected at date of the last item in the list */
     cursor: count === limit ? items[items.length - 1].connected_at.toISOString() : undefined,
-    more: count < total
+    more: !(count !== limit)
   };
 };

package/test/src/__snapshots__/connection-report-storage.test.ts.snap

@@ -227,7 +227,6 @@ exports[`Report storage tests > Should show paginated response of all connection
     },
   ],
   "more": false,
-  "total": 1,
 }
 `;
 
@@ -262,7 +261,6 @@ exports[`Report storage tests > Should show paginated response of all connection
     },
   ],
   "more": true,
-  "total": 8,
 }
 `;
 
@@ -271,18 +269,74 @@ exports[`Report storage tests > Should show paginated response of all connection
   "count": 4,
   "cursor": undefined,
   "items": [
+    {
+      "client_id": "client_three",
+      "sdk": "powersync-js/1.21.2",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_three",
+    },
     {
       "client_id": "client_one",
-      "sdk": "powersync-dart/1.6.4",
-      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
+    {
+      "client_id": "client_month",
+      "sdk": "powersync-js/1.23.6",
+      "user_agent": "powersync-js/1.23.0 powersync-web Firefox/141 linux",
+      "user_id": "user_month",
+    },
+    {
+      "client_id": "client_expired",
+      "sdk": "powersync-js/1.23.7",
+      "user_agent": "powersync-js/1.23.0 powersync-web Firefox/141 linux",
+      "user_id": "user_expired",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 1`] = `
+{
+  "count": 4,
+  "cursor": "<removed-for-snapshot>",
+  "items": [
+    {
+      "client_id": "",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
       "user_id": "user_one",
     },
     {
-      "client_id": "client_four",
-      "sdk": "powersync-js/1.21.4",
+      "client_id": "client_two",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
+    },
+    {
+      "client_id": "client_three",
+      "sdk": "powersync-js/1.21.2",
       "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
-      "user_id": "user_four",
+      "user_id": "user_three",
     },
+    {
+      "client_id": "client_one",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
+  ],
+  "more": true,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 2`] = `
+{
+  "count": 2,
+  "cursor": undefined,
+  "items": [
     {
       "client_id": "",
       "sdk": "unknown",
@@ -295,9 +349,20 @@ exports[`Report storage tests > Should show paginated response of all connection
       "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
       "user_id": "user_two",
     },
+    {
+      "client_id": "client_three",
+      "sdk": "powersync-js/1.21.2",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_three",
+    },
+    {
+      "client_id": "client_one",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
   ],
-  "more": true,
-  "total": 8,
+  "more": false,
 }
 `;
 
@@ -320,7 +385,6 @@ exports[`Report storage tests > Should show paginated response of connections of
     },
   ],
   "more": false,
-  "total": 2,
 }
 `;
 
@@ -367,6 +431,21 @@ exports[`Report storage tests > Should show paginated response of connections ov
     },
   ],
   "more": false,
-  "total": 6,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of connections over a date range of specified client_id and user_id 1`] = `
+{
+  "count": 1,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_one",
+      "sdk": "powersync-dart/1.6.4",
+      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+  ],
+  "more": false,
 }
 `;

package/test/src/storage_compacting.test.ts

@@ -62,6 +62,7 @@ bucket_definitions:
         clearBatchLimit: 200,
         moveBatchLimit: 10,
         moveBatchQueryLimit: 10,
+        minBucketChanges: 1,
         maxOpId: checkpoint,
         signal: null as any
       });