@powersync/service-module-mongodb-storage 0.13.0 → 0.13.2

package/src/storage/implementation/MongoCompactor.ts

@@ -62,6 +62,7 @@ const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
+const DEFAULT_MIN_CHANGE_RATIO = 0.1;
 
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
@@ -75,6 +76,7 @@ export class MongoCompactor {
   private moveBatchQueryLimit: number;
   private clearBatchLimit: number;
   private minBucketChanges: number;
+  private minChangeRatio: number;
   private maxOpId: bigint;
   private buckets: string[] | undefined;
   private signal?: AbortSignal;
@@ -91,6 +93,7 @@ export class MongoCompactor {
     this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
     this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
     this.minBucketChanges = options?.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
+    this.minChangeRatio = options?.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
     this.maxOpId = options?.maxOpId ?? 0n;
     this.buckets = options?.compactBuckets;
     this.signal = options?.signal;
@@ -115,27 +118,19 @@ export class MongoCompactor {
   }
 
   private async compactDirtyBuckets() {
-    while (!this.signal?.aborted) {
-      // Process all buckets with 10 or more changes since last time.
-      // We exclude the last 100 compacted buckets, to avoid repeatedly re-compacting the same buckets over and over
-      // if they are modified while compacting.
-      const TRACK_RECENTLY_COMPACTED_NUMBER = 100;
-
-      let recentlyCompacted: string[] = [];
-      const buckets = await this.dirtyBucketBatch({
-        minBucketChanges: this.minBucketChanges,
-        exclude: recentlyCompacted
-      });
-      if (buckets.length == 0) {
-        // All done
+    for await (let buckets of this.dirtyBucketBatches({
+      minBucketChanges: this.minBucketChanges,
+      minChangeRatio: this.minChangeRatio
+    })) {
+      if (this.signal?.aborted) {
         break;
       }
+      if (buckets.length == 0) {
+        continue;
+      }
+
       for (let { bucket } of buckets) {
         await this.compactSingleBucket(bucket);
-        recentlyCompacted.push(bucket);
-      }
-      if (recentlyCompacted.length > TRACK_RECENTLY_COMPACTED_NUMBER) {
-        recentlyCompacted = recentlyCompacted.slice(-TRACK_RECENTLY_COMPACTED_NUMBER);
       }
     }
   }
@@ -491,13 +486,13 @@ export class MongoCompactor {
   async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
     let count = 0;
     while (!this.signal?.aborted) {
-      const buckets = await this.dirtyBucketBatch(options);
-      if (buckets.length == 0) {
+      const buckets = await this.dirtyBucketBatchForChecksums(options);
+      if (buckets.length == 0 || this.signal?.aborted) {
         // All done
         break;
       }
+
       const start = Date.now();
-      logger.info(`Calculating checksums for batch of ${buckets.length} buckets`);
 
       // Filter batch by estimated bucket size, to reduce possibility of timeouts
       let checkBuckets: typeof buckets = [];
@@ -509,22 +504,97 @@ export class MongoCompactor {
           break;
         }
       }
+      logger.info(
+        `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`
+      );
       await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
       logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
-      count += buckets.length;
+      count += checkBuckets.length;
     }
     return { buckets: count };
   }
 
+  /**
+   * Return batches of dirty buckets.
+   *
+   * Can be used to iterate through all buckets.
+   *
+   * minBucketChanges: minimum number of changes for a bucket to be included in the results.
+   * minChangeRatio: minimum ratio of changes to total ops for a bucket to be included in the results, number between 0 and 1.
+   */
+  private async *dirtyBucketBatches(options: {
+    minBucketChanges: number;
+    minChangeRatio: number;
+  }): AsyncGenerator<{ bucket: string; estimatedCount: number }[]> {
+    // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only buckets with changes.
+    // This works well if there are only a small number of buckets with changes.
+    // However, if buckets are continuosly modified while we are compacting, we get the same buckets over and over again.
+    // This has caused the compact process to re-read the same collection around 5x times in total, which is very inefficient.
+    // To solve this, we now just iterate through all buckets, and filter out the ones with low changes.
+
+    if (options.minBucketChanges <= 0) {
+      throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    }
+    let lastId = { g: this.group_id, b: new mongo.MinKey() as any };
+    const maxId = { g: this.group_id, b: new mongo.MaxKey() as any };
+    while (true) {
+      const batch = await this.db.bucket_state
+        .find(
+          {
+            _id: { $gt: lastId, $lt: maxId },
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+          },
+          {
+            projection: {
+              _id: 1,
+              estimate_since_compact: 1,
+              compacted_state: 1
+            },
+            sort: {
+              _id: 1
+            },
+            limit: 2000,
+            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+          }
+        )
+        .toArray();
+      if (batch.length == 0) {
+        break;
+      }
+      lastId = batch[batch.length - 1]._id;
+      const mapped = batch.map((b) => {
+        const updatedCount = b.estimate_since_compact?.count ?? 0;
+        const totalCount = (b.compacted_state?.count ?? 0) + updatedCount;
+        const updatedBytes = b.estimate_since_compact?.bytes ?? 0;
+        const totalBytes = (b.compacted_state?.bytes ?? 0) + updatedBytes;
+        const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0;
+        const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0;
+        return {
+          bucket: b._id.b,
+          estimatedCount: totalCount,
+          dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes)
+        };
+      });
+      const filtered = mapped.filter(
+        (b) => b.estimatedCount >= options.minBucketChanges && b.dirtyRatio >= options.minChangeRatio
+      );
+      yield filtered;
+    }
+  }
+
   /**
    * Returns a batch of dirty buckets - buckets with most changes first.
    *
    * This cannot be used to iterate on its own - the client is expected to process these buckets and
    * set estimate_since_compact.count: 0 when done, before fetching the next batch.
+   *
+   * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart,
+   * since it is used as the last step for initial replication.
+   *
+   * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing.
    */
-  private async dirtyBucketBatch(options: {
+  private async dirtyBucketBatchForChecksums(options: {
     minBucketChanges: number;
-    exclude?: string[];
   }): Promise<{ bucket: string; estimatedCount: number }[]> {
     if (options.minBucketChanges <= 0) {
       throw new ReplicationAssertionError('minBucketChanges must be >= 1');
@@ -534,8 +604,7 @@ export class MongoCompactor {
       .find(
         {
           '_id.g': this.group_id,
-          'estimate_since_compact.count': { $gte: options.minBucketChanges },
-          '_id.b': { $nin: options.exclude ?? [] }
+          'estimate_since_compact.count': { $gte: options.minBucketChanges }
         },
         {
           projection: {

package/src/storage/implementation/MongoPersistedSyncRules.ts

@@ -1,4 +1,4 @@
-import { SqlSyncRules, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
+import { SyncConfigWithErrors, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
 
 import { storage } from '@powersync/service-core';
 
@@ -7,7 +7,7 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
 
   constructor(
     public readonly id: number,
-    public readonly sync_rules: SqlSyncRules,
+    public readonly sync_rules: SyncConfigWithErrors,
     public readonly checkpoint_lsn: string | null,
     slot_name: string | null
   ) {
@@ -15,6 +15,6 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
   }
 
   hydratedSyncRules(): HydratedSyncRules {
-    return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(this.id) });
+    return this.sync_rules.config.hydrate({ hydrationState: versionedHydrationState(this.id) });
   }
 }

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -15,6 +15,7 @@ import {
   InternalOpId,
   internalToExternalOpId,
   maxLsn,
+  mergeAsyncIterables,
   PopulateChecksumCacheOptions,
   PopulateChecksumCacheResults,
   ProtocolOpId,
@@ -694,53 +695,39 @@ export class MongoSyncBucketStorage
    * Instance-wide watch on the latest available checkpoint (op_id + lsn).
    */
   private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ReplicationCheckpoint> {
-    const stream = this.checkpointChangesStream(signal);
-
     if (signal.aborted) {
       return;
     }
 
+    // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS) before we get another checkpoint,
+    // to avoid stale checkpoint snapshots. This is what checkpointTimeoutStream() is for.
+    // Essentially, even if there are no actual checkpoint changes, we want a new snapshotTime every minute or so,
+    // to ensure that any new clients connecting will get a valid snapshotTime.
+    const stream = mergeAsyncIterables(
+      [this.checkpointChangesStream(signal), this.checkpointTimeoutStream(signal)],
+      signal
+    );
+
     // We only watch changes to the active sync rules.
     // If it changes to inactive, we abort and restart with the new sync rules.
-    try {
-      while (true) {
-        // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS)
-        // before we get another checkpoint, to avoid stale checkpoint snapshots.
-        const timeout = timers
-          .setTimeout(CHECKPOINT_TIMEOUT_MS, { done: false }, { signal })
-          .catch(() => ({ done: true }));
-        try {
-          const result = await Promise.race([stream.next(), timeout]);
-          if (result.done) {
-            break;
-          }
-        } catch (e) {
-          if (e.name == 'AbortError') {
-            break;
-          }
-          throw e;
-        }
-
-        if (signal.aborted) {
-          // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway
-          break;
-        }
-
-        const op = await this.getCheckpointInternal();
-        if (op == null) {
-          // Sync rules have changed - abort and restart.
-          // We do a soft close of the stream here - no error
-          break;
-        }
+    for await (const _ of stream) {
+      if (signal.aborted) {
+        // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway
+        break;
+      }
 
-        // Previously, we only yielded when the checkpoint or lsn changed.
-        // However, we always want to use the latest snapshotTime, so we skip that filtering here.
-        // That filtering could be added in the per-user streams if needed, but in general the capped collection
-        // should already only contain useful changes in most cases.
-        yield op;
+      const op = await this.getCheckpointInternal();
+      if (op == null) {
+        // Sync rules have changed - abort and restart.
+        // We do a soft close of the stream here - no error
+        break;
       }
-    } finally {
-      await stream.return(null);
+
+      // Previously, we only yielded when the checkpoint or lsn changed.
+      // However, we always want to use the latest snapshotTime, so we skip that filtering here.
+      // That filtering could be added in the per-user streams if needed, but in general the capped collection
+      // should already only contain useful changes in most cases.
+      yield op;
     }
   }
 
@@ -900,6 +887,24 @@ export class MongoSyncBucketStorage
     }
   }
 
+  private async *checkpointTimeoutStream(signal: AbortSignal): AsyncGenerator<void> {
+    while (!signal.aborted) {
+      try {
+        await timers.setTimeout(CHECKPOINT_TIMEOUT_MS, undefined, { signal });
+      } catch (e) {
+        if (e.name == 'AbortError') {
+          // This is how we typically abort this stream, when all listeners are done
+          return;
+        }
+        throw e;
+      }
+
+      if (!signal.aborted) {
+        yield;
+      }
+    }
+  }
+
   private async getDataBucketChanges(
     options: GetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {

package/test/src/connection-report-storage.test.ts

@@ -57,7 +57,9 @@ describe('Connection reporting storage', async () => {
       user_agent: userData.user_week.user_agent
     });
 
-    const connection = await factory.db.connection_report_events.find({ user_id: userData.user_week.user_id }).toArray();
+    const connection = await factory.db.connection_report_events
+      .find({ user_id: userData.user_week.user_id })
+      .toArray();
     expect(connection).toHaveLength(2);
     const cleaned = removeVolatileFields(connection);
     expect(cleaned).toMatchSnapshot();
@@ -111,7 +113,9 @@ describe('Connection reporting storage', async () => {
       connected_at: userData.user_three.connected_at
     });
 
-    const connection = await factory.db.connection_report_events.find({ user_id: userData.user_three.user_id }).toArray();
+    const connection = await factory.db.connection_report_events
+      .find({ user_id: userData.user_three.user_id })
+      .toArray();
     expect(connection).toHaveLength(1);
     expect(new Date(connection[0].disconnected_at!)).toEqual(disconnectAt);
     const cleaned = removeVolatileFields(connection);

package/test/src/storage_compacting.test.ts

@@ -63,6 +63,7 @@ bucket_definitions:
         moveBatchLimit: 10,
         moveBatchQueryLimit: 10,
         minBucketChanges: 1,
+        minChangeRatio: 0,
         maxOpId: checkpoint,
         signal: null as any
       });

package/test/tsconfig.json

@@ -1,17 +1,13 @@
 {
-  "extends": "../../../tsconfig.base.json",
+  "extends": "../../../tsconfig.tests.json",
   "compilerOptions": {
-    "rootDir": "src",
     "baseUrl": "./",
-    "noEmit": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "sourceMap": true,
     "paths": {
       "@/*": ["../../../packages/service-core/src/*"],
       "@module/*": ["../src/*"],
       "@core-tests/*": ["../../../packages/service-core/test/src/*"]
-    }
+    },
+    "rootDir": "src"
   },
   "include": ["src"],
   "references": [