@powersync/service-module-mongodb-storage 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223080959

package/src/storage/MongoReportStorage.ts

@@ -2,381 +2,14 @@ import { storage } from '@powersync/service-core';
 import { event_types } from '@powersync/service-types';
 import { PowerSyncMongo } from './implementation/db.js';
 import { logger } from '@powersync/lib-services-framework';
-import { createPaginatedConnectionQuery, createPaginatedSyncCheckpointQuery } from '../utils/util.js';
-import { mongo } from '@powersync/lib-service-mongodb';
-import * as bson from 'bson';
+import { createPaginatedConnectionQuery } from '../utils/util.js';
 
 export class MongoReportStorage implements storage.ReportStorage {
   public readonly db: PowerSyncMongo;
-  private readonly client: mongo.MongoClient;
-  private readonly session: mongo.ClientSession;
 
   constructor(db: PowerSyncMongo) {
     this.db = db;
-    this.client = db.client;
-    this.session = this.client.startSession();
   }
-
-  private async reportSyncCheckpoint(data: event_types.SyncAnalyticsEventData) {
-    const { user_id, client_id, data: checkData, request_streams } = data;
-    const { id, type, last_op_id, buckets, date, streams } = checkData;
-    await this.session.withTransaction(
-      async () => {
-        await this.db.sync_report_events.insertOne(
-          {
-            _id: id,
-            user_id,
-            client_id,
-            type,
-            last_op_id,
-            date,
-            streams,
-            request_streams
-          },
-          { session: this.session }
-        );
-        await this.db.bucket_report_events.insertMany(
-          buckets.map((bucket: any) => {
-            return {
-              state: `synced`,
-              checkpoint_id: id,
-              client_id,
-              user_id,
-              name: bucket.bucket,
-              operations: bucket.count,
-              date: date,
-              subscriptions: bucket.subscriptions,
-              priority: bucket.priority
-            };
-          })
-        );
-      },
-      { session: this.session }
-    );
-  }
-
-  private async reportSyncDiffCheckpoint(data: event_types.SyncAnalyticsEventData) {
-    const { user_id, client_id, data: checkData, request_streams } = data;
-    const { id, type, last_op_id, updated_buckets, removed_buckets, date, streams } = checkData;
-    await this.session.withTransaction(async () => {
-      await this.db.sync_report_events.insertOne(
-        {
-          _id: id,
-          user_id,
-          client_id,
-          type,
-          last_op_id,
-          date,
-          streams,
-          request_streams
-        },
-        { session: this.session }
-      );
-      if (updated_buckets.length > 0) {
-        await this.db.bucket_report_events.insertMany(
-          updated_buckets.map((bucket: any) => {
-            return {
-              state: `updated`,
-              checkpoint_id: id,
-              client_id,
-              user_id,
-              name: bucket.bucket,
-              operations: bucket.count,
-              date: date,
-              subscriptions: bucket.subscriptions,
-              priority: bucket.priority
-            };
-          }),
-          { session: this.session }
-        );
-      }
-      if (removed_buckets.length > 0) {
-        await this.db.bucket_report_events.insertMany(
-          removed_buckets.map((bucket: string) => {
-            return {
-              state: `removed`,
-              checkpoint_id: id,
-              client_id,
-              user_id,
-              name: bucket,
-              operations: null,
-              subscriptions: null,
-              date: date,
-              priority: null
-            };
-          }),
-          { session: this.session }
-        );
-      }
-    });
-  }
-
-  private syncBucketFacetPipeline() {
-    return {
-      $facet: {
-        buckets_removed: [
-          { $match: { state: 'removed' } },
-          { $group: { _id: '$checkpoint_id', total: { $sum: 1 } } },
-          {
-            $project: {
-              _id: 0,
-              checkpoint_id: '$_id',
-              total: 1
-            }
-          },
-          { $sort: { total: -1 } }
-        ],
-        buckets_updated: [
-          { $match: { state: 'updated' } },
-          { $group: { _id: '$checkpoint_id', total: { $sum: 1 } } },
-          {
-            $project: {
-              _id: 0,
-              checkpoint_id: '$_id',
-              total: 1
-            }
-          },
-          { $sort: { total: -1 } }
-        ],
-        buckets_synced: [
-          { $match: { state: 'synced' } },
-          { $group: { _id: '$checkpoint_id', total: { $sum: 1 } } },
-          {
-            $project: {
-              _id: 0,
-              checkpoint_id: '$_id',
-              total: 1
-            }
-          },
-          { $sort: { total: -1 } }
-        ]
-      }
-    };
-  }
-
-  private syncBucketProjectPipeline() {
-    return {
-      $project: {
-        buckets_removed: 1,
-        buckets_updated: 1,
-        buckets_synced: 1
-      }
-    };
-  }
-
-  async getSyncCheckpoint(data: event_types.SyncCheckpointRequest): Promise<event_types.PaginatedResponse<any>> {
-    const { cursor, date_range } = data;
-    const limit = data?.limit || 100;
-    const typeFilter = data.type != null ? { type: data.type } : undefined;
-    const date = { date: { $lte: date_range.end, $gte: date_range.start } };
-    const user_id = data.user_id != null ? { user_id: data.user_id } : undefined;
-    const client_id = data.client_id != null ? { client_id: data.client_id } : undefined;
-    return (await createPaginatedSyncCheckpointQuery(
-      {
-        ...client_id,
-        ...user_id,
-        ...date,
-        ...typeFilter
-      },
-      this.db.sync_report_events,
-      limit,
-      cursor
-    )) as event_types.PaginatedResponse<any>;
-  }
-
-  async getLastSyncReport(data: event_types.LastSyncRequest): Promise<any> {
-    const { client_id, user_id } = data;
-    const checkpoint = await this.db.sync_report_events.findOne(
-      {
-        user_id,
-        client_id
-      },
-      { session: this.session, sort: { date: -1 } }
-    );
-
-    if (!checkpoint) {
-      throw new Error('No checkpoint found for the given user_id and client_id');
-    }
-
-    const sync_streams: string[] = checkpoint.streams.map((stream: any) => stream.name);
-    const subscriptions: string[] =
-      checkpoint.request_streams?.subscriptions.length > 0
-        ? checkpoint.request_streams.subscriptions.map((stream: any) => stream.name)
-        : [];
-
-    const buckets = await this.db.bucket_report_events
-      .aggregate([
-        {
-          $facet: {
-            updated_buckets: [
-              {
-                $match: {
-                  checkpoint_id: checkpoint._id,
-                  state: 'updated'
-                }
-              },
-              {
-                $sort: {
-                  operations: -1
-                }
-              },
-              {
-                $limit: 10
-              },
-              {
-                $project: {
-                  _id: 0,
-                  name: '$name',
-                  operations: '$operations',
-                  subscriptions: '$subscriptions',
-                  priority: '$priority'
-                }
-              }
-            ],
-            removed_buckets: [
-              {
-                $match: {
-                  checkpoint_id: checkpoint._id,
-                  state: 'removed'
-                }
-              },
-              {
-                $group: {
-                  _id: null,
-                  count: { $sum: 1 }
-                }
-              }
-            ],
-            synced_buckets: [
-              {
-                $match: {
-                  checkpoint_id: checkpoint._id,
-                  state: 'synced'
-                }
-              },
-              {
-                $sort: {
-                  operations: -1
-                }
-              },
-              {
-                $limit: 10
-              },
-              {
-                $project: {
-                  _id: 0,
-                  name: '$name',
-                  operations: '$operations',
-                  subscriptions: '$subscriptions',
-                  priority: '$priority'
-                }
-              }
-            ]
-          }
-        }
-      ])
-      .toArray();
-
-    if (buckets.length === 0) {
-      throw new Error('No bucket report events found for the given checkpoint');
-    }
-
-    const syncedBuckets =
-      buckets[0].synced_buckets.length > 0
-        ? buckets[0].synced_buckets.map((bucket: any) => {
-            const subs = {
-              default: [],
-              sub: []
-            };
-            for (const subscription of bucket?.subscriptions ?? []) {
-              if ('default' in subscription) {
-                // @ts-expect-error
-                subs.default.push(sync_streams[subscription.default]);
-              }
-
-              if ('sub' in subscription) {
-                // @ts-expect-error
-                subs.sub.push(subscriptions[subscription.sub]);
-              }
-            }
-            return {
-              name: bucket.name,
-              operations: bucket.operations,
-              priority: bucket.priority,
-              subscriptions: subs
-            };
-          })
-        : [];
-
-    const updatedBuckets =
-      buckets[0].updated_buckets.length > 0
-        ? buckets[0].updated_buckets.map((bucket: any) => {
-            const subs = {
-              default: [],
-              sub: []
-            };
-            for (const subscription of bucket?.subscriptions ?? []) {
-              if ('default' in subscription) {
-                // @ts-expect-error
-                subs.default.push(sync_streams[subscription.default]);
-              }
-
-              if ('sub' in subscription) {
-                // @ts-expect-error
-                subs.sub.push(subscriptions[subscription.sub]);
-              }
-            }
-            return {
-              name: bucket.name,
-              operations: bucket.operations,
-              priority: bucket.priority,
-              subscriptions: subs
-            };
-          })
-        : [];
-
-    return {
-      checkpoint_id: checkpoint._id,
-      last_op_id: checkpoint.last_op_id,
-      date: checkpoint.date,
-      streams: sync_streams,
-      subscriptions,
-      updated_buckets: updatedBuckets,
-      removed_buckets: buckets[0].removed_buckets.length > 0 ? buckets[0].removed_buckets[0].count : 0,
-      synced_buckets: syncedBuckets
-    };
-  }
-
-  async getSyncBucketStats(data: event_types.SyncBucketStatsRequest): Promise<any> {
-    const { date_range } = data;
-    const user_id = data.user_id != null ? { user_id: data.user_id } : undefined;
-    const client_id = data.client_id != null ? { client_id: data.client_id } : undefined;
-    const checkpoint_id =
-      data.checkpoint_id != null ? { checkpoint_id: new bson.ObjectId(data.checkpoint_id) } : undefined;
-    const result = await this.db.bucket_report_events
-      .aggregate([
-        {
-          $match: {
-            ...client_id,
-            ...user_id,
-            ...checkpoint_id,
-            date: { $lte: date_range.end, $gte: date_range.start }
-          }
-        },
-        this.syncBucketFacetPipeline(),
-        this.syncBucketProjectPipeline()
-      ])
-      .toArray();
-    return result[0];
-  }
-
-  async reportSyncAnalyticsEvent(data: event_types.SyncAnalyticsEventData): Promise<void> {
-    if (data.data.type === event_types.SyncEventCheckpointType.FULL) {
-      return this.reportSyncCheckpoint(data);
-    }
-    return this.reportSyncDiffCheckpoint(data);
-  }
-
   async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void> {
     const { date } = data;
     const result = await this.db.connection_report_events.deleteMany({
@@ -393,7 +26,9 @@ export class MongoReportStorage implements storage.ReportStorage {
     }
   }
 
-  async getClientConnectionReports(data: event_types.DateRange): Promise<event_types.ClientConnectionReportResponse> {
+  async getClientConnectionReports(
+    data: event_types.ClientConnectionReportRequest
+  ): Promise<event_types.ClientConnectionReportResponse> {
     const { start, end } = data;
     const result = await this.db.connection_report_events
       .aggregate<event_types.ClientConnectionReportResponse>([

package/src/storage/implementation/MongoChecksums.ts

@@ -13,6 +13,7 @@ import {
   PartialOrFullChecksum
 } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
+import { StorageConfig } from './models.js';
 
 /**
  * Checksum calculation options, primarily for tests.
@@ -27,6 +28,8 @@ export interface MongoChecksumOptions {
    * Limit on the number of documents to calculate a checksum on at a time.
    */
   operationBatchLimit?: number;
+
+  storageConfig: StorageConfig;
 }
 
 const DEFAULT_BUCKET_BATCH_LIMIT = 200;
@@ -43,12 +46,15 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
  */
 export class MongoChecksums {
   private _cache: ChecksumCache | undefined;
+  private readonly storageConfig: StorageConfig;
 
   constructor(
     private db: PowerSyncMongo,
     private group_id: number,
-    private options?: MongoChecksumOptions
-  ) {}
+    private options: MongoChecksumOptions
+  ) {
+    this.storageConfig = options.storageConfig;
+  }
 
   /**
    * Lazy-instantiated cache.
@@ -222,6 +228,11 @@ export class MongoChecksums {
         });
       }
 
+      // Historically, checksum may be stored as 'int' or 'double'.
+      // More recently, this should be a 'long'.
+      // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
+      const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' };
+
       // Aggregate over a max of `batchLimit` operations at a time.
       // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12.
       // Then we'll do three batches:
@@ -245,10 +256,7 @@ export class MongoChecksums {
             {
               $group: {
                 _id: '$_id.b',
-                // Historically, checksum may be stored as 'int' or 'double'.
-                // More recently, this should be a 'long'.
-                // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-                checksum_total: { $sum: { $toLong: '$checksum' } },
+                checksum_total: { $sum: checksumLong },
                 count: { $sum: 1 },
                 has_clear_op: {
                   $max: {

package/src/storage/implementation/MongoCompactor.ts

@@ -62,6 +62,7 @@ const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
 const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
 const DEFAULT_MIN_BUCKET_CHANGES = 10;
+const DEFAULT_MIN_CHANGE_RATIO = 0.1;
 
 /** This default is primarily for tests. */
 const DEFAULT_MEMORY_LIMIT_MB = 64;
@@ -75,6 +76,7 @@ export class MongoCompactor {
   private moveBatchQueryLimit: number;
   private clearBatchLimit: number;
   private minBucketChanges: number;
+  private minChangeRatio: number;
   private maxOpId: bigint;
   private buckets: string[] | undefined;
   private signal?: AbortSignal;
@@ -91,6 +93,7 @@ export class MongoCompactor {
     this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
     this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
     this.minBucketChanges = options?.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
+    this.minChangeRatio = options?.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
     this.maxOpId = options?.maxOpId ?? 0n;
     this.buckets = options?.compactBuckets;
     this.signal = options?.signal;
@@ -115,27 +118,19 @@ export class MongoCompactor {
   }
 
   private async compactDirtyBuckets() {
-    while (!this.signal?.aborted) {
-      // Process all buckets with 10 or more changes since last time.
-      // We exclude the last 100 compacted buckets, to avoid repeatedly re-compacting the same buckets over and over
-      // if they are modified while compacting.
-      const TRACK_RECENTLY_COMPACTED_NUMBER = 100;
-
-      let recentlyCompacted: string[] = [];
-      const buckets = await this.dirtyBucketBatch({
-        minBucketChanges: this.minBucketChanges,
-        exclude: recentlyCompacted
-      });
-      if (buckets.length == 0) {
-        // All done
+    for await (let buckets of this.dirtyBucketBatches({
+      minBucketChanges: this.minBucketChanges,
+      minChangeRatio: this.minChangeRatio
+    })) {
+      if (this.signal?.aborted) {
         break;
       }
+      if (buckets.length == 0) {
+        continue;
+      }
+
       for (let { bucket } of buckets) {
         await this.compactSingleBucket(bucket);
-        recentlyCompacted.push(bucket);
-      }
-      if (recentlyCompacted.length > TRACK_RECENTLY_COMPACTED_NUMBER) {
-        recentlyCompacted = recentlyCompacted.slice(-TRACK_RECENTLY_COMPACTED_NUMBER);
       }
     }
   }
@@ -491,13 +486,13 @@ export class MongoCompactor {
   async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
     let count = 0;
     while (!this.signal?.aborted) {
-      const buckets = await this.dirtyBucketBatch(options);
-      if (buckets.length == 0) {
+      const buckets = await this.dirtyBucketBatchForChecksums(options);
+      if (buckets.length == 0 || this.signal?.aborted) {
         // All done
         break;
       }
+
       const start = Date.now();
-      logger.info(`Calculating checksums for batch of ${buckets.length} buckets`);
 
       // Filter batch by estimated bucket size, to reduce possibility of timeouts
       let checkBuckets: typeof buckets = [];
@@ -509,22 +504,97 @@ export class MongoCompactor {
           break;
         }
       }
+      logger.info(
+        `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`
+      );
       await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket));
       logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
-      count += buckets.length;
+      count += checkBuckets.length;
     }
     return { buckets: count };
   }
 
+  /**
+   * Return batches of dirty buckets.
+   *
+   * Can be used to iterate through all buckets.
+   *
+   * minBucketChanges: minimum number of changes for a bucket to be included in the results.
+   * minChangeRatio: minimum ratio of changes to total ops for a bucket to be included in the results, number between 0 and 1.
+   */
+  private async *dirtyBucketBatches(options: {
+    minBucketChanges: number;
+    minChangeRatio: number;
+  }): AsyncGenerator<{ bucket: string; estimatedCount: number }[]> {
+    // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only buckets with changes.
+    // This works well if there are only a small number of buckets with changes.
+    // However, if buckets are continuosly modified while we are compacting, we get the same buckets over and over again.
+    // This has caused the compact process to re-read the same collection around 5x times in total, which is very inefficient.
+    // To solve this, we now just iterate through all buckets, and filter out the ones with low changes.
+
+    if (options.minBucketChanges <= 0) {
+      throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    }
+    let lastId = { g: this.group_id, b: new mongo.MinKey() as any };
+    const maxId = { g: this.group_id, b: new mongo.MaxKey() as any };
+    while (true) {
+      const batch = await this.db.bucket_state
+        .find(
+          {
+            _id: { $gt: lastId, $lt: maxId },
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+          },
+          {
+            projection: {
+              _id: 1,
+              estimate_since_compact: 1,
+              compacted_state: 1
+            },
+            sort: {
+              _id: 1
+            },
+            limit: 2000,
+            maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+          }
+        )
+        .toArray();
+      if (batch.length == 0) {
+        break;
+      }
+      lastId = batch[batch.length - 1]._id;
+      const mapped = batch.map((b) => {
+        const updatedCount = b.estimate_since_compact?.count ?? 0;
+        const totalCount = (b.compacted_state?.count ?? 0) + updatedCount;
+        const updatedBytes = b.estimate_since_compact?.bytes ?? 0;
+        const totalBytes = (b.compacted_state?.bytes ?? 0) + updatedBytes;
+        const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0;
+        const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0;
+        return {
+          bucket: b._id.b,
+          estimatedCount: totalCount,
+          dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes)
+        };
+      });
+      const filtered = mapped.filter(
+        (b) => b.estimatedCount >= options.minBucketChanges && b.dirtyRatio >= options.minChangeRatio
+      );
+      yield filtered;
+    }
+  }
+
   /**
    * Returns a batch of dirty buckets - buckets with most changes first.
    *
    * This cannot be used to iterate on its own - the client is expected to process these buckets and
    * set estimate_since_compact.count: 0 when done, before fetching the next batch.
+   *
+   * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart,
+   * since it is used as the last step for initial replication.
+   *
+   * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing.
    */
-  private async dirtyBucketBatch(options: {
+  private async dirtyBucketBatchForChecksums(options: {
     minBucketChanges: number;
-    exclude?: string[];
   }): Promise<{ bucket: string; estimatedCount: number }[]> {
     if (options.minBucketChanges <= 0) {
       throw new ReplicationAssertionError('minBucketChanges must be >= 1');
@@ -534,8 +604,7 @@ export class MongoCompactor {
       .find(
         {
           '_id.g': this.group_id,
-          'estimate_since_compact.count': { $gte: options.minBucketChanges },
-          '_id.b': { $nin: options.exclude ?? [] }
+          'estimate_since_compact.count': { $gte: options.minBucketChanges }
         },
         {
           projection: {