@powersync/service-module-mongodb-storage 0.0.0-dev-20250910154512 → 0.0.0-dev-20251030082344

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -16,6 +16,8 @@ import {
   InternalOpId,
   internalToExternalOpId,
   maxLsn,
+  PopulateChecksumCacheOptions,
+  PopulateChecksumCacheResults,
   ProtocolOpId,
   ReplicationCheckpoint,
   storage,
@@ -35,13 +37,23 @@ import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
-import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
-
+import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js';
 
 export interface MongoSyncBucketStorageOptions {
   checksumOptions?: MongoChecksumOptions;
 }
 
+/**
+ * Only keep checkpoints around for a minute, before fetching a fresh one.
+ *
+ * The reason is that we keep a MongoDB snapshot reference (clusterTime) with the checkpoint,
+ * and they expire after 5 minutes by default. This is an issue if the checkpoint stream is idle,
+ * but new clients connect and use an outdated checkpoint snapshot for parameter queries.
+ *
+ * These will be filtered out for existing clients, so should not create significant overhead.
+ */
+const CHECKPOINT_TIMEOUT_MS = 60_000;
+
 export class MongoSyncBucketStorage
   extends BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
@@ -393,7 +405,9 @@ export class MongoSyncBucketStorage
         limit: batchLimit,
         // Increase batch size above the default 101, so that we can fill an entire batch in
         // one go.
-        batchSize: batchLimit,
+        // batchSize is 1 more than limit to auto-close the cursor.
+        // See https://github.com/mongodb/node-mongodb-native/pull/4580
+        batchSize: batchLimit + 1,
         // Raw mode is returns an array of Buffer instead of parsed documents.
         // We use it so that:
         // 1. We can calculate the document size accurately without serializing again.
@@ -653,7 +667,7 @@ export class MongoSyncBucketStorage
     }
   }
 
-  async populatePersistentChecksumCache(options: Required<Pick<CompactOptions, 'signal' | 'maxOpId'>>): Promise<void> {
+  async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
     logger.info(`Populating persistent checksum cache...`);
     const start = Date.now();
     // We do a minimal compact here.
@@ -664,9 +678,14 @@ export class MongoSyncBucketStorage
       memoryLimitMB: 0
     });
 
-    await compactor.populateChecksums();
+    const result = await compactor.populateChecksums({
+      // There are cases with millions of small buckets, in which case it can take very long to
+      // populate the checksums, with minimal benefit. We skip the small buckets here.
+      minBucketChanges: options.minBucketChanges ?? 10
+    });
     const duration = Date.now() - start;
     logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
+    return result;
   }
 
   /**
@@ -681,25 +700,45 @@ export class MongoSyncBucketStorage
 
     // We only watch changes to the active sync rules.
     // If it changes to inactive, we abort and restart with the new sync rules.
-    let lastOp: storage.ReplicationCheckpoint | null = null;
+    try {
+      while (true) {
+        // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS)
+        // before we get another checkpoint, to avoid stale checkpoint snapshots.
+        const timeout = timers
+          .setTimeout(CHECKPOINT_TIMEOUT_MS, { done: false }, { signal })
+          .catch(() => ({ done: true }));
+        try {
+          const result = await Promise.race([stream.next(), timeout]);
+          if (result.done) {
+            break;
+          }
+        } catch (e) {
+          if (e.name == 'AbortError') {
+            break;
+          }
+          throw e;
+        }
 
-    for await (const _ of stream) {
-      if (signal.aborted) {
-        break;
-      }
+        if (signal.aborted) {
+          // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway
+          break;
+        }
 
-      const op = await this.getCheckpointInternal();
-      if (op == null) {
-        // Sync rules have changed - abort and restart.
-        // We do a soft close of the stream here - no error
-        break;
-      }
+        const op = await this.getCheckpointInternal();
+        if (op == null) {
+          // Sync rules have changed - abort and restart.
+          // We do a soft close of the stream here - no error
+          break;
+        }
 
-      // Check for LSN / checkpoint changes - ignore other metadata changes
-      if (lastOp == null || op.lsn != lastOp.lsn || op.checkpoint != lastOp.checkpoint) {
-        lastOp = op;
+        // Previously, we only yielded when the checkpoint or lsn changed.
+        // However, we always want to use the latest snapshotTime, so we skip that filtering here.
+        // That filtering could be added in the per-user streams if needed, but in general the capped collection
+        // should already only contain useful changes in most cases.
         yield op;
       }
+    } finally {
+      await stream.return(null);
     }
   }
 
@@ -875,7 +914,9 @@ export class MongoSyncBucketStorage
             '_id.b': 1
           },
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )
@@ -905,7 +946,9 @@ export class MongoSyncBucketStorage
             lookup: 1
           },
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )

package/src/storage/implementation/MongoTestStorageFactoryGenerator.ts

@@ -0,0 +1,32 @@
+import { TestStorageOptions } from '@powersync/service-core';
+import { MongoBucketStorage } from '../MongoBucketStorage.js';
+import { connectMongoForTests } from './util.js';
+import { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
+
+export type MongoTestStorageOptions = {
+  url: string;
+  isCI: boolean;
+  internalOptions?: MongoSyncBucketStorageOptions;
+};
+
+export const MongoTestStorageFactoryGenerator = (factoryOptions: MongoTestStorageOptions) => {
+  return async (options?: TestStorageOptions) => {
+    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+
+    // None of the tests insert data into this collection, so it was never created
+    if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+      await db.db.createCollection('bucket_parameters');
+    }
+
+    if (!options?.doNotClear) {
+      await db.clear();
+    }
+
+    // Full migrations are not currently run for tests, so we manually create the important ones
+    await db.createCheckpointEventsCollection();
+    await db.createBucketStateIndex();
+    await db.createBucketStateIndex2();
+
+    return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+  };
+};

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -111,7 +111,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
         },
         {
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )
@@ -140,7 +142,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
         },
         {
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )

package/src/storage/implementation/PersistedBatch.ts

@@ -16,7 +16,7 @@ import {
   CurrentDataDocument,
   SourceKey
 } from './models.js';
-import { replicaIdToSubkey } from '../../utils/util.js';
+import { replicaIdToSubkey } from './util.js';
 
 /**
  * Maximum size of operations we write in a single transaction.

package/src/storage/implementation/db.ts

@@ -8,7 +8,6 @@ import {
   BucketParameterDocument,
   BucketStateDocument,
   CheckpointEventDocument,
-  ClientConnectionDocument,
   CurrentDataDocument,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
@@ -38,7 +37,6 @@ export class PowerSyncMongo {
   readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
   readonly bucket_state: mongo.Collection<BucketStateDocument>;
   readonly checkpoint_events: mongo.Collection<CheckpointEventDocument>;
-  readonly connection_report_events: mongo.Collection<ClientConnectionDocument>;
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
@@ -63,7 +61,6 @@ export class PowerSyncMongo {
     this.locks = this.db.collection('locks');
     this.bucket_state = this.db.collection('bucket_state');
     this.checkpoint_events = this.db.collection('checkpoint_events');
-    this.connection_report_events = this.db.collection('connection_report_events');
   }
 
   /**
@@ -134,28 +131,27 @@ export class PowerSyncMongo {
   /**
    * Only use in migrations and tests.
    */
-  async createConnectionReportingCollection() {
-    const existingCollections = await this.db
-      .listCollections({ name: 'connection_report_events' }, { nameOnly: false })
-      .toArray();
-    const collection = existingCollections[0];
-    if (collection != null) {
-      return;
-    }
-    await this.db.createCollection('connection_report_events');
+  async createBucketStateIndex() {
+    // TODO: Implement a better mechanism to use migrations in tests
+    await this.bucket_state.createIndex(
+      {
+        '_id.g': 1,
+        last_op: 1
+      },
+      { name: 'bucket_updates', unique: true }
+    );
   }
-
   /**
    * Only use in migrations and tests.
    */
-  async createBucketStateIndex() {
+  async createBucketStateIndex2() {
     // TODO: Implement a better mechanism to use migrations in tests
     await this.bucket_state.createIndex(
       {
         '_id.g': 1,
-        last_op: 1
+        'estimate_since_compact.count': -1
       },
-      { name: 'bucket_updates', unique: true }
+      { name: 'dirty_count' }
     );
   }
 }

package/src/storage/implementation/models.ts

@@ -1,7 +1,6 @@
 import { InternalOpId, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
-import { event_types } from '@powersync/service-types';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -239,5 +238,3 @@ export interface InstanceDocument {
   // The instance UUID
   _id: string;
 }
-
-export interface ClientConnectionDocument extends event_types.ClientConnection {}

package/src/{utils → storage/implementation}/util.ts

@@ -3,9 +3,11 @@ import * as crypto from 'crypto';
 import * as uuid from 'uuid';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { storage, utils } from '@powersync/service-core';
+import { BucketChecksum, PartialChecksum, PartialOrFullChecksum, storage, utils } from '@powersync/service-core';
+
+import { PowerSyncMongo } from './db.js';
+import { BucketDataDocument } from './models.js';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
-import { BucketDataDocument } from '../storage/implementation/models.js';
 
 export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
   let filter = {
@@ -39,7 +41,7 @@ export function generateSlotName(prefix: string, sync_rules_id: number) {
  * However, that makes `has_more` detection very difficult, since the cursor is always closed
  * after the first batch. Instead, we do a workaround to only fetch a single batch below.
  *
- * For this to be effective, set batchSize = limit in the find command.
+ * For this to be effective, set batchSize = limit + 1 in the find command.
  */
 export async function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
   try {
@@ -103,6 +105,20 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
+/**
+ * Helper for unit tests
+ */
+export const connectMongoForTests = (url: string, isCI: boolean) => {
+  // Short timeout for tests, to fail fast when the server is not available.
+  // Slightly longer timeouts for CI, to avoid arbitrary test failures
+  const client = new mongo.MongoClient(url, {
+    connectTimeoutMS: isCI ? 15_000 : 5_000,
+    socketTimeoutMS: isCI ? 15_000 : 5_000,
+    serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
+  });
+  return new PowerSyncMongo(client);
+};
+
 export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) {
   // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
   if (!session.snapshotEnabled) {

package/src/storage/storage-index.ts

@@ -7,9 +7,8 @@ export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';
 export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/MongoTestStorageFactoryGenerator.js';
 export * from './implementation/OperationBatch.js';
 export * from './implementation/PersistedBatch.js';
-export * from '../utils/util.js';
+export * from './implementation/util.js';
 export * from './MongoBucketStorage.js';
-export * from './MongoReportStorage.js';
-export * as test_utils from '../utils/test-utils.js';

package/test/src/storage.test.ts

@@ -2,7 +2,8 @@ import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 import { env } from './env.js';
-import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js';
+import { MongoChecksumOptions } from '@module/storage/implementation/MongoChecksums.js';
 
 describe('Mongo Sync Bucket Storage - Parameters', () =>
   register.registerDataStorageParameterTests(INITIALIZED_MONGO_STORAGE_FACTORY));
@@ -17,7 +18,7 @@ describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
 describe('Mongo Sync Bucket Storage - split operations', () =>
   register.registerDataStorageDataTests(
-    mongoTestStorageFactoryGenerator({
+    MongoTestStorageFactoryGenerator({
       url: env.MONGO_TEST_URL,
       isCI: env.CI,
       internalOptions: {
@@ -31,7 +32,7 @@ describe('Mongo Sync Bucket Storage - split operations', () =>
 
 describe('Mongo Sync Bucket Storage - split buckets', () =>
   register.registerDataStorageDataTests(
-    mongoTestStorageFactoryGenerator({
+    MongoTestStorageFactoryGenerator({
       url: env.MONGO_TEST_URL,
       isCI: env.CI,
       internalOptions: {
@@ -42,3 +43,50 @@ describe('Mongo Sync Bucket Storage - split buckets', () =>
       }
     })
   ));
+
+describe('Mongo Sync Bucket Storage - checksum calculations', () => {
+  // This test tests 4 buckets x 4 operations in each.
+  // We specifically use operationBatchLimit that does not have factors in common with 4,
+  // as well some that do.
+  const params: MongoChecksumOptions[] = [
+    {
+      bucketBatchLimit: 100,
+      operationBatchLimit: 3
+    },
+
+    {
+      bucketBatchLimit: 10,
+      operationBatchLimit: 7
+    },
+
+    {
+      bucketBatchLimit: 3,
+      operationBatchLimit: 1
+    },
+    {
+      bucketBatchLimit: 1,
+      operationBatchLimit: 3
+    },
+    {
+      bucketBatchLimit: 2,
+      operationBatchLimit: 4
+    },
+    {
+      bucketBatchLimit: 4,
+      operationBatchLimit: 12
+    }
+  ];
+  for (let options of params) {
+    describe(`${options.bucketBatchLimit}|${options.operationBatchLimit}`, () => {
+      register.testChecksumBatching(
+        MongoTestStorageFactoryGenerator({
+          url: env.MONGO_TEST_URL,
+          isCI: env.CI,
+          internalOptions: {
+            checksumOptions: options
+          }
+        })
+      );
+    });
+  }
+});

package/test/src/storage_compacting.test.ts

@@ -97,10 +97,25 @@ bucket_definitions:
       await populate(bucketStorage);
       const { checkpoint } = await bucketStorage.getCheckpoint();
 
-      await bucketStorage.populatePersistentChecksumCache({
+      // Default is to small small numbers - should be a no-op
+      const result0 = await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint
+      });
+      expect(result0.buckets).toEqual(0);
+
+      // This should cache the checksums for the two buckets
+      const result1 = await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint,
+        minBucketChanges: 1
+      });
+      expect(result1.buckets).toEqual(2);
+
+      // This should be a no-op, as the checksums are already cached
+      const result2 = await bucketStorage.populatePersistentChecksumCache({
         maxOpId: checkpoint,
-        signal: new AbortController().signal
+        minBucketChanges: 1
       });
+      expect(result2.buckets).toEqual(0);
 
       const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user2["u1"]', 'by_user2["u2"]']);
       expect(checksumAfter.get('by_user2["u1"]')).toEqual({

package/test/src/util.ts

@@ -1,12 +1,8 @@
 import { env } from './env.js';
-import { mongoTestReportStorageFactoryGenerator, mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 
-export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({
-  url: env.MONGO_TEST_URL,
-  isCI: env.CI
-});
+import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js';
 
-export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFactoryGenerator({
+export const INITIALIZED_MONGO_STORAGE_FACTORY = MongoTestStorageFactoryGenerator({
   url: env.MONGO_TEST_URL,
   isCI: env.CI
 });