@powersync/service-module-mongodb-storage 0.0.0-dev-20251015143910 → 0.0.0-dev-20251030082344

package/src/{utils → storage/implementation}/util.ts

@@ -3,9 +3,11 @@ import * as crypto from 'crypto';
 import * as uuid from 'uuid';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { storage, utils } from '@powersync/service-core';
+import { BucketChecksum, PartialChecksum, PartialOrFullChecksum, storage, utils } from '@powersync/service-core';
+
+import { PowerSyncMongo } from './db.js';
+import { BucketDataDocument } from './models.js';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
-import { BucketDataDocument } from '../storage/implementation/models.js';
 
 export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
   let filter = {
@@ -39,7 +41,7 @@ export function generateSlotName(prefix: string, sync_rules_id: number) {
  * However, that makes `has_more` detection very difficult, since the cursor is always closed
  * after the first batch. Instead, we do a workaround to only fetch a single batch below.
  *
- * For this to be effective, set batchSize = limit in the find command.
+ * For this to be effective, set batchSize = limit + 1 in the find command.
  */
 export async function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
   try {
@@ -103,6 +105,20 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
+/**
+ * Helper for unit tests
+ */
+export const connectMongoForTests = (url: string, isCI: boolean) => {
+  // Short timeout for tests, to fail fast when the server is not available.
+  // Slightly longer timeouts for CI, to avoid arbitrary test failures
+  const client = new mongo.MongoClient(url, {
+    connectTimeoutMS: isCI ? 15_000 : 5_000,
+    socketTimeoutMS: isCI ? 15_000 : 5_000,
+    serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
+  });
+  return new PowerSyncMongo(client);
+};
+
 export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) {
   // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
   if (!session.snapshotEnabled) {

package/src/storage/storage-index.ts

@@ -7,9 +7,8 @@ export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';
 export * from './implementation/MongoSyncRulesLock.js';
+export * from './implementation/MongoTestStorageFactoryGenerator.js';
 export * from './implementation/OperationBatch.js';
 export * from './implementation/PersistedBatch.js';
-export * from '../utils/util.js';
+export * from './implementation/util.js';
 export * from './MongoBucketStorage.js';
-export * from './MongoReportStorage.js';
-export * as test_utils from '../utils/test-utils.js';

package/test/src/storage.test.ts

@@ -2,7 +2,8 @@ import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 import { env } from './env.js';
-import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js';
+import { MongoChecksumOptions } from '@module/storage/implementation/MongoChecksums.js';
 
 describe('Mongo Sync Bucket Storage - Parameters', () =>
   register.registerDataStorageParameterTests(INITIALIZED_MONGO_STORAGE_FACTORY));
@@ -17,7 +18,7 @@ describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
 describe('Mongo Sync Bucket Storage - split operations', () =>
   register.registerDataStorageDataTests(
-    mongoTestStorageFactoryGenerator({
+    MongoTestStorageFactoryGenerator({
       url: env.MONGO_TEST_URL,
       isCI: env.CI,
       internalOptions: {
@@ -31,7 +32,7 @@ describe('Mongo Sync Bucket Storage - split operations', () =>
 
 describe('Mongo Sync Bucket Storage - split buckets', () =>
   register.registerDataStorageDataTests(
-    mongoTestStorageFactoryGenerator({
+    MongoTestStorageFactoryGenerator({
       url: env.MONGO_TEST_URL,
       isCI: env.CI,
       internalOptions: {
@@ -42,3 +43,50 @@ describe('Mongo Sync Bucket Storage - split buckets', () =>
       }
     })
   ));
+
+describe('Mongo Sync Bucket Storage - checksum calculations', () => {
+  // This test tests 4 buckets x 4 operations in each.
+  // We specifically use operationBatchLimit that does not have factors in common with 4,
+  // as well some that do.
+  const params: MongoChecksumOptions[] = [
+    {
+      bucketBatchLimit: 100,
+      operationBatchLimit: 3
+    },
+
+    {
+      bucketBatchLimit: 10,
+      operationBatchLimit: 7
+    },
+
+    {
+      bucketBatchLimit: 3,
+      operationBatchLimit: 1
+    },
+    {
+      bucketBatchLimit: 1,
+      operationBatchLimit: 3
+    },
+    {
+      bucketBatchLimit: 2,
+      operationBatchLimit: 4
+    },
+    {
+      bucketBatchLimit: 4,
+      operationBatchLimit: 12
+    }
+  ];
+  for (let options of params) {
+    describe(`${options.bucketBatchLimit}|${options.operationBatchLimit}`, () => {
+      register.testChecksumBatching(
+        MongoTestStorageFactoryGenerator({
+          url: env.MONGO_TEST_URL,
+          isCI: env.CI,
+          internalOptions: {
+            checksumOptions: options
+          }
+        })
+      );
+    });
+  }
+});

package/test/src/storage_compacting.test.ts

@@ -97,10 +97,25 @@ bucket_definitions:
       await populate(bucketStorage);
       const { checkpoint } = await bucketStorage.getCheckpoint();
 
-      await bucketStorage.populatePersistentChecksumCache({
+      // Default is to small small numbers - should be a no-op
+      const result0 = await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint
+      });
+      expect(result0.buckets).toEqual(0);
+
+      // This should cache the checksums for the two buckets
+      const result1 = await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint,
+        minBucketChanges: 1
+      });
+      expect(result1.buckets).toEqual(2);
+
+      // This should be a no-op, as the checksums are already cached
+      const result2 = await bucketStorage.populatePersistentChecksumCache({
         maxOpId: checkpoint,
-        signal: new AbortController().signal
+        minBucketChanges: 1
       });
+      expect(result2.buckets).toEqual(0);
 
       const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user2["u1"]', 'by_user2["u2"]']);
       expect(checksumAfter.get('by_user2["u1"]')).toEqual({

package/test/src/util.ts

@@ -1,12 +1,8 @@
 import { env } from './env.js';
-import { mongoTestReportStorageFactoryGenerator, mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 
-export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({
-  url: env.MONGO_TEST_URL,
-  isCI: env.CI
-});
+import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js';
 
-export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFactoryGenerator({
+export const INITIALIZED_MONGO_STORAGE_FACTORY = MongoTestStorageFactoryGenerator({
   url: env.MONGO_TEST_URL,
   isCI: env.CI
 });