@powersync/service-module-mongodb-storage 0.12.1 → 0.12.3

package/test/src/storage.test.ts

@@ -1,7 +1,92 @@
 import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { env } from './env.js';
+import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js';
+import { MongoChecksumOptions } from '@module/storage/implementation/MongoChecksums.js';
 
-describe('Mongo Sync Bucket Storage', () => register.registerDataStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+describe('Mongo Sync Bucket Storage - Parameters', () =>
+  register.registerDataStorageParameterTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+
+describe('Mongo Sync Bucket Storage - Data', () =>
+  register.registerDataStorageDataTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+
+describe('Mongo Sync Bucket Storage - Checkpoints', () =>
+  register.registerDataStorageCheckpointTests(INITIALIZED_MONGO_STORAGE_FACTORY));
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
+
+describe('Mongo Sync Bucket Storage - split operations', () =>
+  register.registerDataStorageDataTests(
+    MongoTestStorageFactoryGenerator({
+      url: env.MONGO_TEST_URL,
+      isCI: env.CI,
+      internalOptions: {
+        checksumOptions: {
+          bucketBatchLimit: 100,
+          operationBatchLimit: 1
+        }
+      }
+    })
+  ));
+
+describe('Mongo Sync Bucket Storage - split buckets', () =>
+  register.registerDataStorageDataTests(
+    MongoTestStorageFactoryGenerator({
+      url: env.MONGO_TEST_URL,
+      isCI: env.CI,
+      internalOptions: {
+        checksumOptions: {
+          bucketBatchLimit: 1,
+          operationBatchLimit: 100
+        }
+      }
+    })
+  ));
+
+describe('Mongo Sync Bucket Storage - checksum calculations', () => {
+  // This test tests 4 buckets x 4 operations in each.
+  // We specifically use operationBatchLimit that does not have factors in common with 4,
+  // as well some that do.
+  const params: MongoChecksumOptions[] = [
+    {
+      bucketBatchLimit: 100,
+      operationBatchLimit: 3
+    },
+
+    {
+      bucketBatchLimit: 10,
+      operationBatchLimit: 7
+    },
+
+    {
+      bucketBatchLimit: 3,
+      operationBatchLimit: 1
+    },
+    {
+      bucketBatchLimit: 1,
+      operationBatchLimit: 3
+    },
+    {
+      bucketBatchLimit: 2,
+      operationBatchLimit: 4
+    },
+    {
+      bucketBatchLimit: 4,
+      operationBatchLimit: 12
+    }
+  ];
+  for (let options of params) {
+    describe(`${options.bucketBatchLimit}|${options.operationBatchLimit}`, () => {
+      register.testChecksumBatching(
+        MongoTestStorageFactoryGenerator({
+          url: env.MONGO_TEST_URL,
+          isCI: env.CI,
+          internalOptions: {
+            checksumOptions: options
+          }
+        })
+      );
+    });
+  }
+});

package/test/src/storage_compacting.test.ts

@@ -1,7 +1,122 @@
-import { register } from '@powersync/service-core-tests';
-import { describe } from 'vitest';
+import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { storage, SyncRulesBucketStorage } from '@powersync/service-core';
 
-describe('Mongo Sync Bucket Storage Compact', () => register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY));
-describe('Mongo Sync Parameter Storage Compact', () =>
-  register.registerParameterCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+describe('Mongo Sync Bucket Storage Compact', () => {
+  register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+
+  describe('with blank bucket_state', () => {
+    // This can happen when migrating from older service versions, that did not populate bucket_state yet.
+    const populate = async (bucketStorage: SyncRulesBucketStorage) => {
+      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't1',
+            owner_id: 'u1'
+          },
+          afterReplicaId: test_utils.rid('t1')
+        });
+
+        await batch.save({
+          sourceTable: TEST_TABLE,
+          tag: storage.SaveOperationTag.INSERT,
+          after: {
+            id: 't2',
+            owner_id: 'u2'
+          },
+          afterReplicaId: test_utils.rid('t2')
+        });
+
+        await batch.commit('1/1');
+      });
+
+      return bucketStorage.getCheckpoint();
+    };
+
+    const setup = async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+      const syncRules = await factory.updateSyncRules({
+        content: `
+bucket_definitions:
+  by_user:
+    parameters: select request.user_id() as user_id
+    data: [select * from test where owner_id = bucket.user_id]
+    `
+      });
+      const bucketStorage = factory.getInstance(syncRules);
+      const { checkpoint } = await populate(bucketStorage);
+
+      return { bucketStorage, checkpoint, factory };
+    };
+
+    test('full compact', async () => {
+      const { bucketStorage, checkpoint, factory } = await setup();
+
+      // Simulate bucket_state from old version not being available
+      await factory.db.bucket_state.deleteMany({});
+
+      await bucketStorage.compact({
+        clearBatchLimit: 200,
+        moveBatchLimit: 10,
+        moveBatchQueryLimit: 10,
+        maxOpId: checkpoint,
+        signal: null as any
+      });
+
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user["u1"]', 'by_user["u2"]']);
+      expect(checksumAfter.get('by_user["u1"]')).toEqual({
+        bucket: 'by_user["u1"]',
+        checksum: -659469718,
+        count: 1
+      });
+      expect(checksumAfter.get('by_user["u2"]')).toEqual({
+        bucket: 'by_user["u2"]',
+        checksum: 430217650,
+        count: 1
+      });
+    });
+
+    test('populatePersistentChecksumCache', async () => {
+      // Populate old sync rules version
+      const { factory } = await setup();
+
+      // Not populate another version (bucket definition name changed)
+      const syncRules = await factory.updateSyncRules({
+        content: `
+bucket_definitions:
+  by_user2:
+    parameters: select request.user_id() as user_id
+    data: [select * from test where owner_id = bucket.user_id]
+    `
+      });
+      const bucketStorage = factory.getInstance(syncRules);
+
+      await populate(bucketStorage);
+      const { checkpoint } = await bucketStorage.getCheckpoint();
+
+      await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint,
+        signal: new AbortController().signal
+      });
+
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user2["u1"]', 'by_user2["u2"]']);
+      expect(checksumAfter.get('by_user2["u1"]')).toEqual({
+        bucket: 'by_user2["u1"]',
+        checksum: -659469718,
+        count: 1
+      });
+      expect(checksumAfter.get('by_user2["u2"]')).toEqual({
+        bucket: 'by_user2["u2"]',
+        checksum: 430217650,
+        count: 1
+      });
+    });
+  });
+});
+
+describe('Mongo Sync Parameter Storage Compact', () => {
+  register.registerParameterCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+});