@powersync/service-module-mongodb-storage 0.13.2 → 0.15.0

package/test/src/storage.test.ts

@@ -1,17 +1,19 @@
 import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 import { env } from './env.js';
 import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 
-describe('Mongo Sync Bucket Storage - Parameters', () =>
-  register.registerDataStorageParameterTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+for (let storageVersion of TEST_STORAGE_VERSIONS) {
+  describe(`Mongo Sync Bucket Storage - Parameters - v${storageVersion}`, () =>
+    register.registerDataStorageParameterTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
 
-describe('Mongo Sync Bucket Storage - Data', () =>
-  register.registerDataStorageDataTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+  describe(`Mongo Sync Bucket Storage - Data - v${storageVersion}`, () =>
+    register.registerDataStorageDataTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
 
-describe('Mongo Sync Bucket Storage - Checkpoints', () =>
-  register.registerDataStorageCheckpointTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+  describe(`Mongo Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
+    register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
+}
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 

package/test/src/storage_compacting.test.ts

@@ -1,15 +1,18 @@
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
-import { storage, SyncRulesBucketStorage } from '@powersync/service-core';
 
 describe('Mongo Sync Bucket Storage Compact', () => {
   register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
   describe('with blank bucket_state', () => {
     // This can happen when migrating from older service versions, that did not populate bucket_state yet.
     const populate = async (bucketStorage: SyncRulesBucketStorage) => {
       await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+        await batch.markAllSnapshotDone('1/1');
+
         await batch.save({
           sourceTable: TEST_TABLE,
           tag: storage.SaveOperationTag.INSERT,
@@ -37,23 +40,23 @@ describe('Mongo Sync Bucket Storage Compact', () => {
     };
 
     const setup = async () => {
-      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
-      const syncRules = await factory.updateSyncRules({
-        content: `
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
 bucket_definitions:
   by_user:
     parameters: select request.user_id() as user_id
     data: [select * from test where owner_id = bucket.user_id]
-    `
-      });
+    `)
+      );
       const bucketStorage = factory.getInstance(syncRules);
       const { checkpoint } = await populate(bucketStorage);
 
-      return { bucketStorage, checkpoint, factory };
+      return { bucketStorage, checkpoint, factory, syncRules };
     };
 
     test('full compact', async () => {
-      const { bucketStorage, checkpoint, factory } = await setup();
+      const { bucketStorage, checkpoint, factory, syncRules } = await setup();
 
       // Simulate bucket_state from old version not being available
       await factory.db.bucket_state.deleteMany({});
@@ -68,14 +71,17 @@ bucket_definitions:
         signal: null as any
       });
 
-      const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user["u1"]', 'by_user["u2"]']);
-      expect(checksumAfter.get('by_user["u1"]')).toEqual({
-        bucket: 'by_user["u1"]',
+      const users = ['u1', 'u2'];
+      const userRequests = users.map((user) => bucketRequest(syncRules, `by_user["${user}"]`));
+      const [u1Request, u2Request] = userRequests;
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      expect(checksumAfter.get(u1Request.bucket)).toEqual({
+        bucket: u1Request.bucket,
         checksum: -659469718,
         count: 1
       });
-      expect(checksumAfter.get('by_user["u2"]')).toEqual({
-        bucket: 'by_user["u2"]',
+      expect(checksumAfter.get(u2Request.bucket)).toEqual({
+        bucket: u2Request.bucket,
         checksum: 430217650,
         count: 1
       });
@@ -85,15 +91,15 @@ bucket_definitions:
       // Populate old sync rules version
       const { factory } = await setup();
 
-      // Not populate another version (bucket definition name changed)
-      const syncRules = await factory.updateSyncRules({
-        content: `
+      // Now populate another version (bucket definition name changed)
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
 bucket_definitions:
   by_user2:
     parameters: select request.user_id() as user_id
     data: [select * from test where owner_id = bucket.user_id]
-    `
-      });
+    `)
+      );
       const bucketStorage = factory.getInstance(syncRules);
 
       await populate(bucketStorage);
@@ -119,14 +125,17 @@ bucket_definitions:
       });
       expect(result2.buckets).toEqual(0);
 
-      const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user2["u1"]', 'by_user2["u2"]']);
-      expect(checksumAfter.get('by_user2["u1"]')).toEqual({
-        bucket: 'by_user2["u1"]',
+      const users = ['u1', 'u2'];
+      const userRequests = users.map((user) => bucketRequest(syncRules, `by_user2["${user}"]`));
+      const [u1Request, u2Request] = userRequests;
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      expect(checksumAfter.get(u1Request.bucket)).toEqual({
+        bucket: u1Request.bucket,
         checksum: -659469718,
         count: 1
       });
-      expect(checksumAfter.get('by_user2["u2"]')).toEqual({
-        bucket: 'by_user2["u2"]',
+      expect(checksumAfter.get(u2Request.bucket)).toEqual({
+        bucket: u2Request.bucket,
         checksum: 430217650,
         count: 1
       });

package/test/src/storage_sync.test.ts

@@ -1,26 +1,34 @@
-import { storage } from '@powersync/service-core';
-import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
-describe('sync - mongodb', () => {
-  register.registerSyncTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
+  register.registerSyncTests(storageConfig.factory, {
+    storageVersion,
+    tableIdStrings: storageConfig.tableIdStrings
+  });
+  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageConfig);
 
   // The split of returned results can vary depending on storage drivers
   test('large batch (2)', async () => {
     // Test syncing a batch of data that is small in count,
     // but large enough in size to be split over multiple returned chunks.
     // Similar to the above test, but splits over 1MB chunks.
-    const sync_rules = test_utils.testRules(
-      `
+    await using factory = await storageConfig.factory();
+    const syncRules = await factory.updateSyncRules(
+      updateSyncRulesFromYaml(
+        `
     bucket_definitions:
       global:
         data:
           - SELECT id, description FROM "%"
-    `
+    `,
+        { storageVersion }
+      )
     );
-    await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
-    const bucketStorage = factory.getInstance(sync_rules);
+    const bucketStorage = factory.getInstance(syncRules);
+    const globalBucket = bucketRequest(syncRules, 'global[]');
 
     const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
       const sourceTable = TEST_TABLE;
@@ -72,9 +80,8 @@ describe('sync - mongodb', () => {
     const checkpoint = result!.flushed_op;
 
     const options: storage.BucketDataBatchOptions = {};
-
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)], options)
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -89,7 +96,7 @@ describe('sync - mongodb', () => {
     const batch2 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch1[0].chunkData.next_after)]]),
+        [bucketRequest(syncRules, 'global[]', batch1[0].chunkData.next_after)],
         options
       )
     );
@@ -105,7 +112,7 @@ describe('sync - mongodb', () => {
     const batch3 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([['global[]', BigInt(batch2[0].chunkData.next_after)]]),
+        [bucketRequest(syncRules, 'global[]', batch2[0].chunkData.next_after)],
         options
       )
     );
@@ -120,9 +127,18 @@ describe('sync - mongodb', () => {
 
     // Test that the checksum type is correct.
     // Specifically, test that it never persisted as double.
-    const checksumTypes = await factory.db.bucket_data
+    const mongoFactory = factory as any;
+    const checksumTypes = await mongoFactory.db.bucket_data
       .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }])
       .toArray();
     expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]);
   });
+}
+
+describe('sync - mongodb', () => {
+  for (const storageVersion of TEST_STORAGE_VERSIONS) {
+    describe(`storage v${storageVersion}`, () => {
+      registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
+    });
+  }
 });

package/test/src/util.ts

@@ -1,5 +1,6 @@
-import { env } from './env.js';
 import { mongoTestReportStorageFactoryGenerator, mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core';
+import { env } from './env.js';
 
 export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({
   url: env.MONGO_TEST_URL,
@@ -10,3 +11,5 @@ export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFa
   url: env.MONGO_TEST_URL,
   isCI: env.CI
 });
+
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;