@powersync/service-module-mongodb-storage 0.14.0 → 0.15.1

package/test/src/storage.test.ts

@@ -1,17 +1,19 @@
 import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 import { env } from './env.js';
 import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 
-describe('Mongo Sync Bucket Storage - Parameters', () =>
-  register.registerDataStorageParameterTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+for (let storageVersion of TEST_STORAGE_VERSIONS) {
+  describe(`Mongo Sync Bucket Storage - Parameters - v${storageVersion}`, () =>
+    register.registerDataStorageParameterTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
 
-describe('Mongo Sync Bucket Storage - Data', () =>
-  register.registerDataStorageDataTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+  describe(`Mongo Sync Bucket Storage - Data - v${storageVersion}`, () =>
+    register.registerDataStorageDataTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
 
-describe('Mongo Sync Bucket Storage - Checkpoints', () =>
-  register.registerDataStorageCheckpointTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+  describe(`Mongo Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
+    register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
+}
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 

package/test/src/storage_compacting.test.ts

@@ -1,43 +1,53 @@
-import { bucketRequest, bucketRequests, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
+import { MongoCompactor } from '../../src/storage/implementation/MongoCompactor.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
-import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
 
 describe('Mongo Sync Bucket Storage Compact', () => {
   register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 
   describe('with blank bucket_state', () => {
     // This can happen when migrating from older service versions, that did not populate bucket_state yet.
-    const populate = async (bucketStorage: SyncRulesBucketStorage) => {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        await batch.save({
-          sourceTable: TEST_TABLE,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 't1',
-            owner_id: 'u1'
-          },
-          afterReplicaId: test_utils.rid('t1')
-        });
-
-        await batch.save({
-          sourceTable: TEST_TABLE,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 't2',
-            owner_id: 'u2'
-          },
-          afterReplicaId: test_utils.rid('t2')
-        });
-
-        await batch.commit('1/1');
+    const populate = async (bucketStorage: SyncRulesBucketStorage, sourceTableIndex: number) => {
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+
+      const sourceTable = await test_utils.resolveTestTable(
+        writer,
+        'test',
+        ['id'],
+        INITIALIZED_MONGO_STORAGE_FACTORY,
+        sourceTableIndex
+      );
+      await writer.markAllSnapshotDone('1/1');
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          owner_id: 'u1'
+        },
+        afterReplicaId: test_utils.rid('t1')
       });
 
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't2',
+          owner_id: 'u2'
+        },
+        afterReplicaId: test_utils.rid('t2')
+      });
+
+      await writer.commit('1/1');
+
       return bucketStorage.getCheckpoint();
     };
 
     const setup = async () => {
-      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -47,7 +57,7 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
-      const { checkpoint } = await populate(bucketStorage);
+      const { checkpoint } = await populate(bucketStorage, 1);
 
       return { bucketStorage, checkpoint, factory, syncRules };
     };
@@ -68,17 +78,17 @@ bucket_definitions:
         signal: null as any
       });
 
-      const checksumAfter = await bucketStorage.getChecksums(
-        checkpoint,
-        bucketRequests(syncRules, ['by_user["u1"]', 'by_user["u2"]'])
-      );
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user["u1"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user["u1"]'),
+      const users = ['u1', 'u2'];
+      const userRequests = users.map((user) => bucketRequest(syncRules, `by_user["${user}"]`));
+      const [u1Request, u2Request] = userRequests;
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      expect(checksumAfter.get(u1Request.bucket)).toEqual({
+        bucket: u1Request.bucket,
         checksum: -659469718,
         count: 1
       });
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user["u2"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user["u2"]'),
+      expect(checksumAfter.get(u2Request.bucket)).toEqual({
+        bucket: u2Request.bucket,
         checksum: 430217650,
         count: 1
       });
@@ -88,7 +98,7 @@ bucket_definitions:
       // Populate old sync rules version
       const { factory } = await setup();
 
-      // Not populate another version (bucket definition name changed)
+      // Now populate another version (bucket definition name changed)
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -99,7 +109,7 @@ bucket_definitions:
       );
       const bucketStorage = factory.getInstance(syncRules);
 
-      await populate(bucketStorage);
+      await populate(bucketStorage, 2);
       const { checkpoint } = await bucketStorage.getCheckpoint();
 
       // Default is to small small numbers - should be a no-op
@@ -122,21 +132,83 @@ bucket_definitions:
       });
       expect(result2.buckets).toEqual(0);
 
-      const checksumAfter = await bucketStorage.getChecksums(
-        checkpoint,
-        bucketRequests(syncRules, ['by_user2["u1"]', 'by_user2["u2"]'])
-      );
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user2["u1"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user2["u1"]'),
+      const users = ['u1', 'u2'];
+      const userRequests = users.map((user) => bucketRequest(syncRules, `by_user2["${user}"]`));
+      const [u1Request, u2Request] = userRequests;
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      expect(checksumAfter.get(u1Request.bucket)).toEqual({
+        bucket: u1Request.bucket,
         checksum: -659469718,
         count: 1
       });
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user2["u2"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user2["u2"]'),
+      expect(checksumAfter.get(u2Request.bucket)).toEqual({
+        bucket: u2Request.bucket,
         checksum: 430217650,
         count: 1
       });
     });
+
+    test('dirty bucket discovery handles bigint bucket_state bytes', async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `)
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+
+      // This simulates bucket_state created using bigint bytes.
+      // This typically happens when buckets get very large (> 2GiB). We don't want to create that much
+      // data in the tests, so we directly insert the bucket_state here.
+      await factory.db.bucket_state.insertOne({
+        _id: {
+          g: bucketStorage.group_id,
+          b: 'global[]'
+        },
+        last_op: 5n,
+        compacted_state: {
+          op_id: 3n,
+          count: 3,
+          checksum: 0n,
+          bytes: 7n
+        },
+        estimate_since_compact: {
+          count: 2,
+          bytes: 5n
+        }
+      });
+
+      // This test uses a couple of internal APIs of the compactor - there is no simple way
+      // to test this using the current public APIs.
+      const compactor = new MongoCompactor(bucketStorage, (bucketStorage as any).db, {
+        maxOpId: 5n
+      });
+
+      const dirtyBuckets = (compactor as any).dirtyBucketBatches({
+        minBucketChanges: 1,
+        minChangeRatio: 0.39
+      });
+      const firstBatch = await dirtyBuckets.next();
+
+      expect(firstBatch.done).toBe(false);
+      expect(firstBatch.value).toHaveLength(1);
+      expect(firstBatch.value[0].bucket).toBe('global[]');
+      expect(firstBatch.value[0].estimatedCount).toBe(5);
+      expect(typeof firstBatch.value[0].estimatedCount).toBe('number');
+      expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12);
+
+      const checksumBuckets = await (compactor as any).dirtyBucketBatchForChecksums({
+        minBucketChanges: 1
+      });
+      expect(checksumBuckets).toEqual([
+        {
+          bucket: 'global[]',
+          estimatedCount: 5
+        }
+      ]);
+    });
   });
 });
 

package/test/src/storage_sync.test.ts

@@ -1,17 +1,19 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { bucketRequest, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
-function registerSyncStorageTests(storageFactory: storage.TestStorageFactory, storageVersion: number) {
-  register.registerSyncTests(storageFactory, { storageVersion });
-
+function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
+  register.registerSyncTests(storageConfig.factory, {
+    storageVersion,
+    tableIdStrings: storageConfig.tableIdStrings
+  });
   // The split of returned results can vary depending on storage drivers
   test('large batch (2)', async () => {
     // Test syncing a batch of data that is small in count,
     // but large enough in size to be split over multiple returned chunks.
     // Similar to the above test, but splits over 1MB chunks.
-    await using factory = await storageFactory();
+    await using factory = await storageConfig.factory();
     const syncRules = await factory.updateSyncRules(
       updateSyncRulesFromYaml(
         `
@@ -24,61 +26,61 @@ function registerSyncStorageTests(storageFactory: storage.TestStorageFactory, st
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const globalBucket = bucketRequest(syncRules, 'global[]');
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      const sourceTable = TEST_TABLE;
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
 
-      const largeDescription = '0123456789'.repeat(2_000_00);
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test1',
-          description: 'test1'
-        },
-        afterReplicaId: test_utils.rid('test1')
-      });
+    const largeDescription = '0123456789'.repeat(2_000_00);
+
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'test1',
+        description: 'test1'
+      },
+      afterReplicaId: test_utils.rid('test1')
+    });
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large1',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large1')
-      });
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'large1',
+        description: largeDescription
+      },
+      afterReplicaId: test_utils.rid('large1')
+    });
 
-      // Large enough to split the returned batch
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large2',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large2')
-      });
+    // Large enough to split the returned batch
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'large2',
+        description: largeDescription
+      },
+      afterReplicaId: test_utils.rid('large2')
+    });
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test3',
-          description: 'test3'
-        },
-        afterReplicaId: test_utils.rid('test3')
-      });
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'test3',
+        description: 'test3'
+      },
+      afterReplicaId: test_utils.rid('test3')
     });
 
-    const checkpoint = result!.flushed_op;
+    const flushResult = await writer.flush();
 
-    const options: storage.BucketDataBatchOptions = {};
+    const checkpoint = flushResult!.flushed_op;
 
+    const options: storage.BucketDataBatchOptions = {};
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([[globalBucket, 0n]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)], options)
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -93,7 +95,7 @@ function registerSyncStorageTests(storageFactory: storage.TestStorageFactory, st
     const batch2 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([[globalBucket, BigInt(batch1[0].chunkData.next_after)]]),
+        [bucketRequest(syncRules, 'global[]', batch1[0].chunkData.next_after)],
         options
       )
     );
@@ -109,7 +111,7 @@ function registerSyncStorageTests(storageFactory: storage.TestStorageFactory, st
     const batch3 = await test_utils.fromAsync(
       bucketStorage.getBucketDataBatch(
         checkpoint,
-        new Map([[globalBucket, BigInt(batch2[0].chunkData.next_after)]]),
+        [bucketRequest(syncRules, 'global[]', batch2[0].chunkData.next_after)],
         options
       )
     );

package/test/src/util.ts

@@ -1,6 +1,6 @@
-import { env } from './env.js';
 import { mongoTestReportStorageFactoryGenerator, mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
-import { CURRENT_STORAGE_VERSION, LEGACY_STORAGE_VERSION } from '@powersync/service-core';
+import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core';
+import { env } from './env.js';
 
 export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({
   url: env.MONGO_TEST_URL,
@@ -12,4 +12,4 @@ export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFa
   isCI: env.CI
 });
 
-export const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION, CURRENT_STORAGE_VERSION];
+export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;