@powersync/service-module-mongodb-storage 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/test/src/connection-report-storage.test.ts

@@ -1,8 +1,8 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js';
+import { MongoReportStorage } from '@module/storage/MongoReportStorage.js';
 import { register, ReportUserData } from '@powersync/service-core-tests';
 import { event_types } from '@powersync/service-types';
-import { MongoReportStorage } from '@module/storage/MongoReportStorage.js';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js';
 
 const userData = register.REPORT_TEST_USERS;
 const dates = register.REPORT_TEST_DATES;

package/test/src/setup.ts

@@ -1,6 +1,6 @@
 import { container } from '@powersync/lib-services-framework';
-import { beforeAll, beforeEach } from 'vitest';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { beforeAll, beforeEach } from 'vitest';
 
 beforeAll(async () => {
   // Executes for every test file

package/test/src/storage.test.ts

@@ -1,17 +1,19 @@
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 import { env } from './env.js';
-import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
-describe('Mongo Sync Bucket Storage - Parameters', () =>
-  register.registerDataStorageParameterTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+for (let storageVersion of TEST_STORAGE_VERSIONS) {
+  describe(`Mongo Sync Bucket Storage - Parameters - v${storageVersion}`, () =>
+    register.registerDataStorageParameterTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
 
-describe('Mongo Sync Bucket Storage - Data', () =>
-  register.registerDataStorageDataTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+  describe(`Mongo Sync Bucket Storage - Data - v${storageVersion}`, () =>
+    register.registerDataStorageDataTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
 
-describe('Mongo Sync Bucket Storage - Checkpoints', () =>
-  register.registerDataStorageCheckpointTests(INITIALIZED_MONGO_STORAGE_FACTORY));
+  describe(`Mongo Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
+    register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion }));
+}
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 

package/test/src/storage_compacting.test.ts

@@ -1,43 +1,53 @@
-import { bucketRequest, bucketRequests, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
+import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
+import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
+import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
-import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
 
 describe('Mongo Sync Bucket Storage Compact', () => {
   register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
 
   describe('with blank bucket_state', () => {
     // This can happen when migrating from older service versions, that did not populate bucket_state yet.
-    const populate = async (bucketStorage: SyncRulesBucketStorage) => {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        await batch.save({
-          sourceTable: TEST_TABLE,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 't1',
-            owner_id: 'u1'
-          },
-          afterReplicaId: test_utils.rid('t1')
-        });
-
-        await batch.save({
-          sourceTable: TEST_TABLE,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 't2',
-            owner_id: 'u2'
-          },
-          afterReplicaId: test_utils.rid('t2')
-        });
+    const populate = async (bucketStorage: SyncRulesBucketStorage, sourceTableIndex: number) => {
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+
+      const sourceTable = await test_utils.resolveTestTable(
+        writer,
+        'test',
+        ['id'],
+        INITIALIZED_MONGO_STORAGE_FACTORY,
+        sourceTableIndex
+      );
+      await writer.markAllSnapshotDone('1/1');
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          owner_id: 'u1'
+        },
+        afterReplicaId: test_utils.rid('t1')
+      });
 
-        await batch.commit('1/1');
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't2',
+          owner_id: 'u2'
+        },
+        afterReplicaId: test_utils.rid('t2')
       });
 
+      await writer.commit('1/1');
+
       return bucketStorage.getCheckpoint();
     };
 
     const setup = async () => {
-      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY();
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -47,16 +57,23 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
-      const { checkpoint } = await populate(bucketStorage);
+      const { checkpoint } = await populate(bucketStorage, 1);
 
       return { bucketStorage, checkpoint, factory, syncRules };
     };
 
     test('full compact', async () => {
       const { bucketStorage, checkpoint, factory, syncRules } = await setup();
+      const storageDb = bucketStorage.db;
 
       // Simulate bucket_state from old version not being available
-      await factory.db.bucket_state.deleteMany({});
+      if (storageDb.storageConfig.incrementalReprocessing) {
+        // This should actually never happen on V3, but we test this anyway.
+        // Can remove this if it causes issues in the future.
+        await (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id).deleteMany({});
+      } else {
+        await factory.db.bucket_state.deleteMany({});
+      }
 
       await bucketStorage.compact({
         clearBatchLimit: 200,
@@ -68,27 +85,27 @@ bucket_definitions:
         signal: null as any
       });
 
-      const checksumAfter = await bucketStorage.getChecksums(
-        checkpoint,
-        bucketRequests(syncRules, ['by_user["u1"]', 'by_user["u2"]'])
-      );
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user["u1"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user["u1"]'),
+      const users = ['u1', 'u2'];
+      const userRequests = users.map((user) => bucketRequest(syncRules, `by_user["${user}"]`));
+      const [u1Request, u2Request] = userRequests;
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      expect(checksumAfter.get(u1Request.bucket)).toEqual({
+        bucket: u1Request.bucket,
         checksum: -659469718,
         count: 1
       });
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user["u2"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user["u2"]'),
+      expect(checksumAfter.get(u2Request.bucket)).toEqual({
+        bucket: u2Request.bucket,
         checksum: 430217650,
         count: 1
       });
     });
 
     test('populatePersistentChecksumCache', async () => {
-      // Populate old sync rules version
+      // Populate old replication stream
       const { factory } = await setup();
 
-      // Not populate another version (bucket definition name changed)
+      // Now populate another replication stream (bucket definition name changed)
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -98,8 +115,9 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
+      const storageDb = (bucketStorage as any).db;
 
-      await populate(bucketStorage);
+      await populate(bucketStorage, 2);
       const { checkpoint } = await bucketStorage.getCheckpoint();
 
       // Default is to small small numbers - should be a no-op
@@ -122,21 +140,103 @@ bucket_definitions:
       });
       expect(result2.buckets).toEqual(0);
 
-      const checksumAfter = await bucketStorage.getChecksums(
-        checkpoint,
-        bucketRequests(syncRules, ['by_user2["u1"]', 'by_user2["u2"]'])
-      );
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user2["u1"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user2["u1"]'),
+      const users = ['u1', 'u2'];
+      const userRequests = users.map((user) => bucketRequest(syncRules, `by_user2["${user}"]`));
+      const [u1Request, u2Request] = userRequests;
+      const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
+      expect(checksumAfter.get(u1Request.bucket)).toEqual({
+        bucket: u1Request.bucket,
         checksum: -659469718,
         count: 1
       });
-      expect(checksumAfter.get(bucketRequest(syncRules, 'by_user2["u2"]'))).toEqual({
-        bucket: bucketRequest(syncRules, 'by_user2["u2"]'),
+      expect(checksumAfter.get(u2Request.bucket)).toEqual({
+        bucket: u2Request.bucket,
         checksum: 430217650,
         count: 1
       });
     });
+
+    test('dirty bucket discovery handles bigint bucket_state bytes', async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `)
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const storageDb = bucketStorage.db;
+
+      // This simulates bucket_state created using bigint bytes.
+      // This typically happens when buckets get very large (> 2GiB). We don't want to create that much
+      // data in the tests, so we directly insert the bucket_state here.
+      if (storageDb.storageConfig.incrementalReprocessing) {
+        const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id);
+        await bucketStateCollection.insertOne({
+          _id: {
+            d: '1',
+            b: 'global[]'
+          },
+          last_op: 5n,
+          compacted_state: {
+            op_id: 3n,
+            count: 3,
+            checksum: 0n,
+            bytes: 7n
+          },
+          estimate_since_compact: {
+            count: 2,
+            bytes: 5n
+          }
+        });
+      } else {
+        await factory.db.bucket_state.insertOne({
+          _id: {
+            g: bucketStorage.group_id,
+            b: 'global[]'
+          },
+          last_op: 5n,
+          compacted_state: {
+            op_id: 3n,
+            count: 3,
+            checksum: 0n,
+            bytes: 7n
+          },
+          estimate_since_compact: {
+            count: 2,
+            bytes: 5n
+          }
+        });
+      }
+
+      // This test uses a couple of "internal" APIs of the compactor.
+      const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n });
+
+      const dirtyBuckets = compactor.dirtyBucketBatches({
+        minBucketChanges: 1,
+        minChangeRatio: 0.39
+      });
+      const firstBatch = await dirtyBuckets.next();
+
+      expect(firstBatch.done).toBe(false);
+      expect(firstBatch.value).toHaveLength(1);
+      expect(firstBatch.value[0].bucket).toBe('global[]');
+      expect(firstBatch.value[0].estimatedCount).toBe(5);
+      expect(typeof firstBatch.value[0].estimatedCount).toBe('number');
+      expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12);
+
+      const checksumBuckets = await (compactor as any).dirtyBucketBatchForChecksums({
+        minBucketChanges: 1
+      });
+      expect(checksumBuckets).toEqual([
+        {
+          bucket: 'global[]',
+          definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null,
+          estimatedCount: 5
+        }
+      ]);
+    });
   });
 });