@powersync/service-module-mongodb-storage 0.0.0-dev-20260313100403 → 0.0.0-dev-20260515144844

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -1,5 +1,99 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`sync - mongodb > storage v1 > can override priority when subscribing to stream 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#todos|0["a"]",
+          "checksum": -1712802421,
+          "count": 1,
+          "priority": 0,
+          "subscriptions": [
+            {
+              "sub": 0,
+            },
+            {
+              "sub": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#todos|0["b"]",
+          "checksum": -1291414318,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "sub": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": false,
+          "name": "todos",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["a"]",
+      "data": [
+        {
+          "checksum": 2582164875,
+          "data": "{"id":"a","description":"Test 1"}",
+          "object_id": "a",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 0,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["b"]",
+      "data": [
+        {
+          "checksum": 3003552978,
+          "data": "{"id":"b","description":"Test 2"}",
+          "object_id": "b",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > storage v1 > compacting data - invalidate checkpoint 1`] = `
 [
   {
@@ -1095,6 +1189,100 @@ exports[`sync - mongodb > storage v1 > sync updates to parameter query only 2`]
 ]
 `;
 
+exports[`sync - mongodb > storage v2 > can override priority when subscribing to stream 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#todos|0["a"]",
+          "checksum": -1712802421,
+          "count": 1,
+          "priority": 0,
+          "subscriptions": [
+            {
+              "sub": 0,
+            },
+            {
+              "sub": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#todos|0["b"]",
+          "checksum": -1291414318,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "sub": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": false,
+          "name": "todos",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["a"]",
+      "data": [
+        {
+          "checksum": 2582164875,
+          "data": "{"id":"a","description":"Test 1"}",
+          "object_id": "a",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 0,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["b"]",
+      "data": [
+        {
+          "checksum": 3003552978,
+          "data": "{"id":"b","description":"Test 2"}",
+          "object_id": "b",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > storage v2 > compacting data - invalidate checkpoint 1`] = `
 [
   {
@@ -2190,6 +2378,100 @@ exports[`sync - mongodb > storage v2 > sync updates to parameter query only 2`]
 ]
 `;
 
+exports[`sync - mongodb > storage v3 > can override priority when subscribing to stream 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#todos|0["a"]",
+          "checksum": -1712802421,
+          "count": 1,
+          "priority": 0,
+          "subscriptions": [
+            {
+              "sub": 0,
+            },
+            {
+              "sub": 1,
+            },
+          ],
+        },
+        {
+          "bucket": "1#todos|0["b"]",
+          "checksum": -1291414318,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "sub": 1,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": false,
+          "name": "todos",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["a"]",
+      "data": [
+        {
+          "checksum": 2582164875,
+          "data": "{"id":"a","description":"Test 1"}",
+          "object_id": "a",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "partial_checkpoint_complete": {
+      "last_op_id": "2",
+      "priority": 0,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#todos|0["b"]",
+      "data": [
+        {
+          "checksum": 3003552978,
+          "data": "{"id":"b","description":"Test 2"}",
+          "object_id": "b",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0dfe86bd-d15b-5fd0-9c7b-a31693030ee0",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > storage v3 > compacting data - invalidate checkpoint 1`] = `
 [
   {

package/test/src/{client-connections-storage.test.ts → connection-report-storage.test.ts}

@@ -1,8 +1,8 @@
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js';
+import { MongoReportStorage } from '@module/storage/MongoReportStorage.js';
 import { register, ReportUserData } from '@powersync/service-core-tests';
 import { event_types } from '@powersync/service-types';
-import { MongoReportStorage } from '@module/storage/MongoReportStorage.js';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js';
 
 const userData = register.REPORT_TEST_USERS;
 const dates = register.REPORT_TEST_DATES;
@@ -128,7 +128,7 @@ describe('Connection reporting storage', async () => {
     await factory.deleteOldConnectionData({
       date: dates.weekAgo
     });
-    const connection = await factory.getClientConnectionsSummary({
+    const connection = await factory.getClientConnectionReports({
       start: dates.monthAgo,
       end: dates.now
     });

package/test/src/setup.ts

@@ -1,6 +1,6 @@
 import { container } from '@powersync/lib-services-framework';
-import { beforeAll, beforeEach } from 'vitest';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
+import { beforeAll, beforeEach } from 'vitest';
 
 beforeAll(async () => {
   // Executes for every test file

package/test/src/storage.test.ts

@@ -1,8 +1,8 @@
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { register } from '@powersync/service-core-tests';
 import { describe } from 'vitest';
-import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 import { env } from './env.js';
-import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
 
 for (let storageVersion of TEST_STORAGE_VERSIONS) {
   describe(`Mongo Sync Bucket Storage - Parameters - v${storageVersion}`, () =>

package/test/src/storage_compacting.test.ts

@@ -1,3 +1,4 @@
+import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
 import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
@@ -63,9 +64,16 @@ bucket_definitions:
 
     test('full compact', async () => {
       const { bucketStorage, checkpoint, factory, syncRules } = await setup();
+      const storageDb = bucketStorage.db;
 
       // Simulate bucket_state from old version not being available
-      await factory.db.bucket_state.deleteMany({});
+      if (storageDb.storageConfig.incrementalReprocessing) {
+        // This should actually never happen on V3, but we test this anyway.
+        // Can remove this if it causes issues in the future.
+        await (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id).deleteMany({});
+      } else {
+        await factory.db.bucket_state.deleteMany({});
+      }
 
       await bucketStorage.compact({
         clearBatchLimit: 200,
@@ -94,10 +102,10 @@ bucket_definitions:
     });
 
     test('populatePersistentChecksumCache', async () => {
-      // Populate old sync rules version
+      // Populate old replication stream
       const { factory } = await setup();
 
-      // Now populate another version (bucket definition name changed)
+      // Now populate another replication stream (bucket definition name changed)
       const syncRules = await factory.updateSyncRules(
         updateSyncRulesFromYaml(`
 bucket_definitions:
@@ -107,6 +115,7 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
+      const storageDb = (bucketStorage as any).db;
 
       await populate(bucketStorage, 2);
       const { checkpoint } = await bucketStorage.getCheckpoint();
@@ -146,6 +155,88 @@ bucket_definitions:
         count: 1
       });
     });
+
+    test('dirty bucket discovery handles bigint bucket_state bytes', async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `)
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+      const storageDb = bucketStorage.db;
+
+      // This simulates bucket_state created using bigint bytes.
+      // This typically happens when buckets get very large (> 2GiB). We don't want to create that much
+      // data in the tests, so we directly insert the bucket_state here.
+      if (storageDb.storageConfig.incrementalReprocessing) {
+        const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id);
+        await bucketStateCollection.insertOne({
+          _id: {
+            d: '1',
+            b: 'global[]'
+          },
+          last_op: 5n,
+          compacted_state: {
+            op_id: 3n,
+            count: 3,
+            checksum: 0n,
+            bytes: 7n
+          },
+          estimate_since_compact: {
+            count: 2,
+            bytes: 5n
+          }
+        });
+      } else {
+        await factory.db.bucket_state.insertOne({
+          _id: {
+            g: bucketStorage.group_id,
+            b: 'global[]'
+          },
+          last_op: 5n,
+          compacted_state: {
+            op_id: 3n,
+            count: 3,
+            checksum: 0n,
+            bytes: 7n
+          },
+          estimate_since_compact: {
+            count: 2,
+            bytes: 5n
+          }
+        });
+      }
+
+      // This test uses a couple of "internal" APIs of the compactor.
+      const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n });
+
+      const dirtyBuckets = compactor.dirtyBucketBatches({
+        minBucketChanges: 1,
+        minChangeRatio: 0.39
+      });
+      const firstBatch = await dirtyBuckets.next();
+
+      expect(firstBatch.done).toBe(false);
+      expect(firstBatch.value).toHaveLength(1);
+      expect(firstBatch.value[0].bucket).toBe('global[]');
+      expect(firstBatch.value[0].estimatedCount).toBe(5);
+      expect(typeof firstBatch.value[0].estimatedCount).toBe('number');
+      expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12);
+
+      const checksumBuckets = await (compactor as any).dirtyBucketBatchForChecksums({
+        minBucketChanges: 1
+      });
+      expect(checksumBuckets).toEqual([
+        {
+          bucket: 'global[]',
+          definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null,
+          estimatedCount: 5
+        }
+      ]);
+    });
   });
 });