@powersync/service-module-mongodb-storage 0.15.0 → 0.15.1

package/test/src/storage_compacting.test.ts

@@ -1,41 +1,48 @@
 import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core';
 import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
+import { MongoCompactor } from '../../src/storage/implementation/MongoCompactor.js';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
 
 describe('Mongo Sync Bucket Storage Compact', () => {
   register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
-  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
   describe('with blank bucket_state', () => {
     // This can happen when migrating from older service versions, that did not populate bucket_state yet.
-    const populate = async (bucketStorage: SyncRulesBucketStorage) => {
-      await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-        await batch.markAllSnapshotDone('1/1');
-
-        await batch.save({
-          sourceTable: TEST_TABLE,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 't1',
-            owner_id: 'u1'
-          },
-          afterReplicaId: test_utils.rid('t1')
-        });
-
-        await batch.save({
-          sourceTable: TEST_TABLE,
-          tag: storage.SaveOperationTag.INSERT,
-          after: {
-            id: 't2',
-            owner_id: 'u2'
-          },
-          afterReplicaId: test_utils.rid('t2')
-        });
-
-        await batch.commit('1/1');
+    const populate = async (bucketStorage: SyncRulesBucketStorage, sourceTableIndex: number) => {
+      await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+
+      const sourceTable = await test_utils.resolveTestTable(
+        writer,
+        'test',
+        ['id'],
+        INITIALIZED_MONGO_STORAGE_FACTORY,
+        sourceTableIndex
+      );
+      await writer.markAllSnapshotDone('1/1');
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't1',
+          owner_id: 'u1'
+        },
+        afterReplicaId: test_utils.rid('t1')
+      });
+
+      await writer.save({
+        sourceTable,
+        tag: storage.SaveOperationTag.INSERT,
+        after: {
+          id: 't2',
+          owner_id: 'u2'
+        },
+        afterReplicaId: test_utils.rid('t2')
       });
 
+      await writer.commit('1/1');
+
       return bucketStorage.getCheckpoint();
     };
 
@@ -50,7 +57,7 @@ bucket_definitions:
     `)
       );
       const bucketStorage = factory.getInstance(syncRules);
-      const { checkpoint } = await populate(bucketStorage);
+      const { checkpoint } = await populate(bucketStorage, 1);
 
       return { bucketStorage, checkpoint, factory, syncRules };
     };
@@ -102,7 +109,7 @@ bucket_definitions:
       );
       const bucketStorage = factory.getInstance(syncRules);
 
-      await populate(bucketStorage);
+      await populate(bucketStorage, 2);
       const { checkpoint } = await bucketStorage.getCheckpoint();
 
       // Default is to small small numbers - should be a no-op
@@ -140,6 +147,68 @@ bucket_definitions:
         count: 1
       });
     });
+
+    test('dirty bucket discovery handles bigint bucket_state bytes', async () => {
+      await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
+      const syncRules = await factory.updateSyncRules(
+        updateSyncRulesFromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `)
+      );
+      const bucketStorage = factory.getInstance(syncRules);
+
+      // This simulates bucket_state created using bigint bytes.
+      // This typically happens when buckets get very large (> 2GiB). We don't want to create that much
+      // data in the tests, so we directly insert the bucket_state here.
+      await factory.db.bucket_state.insertOne({
+        _id: {
+          g: bucketStorage.group_id,
+          b: 'global[]'
+        },
+        last_op: 5n,
+        compacted_state: {
+          op_id: 3n,
+          count: 3,
+          checksum: 0n,
+          bytes: 7n
+        },
+        estimate_since_compact: {
+          count: 2,
+          bytes: 5n
+        }
+      });
+
+      // This test uses a couple of internal APIs of the compactor - there is no simple way
+      // to test this using the current public APIs.
+      const compactor = new MongoCompactor(bucketStorage, (bucketStorage as any).db, {
+        maxOpId: 5n
+      });
+
+      const dirtyBuckets = (compactor as any).dirtyBucketBatches({
+        minBucketChanges: 1,
+        minChangeRatio: 0.39
+      });
+      const firstBatch = await dirtyBuckets.next();
+
+      expect(firstBatch.done).toBe(false);
+      expect(firstBatch.value).toHaveLength(1);
+      expect(firstBatch.value[0].bucket).toBe('global[]');
+      expect(firstBatch.value[0].estimatedCount).toBe(5);
+      expect(typeof firstBatch.value[0].estimatedCount).toBe('number');
+      expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12);
+
+      const checksumBuckets = await (compactor as any).dirtyBucketBatchForChecksums({
+        minBucketChanges: 1
+      });
+      expect(checksumBuckets).toEqual([
+        {
+          bucket: 'global[]',
+          estimatedCount: 5
+        }
+      ]);
+    });
   });
 });
 

package/test/src/storage_sync.test.ts

@@ -8,8 +8,6 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
     storageVersion,
     tableIdStrings: storageConfig.tableIdStrings
   });
-  const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageConfig);
-
   // The split of returned results can vary depending on storage drivers
   test('large batch (2)', async () => {
     // Test syncing a batch of data that is small in count,
@@ -28,56 +26,57 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
       )
     );
     const bucketStorage = factory.getInstance(syncRules);
-    const globalBucket = bucketRequest(syncRules, 'global[]');
 
-    const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
-      const sourceTable = TEST_TABLE;
+    await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
+
+    const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
-      const largeDescription = '0123456789'.repeat(2_000_00);
+    const largeDescription = '0123456789'.repeat(2_000_00);
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test1',
-          description: 'test1'
-        },
-        afterReplicaId: test_utils.rid('test1')
-      });
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'test1',
+        description: 'test1'
+      },
+      afterReplicaId: test_utils.rid('test1')
+    });
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large1',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large1')
-      });
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'large1',
+        description: largeDescription
+      },
+      afterReplicaId: test_utils.rid('large1')
+    });
 
-      // Large enough to split the returned batch
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'large2',
-          description: largeDescription
-        },
-        afterReplicaId: test_utils.rid('large2')
-      });
+    // Large enough to split the returned batch
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'large2',
+        description: largeDescription
+      },
+      afterReplicaId: test_utils.rid('large2')
+    });
 
-      await batch.save({
-        sourceTable,
-        tag: storage.SaveOperationTag.INSERT,
-        after: {
-          id: 'test3',
-          description: 'test3'
-        },
-        afterReplicaId: test_utils.rid('test3')
-      });
+    await writer.save({
+      sourceTable,
+      tag: storage.SaveOperationTag.INSERT,
+      after: {
+        id: 'test3',
+        description: 'test3'
+      },
+      afterReplicaId: test_utils.rid('test3')
     });
 
-    const checkpoint = result!.flushed_op;
+    const flushResult = await writer.flush();
+
+    const checkpoint = flushResult!.flushed_op;
 
     const options: storage.BucketDataBatchOptions = {};
     const batch1 = await test_utils.fromAsync(