@powersync/service-module-mongodb-storage 0.22.0 → 0.23.0

package/src/utils/test-utils.ts

@@ -7,12 +7,13 @@ import { PowerSyncMongo } from '../storage/implementation/db.js';
 export type MongoTestStorageOptions = {
   url: string;
   isCI: boolean;
+  monitorCommands?: boolean;
 } & Omit<MongoBucketStorageOptions, 'replicationStreamNamePrefix'>;
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
   return {
     factory: async (options?: TestStorageOptions) => {
-      const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+      const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.monitorCommands);
 
       // None of the tests insert data into this collection, so it was never created
       if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
@@ -41,7 +42,7 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
 
 export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
   return async (options?: TestStorageOptions) => {
-    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.monitorCommands);
 
     await db.createConnectionReportingCollection();
 
@@ -53,10 +54,11 @@ export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTest
   };
 }
 
-export const connectMongoForTests = (url: string, isCI: boolean) => {
+export const connectMongoForTests = (url: string, isCI: boolean, monitorCommands = false) => {
   // Short timeout for tests, to fail fast when the server is not available.
   // Slightly longer timeouts for CI, to avoid arbitrary test failures
   const client = new mongo.MongoClient(url, {
+    monitorCommands,
     connectTimeoutMS: isCI ? 15_000 : 5_000,
     socketTimeoutMS: isCI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: isCI ? 15_000 : 2_500

package/test/src/__snapshots__/storage.test.ts.snap

@@ -100,7 +100,7 @@ exports[`Mongo Sync Bucket Storage - Data - v4 > (insert, delete, insert), (dele
 ]
 `;
 
-exports[`Mongo Sync Bucket Storage - split buckets > (insert, delete, insert), (delete) 1`] = `
+exports[`Mongo Sync Bucket Storage - split buckets - v1 > (insert, delete, insert), (delete) 1`] = `
 [
   {
     "checksum": 2871785649,
@@ -125,7 +125,107 @@ exports[`Mongo Sync Bucket Storage - split buckets > (insert, delete, insert), (
 ]
 `;
 
-exports[`Mongo Sync Bucket Storage - split operations > (insert, delete, insert), (delete) 1`] = `
+exports[`Mongo Sync Bucket Storage - split buckets - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split buckets - v4 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split operations - v1 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split operations - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split operations - v4 > (insert, delete, insert), (delete) 1`] = `
 [
   {
     "checksum": 2871785649,

package/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -2,7 +2,7 @@
 
 exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting (2) 1`] = `
 {
-  "bucket": "1#global[]",
+  "bucket": "global.1.1[]",
   "checksum": -1481659821,
   "count": 1,
 }
@@ -10,7 +10,7 @@ exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting
 
 exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting 1`] = `
 {
-  "bucket": "1#global[]",
+  "bucket": "global.1.1[]",
   "checksum": 1874612650,
   "count": 4,
 }

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -270,7 +270,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for stre
           ],
         },
       ],
-      "last_op_id": "2",
+      "last_op_id": "65537",
       "streams": [
         {
           "errors": [],
@@ -292,17 +292,17 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for stre
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
-          "op_id": "2",
+          "op_id": "65537",
           "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56",
         },
       ],
       "has_more": false,
-      "next_after": "2",
+      "next_after": "65537",
     },
   },
   {
     "checkpoint_complete": {
-      "last_op_id": "2",
+      "last_op_id": "65537",
     },
   },
 ]
@@ -1265,7 +1265,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for stre
           ],
         },
       ],
-      "last_op_id": "2",
+      "last_op_id": "65537",
       "streams": [
         {
           "errors": [],
@@ -1287,17 +1287,17 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for stre
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
-          "op_id": "2",
+          "op_id": "65537",
           "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56",
         },
       ],
       "has_more": false,
-      "next_after": "2",
+      "next_after": "65537",
     },
   },
   {
     "checkpoint_complete": {
-      "last_op_id": "2",
+      "last_op_id": "65537",
     },
   },
 ]
@@ -2849,7 +2849,7 @@ exports[`sync - mongodb > storage v4 > encodes sync rules id in buckets for stre
           ],
         },
       ],
-      "last_op_id": "2",
+      "last_op_id": "65537",
       "streams": [
         {
           "errors": [],
@@ -2871,17 +2871,17 @@ exports[`sync - mongodb > storage v4 > encodes sync rules id in buckets for stre
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
-          "op_id": "2",
+          "op_id": "65537",
           "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56",
         },
       ],
       "has_more": false,
-      "next_after": "2",
+      "next_after": "65537",
     },
   },
   {
     "checkpoint_complete": {
-      "last_op_id": "2",
+      "last_op_id": "65537",
     },
   },
 ]

package/test/src/cleanup-stopped-sync-configs.test.ts

@@ -2,7 +2,7 @@ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/Mong
 import { ObjectStorageLifecycle } from '@module/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js';
 import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { test_utils } from '@powersync/service-core-tests';
+import { getTestStorage, test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
@@ -78,7 +78,7 @@ streams:
         { storageVersion: 3 }
       )
     );
-    const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
+    const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
     const db = firstStorage.db as VersionedPowerSyncMongoV3;
     await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
 
@@ -155,7 +155,7 @@ streams:
     );
     expect(second.replicationStreamId).toBe(first.replicationStreamId);
     const replicatingStreams = await factory.getReplicatingReplicationStreams();
-    const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
+    const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
     await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
     await secondWriter.markAllSnapshotDone('2/1');
     await secondWriter.commit('2/1');
@@ -218,7 +218,7 @@ streams:
         { storageVersion: 3 }
       )
     );
-    const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
+    const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
     const db = firstStorage.db as VersionedPowerSyncMongoV3;
     await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
     const resolved = await firstWriter.resolveTables({
@@ -266,7 +266,7 @@ streams:
     expect(secondDefinitionId).toBe(projectDefinitionId);
 
     const replicatingStreams = await factory.getReplicatingReplicationStreams();
-    const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
+    const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
     await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
     await secondWriter.markAllSnapshotDone('2/1');
     await secondWriter.commit('2/1');
@@ -289,7 +289,10 @@ streams:
     expect(await collectionExists(db, ownerBucketDataCollection)).toBe(false);
     expect(await collectionExists(db, sourceRecordsCollection)).toBe(true);
 
-    const activeStorage = (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3;
+    const activeStorage = (await getTestStorage(
+      factory,
+      (await factory.getActiveSyncConfig())!.replicationStream
+    )) as MongoSyncBucketStorageV3;
     await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
     const activeTable = (
       await activeWriter.resolveTables({
@@ -343,7 +346,7 @@ event_definitions:
         { storageVersion: 3 }
       )
     );
-    const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
+    const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
     const db = firstStorage.db as VersionedPowerSyncMongoV3;
     await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
     const resolved = await firstWriter.resolveTables({
@@ -402,7 +405,7 @@ event_definitions:
     expect(second.replicationStreamId).toBe(first.replicationStreamId);
 
     const replicatingStreams = await factory.getReplicatingReplicationStreams();
-    const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
+    const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
     await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
     await secondWriter.markAllSnapshotDone('2/1');
     await secondWriter.commit('2/1');
@@ -466,7 +469,7 @@ event_definitions:
         { storageVersion: 3 }
       )
     );
-    const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
+    const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
     const db = firstStorage.db as VersionedPowerSyncMongoV3;
     await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
 
@@ -538,7 +541,7 @@ streams:
     expect(second.replicationStreamId).toBe(first.replicationStreamId);
 
     const replicatingStreams = await factory.getReplicatingReplicationStreams();
-    const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
+    const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
     await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
     await secondWriter.markAllSnapshotDone('2/1');
     await secondWriter.commit('2/1');
@@ -599,7 +602,7 @@ streams:
         { storageVersion: 3 }
       )
     );
-    const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
+    const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
     const db = firstStorage.db as VersionedPowerSyncMongoV3;
     await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
     const resolved = await firstWriter.resolveTables({
@@ -652,7 +655,7 @@ streams:
     expect(secondIndexId).toBe(roleIndexId);
 
     const replicatingStreams = await factory.getReplicatingReplicationStreams();
-    const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
+    const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
     await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
     await secondWriter.markAllSnapshotDone('2/1');
     await secondWriter.commit('2/1');
@@ -664,7 +667,10 @@ streams:
     expect(await collectionExists(db, orgParameterIndexCollection)).toBe(false);
     expect(await collectionExists(db, sourceRecordsCollection)).toBe(true);
 
-    const activeStorage = (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3;
+    const activeStorage = (await getTestStorage(
+      factory,
+      (await factory.getActiveSyncConfig())!.replicationStream
+    )) as MongoSyncBucketStorageV3;
     await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
     const activeTable = (
       await activeWriter.resolveTables({

package/test/src/compact-utils.test.ts

@@ -3,12 +3,12 @@ import {
   applyStatsReplacement,
   bucketStats,
   chooseCompactionKind,
+  chooseRequestedCompactionKind,
   combineAdjacentStats,
   combineChunkStats,
   CompactIntervalConfig,
   CompactionKind,
   emptyBucketStats,
-  forcedCompactionKind,
   fullCompactionCheckAt,
   statsForDocument,
   statsForDocuments
@@ -130,44 +130,69 @@ describe('V3 compact utilities', () => {
   test.each([
     {
       name: 'does not force an unspecified kind',
-      forceKind: undefined,
+      requestedKind: undefined,
       compactedOpId: undefined,
       maxOpIdCap: undefined,
       expected: null
     },
     {
       name: 'forces work without previous compacted state',
-      forceKind: CompactionKind.Chunks,
+      requestedKind: CompactionKind.Chunks,
       compactedOpId: undefined,
       maxOpIdCap: undefined,
       expected: CompactionKind.Chunks
     },
     {
       name: 'skips work already compacted through the cap',
-      forceKind: CompactionKind.Chunks,
+      requestedKind: CompactionKind.Chunks,
       compactedOpId: 5n,
       maxOpIdCap: 5n,
       expected: null
     },
     {
       name: 'skips work already compacted through the bucket head',
-      forceKind: CompactionKind.Chunks,
+      requestedKind: CompactionKind.Chunks,
       compactedOpId: 10n,
       maxOpIdCap: undefined,
       expected: null
     },
     {
       name: 'forces work beyond the compacted state',
-      forceKind: CompactionKind.Chunks,
+      requestedKind: CompactionKind.Chunks,
       compactedOpId: 5n,
       maxOpIdCap: 6n,
       expected: CompactionKind.Chunks
     }
-  ])('$name', ({ forceKind, compactedOpId, maxOpIdCap, expected }) => {
+  ])('$name', ({ requestedKind, compactedOpId, maxOpIdCap, expected }) => {
     const state = bucketState({
+      bucket_stats: { count: 10, bytes: 100n, chunks: 9 },
       compacted_state: compactedOpId == null ? undefined : compactedState({ op_id: compactedOpId })
     });
-    expect(forcedCompactionKind(state, forceKind, { maxOpIdCap })).toBe(expected);
+    expect(chooseRequestedCompactionKind(state, requestedKind, { maxOpIdCap })).toBe(expected);
+  });
+
+  test.each([undefined, compactedState({ at: new Date() })])(
+    'initial chunk compaction requires eight new chunks, regardless of the interval: %o',
+    (compacted) => {
+      for (const newChunks of [1, 7, 8]) {
+        const state = bucketState({
+          bucket_stats: { count: 10, bytes: 100n, chunks: (compacted?.chunks ?? 0) + newChunks },
+          compacted_state: compacted
+        });
+        expect(chooseRequestedCompactionKind(state, CompactionKind.Chunks, { maxOpIdCap: undefined })).toBe(
+          newChunks < 8 ? null : CompactionKind.Chunks
+        );
+      }
+    }
+  );
+
+  test('the testing override bypasses the chunk threshold but preserves the compaction watermark', () => {
+    const state = bucketState();
+    const config = { maxOpIdCap: undefined, forceChunkCompaction: true };
+    expect(chooseRequestedCompactionKind(state, CompactionKind.Chunks, config)).toBe(CompactionKind.Chunks);
+    expect(chooseRequestedCompactionKind(state, undefined, config)).toBeNull();
+    state.compacted_state = compactedState({ op_id: state.last_op });
+    expect(chooseRequestedCompactionKind(state, CompactionKind.Chunks, config)).toBeNull();
   });
 
   test('derives and combines bucket statistics', () => {