@powersync/service-module-mongodb-storage 0.22.0 → 0.23.1

package/test/src/object_storage_usage.test.ts

@@ -8,6 +8,7 @@ import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/Ver
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { updateSyncRulesFromYaml } from '@powersync/service-core';
+import { getTestStorage } from '@powersync/service-core-tests';
 import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
@@ -33,7 +34,7 @@ type UsageEntry = {
 async function withUsageContext<T>(callback: (context: UsageContext) => Promise<T>): Promise<T> {
   await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
   const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
-  const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
+  const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage;
   const db = bucketStorage.db as VersionedPowerSyncMongoV3;
   const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
   return callback({ db, bucketStorage, definitionId });

package/test/src/parameter_compacting_v1.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { test_utils } from '@powersync/service-core-tests';
+import { getTestStorage, test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
 import type { SyncRuleDocumentV1 } from '../../src/storage/implementation/v1/models.js';
@@ -20,7 +20,7 @@ async function createActiveStorage() {
   const syncRules = await factory.updateSyncRules(
     updateSyncRulesFromYaml(PARAMETER_RULES, { storageVersion: storage.STORAGE_VERSION_2 })
   );
-  const processingStorage = factory.getInstance(syncRules);
+  const processingStorage = await getTestStorage(factory, syncRules);
   await using writer = await processingStorage.createWriter(test_utils.BATCH_OPTIONS);
   await writer.markAllSnapshotDone('1/1');
   await writer.commit('1/1');

package/test/src/parameter_compacting_v3.test.ts

@@ -1,5 +1,5 @@
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { test_utils } from '@powersync/service-core-tests';
+import { getTestStorage, test_utils } from '@powersync/service-core-tests';
 import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
 import { MongoParameterCompactorV3 } from '../../src/storage/implementation/v3/MongoParameterCompactorV3.js';
@@ -31,7 +31,7 @@ async function createActiveStorage(rules = PARAMETER_RULES) {
   const syncRules = await factory.updateSyncRules(
     updateSyncRulesFromYaml(rules, { storageVersion: storage.STORAGE_VERSION_3 })
   );
-  const processingStorage = factory.getInstance(syncRules);
+  const processingStorage = await getTestStorage(factory, syncRules);
   await using writer = await processingStorage.createWriter(test_utils.BATCH_OPTIONS);
   await writer.markAllSnapshotDone('1/1');
   await writer.commit('1/1');

package/test/src/parameter_compaction_fence.test.ts

@@ -1,6 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { InternalOpId, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { test_utils } from '@powersync/service-core-tests';
+import { getTestStorage, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import type { VersionedPowerSyncMongo } from '../../src/storage/implementation/db.js';
 import type { SyncRuleDocumentBase } from '../../src/storage/implementation/models.js';
@@ -83,7 +83,7 @@ describe('parameter compaction invalidation fence', () => {
        */
       async function replicateParameterHistory(factory: storage.BucketStorageFactory) {
         const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(PARAMETER_RULES, { storageVersion }));
-        const processingStorage = factory.getInstance(syncRules);
+        const processingStorage = await getTestStorage(factory, syncRules);
         const writer = await processingStorage.createWriter(test_utils.BATCH_OPTIONS);
         const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
 
@@ -196,7 +196,7 @@ describe('parameter compaction invalidation fence', () => {
         // which is the only record of t2's lookup.
         //
         // A separate storage instance is used to get a cold checkpoint-changes cache.
-        const coldStorage = (factory as MongoBucketStorage).getInstance(replicationStream);
+        const coldStorage = await getTestStorage(factory as MongoBucketStorage, replicationStream);
         const atSnapshot = await coldStorage.getCheckpointChanges({
           lastCheckpoint: checkpoint1,
           nextCheckpoint: checkpoint3

package/test/src/source_table_config.test.ts

@@ -0,0 +1,97 @@
+import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
+import { updateSyncRulesFromConfig } from '@powersync/service-core';
+import { test_utils } from '@powersync/service-core-tests';
+import { PrecompiledSyncConfig, SqlSyncRules } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+import { env } from './env.js';
+import { TEST_STORAGE_VERSIONS } from './util.js';
+
+const STREAMS = /* yaml */ `
+  # sync config fixture.
+  config:
+    edition: 3
+  streams:
+    orders:
+      query: SELECT * FROM orders
+`;
+const CONFIGURED = /* yaml */ `
+  # sync config fixture.
+  config:
+    edition: 3
+    source_table_options:
+      orders:
+        mongodb_filter_expression: { $eq: ['$$doc.active', true] }
+  streams:
+    orders:
+      query: SELECT * FROM orders
+`;
+
+function storageFactory() {
+  return mongoTestStorageFactoryGenerator({
+    url: env.MONGO_TEST_URL,
+    isCI: env.CI,
+    supportsMultipleSyncConfigs: true
+  });
+}
+
+describe.for(TEST_STORAGE_VERSIONS)('source-table config with storage v%i', (storageVersion) => {
+  test('preserves source-table options across closing storage and loading its compiled plan', async () => {
+    let expected;
+    {
+      await using factory = await storageFactory().factory();
+      const parsed = SqlSyncRules.fromYaml(CONFIGURED, test_utils.PARSE_OPTIONS);
+      expected = (parsed.config as PrecompiledSyncConfig).plan.sourceTableConfig;
+      await factory.updateSyncRules(updateSyncRulesFromConfig(parsed, { storageVersion }));
+    }
+
+    // Reopen the same database without clearing it, as a service restart would.
+    await using reopened = await storageFactory().factory({ doNotClear: true });
+    const deploying = (await reopened.getDeployingSyncConfig())!;
+    expect(deploying.content.compiled_plan?.plan.version).toBe(3);
+    const parsed = deploying.content.parsed(test_utils.PARSE_OPTIONS);
+    expect((parsed.syncConfigs[0].config as PrecompiledSyncConfig).plan.sourceTableConfig).toEqual(expected);
+    expect(parsed.hydratedSyncConfig.sourceTableConfig).toEqual(expected);
+  });
+
+  test('continues loading unfiltered plans without source options', async () => {
+    await using factory = await storageFactory().factory();
+    await factory.updateSyncRules(
+      updateSyncRulesFromConfig(SqlSyncRules.fromYaml(STREAMS, test_utils.PARSE_OPTIONS), {
+        storageVersion
+      })
+    );
+    const deploying = (await factory.getDeployingSyncConfig())!;
+    expect(deploying.content.compiled_plan?.plan.version).toBe(1);
+    expect(deploying.content.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig.sourceTableConfig).toEqual({});
+  });
+
+  test('starts replacement processing when source-table options change', async () => {
+    await using factory = await storageFactory().factory();
+    const deploy = (yaml: string) =>
+      factory.updateSyncRules(
+        updateSyncRulesFromConfig(SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS), {
+          storageVersion
+        })
+      );
+    const first = await deploy(CONFIGURED);
+    {
+      // Activate the first config so compatibility is checked against data already being served.
+      const storage = await test_utils.getTestStorage(factory, first);
+      await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
+      await writer.markAllSnapshotDone('1/1');
+      await writer.commit('1/1');
+    }
+    if (storageVersion >= 3) {
+      // SQL-only edits can share source data when the source-table options are identical.
+      const sameOptions = await deploy(CONFIGURED.replace('SELECT * FROM orders', 'SELECT id FROM orders'));
+      expect(sameOptions.replicationStreamId).toBe(first.replicationStreamId);
+      const storage = await test_utils.getTestStorage(factory, sameOptions);
+      await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS);
+      await writer.markAllSnapshotDone('2/1');
+      await writer.commit('2/1');
+    }
+    const changed = await deploy(CONFIGURED.replace('true', 'false'));
+    expect(changed.replicationStreamId).not.toBe(first.replicationStreamId);
+    expect((await factory.getActiveSyncConfig())?.content.replicationStreamId).toBe(first.replicationStreamId);
+  });
+});

package/test/src/storage.test.ts

@@ -1,7 +1,7 @@
 import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
-import { compactActive, register, test_utils } from '@powersync/service-core-tests';
+import { compactActive, getTestStorage, register, test_utils } from '@powersync/service-core-tests';
 import { describe, expect, test } from 'vitest';
 import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
 import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
@@ -35,7 +35,7 @@ bucket_definitions:
           { storageVersion }
         )
       );
-      const bucketStorage = factory.getInstance(syncRules);
+      const bucketStorage = await getTestStorage(factory, syncRules);
 
       // user1 has no existing row, so this covers updateMany with upsert.
       // The initial request stores checkpoint id 42 at source head 5/0.
@@ -144,7 +144,8 @@ bucket_definitions:
           user_id: 'custom1',
           checkpoint: 52n
         });
-        await writer.flush();
+        // A custom-checkpoint-only commit uses the combined final flush path too.
+        await writer.commit('8/1');
         const customGenerated = await factory.db.custom_write_checkpoints.findOne({ user_id: 'custom1' });
         expect(customGenerated).not.toBeNull();
         expect(customGenerated?.checkpoint_requested_at).toBeUndefined();
@@ -179,7 +180,7 @@ event_definitions:
             { storageVersion }
           )
         );
-        const bucketStorage = factory.getInstance(syncRules);
+        const bucketStorage = await getTestStorage(factory, syncRules);
         const db = bucketStorage.db as VersionedPowerSyncMongoV3;
         const activeSyncConfig = bucketStorage.getParsedSyncRules({ defaultSchema: 'public' });
         const eventA = activeSyncConfig.eventDescriptors.find((event) => event.name == 'checkpoint_a')!;
@@ -228,7 +229,7 @@ event_definitions:
 
         writer.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 5n, event_id: eventAId });
         writer.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 8n, event_id: eventBId });
-        await writer.flush();
+        // Collections must be prepared before the combined flush/checkpoint transaction.
         await writer.keepalive('5/0');
 
         const eventACollection = db.customCheckpointRequests({
@@ -296,7 +297,7 @@ event_definitions:
         // A fresh storage instance has an empty initialization cache, just like
         // one created after a service restart. Recreating the existing indexes
         // must be idempotent before another checkpoint is written.
-        const freshBucketStorage = factory.getInstance(syncRules);
+        const freshBucketStorage = await getTestStorage(factory, syncRules);
         await using freshWriter = await freshBucketStorage.createWriter(test_utils.BATCH_OPTIONS);
         freshWriter.addCustomWriteCheckpoint({
           user_id: 'after-restart',
@@ -336,7 +337,7 @@ event_definitions:
           updateSyncRulesFromYaml(syncConfigYaml("kind = 'write'", false), { storageVersion })
         );
         const firstEventId = first.syncConfigContent[0].mapping.eventDefinitionIdByName('write_checkpoints');
-        const firstStorage = factory.getInstance(first);
+        const firstStorage = await getTestStorage(factory, first);
         await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
         firstWriter.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 5n, event_id: firstEventId });
         await firstWriter.markAllSnapshotDone('1/1');
@@ -348,7 +349,9 @@ event_definitions:
         const unchangedEventId = unchanged.syncConfigContent[1].mapping.eventDefinitionIdByName('write_checkpoints');
         expect(unchangedEventId).toBe(firstEventId);
 
-        await using unchangedWriter = await factory.getInstance(unchanged).createWriter(test_utils.BATCH_OPTIONS);
+        await using unchangedWriter = await (
+          await getTestStorage(factory, unchanged)
+        ).createWriter(test_utils.BATCH_OPTIONS);
         await unchangedWriter.markAllSnapshotDone('2/1');
         await unchangedWriter.commit('2/1');
 
@@ -369,7 +372,9 @@ event_definitions:
         const changedEventId = changed.syncConfigContent[1].mapping.eventDefinitionIdByName('write_checkpoints');
         expect(changedEventId).not.toBe(unchangedEventId);
 
-        await using changedWriter = await factory.getInstance(changed).createWriter(test_utils.BATCH_OPTIONS);
+        await using changedWriter = await (
+          await getTestStorage(factory, changed)
+        ).createWriter(test_utils.BATCH_OPTIONS);
         changedWriter.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 9n, event_id: changedEventId });
         await changedWriter.flush();
 
@@ -415,7 +420,7 @@ bucket_definitions:
           { storageVersion }
         )
       );
-      const bucketStorage = factory.getInstance(syncRules);
+      const bucketStorage = await getTestStorage(factory, syncRules);
 
       // Replicate an extremely rare possibility where there are multiple records
       await factory.db.write_checkpoints.insertMany([
@@ -486,26 +491,20 @@ bucket_definitions:
 
 describe('Sync Bucket Validation', register.registerBucketValidationTests);
 
-describe('Mongo Sync Bucket Storage - split operations', () =>
-  register.registerDataStorageDataTests(
-    mongoTestStorageFactoryGenerator({
-      url: env.MONGO_TEST_URL,
-      isCI: env.CI,
-      checksumOptions: {
-        bucketBatchLimit: 100,
-        operationBatchLimit: 1
-      }
-    })
-  ));
-
-describe('Mongo Sync Bucket Storage - split buckets', () =>
-  register.registerDataStorageDataTests(
-    mongoTestStorageFactoryGenerator({
-      url: env.MONGO_TEST_URL,
-      isCI: env.CI,
-      checksumOptions: {
-        bucketBatchLimit: 1,
-        operationBatchLimit: 100
-      }
-    })
-  ));
+for (const storageVersion of TEST_STORAGE_VERSIONS) {
+  for (const [name, checksumOptions] of [
+    ['split operations', { bucketBatchLimit: 100, operationBatchLimit: 1 }],
+    ['split buckets', { bucketBatchLimit: 1, operationBatchLimit: 100 }]
+  ] as const) {
+    describe(`Mongo Sync Bucket Storage - ${name} - v${storageVersion}`, () =>
+      register.registerDataStorageDataTests({
+        ...mongoTestStorageFactoryGenerator({
+          url: env.MONGO_TEST_URL,
+          isCI: env.CI,
+          checksumOptions
+        }),
+        storageVersion,
+        compressedBucketStorage: storageVersion >= 3
+      }));
+  }
+}