@powersync/service-module-mongodb 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/test/src/change_stream.test.ts

@@ -3,12 +3,14 @@ import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test, vi } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { createWriteCheckpoint, storage } from '@powersync/service-core';
+import { createWriteCheckpoint, storage, WriteCheckpointBatcher } from '@powersync/service-core';
 import { test_utils } from '@powersync/service-core-tests';
 
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
 import { PostImagesOption } from '@module/types/types.js';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { testTimeout } from './test-timeouts.js';
 import { describeWithStorage, StorageVersionTestContext, TEST_CONNECTION_OPTIONS } from './util.js';
 
 const BASIC_SYNC_RULES = `
@@ -19,7 +21,7 @@ bucket_definitions:
 `;
 
 describe('change stream', () => {
-  describeWithStorage({ timeout: 20_000 }, defineChangeStreamTests);
+  describeWithStorage({ timeout: testTimeout(20_000, { cloudOverride: 40_000 }) }, defineChangeStreamTests);
 });
 
 function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
@@ -28,7 +30,8 @@ function defineChangeStreamTests({ factory, storageVersion }: StorageVersionTest
   const openContext = (options?: Parameters<typeof ChangeStreamTestContext.open>[1]) => {
     return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
   };
-  test('replicating basic values', async () => {
+  // DocumentDB does not support changeStreamPreAndPostImages, which this test requires via postImages: READ_ONLY.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating basic values', async () => {
     await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.READ_ONLY }
     });
@@ -74,7 +77,10 @@ bucket_definitions:
       - SELECT _id as id, description, num FROM "test_%"`);
 
     await db.createCollection('test_data', {
-      changeStreamPreAndPostImages: { enabled: false }
+      // DocumentDB does not support changeStreamPreAndPostImages, even when
+      // explicitly disabled. This test only needs a collection to exercise
+      // fatal error recovery, so omit the option in DocumentDB mode.
+      ...(DATABASE_TYPE == DatabaseType.MONGODB ? { changeStreamPreAndPostImages: { enabled: false } } : {})
     });
     const collection = db.collection('test_data');
 
@@ -152,7 +158,7 @@ bucket_definitions:
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
     await db.createCollection('test_data', {
-      changeStreamPreAndPostImages: { enabled: false }
+      changeStreamPreAndPostImages: DATABASE_TYPE == DatabaseType.MONGODB ? { enabled: false } : undefined
     });
     testId = new mongo.ObjectId();
     collection = db.collection('test_data');
@@ -167,7 +173,9 @@ bucket_definitions:
     expect(test_utils.reduceBucket(data).slice(1)).toEqual([]);
   });
 
-  test('updateLookup - no fullDocument available', async () => {
+  // DocumentDB always materializes a write-time post-image for updateLookup, so the 'fullDocument unavailable'
+  // case this test exercises (update treated as a remove) cannot occur there.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('updateLookup - no fullDocument available', async () => {
     await using context = await openContext({
       mongoOptions: { postImages: PostImagesOption.OFF }
     });
@@ -179,6 +187,9 @@ bucket_definitions:
       - SELECT _id as id, description, num FROM "test_data"`);
 
     await db.createCollection('test_data', {
+      // DocumentDB does not support changeStreamPreAndPostImages, even when
+      // explicitly disabled. This test only needs a collection to exercise
+      // fatal error recovery, so omit the option in DocumentDB mode.
       changeStreamPreAndPostImages: { enabled: false }
     });
     const collection = db.collection('test_data');
@@ -211,7 +222,8 @@ bucket_definitions:
     ]);
   });
 
-  test('postImages - autoConfigure', async () => {
+  // DocumentDB does not support changeStreamPreAndPostImages (post-images).
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('postImages - autoConfigure', async () => {
     // Similar to the above test, but with postImages enabled.
     // This resolves the consistency issue.
     await using context = await openContext({
@@ -259,7 +271,8 @@ bucket_definitions:
     ]);
   });
 
-  test('postImages - on', async () => {
+  // DocumentDB does not support changeStreamPreAndPostImages (post-images).
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('postImages - on', async () => {
     // Similar to postImages - autoConfigure, but does not auto-configure.
     // changeStreamPreAndPostImages must be manually configured.
     await using context = await openContext({
@@ -408,7 +421,9 @@ bucket_definitions:
     ]);
   });
 
-  test('replicating dropCollection', async () => {
+  // Collection drop events are not currently replicated on DocumentDB: they are not delivered through the
+  // cluster-level change stream the way standard MongoDB delivers them, so the drop is not applied as a remove.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating dropCollection', async () => {
     await using context = await openContext();
     const { db } = context;
     const syncRuleContent = `
@@ -440,7 +455,9 @@ bucket_definitions:
     ]);
   });
 
-  test('replicating renameCollection', async () => {
+  // Collection rename events are not currently replicated on DocumentDB: they are not delivered through the
+  // cluster-level change stream the way standard MongoDB delivers them.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating renameCollection', async () => {
     await using context = await openContext();
     const { db } = context;
     const syncRuleContent = `
@@ -471,7 +488,9 @@ bucket_definitions:
     ]);
   });
 
-  test.runIf(supportsConcurrentSnapshots)(
+  // Concurrent-snapshot regression test driven by collection drop/recreate DDL events, which DocumentDB does
+  // not deliver through the cluster-level change stream the way standard MongoDB does.
+  test.runIf(supportsConcurrentSnapshots && DATABASE_TYPE != DatabaseType.DOCUMENTDB)(
     'collection recreated while queued snapshot is waiting does not stall checkpoints',
     async () => {
       // This is a regression test for a specific timing issue in concurrent snapshot logic.
@@ -603,6 +622,79 @@ bucket_definitions:
     expect(data).toMatchObject([test_utils.putOp('test_data', { id: test_id, description: 'test1' })]);
   });
 
+  test.runIf(storageVersion >= 3)(
+    'does not resnapshot an unchanged event definition during incremental reprocessing',
+    async (testContext) => {
+      const eventDefinition = `
+event_definitions:
+  checkpoint_requests:
+    payloads:
+      - SELECT user_id, checkpoint, client_id, CASE WHEN checkpoint_requested_at IS NULL THEN true ELSE false END AS is_legacy FROM checkpoints
+`;
+      const syncConfig = (deploymentComment: string) => `
+config:
+  edition: 3
+  storage_version: 3
+  # ${deploymentComment}
+
+${eventDefinition}
+streams:
+  global:
+    auto_subscribe: true
+    queries:
+      - SELECT *, _id AS id FROM lists
+      - SELECT *, _id AS id FROM todos
+`;
+
+      let replicationStreamId: number;
+      {
+        await using context = await openContext();
+        const firstStorage = await context.updateSyncRules(syncConfig('initial deployment'));
+        if (
+          !('incrementalReprocessing' in firstStorage.storageConfig) ||
+          firstStorage.storageConfig.incrementalReprocessing !== true
+        ) {
+          testContext.skip('storage does not support incremental reprocessing');
+        }
+
+        const { db } = context;
+        await db.collection('lists').insertOne({ name: 'list' });
+        await db.collection('todos').insertOne({ description: 'todo' });
+        await db.collection('checkpoints').insertOne({
+          user_id: 'user-1',
+          checkpoint: 1n,
+          client_id: 'client-1',
+          checkpoint_requested_at: null
+        });
+
+        replicationStreamId = firstStorage.replicationStreamId;
+        await context.replicateSnapshot();
+        context.startStreaming();
+        await context.getCheckpoint();
+      }
+
+      const resnapshottedCollections: string[] = [];
+      await using context = await openContext({
+        doNotClear: true,
+        streamOptions: {
+          snapshotHooks: {
+            beforeSnapshotStarted: async (table) => {
+              resnapshottedCollections.push(table.name);
+            }
+          }
+        }
+      });
+
+      const processingStorage = await context.updateSyncRules(syncConfig('redeployed without behavioral changes'));
+      expect(processingStorage.replicationStreamId).toBe(replicationStreamId);
+      expect(await context.factory.getDeployingSyncConfig()).not.toBeNull();
+
+      await context.replicateSnapshot();
+
+      expect(resnapshottedCollections).toEqual([]);
+    }
+  );
+
   test('coalesces standalone checkpoints when backlog is buffered', async () => {
     await using context = await openContext();
     await context.updateSyncRules(BASIC_SYNC_RULES);
@@ -625,6 +717,10 @@ bucket_definitions:
       type: 'mongodb',
       ...TEST_CONNECTION_OPTIONS
     });
+    const writeCheckpointBatcher = new WriteCheckpointBatcher(
+      () => api,
+      () => context.factory
+    );
 
     context.startStreaming();
 
@@ -641,8 +737,7 @@ bucket_definitions:
         createWriteCheckpoint({
           userId: 'test_user',
           clientId: 'test_client' + i,
-          api,
-          storage: context.factory
+          batcher: writeCheckpointBatcher
         })
       )
     );
@@ -652,12 +747,14 @@ bucket_definitions:
 
     // We need at least 1 commit.
     expect(commitCount).toBeGreaterThan(0);
-    // The previous implementation greated 1 commit per checkpoint, which is bad for performance.
+    // The previous implementation created 1 commit per checkpoint, which is bad for performance.
     // We expect a small number here - typically 2-10, but allow for anything less than the total number of checkpoints.
     expect(commitCount).toBeLessThan(checkpointCount + 1);
   });
 
-  test('large record', async () => {
+  test
+    // DocumentDBDB does not support changeStreamSplitLargeEvent
+    .skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('large record', async () => {
     // Test a large update.
 
     // Without $changeStreamSplitLargeEvent, we get this error:
@@ -729,75 +826,83 @@ bucket_definitions:
     expect(data).toMatchObject([]);
   });
 
-  test('postImages - new collection with postImages enabled', async () => {
-    await using context = await openContext({
-      mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
-    });
-    const { db } = context;
-    await context.updateSyncRules(`
+  // DocumentDB does not support changeStreamPreAndPostImages (post-images).
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
+    'postImages - new collection with postImages enabled',
+    async () => {
+      await using context = await openContext({
+        mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
+      });
+      const { db } = context;
+      await context.updateSyncRules(`
 bucket_definitions:
   global:
     data:
       - SELECT _id as id, description FROM "test_%"`);
 
-    await context.replicateSnapshot();
+      await context.replicateSnapshot();
 
-    await db.createCollection('test_data', {
-      // enabled: true here - everything should work
-      changeStreamPreAndPostImages: { enabled: true }
-    });
-    const collection = db.collection('test_data');
-    const result = await collection.insertOne({ description: 'test1' });
-    const test_id = result.insertedId;
-    await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
+      await db.createCollection('test_data', {
+        // enabled: true here - everything should work
+        changeStreamPreAndPostImages: { enabled: true }
+      });
+      const collection = db.collection('test_data');
+      const result = await collection.insertOne({ description: 'test1' });
+      const test_id = result.insertedId;
+      await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
 
-    context.startStreaming();
+      context.startStreaming();
 
-    const data = await context.getBucketData('global[]');
-    if (data.length == 3) {
-      expect(data).toMatchObject([
-        // An extra op here, since this triggers a snapshot in addition to getting the event.
-        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
-        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
-      ]);
-    } else {
-      expect(data).toMatchObject([
-        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
-        test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
-      ]);
+      const data = await context.getBucketData('global[]');
+      if (data.length == 3) {
+        expect(data).toMatchObject([
+          // An extra op here, since this triggers a snapshot in addition to getting the event.
+          test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' }),
+          test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+          test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+        ]);
+      } else {
+        expect(data).toMatchObject([
+          test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test1' }),
+          test_utils.putOp('test_data', { id: test_id!.toHexString(), description: 'test2' })
+        ]);
+      }
     }
-  });
+  );
 
-  test('postImages - new collection with postImages disabled', async () => {
-    await using context = await openContext({
-      mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
-    });
-    const { db } = context;
-    await context.updateSyncRules(`
+  // DocumentDB does not support changeStreamPreAndPostImages (post-images).
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
+    'postImages - new collection with postImages disabled',
+    async () => {
+      await using context = await openContext({
+        mongoOptions: { postImages: PostImagesOption.AUTO_CONFIGURE }
+      });
+      const { db } = context;
+      await context.updateSyncRules(`
 bucket_definitions:
   global:
     data:
       - SELECT _id as id, description FROM "test_data%"`);
 
-    await context.replicateSnapshot();
+      await context.replicateSnapshot();
 
-    await db.createCollection('test_data', {
-      // enabled: false here, but autoConfigure will enable it.
-      // Unfortunately, that is too late, and replication must be restarted.
-      changeStreamPreAndPostImages: { enabled: false }
-    });
-    const collection = db.collection('test_data');
-    const result = await collection.insertOne({ description: 'test1' });
-    const test_id = result.insertedId;
-    await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
+      await db.createCollection('test_data', {
+        // enabled: false here, but autoConfigure will enable it.
+        // Unfortunately, that is too late, and replication must be restarted.
+        changeStreamPreAndPostImages: { enabled: false }
+      });
+      const collection = db.collection('test_data');
+      const result = await collection.insertOne({ description: 'test1' });
+      const test_id = result.insertedId;
+      await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
 
-    context.startStreaming();
+      context.startStreaming();
 
-    await expect(() => context.getBucketData('global[]')).rejects.toMatchObject({
-      message: expect.stringContaining('stream was configured to require a post-image for all update events')
-    });
-  });
+      await expect(() => context.getBucketData('global[]')).rejects.toMatchObject({
+        message: expect.stringContaining('stream was configured to require a post-image for all update events')
+      });
+    }
+  );
 
   test('recover from error', async () => {
     await using context = await openContext();
@@ -809,7 +914,10 @@ bucket_definitions:
       - SELECT _id as id, description, num FROM "test_data"`);
 
     await db.createCollection('test_data', {
-      changeStreamPreAndPostImages: { enabled: false }
+      // DocumentDB does not support changeStreamPreAndPostImages, even when
+      // explicitly disabled. This test only needs a collection to exercise
+      // fatal error recovery, so omit the option in DocumentDB mode.
+      changeStreamPreAndPostImages: DATABASE_TYPE == DatabaseType.MONGODB ? { enabled: false } : undefined
     });
 
     const collection = db.collection('test_data');
@@ -820,9 +928,9 @@ bucket_definitions:
 
     // Simulate an error
     await context.storage!.reportError(new Error('simulated error'));
-    const syncRules = await context.factory.getActiveSyncRulesContent();
+    const syncRules = (await context.factory.getActiveSyncConfig())?.content;
     expect(syncRules).toBeTruthy();
-    expect(syncRules?.last_fatal_error).toEqual('simulated error');
+    expect((await syncRules?.getSyncConfigStatus())?.last_fatal_error).toEqual('simulated error');
 
     // The next checkpoint should clear the error.
     await context.getCheckpoint();
@@ -830,10 +938,11 @@ bucket_definitions:
     // Just wait, and check that the error is cleared automatically.
     await vi.waitUntil(
       async () => {
-        const error = (await context.factory.getActiveSyncRulesContent())?.last_fatal_error;
+        const syncRules = (await context.factory.getActiveSyncConfig())?.content;
+        const error = (await syncRules?.getSyncConfigStatus())?.last_fatal_error;
         return error == null;
       },
-      { timeout: 2_000 }
+      { timeout: testTimeout(2_000, { cloudOverride: 5_000 }) }
     );
   });
 }