@powersync/service-module-mongodb 0.18.2 → 0.19.0

package/test/src/mongo_test.test.ts

@@ -12,8 +12,13 @@ import { parseDocumentId } from '@module/replication/bufferToSqlite.js';
 import { ChangeStreamBatch, parseChangeDocument, rawChangeStream } from '@module/replication/RawChangeStream.js';
 import { DirectSourceRowConverter } from '@module/replication/SourceRowConverter.js';
 import { PostImagesOption } from '@module/types/types.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
 import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js';
 
+// DocumentDB does not support the (deprecated) mapReduce command, which
+// insertUndefined() relies on to produce BSON `undefined` values. It also does
+// not support database-level change streams (production always uses
+// cluster-level streams on DocumentDB). Tests depending on either are skipped.
 describe('mongo data types', () => {
   // These test the full data cycle by writing to mongodb, then checking the change stream and direct collection queries.
   // More direct tests directly on the BSON values are in buffer_to_sqlite.test.ts.
@@ -244,7 +249,8 @@ describe('mongo data types', () => {
     });
   }
 
-  test('test direct queries', async () => {
+  // DocumentDB: requires mapReduce (insertUndefined)
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('test direct queries', async () => {
     const { db, client } = await connectMongoData();
 
     const collection = db.collection('test_data');
@@ -265,7 +271,8 @@ describe('mongo data types', () => {
     }
   });
 
-  test('test direct queries - arrays', async () => {
+  // DocumentDB: requires mapReduce (insertUndefined)
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('test direct queries - arrays', async () => {
     const { db, client } = await connectMongoData();
     const collection = db.collection('test_data_arrays');
     try {
@@ -286,7 +293,8 @@ describe('mongo data types', () => {
     }
   });
 
-  test('test replication', async () => {
+  // DocumentDB: requires a database-level change stream and mapReduce
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('test replication', async () => {
     // With MongoDB, replication uses the exact same document format
     // as normal queries. We test it anyway.
     const { db, client } = await connectMongoData();
@@ -311,7 +319,8 @@ describe('mongo data types', () => {
     }
   });
 
-  test('test replication - arrays', async () => {
+  // DocumentDB: requires a database-level change stream and mapReduce
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('test replication - arrays', async () => {
     const { db, client } = await connectMongoData();
     const collection = db.collection('test_data');
     try {
@@ -334,7 +343,8 @@ describe('mongo data types', () => {
     }
   });
 
-  test('connection schema', async () => {
+  // DocumentDB: requires mapReduce (insertUndefined)
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('connection schema', async () => {
     await using adapter = new MongoRouteAPIAdapter({
       type: 'mongodb',
       ...TEST_CONNECTION_OPTIONS

package/test/src/raw_change_stream.test.ts

@@ -4,19 +4,28 @@ import { ChangeStreamBatch, namespaceCollection, rawChangeStream } from '@module
 import { getCursorBatchBytes } from '@module/replication/replication-index.js';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { bson } from '@powersync/service-core';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+import { testTimeout } from './test-timeouts.js';
 import { clearTestDb, connectMongoData, requireFailCommand } from './util.js';
 
+// DocumentDB only supports cluster-level change streams — collection- and
+// database-level streams fail with NamespaceNotFound ("Collection not found"),
+// which is why production always uses cluster-level streams on DocumentDB.
+// The cluster-level size tracking test is also skipped: its strict batching
+// assertions (exact batch counts, immediate event delivery per readAll pass)
+// are racy against a remote DocumentDB cluster with multi-second event latency.
+// Byte tracking on DocumentDB is exercised indirectly by documentdb_mode.test.ts.
 describe('internal mongodb utils', () => {
   // The implementation relies on internal APIs, so we verify this works as expected for various types of change streams.
-  test('collection change stream size tracking', async () => {
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('collection change stream size tracking', async () => {
     await testChangeStreamBsonBytes('collection');
   });
 
-  test('db change stream size tracking', async () => {
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('db change stream size tracking', async () => {
     await testChangeStreamBsonBytes('db');
   });
 
-  test('cluster change stream size tracking', async () => {
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('cluster change stream size tracking', async () => {
     await testChangeStreamBsonBytes('cluster');
   });
 
@@ -44,144 +53,215 @@ describe('internal mongodb utils', () => {
     expect(totalBytes).toBeLessThan(1200);
   });
 
-  test('uses separate aggregate and getMore command options', async () => {
-    const { db, client } = await connectMongoData({ monitorCommands: true });
-    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
-    await clearTestDb(db);
-    const collection = db.collection('test_data');
-
-    const started: any[] = [];
-    client.on('commandStarted', (event) => {
-      if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
-        started.push(event);
-      }
-    });
+  // DocumentDB does not support database-level change streams (rawChangeStream on a db); only cluster-level.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
+    'uses separate aggregate and getMore command options',
+    async () => {
+      const { db, client } = await connectMongoData({ monitorCommands: true });
+      await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+      await clearTestDb(db);
+      const collection = db.collection('test_data');
+
+      const started: any[] = [];
+      client.on('commandStarted', (event) => {
+        if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+          started.push(event);
+        }
+      });
 
-    const stream = rawChangeStream(
-      db,
-      [
-        {
-          $changeStream: {
-            fullDocument: 'updateLookup'
+      const stream = rawChangeStream(
+        db,
+        [
+          {
+            $changeStream: {
+              fullDocument: 'updateLookup'
+            }
           }
+        ],
+        {
+          batchSize: 10,
+          maxAwaitTimeMS: 50,
+          maxTimeMS: 1_000
         }
-      ],
-      {
-        batchSize: 10,
-        maxAwaitTimeMS: 50,
-        maxTimeMS: 1_000
-      }
-    );
+      );
 
-    await stream.next();
-    await collection.insertOne({ test: 1 });
-    const nextBatch = await readUntilNonEmptyBatch(stream);
-    await stream.return?.();
+      await stream.next();
+      await collection.insertOne({ test: 1 });
+      const nextBatch = await readUntilNonEmptyBatch(stream);
+      await stream.return?.();
 
-    expect(nextBatch.events).toHaveLength(1);
+      expect(nextBatch.events).toHaveLength(1);
 
-    const aggregate = started.find((event) => event.commandName == 'aggregate');
-    const getMore = started.find((event) => event.commandName == 'getMore');
+      const aggregate = started.find((event) => event.commandName == 'aggregate');
+      const getMore = started.find((event) => event.commandName == 'getMore');
 
-    expect(aggregate?.command.cursor?.batchSize).toEqual(1);
-    expect(aggregate?.command.maxTimeMS).toEqual(1_000);
-    expect(getMore?.command.batchSize).toEqual(10);
-    expect(getMore?.command.maxTimeMS).toEqual(50);
-  });
+      expect(aggregate?.command.cursor?.batchSize).toEqual(1);
+      expect(aggregate?.command.maxTimeMS).toEqual(1_000);
+      expect(getMore?.command.batchSize).toEqual(10);
+      expect(getMore?.command.maxTimeMS).toEqual(50);
+    }
+  );
+
+  test(
+    'keeps getMore maxTimeMS when client-side maxAwaitTimeMS is enabled',
+    { timeout: testTimeout(30_000, { cloudOverride: 150_000 }) },
+    async () => {
+      const { db, client } = await connectMongoData({ monitorCommands: true });
+      await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+      await clearTestDb(db);
+      const collection = db.collection('test_data');
+      // Keep the local Mongo path fast, but give Azure DocumentDB enough time
+      // for slow cloud change-stream delivery when maxTimeMS is sent to getMore.
+      const maxAwaitTimeMS = testTimeout(50, { cloudOverride: 10_000 });
+
+      const started: any[] = [];
+      client.on('commandStarted', (event) => {
+        if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+          started.push(event);
+        }
+      });
 
-  test('should resume on MaxTimeMSExpired from getMore', async (ctx) => {
-    const { db, client } = await connectMongoData({ monitorCommands: true });
-    await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
-    await using failCommand = await requireFailCommand(client, ctx);
+      const stream = rawChangeStream(
+        DATABASE_TYPE == DatabaseType.DOCUMENTDB ? client.db('admin') : db,
+        [
+          {
+            $changeStream: {
+              fullDocument: 'updateLookup',
+              ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB ? { allChangesForCluster: true } : {})
+            }
+          },
+          ...(DATABASE_TYPE == DatabaseType.DOCUMENTDB
+            ? [
+                {
+                  $match: {
+                    'ns.db': db.databaseName,
+                    'ns.coll': collection.collectionName
+                  }
+                }
+              ]
+            : [])
+        ],
+        {
+          batchSize: 10,
+          maxAwaitTimeMS,
+          clientSideMaxAwaitTimeMS: true,
+          maxTimeMS: 1_000
+        }
+      );
 
-    await clearTestDb(db);
-    const collection = db.collection('test_data');
+      await stream.next();
+      await collection.insertOne({ test: 1 });
+      const nextBatch = await readUntilNonEmptyBatch(stream);
+      await stream.return?.();
 
-    const started: any[] = [];
-    client.on('commandStarted', (event) => {
-      if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
-        started.push(event);
-      }
-    });
+      expect(nextBatch.events).toHaveLength(1);
 
-    const stream = rawChangeStream(
-      db,
-      [
-        {
-          $changeStream: {
-            fullDocument: 'updateLookup'
-          }
+      const aggregate = started.find((event) => event.commandName == 'aggregate');
+      const getMore = started.find((event) => event.commandName == 'getMore');
+
+      expect(aggregate?.command.maxTimeMS).toEqual(1_000);
+      expect(getMore?.command.maxTimeMS).toEqual(maxAwaitTimeMS);
+    }
+  );
+
+  // This test uses configureFailPoint to inject a getMore timeout. Azure
+  // DocumentDB does not support configureFailPoint.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)(
+    'should resume on MaxTimeMSExpired from getMore',
+    async (ctx) => {
+      const { db, client } = await connectMongoData({ monitorCommands: true });
+      await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
+      await using failCommand = await requireFailCommand(client, ctx);
+
+      await clearTestDb(db);
+      const collection = db.collection('test_data');
+
+      const started: any[] = [];
+      client.on('commandStarted', (event) => {
+        if (event.commandName == 'aggregate' || event.commandName == 'getMore') {
+          started.push(event);
         }
-      ],
-      {
-        batchSize: 3,
-        maxAwaitTimeMS: 50,
-        maxTimeMS: 1_000
-        // To get more details when debugging, enable the logger:
-        // logger: logger
-      }
-    );
+      });
 
-    await stream.next();
+      const stream = rawChangeStream(
+        db,
+        [
+          {
+            $changeStream: {
+              fullDocument: 'updateLookup'
+            }
+          }
+        ],
+        {
+          batchSize: 3,
+          maxAwaitTimeMS: 50,
+          maxTimeMS: 1_000
+          // To get more details when debugging, enable the logger:
+          // logger: logger
+        }
+      );
 
-    await failCommand.configure({
-      mode: { times: 1 },
-      data: {
-        failCommands: ['getMore'],
-        errorCode: 50 // MaxTimeMSExpired
-      }
-    });
+      await stream.next();
 
-    for (let i = 1; i <= 8; i++) {
-      await collection.insertOne({ test: i });
-    }
+      await failCommand.configure({
+        mode: { times: 1 },
+        data: {
+          failCommands: ['getMore'],
+          errorCode: 50 // MaxTimeMSExpired
+        }
+      });
 
-    // Test the exponentially-increasing batch size after the retry
-    {
-      // This will fail the getMore, then retry with aggregate with batchSize 1
-      const batch = await readUntilNonEmptyBatch(stream, 10);
-      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
-        { test: 1 }
-      ]);
-    }
-    {
-      // This will be a getMore with batchSize 2
-      const batch = await readUntilNonEmptyBatch(stream, 10);
-      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
-        { test: 2 },
-        { test: 3 }
-      ]);
-    }
-    {
-      // At this point, this batch size is at the original size of 3 again.
-      const batch = await readUntilNonEmptyBatch(stream, 10);
-      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
-        { test: 4 },
-        { test: 5 },
-        { test: 6 }
-      ]);
-    }
-    {
-      const batch = await readUntilNonEmptyBatch(stream, 10);
-      expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
-        { test: 7 },
-        { test: 8 }
-      ]);
-    }
+      for (let i = 1; i <= 8; i++) {
+        await collection.insertOne({ test: i });
+      }
 
-    const aggregateCommands = started.filter((event) => event.commandName == 'aggregate');
-    expect(aggregateCommands.length).toBeGreaterThanOrEqual(2);
-    expect(aggregateCommands[0].command.pipeline).toEqual([
+      // Test the exponentially-increasing batch size after the retry
       {
-        $changeStream: {
-          fullDocument: 'updateLookup'
-        }
+        // This will fail the getMore, then retry with aggregate with batchSize 1
+        const batch = await readUntilNonEmptyBatch(stream, 10);
+        expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+          { test: 1 }
+        ]);
+      }
+      {
+        // This will be a getMore with batchSize 2
+        const batch = await readUntilNonEmptyBatch(stream, 10);
+        expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+          { test: 2 },
+          { test: 3 }
+        ]);
+      }
+      {
+        // At this point, this batch size is at the original size of 3 again.
+        const batch = await readUntilNonEmptyBatch(stream, 10);
+        expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+          { test: 4 },
+          { test: 5 },
+          { test: 6 }
+        ]);
+      }
+      {
+        const batch = await readUntilNonEmptyBatch(stream, 10);
+        expect(batch.events.map((e) => bson.deserialize(e, { useBigInt64: true }).fullDocument)).toMatchObject([
+          { test: 7 },
+          { test: 8 }
+        ]);
       }
-    ]);
-    expect(aggregateCommands[1].command.pipeline[0]?.$changeStream?.resumeAfter).toBeDefined();
 
-    await stream?.return().catch(() => {});
-  });
+      const aggregateCommands = started.filter((event) => event.commandName == 'aggregate');
+      expect(aggregateCommands.length).toBeGreaterThanOrEqual(2);
+      expect(aggregateCommands[0].command.pipeline).toEqual([
+        {
+          $changeStream: {
+            fullDocument: 'updateLookup'
+          }
+        }
+      ]);
+      expect(aggregateCommands[1].command.pipeline[0]?.$changeStream?.resumeAfter).toBeDefined();
+
+      await stream?.return().catch(() => {});
+    }
+  );
 
   async function testChangeStreamBsonBytes(type: 'db' | 'collection' | 'cluster') {
     const { db, client } = await connectMongoData();
@@ -260,7 +340,8 @@ describe('internal mongodb utils', () => {
     expect(totalBytes).toBeLessThan(8000);
   }
 
-  test('should resume on missing cursor (1)', async () => {
+  // DocumentDB: database-level change streams and $currentOp are not supported
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('should resume on missing cursor (1)', async () => {
     // Many resumable errors are difficult to simulate, but CursorNotFound is easy.
 
     const { db, client } = await connectMongoData();
@@ -313,7 +394,8 @@ describe('internal mongodb utils', () => {
     expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }, { test: 2 }, { test: 3 }, { test: 4 }]);
   });
 
-  test('should resume on missing cursor (2)', async () => {
+  // DocumentDB: database-level change streams and $currentOp are not supported
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('should resume on missing cursor (2)', async () => {
     const { db, client } = await connectMongoData();
     await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
     await clearTestDb(db);
@@ -387,7 +469,8 @@ describe('internal mongodb utils', () => {
     expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }, { test: 2 }, { test: 3 }, { test: 4 }]);
   });
 
-  test('should cleanly abort a stream between events', async () => {
+  // DocumentDB: database-level change streams are not supported
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('should cleanly abort a stream between events', async () => {
     const { db, client } = await connectMongoData();
     const abortController = new AbortController();
     await using _ = { [Symbol.asyncDispose]: async () => await client.close() };
@@ -437,7 +520,8 @@ describe('internal mongodb utils', () => {
     expect(readDocs.map((doc) => doc.fullDocument)).toMatchObject([{ test: 1 }, { test: 2 }]);
   });
 
-  test('should cleanly abort a stream in an event', async () => {
+  // DocumentDB: database-level change streams are not supported
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('should cleanly abort a stream in an event', async () => {
     const { db, client } = await connectMongoData();
     const abortController = new AbortController();
     await using _ = { [Symbol.asyncDispose]: async () => await client.close() };

package/test/src/resume_token.test.ts

@@ -1,4 +1,5 @@
 import { MongoLSN, parseResumeTokenTimestamp, ZERO_LSN } from '@module/common/MongoLSN.js';
+import { SentinelLSN } from '@module/common/SentinelLSN.js';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { describe, expect, it, test } from 'vitest';
 
@@ -109,3 +110,32 @@ describe('mongo lsn', () => {
     ).toBe(true);
   });
 });
+
+describe('documentdb lsn', () => {
+  test('uses sentinel as the comparable prefix', () => {
+    expect(new SentinelLSN({ sentinel: 9n }).comparable < new SentinelLSN({ sentinel: 10n }).comparable).toBe(true);
+    expect(new SentinelLSN({ sentinel: 10n }).comparable > new SentinelLSN({ sentinel: 9n }).comparable).toBe(true);
+  });
+
+  test('resume token round-trips without changing the sentinel', () => {
+    const lsn = new SentinelLSN({
+      sentinel: 42n,
+      resume_token: { _data: 'MzowOjM1MjY4MTIwOTEyOjA6MzUyNjgxMjA5MTI6MTowOg==' }
+    });
+
+    const parsed = SentinelLSN.fromSerialized(lsn.comparable);
+
+    expect(parsed.sentinel).toEqual(42n);
+    expect(parsed.resumeToken).toEqual({ _data: 'MzowOjM1MjY4MTIwOTEyOjA6MzUyNjgxMjA5MTI6MTowOg==' });
+  });
+
+  test('write checkpoint LSN without resume token compares below observed stream LSN for the same sentinel', () => {
+    const writeCheckpointHead = new SentinelLSN({ sentinel: 42n }).comparable;
+    const observedStreamHead = new SentinelLSN({
+      sentinel: 42n,
+      resume_token: { _data: 'resume' }
+    }).comparable;
+
+    expect(observedStreamHead >= writeCheckpointHead).toBe(true);
+  });
+});

package/test/src/slow_tests.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, test } from 'vitest';
 
 import { mongo } from '@powersync/lib-service-mongodb';
 import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js';
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
 import { env } from './env.js';
 import { describeWithStorage, StorageVersionTestContext } from './util.js';
 
@@ -15,7 +16,9 @@ function defineSlowTests({ factory, storageVersion }: StorageVersionTestContext)
     return ChangeStreamTestContext.open(factory, { ...options, storageVersion });
   };
 
-  test('replicating snapshot with lots of data', async () => {
+  // This test uses getParameter/setParameter for minSnapshotHistoryWindowInSeconds,
+  // which Azure DocumentDB does not support.
+  test.skipIf(DATABASE_TYPE == DatabaseType.DOCUMENTDB)('replicating snapshot with lots of data', async () => {
     await using context = await openContext();
     // Test with low minSnapshotHistoryWindowInSeconds, to trigger:
     // > Read timestamp .. is older than the oldest available timestamp.

package/test/src/test-timeouts.ts

@@ -0,0 +1,23 @@
+import { DATABASE_TYPE, DatabaseType } from './DatabaseType.js';
+
+/**
+ * Scale factor applied to a default test timeout on cloud DocumentDB when no
+ * explicit cloud override is given to {@link testTimeout}. Cloud DocumentDB has
+ * 10-30s change-stream propagation spikes, so timeouts there need headroom.
+ * Override with the `TEST_TIMEOUT_MULTIPLIER` env var; defaults to 6.
+ */
+export const TEST_TIMEOUT_MULTIPLIER = Number(process.env.TEST_TIMEOUT_MULTIPLIER ?? '6');
+
+/**
+ * Resolve a test timeout in milliseconds for the current backend.
+ *
+ * - Fast backend (local / standard MongoDB): uses `defaultMs` as-is.
+ * - Cloud DocumentDB: uses `cloudOverride` when provided, otherwise scales
+ *   `defaultMs` by {@link TEST_TIMEOUT_MULTIPLIER}.
+ */
+export function testTimeout(defaultMs: number, options?: { cloudOverride?: number }): number {
+  if (DATABASE_TYPE !== DatabaseType.DOCUMENTDB) {
+    return defaultMs;
+  }
+  return options?.cloudOverride ?? Math.ceil(defaultMs * TEST_TIMEOUT_MULTIPLIER);
+}

package/test/src/util.ts

@@ -71,8 +71,7 @@ export async function connectMongoData(options: mongo.MongoClientOptions = {}) {
     ...BSON_DESERIALIZE_DATA_OPTIONS,
     ...options
   });
-  const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1);
-  return { client, db: client.db(dbname) };
+  return { client, db: client.db(TEST_CONNECTION_OPTIONS.database) };
 }
 
 /**