@powersync/service-module-mongodb 0.0.0-dev-20250122110924 → 0.0.0-dev-20250227082606

package/test/src/mongo_test.test.ts

@@ -47,6 +47,13 @@ describe('mongo data types', () => {
           'mydb',
           { foo: 'bar' }
         )
+      },
+      {
+        _id: 6 as any,
+        int4: -1,
+        int8: -9007199254740993n,
+        float: -3.14,
+        decimal: new mongo.Decimal128('-3.14')
       }
     ]);
   }
@@ -109,6 +116,13 @@ describe('mongo data types', () => {
       },
       { _id: 2 as any, nested: [{ test: 'thing' }] },
       { _id: 3 as any, date: [new Date('2023-03-06 15:47+02')] },
+      {
+        _id: 6 as any,
+        int4: [-1],
+        int8: [-9007199254740993n],
+        float: [-3.14],
+        decimal: [new mongo.Decimal128('-3.14')]
+      },
       {
         _id: 10 as any,
         timestamp: [mongo.Timestamp.fromBits(123, 456)],
@@ -164,6 +178,14 @@ describe('mongo data types', () => {
 
     // This must specifically be null, and not undefined.
     expect(transformed[4].undefined).toBeNull();
+
+    expect(transformed[5]).toMatchObject({
+      _id: 6n,
+      int4: -1n,
+      int8: -9007199254740993n,
+      float: -3.14,
+      decimal: '-3.14'
+    });
   }
 
   function checkResultsNested(transformed: Record<string, any>[]) {
@@ -193,6 +215,19 @@ describe('mongo data types', () => {
     });
 
     expect(transformed[3]).toMatchObject({
+      _id: 5n,
+      undefined: '[null]'
+    });
+
+    expect(transformed[4]).toMatchObject({
+      _id: 6n,
+      int4: '[-1]',
+      int8: '[-9007199254740993]',
+      float: '[-3.14]',
+      decimal: '["-3.14"]'
+    });
+
+    expect(transformed[5]).toMatchObject({
       _id: 10n,
       objectId: '["66e834cc91d805df11fa0ecb"]',
       timestamp: '[1958505087099]',
@@ -203,10 +238,6 @@ describe('mongo data types', () => {
       minKey: '[null]',
       maxKey: '[null]'
     });
-
-    expect(transformed[4]).toMatchObject({
-      undefined: '[null]'
-    });
   }
 
   test('test direct queries', async () => {
@@ -218,11 +249,13 @@ describe('mongo data types', () => {
       await insert(collection);
       await insertUndefined(db, 'test_data');
 
-      const rawResults = await db.collection('test_data').find().toArray();
+      const rawResults = await db
+        .collection('test_data')
+        .find({}, { sort: { _id: 1 } })
+        .toArray();
       // It is tricky to save "undefined" with mongo, so we check that it succeeded.
       expect(rawResults[4].undefined).toBeUndefined();
       const transformed = [...ChangeStream.getQueryData(rawResults)];
-
       checkResults(transformed);
     } finally {
       await client.close();
@@ -238,8 +271,11 @@ describe('mongo data types', () => {
       await insertNested(collection);
       await insertUndefined(db, 'test_data_arrays', true);
 
-      const rawResults = await db.collection('test_data_arrays').find().toArray();
-      expect(rawResults[4].undefined).toEqual([undefined]);
+      const rawResults = await db
+        .collection('test_data_arrays')
+        .find({}, { sort: { _id: 1 } })
+        .toArray();
+      expect(rawResults[3].undefined).toEqual([undefined]);
       const transformed = [...ChangeStream.getQueryData(rawResults)];
 
       checkResultsNested(transformed);
@@ -257,7 +293,6 @@ describe('mongo data types', () => {
       await setupTable(db);
 
       const stream = db.watch([], {
-        useBigInt64: true,
         maxAwaitTimeMS: 50,
         fullDocument: 'updateLookup'
       });
@@ -267,7 +302,7 @@ describe('mongo data types', () => {
       await insert(collection);
       await insertUndefined(db, 'test_data');
 
-      const transformed = await getReplicationTx(stream, 5);
+      const transformed = await getReplicationTx(stream, 6);
 
       checkResults(transformed);
     } finally {
@@ -282,7 +317,6 @@ describe('mongo data types', () => {
       await setupTable(db);
 
       const stream = db.watch([], {
-        useBigInt64: true,
         maxAwaitTimeMS: 50,
         fullDocument: 'updateLookup'
       });
@@ -292,7 +326,7 @@ describe('mongo data types', () => {
       await insertNested(collection);
       await insertUndefined(db, 'test_data_arrays', true);
 
-      const transformed = await getReplicationTx(stream, 5);
+      const transformed = await getReplicationTx(stream, 6);
 
       checkResultsNested(transformed);
     } finally {
@@ -505,5 +539,6 @@ async function getReplicationTx(replicationStream: mongo.ChangeStream, count: nu
       break;
     }
   }
+  transformed.sort((a, b) => Number(a._id) - Number(b._id));
   return transformed;
 }

package/test/src/resume.test.ts

@@ -0,0 +1,152 @@
+import { MongoLSN, ZERO_LSN } from '@module/common/MongoLSN.js';
+
+import { MongoManager } from '@module/replication/MongoManager.js';
+import { normalizeConnectionConfig } from '@module/types/types.js';
+import { isMongoServerError, mongo } from '@powersync/lib-service-mongodb';
+import { BucketStorageFactory, TestStorageOptions } from '@powersync/service-core';
+import { describe, expect, test, vi } from 'vitest';
+import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { env } from './env.js';
+import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js';
+
+describe('mongo lsn', () => {
+  test('LSN with resume tokens should be comparable', () => {
+    // Values without a resume token should be comparable
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1)
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10)
+        }).comparable
+    ).true;
+
+    // Values with resume tokens should correctly compare
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1),
+        resume_token: { _data: 'resume1' }
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10),
+          resume_token: { _data: 'resume2' }
+        }).comparable
+    ).true;
+
+    // The resume token should not affect comparison
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1),
+        resume_token: { _data: '2' }
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10),
+          resume_token: { _data: '1' }
+        }).comparable
+    ).true;
+
+    // Resume token should not be required for comparison
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(10),
+        resume_token: { _data: '2' }
+      }).comparable > // Switching the order to test this case
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(9)
+        }).comparable
+    ).true;
+
+    // Comparison should be backwards compatible with old LSNs
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(10),
+        resume_token: { _data: '2' }
+      }).comparable > ZERO_LSN
+    ).true;
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(10),
+        resume_token: { _data: '2' }
+      }).comparable >
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(1)
+        }).comparable.split('|')[0] // Simulate an old LSN
+    ).true;
+    expect(
+      new MongoLSN({
+        timestamp: mongo.Timestamp.fromNumber(1),
+        resume_token: { _data: '2' }
+      }).comparable <
+        new MongoLSN({
+          timestamp: mongo.Timestamp.fromNumber(10)
+        }).comparable.split('|')[0] // Simulate an old LSN
+    ).true;
+  });
+});
+
+describe.skipIf(!env.TEST_MONGO_STORAGE)('MongoDB resume - mongo storage', () => {
+  defineResumeTest(INITIALIZED_MONGO_STORAGE_FACTORY);
+});
+
+describe.skipIf(!env.TEST_POSTGRES_STORAGE)('MongoDB resume - postgres storage', () => {
+  defineResumeTest(INITIALIZED_POSTGRES_STORAGE_FACTORY);
+});
+
+function defineResumeTest(factoryGenerator: (options?: TestStorageOptions) => Promise<BucketStorageFactory>) {
+  test('resuming with a different source database', async () => {
+    await using context = await ChangeStreamTestContext.open(factoryGenerator);
+    const { db } = context;
+
+    await context.updateSyncRules(/* yaml */
+    ` bucket_definitions:
+        global:
+          data:
+            - SELECT _id as id, description, num FROM "test_data"`);
+
+    await context.replicateSnapshot();
+
+    context.startStreaming();
+
+    const collection = db.collection('test_data');
+    await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
+
+    // Wait for the item above to be replicated. The commit should store a resume token.
+    await vi.waitFor(
+      async () => {
+        const checkpoint = await context.storage?.getCheckpoint();
+        expect(MongoLSN.fromSerialized(checkpoint!.lsn!).resumeToken).exist;
+      },
+      { timeout: 5000 }
+    );
+
+    // Done with this context for now
+    await context.dispose();
+
+    // Use the provided MongoDB url to connect to a different source database
+    const originalUrl = env.MONGO_TEST_URL;
+    // Change this to a different database
+    const url = new URL(originalUrl);
+    const parts = url.pathname.split('/');
+    parts[1] = 'differentDB'; // Replace the database name
+    url.pathname = parts.join('/');
+
+    // Point to a new source DB
+    const connectionManager = new MongoManager(
+      normalizeConnectionConfig({
+        type: 'mongodb',
+        uri: url.toString()
+      })
+    );
+    const factory = await factoryGenerator({ doNotClear: true });
+
+    // Create a new context without updating the sync rules
+    await using context2 = new ChangeStreamTestContext(factory, connectionManager);
+    const activeContent = await factory.getActiveSyncRulesContent();
+    context2.storage = factory.getInstance(activeContent!);
+
+    const error = await context2.startStreaming().catch((ex) => ex);
+    expect(error).exist;
+    // The ChangeStreamReplicationJob will detect this and throw a ChangeStreamInvalidatedError
+    expect(isMongoServerError(error) && error.hasErrorLabel('NonResumableChangeStreamError'));
+  });
+}

package/test/src/util.ts

@@ -4,6 +4,7 @@ import * as postgres_storage from '@powersync/service-module-postgres-storage';
 
 import * as types from '@module/types/types.js';
 import { env } from './env.js';
+import { BSON_DESERIALIZE_DATA_OPTIONS } from '@powersync/service-core';
 
 export const TEST_URI = env.MONGO_TEST_DATA_URL;
 
@@ -30,7 +31,7 @@ export async function connectMongoData() {
     connectTimeoutMS: env.CI ? 15_000 : 5_000,
     socketTimeoutMS: env.CI ? 15_000 : 5_000,
     serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500,
-    useBigInt64: true
+    ...BSON_DESERIALIZE_DATA_OPTIONS
   });
   const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1);
   return { client, db: client.db(dbname) };