@powersync/service-module-mongodb 0.10.4 → 0.12.0

package/src/replication/replication-utils.ts

@@ -52,7 +52,7 @@ export async function checkSourceConfiguration(connectionManager: MongoManager):
       const fullName = `${db.databaseName}.${CHECKPOINTS_COLLECTION}`;
       throw new ServiceError(
         ErrorCode.PSYNC_S1307,
-        `MongoDB user does not have the required ${missingCheckpointActions.map((a) => `"${a}"`).join(', ')} priviledge(s) on "${fullName}".`
+        `MongoDB user does not have the required ${missingCheckpointActions.map((a) => `"${a}"`).join(', ')} privilege(s) on "${fullName}".`
       );
     }
 
@@ -62,14 +62,14 @@ export async function checkSourceConfiguration(connectionManager: MongoManager):
       if (!anyCollectionActions.has('collMod')) {
         throw new ServiceError(
           ErrorCode.PSYNC_S1307,
-          `MongoDB user does not have the required "collMod" priviledge on "${db.databaseName}", required for "post_images: auto_configure".`
+          `MongoDB user does not have the required "collMod" privilege on "${db.databaseName}", required for "post_images: auto_configure".`
         );
       }
     }
     if (!anyCollectionActions.has('listCollections')) {
       throw new ServiceError(
         ErrorCode.PSYNC_S1307,
-        `MongoDB user does not have the required "listCollections" priviledge on "${db.databaseName}".`
+        `MongoDB user does not have the required "listCollections" privilege on "${db.databaseName}".`
       );
     }
   } else {

package/test/src/change_stream.test.ts

@@ -23,7 +23,9 @@ describe('change stream', () => {
 
 function defineChangeStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await ChangeStreamTestContext.open(factory, {
+      mongoOptions: { postImages: PostImagesOption.READ_ONLY }
+    });
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -32,7 +34,7 @@ bucket_definitions:
       - SELECT _id as id, description, num FROM "test_data"`);
 
     await db.createCollection('test_data', {
-      changeStreamPreAndPostImages: { enabled: false }
+      changeStreamPreAndPostImages: { enabled: true }
     });
     const collection = db.collection('test_data');
 
@@ -42,11 +44,8 @@ bucket_definitions:
 
     const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
     const test_id = result.insertedId;
-    await setTimeout(30);
     await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
-    await setTimeout(30);
     await collection.replaceOne({ _id: test_id }, { description: 'test3' });
-    await setTimeout(30);
     await collection.deleteOne({ _id: test_id });
 
     const data = await context.getBucketData('global[]');
@@ -354,6 +353,9 @@ bucket_definitions:
     const test_id = result.insertedId.toHexString();
 
     await context.replicateSnapshot();
+    // Note: snapshot is only consistent some time into the streaming request.
+    // At the point that we get the first acknowledged checkpoint, as is required
+    // for getBucketData(), the data should be consistent.
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
@@ -512,10 +514,13 @@ bucket_definitions:
     await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
 
     await context.replicateSnapshot();
+    await context.markSnapshotConsistent();
 
     // Simulate an error
     await context.storage!.reportError(new Error('simulated error'));
-    expect((await context.factory.getActiveSyncRulesContent())?.last_fatal_error).toEqual('simulated error');
+    const syncRules = await context.factory.getActiveSyncRulesContent();
+    expect(syncRules).toBeTruthy();
+    expect(syncRules?.last_fatal_error).toEqual('simulated error');
 
     // startStreaming() should automatically clear the error.
     context.startStreaming();

package/test/src/change_stream_utils.ts

@@ -17,7 +17,7 @@ import { MongoManager } from '@module/replication/MongoManager.js';
 import { createCheckpoint, STANDALONE_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
 import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
 
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
+import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
@@ -119,7 +119,20 @@ export class ChangeStreamTestContext {
 
   async replicateSnapshot() {
     await this.walStream.initReplication();
-    await this.storage!.autoActivate();
+  }
+
+  /**
+   * A snapshot is not consistent until streaming replication has caught up.
+   * We simulate that for tests.
+   * Do not use if there are any writes performed while doing the snapshot, as that
+   * would result in inconsistent data.
+   */
+  async markSnapshotConsistent() {
+    const checkpoint = await createCheckpoint(this.client, this.db, STANDALONE_CHECKPOINT_ID);
+
+    await this.storage!.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive(checkpoint);
+    });
   }
 
   startStreaming() {
@@ -195,12 +208,11 @@ export async function getClientCheckpoint(
   while (Date.now() - start < timeout) {
     const storage = await storageFactory.getActiveStorage();
     const cp = await storage?.getCheckpoint();
-    if (cp == null) {
-      throw new Error('No sync rules available');
-    }
-    lastCp = cp;
-    if (cp.lsn && cp.lsn >= lsn) {
-      return cp.checkpoint;
+    if (cp != null) {
+      lastCp = cp;
+      if (cp.lsn && cp.lsn >= lsn) {
+        return cp.checkpoint;
+      }
     }
     await new Promise((resolve) => setTimeout(resolve, 30));
   }

package/test/src/mongo_test.test.ts

@@ -1,5 +1,11 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules';
+import {
+  applyRowContext,
+  CompatibilityContext,
+  CompatibilityEdition,
+  SqliteInputRow,
+  SqlSyncRules
+} from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 
 import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
@@ -138,8 +144,10 @@ describe('mongo data types', () => {
     ]);
   }
 
-  function checkResults(transformed: Record<string, any>[]) {
-    expect(transformed[0]).toMatchObject({
+  function checkResults(transformed: SqliteInputRow[]) {
+    const sqliteValue = transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY));
+
+    expect(sqliteValue[0]).toMatchObject({
       _id: 1n,
       text: 'text',
       uuid: 'baeb2514-4c57-436d-b3cc-c1256211656d',
@@ -152,17 +160,17 @@ describe('mongo data types', () => {
       null: null,
       decimal: '3.14'
     });
-    expect(transformed[1]).toMatchObject({
+    expect(sqliteValue[1]).toMatchObject({
       _id: 2n,
       nested: '{"test":"thing"}'
     });
 
-    expect(transformed[2]).toMatchObject({
+    expect(sqliteValue[2]).toMatchObject({
       _id: 3n,
       date: '2023-03-06 13:47:00.000Z'
     });
 
-    expect(transformed[3]).toMatchObject({
+    expect(sqliteValue[3]).toMatchObject({
       _id: 4n,
       objectId: '66e834cc91d805df11fa0ecb',
       timestamp: 1958505087099n,
@@ -177,9 +185,9 @@ describe('mongo data types', () => {
     });
 
     // This must specifically be null, and not undefined.
-    expect(transformed[4].undefined).toBeNull();
+    expect(sqliteValue[4].undefined).toBeNull();
 
-    expect(transformed[5]).toMatchObject({
+    expect(sqliteValue[5]).toMatchObject({
       _id: 6n,
       int4: -1n,
       int8: -9007199254740993n,
@@ -188,8 +196,10 @@ describe('mongo data types', () => {
     });
   }
 
-  function checkResultsNested(transformed: Record<string, any>[]) {
-    expect(transformed[0]).toMatchObject({
+  function checkResultsNested(transformed: SqliteInputRow[]) {
+    const sqliteValue = transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY));
+
+    expect(sqliteValue[0]).toMatchObject({
       _id: 1n,
       text: `["text"]`,
       uuid: '["baeb2514-4c57-436d-b3cc-c1256211656d"]',
@@ -204,22 +214,22 @@ describe('mongo data types', () => {
 
     // Note: Depending on to what extent we use the original postgres value, the whitespace may change, and order may change.
     // We do expect that decimals and big numbers are preserved.
-    expect(transformed[1]).toMatchObject({
+    expect(sqliteValue[1]).toMatchObject({
       _id: 2n,
       nested: '[{"test":"thing"}]'
     });
 
-    expect(transformed[2]).toMatchObject({
+    expect(sqliteValue[2]).toMatchObject({
       _id: 3n,
       date: '["2023-03-06 13:47:00.000Z"]'
     });
 
-    expect(transformed[3]).toMatchObject({
+    expect(sqliteValue[3]).toMatchObject({
       _id: 5n,
       undefined: '[null]'
     });
 
-    expect(transformed[4]).toMatchObject({
+    expect(sqliteValue[4]).toMatchObject({
       _id: 6n,
       int4: '[-1]',
       int8: '[-9007199254740993]',
@@ -227,7 +237,7 @@ describe('mongo data types', () => {
       decimal: '["-3.14"]'
     });
 
-    expect(transformed[5]).toMatchObject({
+    expect(sqliteValue[5]).toMatchObject({
       _id: 10n,
       objectId: '["66e834cc91d805df11fa0ecb"]',
       timestamp: '[1958505087099]',
@@ -522,13 +532,45 @@ bucket_definitions:
       errors: []
     });
   });
+
+  test('date format', async () => {
+    const { db, client } = await connectMongoData();
+    const collection = db.collection('test_data');
+    try {
+      await setupTable(db);
+      await collection.insertOne({
+        fraction: new Date('2023-03-06 15:47:01.123+02'),
+        noFraction: new Date('2023-03-06 15:47:01+02')
+      });
+
+      const rawResults = await db
+        .collection('test_data')
+        .find({}, { sort: { _id: 1 } })
+        .toArray();
+      const [row] = [...ChangeStream.getQueryData(rawResults)];
+
+      const oldFormat = applyRowContext(row, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY);
+      expect(oldFormat).toMatchObject({
+        fraction: '2023-03-06 13:47:01.123Z',
+        noFraction: '2023-03-06 13:47:01.000Z'
+      });
+
+      const newFormat = applyRowContext(row, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
+      expect(newFormat).toMatchObject({
+        fraction: '2023-03-06T13:47:01.123Z',
+        noFraction: '2023-03-06T13:47:01.000Z'
+      });
+    } finally {
+      await client.close();
+    }
+  });
 });
 
 /**
  * Return all the inserts from the first transaction in the replication stream.
  */
 async function getReplicationTx(replicationStream: mongo.ChangeStream, count: number) {
-  let transformed: SqliteRow[] = [];
+  let transformed: SqliteInputRow[] = [];
   for await (const doc of replicationStream) {
     // Specifically filter out map_input / map_output collections
     if (!(doc as any)?.ns?.coll?.startsWith('test_data')) {