@powersync/service-module-mongodb 0.10.3 → 0.11.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.10.3",
+  "version": "0.11.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -25,17 +25,17 @@
     "bson": "^6.10.3",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-mongodb": "0.6.1",
-    "@powersync/lib-services-framework": "0.7.0",
-    "@powersync/service-core": "1.13.3",
+    "@powersync/lib-service-mongodb": "0.6.3",
+    "@powersync/lib-services-framework": "0.7.2",
+    "@powersync/service-core": "1.14.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.27.0",
-    "@powersync/service-types": "0.12.0"
+    "@powersync/service-sync-rules": "0.28.0",
+    "@powersync/service-types": "0.12.1"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.10.3",
-    "@powersync/service-module-mongodb-storage": "0.10.3",
-    "@powersync/service-module-postgres-storage": "0.8.3"
+    "@powersync/service-core-tests": "0.11.0",
+    "@powersync/service-module-mongodb-storage": "0.11.0",
+    "@powersync/service-module-postgres-storage": "0.9.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/api/MongoRouteAPIAdapter.ts

@@ -5,7 +5,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as service_types from '@powersync/service-types';
 
 import { MongoManager } from '../replication/MongoManager.js';
-import { constructAfterRecord, createCheckpoint, STANDALONE_CHECKPOINT_ID } from '../replication/MongoRelation.js';
+import { constructAfterRecord, STANDALONE_CHECKPOINT_ID } from '../replication/MongoRelation.js';
 import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js';
 import * as types from '../types/types.js';
 import { escapeRegExp } from '../utils.js';
@@ -137,15 +137,15 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
       if (tablePattern.isWildcard) {
         patternResult.tables = [];
         for (let collection of collections) {
-          const sourceTable = new SourceTable(
-            0,
-            this.connectionTag,
-            collection.name,
-            schema,
-            collection.name,
-            [],
-            true
-          );
+          const sourceTable = new SourceTable({
+            id: 0,
+            connectionTag: this.connectionTag,
+            objectId: collection.name,
+            schema: schema,
+            name: collection.name,
+            replicaIdColumns: [],
+            snapshotComplete: true
+          });
           let errors: service_types.ReplicationError[] = [];
           if (collection.type == 'view') {
             errors.push({ level: 'warning', message: `Collection ${schema}.${tablePattern.name} is a view` });
@@ -164,15 +164,15 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
           });
         }
       } else {
-        const sourceTable = new SourceTable(
-          0,
-          this.connectionTag,
-          tablePattern.name,
-          schema,
-          tablePattern.name,
-          [],
-          true
-        );
+        const sourceTable = new SourceTable({
+          id: 0,
+          connectionTag: this.connectionTag,
+          objectId: tablePattern.name,
+          schema: schema,
+          name: tablePattern.name,
+          replicaIdColumns: [],
+          snapshotComplete: true
+        });
 
         const syncData = sqlSyncRules.tableSyncsData(sourceTable);
         const syncParameters = sqlSyncRules.tableSyncsParameters(sourceTable);

package/src/replication/ChangeStream.ts

@@ -215,9 +215,14 @@ export class ChangeStream {
 
   async estimatedCountNumber(table: storage.SourceTable): Promise<number> {
     const db = this.client.db(table.schema);
-    return await db.collection(table.table).estimatedDocumentCount();
+    return await db.collection(table.name).estimatedDocumentCount();
   }
 
+  /**
+   * This gets a LSN before starting a snapshot, which we can resume streaming from after the snapshot.
+   *
+   * This LSN can survive initial replication restarts.
+   */
   private async getSnapshotLsn(): Promise<string> {
     const hello = await this.defaultDb.command({ hello: 1 });
     // Basic sanity check
@@ -292,6 +297,9 @@ export class ChangeStream {
     );
   }
 
+  /**
+   * Given a snapshot LSN, validate that we can read from it, by opening a change stream.
+   */
   private async validateSnapshotLsn(lsn: string) {
     await using streamManager = this.openChangeStream({ lsn: lsn, maxAwaitTimeMs: 0 });
     const { stream } = streamManager;
@@ -321,7 +329,7 @@ export class ChangeStream {
         if (snapshotLsn == null) {
           // First replication attempt - get a snapshot and store the timestamp
           snapshotLsn = await this.getSnapshotLsn();
-          await batch.setSnapshotLsn(snapshotLsn);
+          await batch.setResumeLsn(snapshotLsn);
           this.logger.info(`Marking snapshot at ${snapshotLsn}`);
         } else {
           this.logger.info(`Resuming snapshot at ${snapshotLsn}`);
@@ -359,11 +367,20 @@ export class ChangeStream {
           await this.snapshotTable(batch, table);
           await batch.markSnapshotDone([table], MongoLSN.ZERO.comparable);
 
-          await touch();
+          this.touch();
         }
 
-        this.logger.info(`Snapshot commit at ${snapshotLsn}`);
+        // The checkpoint here is a marker - we need to replicate up to at least this
+        // point before the data can be considered consistent.
+        // We could do this for each individual table, but may as well just do it once for the entire snapshot.
+        const checkpoint = await createCheckpoint(this.client, this.defaultDb, STANDALONE_CHECKPOINT_ID);
+        await batch.markSnapshotDone([], checkpoint);
+
+        // This will not create a consistent checkpoint yet, but will persist the op.
+        // Actual checkpoint will be created when streaming replication caught up.
         await batch.commit(snapshotLsn);
+
+        this.logger.info(`Snapshot done. Need to replicate from ${snapshotLsn} to ${checkpoint} to be consistent`);
       }
     );
   }
@@ -432,7 +449,7 @@ export class ChangeStream {
     const totalEstimatedCount = await this.estimatedCountNumber(table);
     let at = table.snapshotStatus?.replicatedCount ?? 0;
     const db = this.client.db(table.schema);
-    const collection = db.collection(table.table);
+    const collection = db.collection(table.name);
     await using query = new ChunkedSnapshotQuery({
       collection,
       key: table.snapshotStatus?.lastKey,
@@ -492,7 +509,7 @@ export class ChangeStream {
       this.logger.info(
         `Replicating ${table.qualifiedName} ${table.formatSnapshotProgress()} in ${duration.toFixed(0)}ms`
       );
-      await touch();
+      this.touch();
     }
     // In case the loop was interrupted, make sure we await the last promise.
     await nextChunkPromise;
@@ -656,7 +673,6 @@ export class ChangeStream {
     try {
       // If anything errors here, the entire replication process is halted, and
       // all connections automatically closed, including this one.
-
       await this.initReplication();
       await this.streamChanges();
     } catch (e) {
@@ -757,19 +773,20 @@ export class ChangeStream {
   }
 
   async streamChangesInternal() {
-    // Auto-activate as soon as initial replication is done
-    await this.storage.autoActivate();
-
     await this.storage.startBatch(
       {
         logger: this.logger,
         zeroLSN: MongoLSN.ZERO.comparable,
         defaultSchema: this.defaultDb.databaseName,
+        // We get a complete postimage for every change, so we don't need to store the current data.
         storeCurrentData: false
       },
       async (batch) => {
-        const { lastCheckpointLsn } = batch;
-        const lastLsn = MongoLSN.fromSerialized(lastCheckpointLsn!);
+        const { resumeFromLsn } = batch;
+        if (resumeFromLsn == null) {
+          throw new ReplicationAssertionError(`No LSN found to resume from`);
+        }
+        const lastLsn = MongoLSN.fromSerialized(resumeFromLsn);
         const startAfter = lastLsn?.timestamp;
 
         // It is normal for this to be a minute or two old when there is a low volume
@@ -778,7 +795,7 @@ export class ChangeStream {
 
         this.logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}  | Token age: ${tokenAgeSeconds}s`);
 
-        await using streamManager = this.openChangeStream({ lsn: lastCheckpointLsn });
+        await using streamManager = this.openChangeStream({ lsn: resumeFromLsn });
         const { stream, filters } = streamManager;
         if (this.abort_signal.aborted) {
           await stream.close();
@@ -797,6 +814,7 @@ export class ChangeStream {
         let splitDocument: mongo.ChangeStreamDocument | null = null;
 
         let flexDbNameWorkaroundLogged = false;
+        let changesSinceLastCheckpoint = 0;
 
         let lastEmptyResume = performance.now();
 
@@ -831,7 +849,7 @@ export class ChangeStream {
             if (waitForCheckpointLsn == null && performance.now() - lastEmptyResume > 60_000) {
               const { comparable: lsn, timestamp } = MongoLSN.fromResumeToken(stream.resumeToken);
               await batch.keepalive(lsn);
-              await touch();
+              this.touch();
               lastEmptyResume = performance.now();
               // Log the token update. This helps as a general "replication is still active" message in the logs.
               // This token would typically be around 10s behind.
@@ -843,7 +861,7 @@ export class ChangeStream {
             continue;
           }
 
-          await touch();
+          this.touch();
 
           if (startAfter != null && originalChangeDocument.clusterTime?.lte(startAfter)) {
             continue;
@@ -947,6 +965,16 @@ export class ChangeStream {
               timestamp: changeDocument.clusterTime!,
               resume_token: changeDocument._id
             });
+            if (batch.lastCheckpointLsn != null && lsn < batch.lastCheckpointLsn) {
+              // Checkpoint out of order - should never happen with MongoDB.
+              // If it does happen, we throw an error to stop the replication - restarting should recover.
+              // Since we use batch.lastCheckpointLsn for the next resumeAfter, this should not result in an infinite loop.
+              // This is a workaround for the issue below, but we can keep this as a safety-check even if the issue is fixed.
+              // Driver issue report: https://jira.mongodb.org/browse/NODE-7042
+              throw new ReplicationAssertionError(
+                `Change resumeToken ${(changeDocument._id as any)._data} (${timestampToDate(changeDocument.clusterTime!).toISOString()}) is less than last checkpoint LSN ${batch.lastCheckpointLsn}. Restarting replication.`
+              );
+            }
 
             if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) {
               waitForCheckpointLsn = null;
@@ -956,6 +984,7 @@ export class ChangeStream {
             if (didCommit) {
               this.oldestUncommittedChange = null;
               this.isStartingReplication = false;
+              changesSinceLastCheckpoint = 0;
             }
           } else if (
             changeDocument.operationType == 'insert' ||
@@ -978,7 +1007,21 @@ export class ChangeStream {
               if (this.oldestUncommittedChange == null && changeDocument.clusterTime != null) {
                 this.oldestUncommittedChange = timestampToDate(changeDocument.clusterTime);
               }
-              await this.writeChange(batch, table, changeDocument);
+              const flushResult = await this.writeChange(batch, table, changeDocument);
+              changesSinceLastCheckpoint += 1;
+              if (flushResult != null && changesSinceLastCheckpoint >= 20_000) {
+                // When we are catching up replication after an initial snapshot, there may be a very long delay
+                // before we do a commit(). In that case, we need to periodically persist the resume LSN, so
+                // we don't restart from scratch if we restart replication.
+                // The same could apply if we need to catch up on replication after some downtime.
+                const { comparable: lsn } = new MongoLSN({
+                  timestamp: changeDocument.clusterTime!,
+                  resume_token: changeDocument._id
+                });
+                this.logger.info(`Updating resume LSN to ${lsn} after ${changesSinceLastCheckpoint} changes`);
+                await batch.setResumeLsn(lsn);
+                changesSinceLastCheckpoint = 0;
+              }
             }
           } else if (changeDocument.operationType == 'drop') {
             const rel = getMongoRelation(changeDocument.ns);
@@ -1026,13 +1069,18 @@ export class ChangeStream {
     }
     return Date.now() - this.oldestUncommittedChange.getTime();
   }
-}
 
-async function touch() {
-  // FIXME: The hosted Kubernetes probe does not actually check the timestamp on this.
-  // FIXME: We need a timeout of around 5+ minutes in Kubernetes if we do start checking the timestamp,
-  // or reduce PING_INTERVAL here.
-  return container.probes.touch();
+  private lastTouchedAt = performance.now();
+
+  private touch() {
+    if (performance.now() - this.lastTouchedAt > 1_000) {
+      this.lastTouchedAt = performance.now();
+      // Update the probes, but don't wait for it
+      container.probes.touch().catch((e) => {
+        this.logger.error(`Failed to touch the container probe: ${e.message}`, e);
+      });
+    }
+  }
 }
 
 function mapChangeStreamError(e: any) {

package/src/replication/MongoErrorRateLimiter.ts

@@ -28,7 +28,7 @@ export class MongoErrorRateLimiter implements ErrorRateLimiter {
       // Could be fail2ban or similar
       this.setDelay(120_000);
     } else {
-      this.setDelay(30_000);
+      this.setDelay(5_000);
     }
   }
 

package/src/replication/MongoManager.ts

@@ -1,8 +1,11 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 
 import { NormalizedMongoConnectionConfig } from '../types/types.js';
-import { BSON_DESERIALIZE_DATA_OPTIONS } from '@powersync/service-core';
+import { BSON_DESERIALIZE_DATA_OPTIONS, POWERSYNC_VERSION } from '@powersync/service-core';
 
+/**
+ * Manage a MongoDB source database connection.
+ */
 export class MongoManager {
   public readonly client: mongo.MongoClient;
   public readonly db: mongo.Db;
@@ -26,6 +29,14 @@ export class MongoManager {
       // How long to wait for new primary selection
       serverSelectionTimeoutMS: 30_000,
 
+      // Identify the client
+      appName: `powersync ${POWERSYNC_VERSION}`,
+      driverInfo: {
+        // This is merged with the node driver info.
+        name: 'powersync',
+        version: POWERSYNC_VERSION
+      },
+
       // Avoid too many connections:
       // 1. It can overwhelm the source database.
       // 2. Processing too many queries in parallel can cause the process to run out of memory.

package/src/replication/MongoRelation.ts

@@ -13,7 +13,7 @@ export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.S
     schema: source.db,
     // Not relevant for MongoDB - we use db + coll name as the identifier
     objectId: undefined,
-    replicationColumns: [{ name: '_id' }]
+    replicaIdColumns: [{ name: '_id' }]
   } satisfies storage.SourceEntityDescriptor;
 }
 
@@ -22,7 +22,7 @@ export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.S
  */
 export function getCacheIdentifier(source: storage.SourceEntityDescriptor | storage.SourceTable): string {
   if (source instanceof storage.SourceTable) {
-    return `${source.schema}.${source.table}`;
+    return `${source.schema}.${source.name}`;
   }
   return `${source.schema}.${source.name}`;
 }

package/test/src/change_stream.test.ts

@@ -23,7 +23,9 @@ describe('change stream', () => {
 
 function defineChangeStreamTests(factory: storage.TestStorageFactory) {
   test('replicating basic values', async () => {
-    await using context = await ChangeStreamTestContext.open(factory);
+    await using context = await ChangeStreamTestContext.open(factory, {
+      mongoOptions: { postImages: PostImagesOption.READ_ONLY }
+    });
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -32,7 +34,7 @@ bucket_definitions:
       - SELECT _id as id, description, num FROM "test_data"`);
 
     await db.createCollection('test_data', {
-      changeStreamPreAndPostImages: { enabled: false }
+      changeStreamPreAndPostImages: { enabled: true }
     });
     const collection = db.collection('test_data');
 
@@ -42,11 +44,8 @@ bucket_definitions:
 
     const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
     const test_id = result.insertedId;
-    await setTimeout(30);
     await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } });
-    await setTimeout(30);
     await collection.replaceOne({ _id: test_id }, { description: 'test3' });
-    await setTimeout(30);
     await collection.deleteOne({ _id: test_id });
 
     const data = await context.getBucketData('global[]');
@@ -354,6 +353,9 @@ bucket_definitions:
     const test_id = result.insertedId.toHexString();
 
     await context.replicateSnapshot();
+    // Note: snapshot is only consistent some time into the streaming request.
+    // At the point that we get the first acknowledged checkpoint, as is required
+    // for getBucketData(), the data should be consistent.
     context.startStreaming();
 
     const data = await context.getBucketData('global[]');
@@ -512,10 +514,13 @@ bucket_definitions:
     await collection.insertOne({ description: 'test1', num: 1152921504606846976n });
 
     await context.replicateSnapshot();
+    await context.markSnapshotConsistent();
 
     // Simulate an error
     await context.storage!.reportError(new Error('simulated error'));
-    expect((await context.factory.getActiveSyncRulesContent())?.last_fatal_error).toEqual('simulated error');
+    const syncRules = await context.factory.getActiveSyncRulesContent();
+    expect(syncRules).toBeTruthy();
+    expect(syncRules?.last_fatal_error).toEqual('simulated error');
 
     // startStreaming() should automatically clear the error.
     context.startStreaming();

package/test/src/change_stream_utils.ts

@@ -17,7 +17,7 @@ import { MongoManager } from '@module/replication/MongoManager.js';
 import { createCheckpoint, STANDALONE_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
 import { NormalizedMongoConnectionConfig } from '@module/types/types.js';
 
-import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js';
+import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js';
 
 export class ChangeStreamTestContext {
   private _walStream?: ChangeStream;
@@ -119,7 +119,20 @@ export class ChangeStreamTestContext {
 
   async replicateSnapshot() {
     await this.walStream.initReplication();
-    await this.storage!.autoActivate();
+  }
+
+  /**
+   * A snapshot is not consistent until streaming replication has caught up.
+   * We simulate that for tests.
+   * Do not use if there are any writes performed while doing the snapshot, as that
+   * would result in inconsistent data.
+   */
+  async markSnapshotConsistent() {
+    const checkpoint = await createCheckpoint(this.client, this.db, STANDALONE_CHECKPOINT_ID);
+
+    await this.storage!.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
+      await batch.keepalive(checkpoint);
+    });
   }
 
   startStreaming() {
@@ -195,12 +208,11 @@ export async function getClientCheckpoint(
   while (Date.now() - start < timeout) {
     const storage = await storageFactory.getActiveStorage();
     const cp = await storage?.getCheckpoint();
-    if (cp == null) {
-      throw new Error('No sync rules available');
-    }
-    lastCp = cp;
-    if (cp.lsn && cp.lsn >= lsn) {
-      return cp.checkpoint;
+    if (cp != null) {
+      lastCp = cp;
+      if (cp.lsn && cp.lsn >= lsn) {
+        return cp.checkpoint;
+      }
     }
     await new Promise((resolve) => setTimeout(resolve, 30));
   }